| package org.unicode.cldr.unittest; |
| |
| import com.google.common.base.Joiner; |
| import com.google.common.base.Objects; |
| import com.google.common.collect.ImmutableMultimap; |
| import com.google.common.collect.ImmutableSet; |
| import com.google.common.collect.Multimap; |
| import com.google.common.collect.TreeMultimap; |
| import com.ibm.icu.impl.Relation; |
| import com.ibm.icu.impl.Row; |
| import com.ibm.icu.impl.Row.R2; |
| import com.ibm.icu.impl.Row.R3; |
| import com.ibm.icu.impl.Utility; |
| import com.ibm.icu.lang.UCharacter; |
| import com.ibm.icu.text.Collator; |
| import com.ibm.icu.text.DecimalFormat; |
| import com.ibm.icu.text.Normalizer; |
| import com.ibm.icu.text.NumberFormat; |
| import com.ibm.icu.text.UTF16; |
| import com.ibm.icu.text.UnicodeSet; |
| import com.ibm.icu.text.UnicodeSetIterator; |
| import com.ibm.icu.util.Currency; |
| import com.ibm.icu.util.ULocale; |
| import java.io.File; |
| import java.io.IOException; |
| import java.io.InputStream; |
| import java.io.PrintWriter; |
| import java.io.StringWriter; |
| import java.util.ArrayList; |
| import java.util.Arrays; |
| import java.util.Collection; |
| import java.util.Collections; |
| import java.util.Comparator; |
| import java.util.EnumSet; |
| import java.util.HashSet; |
| import java.util.Iterator; |
| import java.util.LinkedHashSet; |
| import java.util.List; |
| import java.util.Map; |
| import java.util.Map.Entry; |
| import java.util.Set; |
| import java.util.TreeMap; |
| import java.util.TreeSet; |
| import org.unicode.cldr.test.DisplayAndInputProcessor; |
| import org.unicode.cldr.tool.CldrVersion; |
| import org.unicode.cldr.tool.LikelySubtags; |
| import org.unicode.cldr.util.Builder; |
| import org.unicode.cldr.util.CLDRConfig; |
| import org.unicode.cldr.util.CLDRFile; |
| import org.unicode.cldr.util.CLDRFile.DraftStatus; |
| import org.unicode.cldr.util.CLDRFile.Status; |
| import org.unicode.cldr.util.CLDRFile.WinningChoice; |
| import org.unicode.cldr.util.CLDRPaths; |
| import org.unicode.cldr.util.ChainedMap; |
| import org.unicode.cldr.util.ChainedMap.M4; |
| import org.unicode.cldr.util.CharacterFallbacks; |
| import org.unicode.cldr.util.CldrUtility; |
| import org.unicode.cldr.util.Counter; |
| import org.unicode.cldr.util.DiscreteComparator; |
| import org.unicode.cldr.util.DiscreteComparator.Ordering; |
| import org.unicode.cldr.util.DoctypeXmlStreamWrapper; |
| import org.unicode.cldr.util.DtdData; |
| import org.unicode.cldr.util.DtdData.Attribute; |
| import org.unicode.cldr.util.DtdData.Element; |
| import org.unicode.cldr.util.DtdData.ElementType; |
| import org.unicode.cldr.util.DtdType; |
| import org.unicode.cldr.util.DtdType.DtdStatus; |
| import org.unicode.cldr.util.ElementAttributeInfo; |
| import org.unicode.cldr.util.Factory; |
| import org.unicode.cldr.util.InputStreamFactory; |
| import org.unicode.cldr.util.LanguageTagParser; |
| import org.unicode.cldr.util.Level; |
| import org.unicode.cldr.util.LocaleIDParser; |
| import org.unicode.cldr.util.Pair; |
| import org.unicode.cldr.util.PathHeader; |
| import org.unicode.cldr.util.PathUtilities; |
| import org.unicode.cldr.util.StandardCodes; |
| import org.unicode.cldr.util.SupplementalDataInfo; |
| import org.unicode.cldr.util.SupplementalDataInfo.PluralInfo; |
| import org.unicode.cldr.util.SupplementalDataInfo.PluralType; |
| import org.unicode.cldr.util.TestCLDRPaths; |
| import org.unicode.cldr.util.XMLFileReader; |
| import org.unicode.cldr.util.XPathParts; |
| import org.xml.sax.ErrorHandler; |
| import org.xml.sax.InputSource; |
| import org.xml.sax.SAXException; |
| import org.xml.sax.SAXParseException; |
| import org.xml.sax.XMLReader; |
| |
| public class TestBasic extends TestFmwkPlus { |
| |
| private static final boolean DEBUG = false; |
| |
| static CLDRConfig testInfo = CLDRConfig.getInstance(); |
| |
| private static final SupplementalDataInfo SUPPLEMENTAL_DATA_INFO = |
| testInfo.getSupplementalDataInfo(); |
| |
| private static final ImmutableSet<Pair<String, String>> knownElementExceptions = |
| ImmutableSet.of(Pair.of("ldml", "usesMetazone"), Pair.of("ldmlICU", "usesMetazone")); |
| |
| private static final ImmutableSet<Pair<String, String>> knownAttributeExceptions = |
| ImmutableSet.of( |
| Pair.of("ldml", "version"), |
| Pair.of("supplementalData", "version"), |
| Pair.of("ldmlICU", "version"), |
| Pair.of("layout", "standard"), |
| Pair.of("currency", "id"), // for v1.1.1 |
| Pair.of("monthNames", "type"), // for v1.1.1 |
| Pair.of("alias", "type") // for v1.1.1 |
| ); |
| |
| private static final ImmutableSet<Pair<String, String>> knownChildExceptions = |
| ImmutableSet.of( |
| Pair.of("abbreviationFallback", "special"), |
| Pair.of("inList", "special"), |
| Pair.of("preferenceOrdering", "special")); |
| |
| /** |
| * Simple test that loads each file in the cldr directory, thus verifying that the DTD works, |
| * and also checks that the PrettyPaths work. |
| * |
| * @author markdavis |
| */ |
| public static void main(String[] args) { |
| new TestBasic().run(args); |
| } |
| |
| private static final ImmutableSet<String> skipAttributes = |
| ImmutableSet.of("alt", "draft", "references"); |
| |
| private final ImmutableSet<String> eightPointLocales = |
| ImmutableSet.of( |
| "ar", "ca", "cs", "da", "de", "el", "es", "fi", "fr", "he", "hi", "hr", "hu", |
| "id", "it", "ja", "ko", "lt", "lv", "nl", "no", "pl", "pt", "pt_PT", "ro", "ru", |
| "sk", "sl", "sr", "sv", "th", "tr", "uk", "vi", "zh", "zh_Hant"); |
| |
| // private final boolean showForceZoom = Utility.getProperty("forcezoom", |
| // false); |
| |
| private final boolean resolved = CldrUtility.getProperty("resolved", false); |
| |
| private final Exception[] internalException = new Exception[1]; |
| |
| public void TestDtds() throws IOException { |
| Relation<Row.R2<DtdType, String>, String> foundAttributes = |
| Relation.of(new TreeMap<Row.R2<DtdType, String>, Set<String>>(), TreeSet.class); |
| final CLDRConfig config = CLDRConfig.getInstance(); |
| final File basedir = config.getCldrBaseDirectory(); |
| List<TimingInfo> data = new ArrayList<>(); |
| |
| for (String subdir : CLDRConfig.getCLDRDataDirectories()) { |
| checkDtds(new File(basedir, subdir), 0, foundAttributes, data); |
| } |
| if (foundAttributes.size() > 0) { |
| showFoundElements(foundAttributes); |
| } |
| if (isVerbose()) { |
| long totalBytes = 0; |
| long totalNanos = 0; |
| for (TimingInfo i : data) { |
| long length = i.file.length(); |
| totalBytes += length; |
| totalNanos += i.nanos; |
| logln(i.nanos + "\t" + length + "\t" + i.file); |
| } |
| logln(totalNanos + "\t" + totalBytes); |
| } |
| } |
| |
| private void checkDtds( |
| File directoryFile, |
| int level, |
| Relation<R2<DtdType, String>, String> foundAttributes, |
| List<TimingInfo> data) |
| throws IOException { |
| boolean deepCheck = getInclusion() >= 10; |
| if (directoryFile.getName().equals("import") |
| && directoryFile.getParentFile().getName().equals("keyboards")) { |
| return; // skip imports |
| } |
| File[] listFiles = directoryFile.listFiles(); |
| String normalizedPath = PathUtilities.getNormalizedPathString(directoryFile); |
| String indent = Utility.repeat("\t", level); |
| if (listFiles == null) { |
| throw new IllegalArgumentException(indent + "Empty directory: " + normalizedPath); |
| } |
| logln("Checking files for DTD errors in: " + indent + normalizedPath); |
| for (File fileName : listFiles) { |
| String name = fileName.getName(); |
| if (CLDRConfig.isJunkFile(name)) { |
| continue; |
| } else if (fileName.isDirectory()) { |
| checkDtds(fileName, level + 1, foundAttributes, data); |
| } else if (name.endsWith(".xml")) { |
| data.add(check(fileName)); |
| if (deepCheck // takes too long to do all the time |
| ) { |
| CLDRFile cldrfile = |
| CLDRFile.loadFromFile(fileName, "temp", DraftStatus.unconfirmed); |
| for (String xpath : cldrfile) { |
| String fullPath = cldrfile.getFullXPath(xpath); |
| if (fullPath == null) { |
| fullPath = cldrfile.getFullXPath(xpath); |
| assertNotNull("", fullPath); |
| continue; |
| } |
| XPathParts parts = XPathParts.getFrozenInstance(fullPath); |
| DtdType type = parts.getDtdData().dtdType; |
| for (int i = 0; i < parts.size(); ++i) { |
| String element = parts.getElement(i); |
| R2<DtdType, String> typeElement = Row.of(type, element); |
| if (parts.getAttributeCount(i) == 0) { |
| foundAttributes.put(typeElement, "NONE"); |
| } else { |
| for (String attribute : parts.getAttributeKeys(i)) { |
| foundAttributes.put(typeElement, attribute); |
| } |
| } |
| } |
| } |
| } |
| } |
| } |
| } |
| |
| public void showFoundElements(Relation<Row.R2<DtdType, String>, String> foundAttributes) { |
| Relation<Row.R2<DtdType, String>, String> theoryAttributes = |
| Relation.of(new TreeMap<Row.R2<DtdType, String>, Set<String>>(), TreeSet.class); |
| for (DtdType type : DtdType.values()) { |
| if (type.getStatus() != DtdType.DtdStatus.active) { |
| continue; |
| } |
| DtdData dtdData = DtdData.getInstance(type); |
| for (Element element : dtdData.getElementFromName().values()) { |
| String name = element.getName(); |
| Set<Attribute> attributes = element.getAttributes().keySet(); |
| R2<DtdType, String> typeElement = Row.of(type, name); |
| if (attributes.isEmpty()) { |
| theoryAttributes.put(typeElement, "NONE"); |
| } else { |
| for (Attribute attribute : attributes) { |
| theoryAttributes.put(typeElement, attribute.name); |
| } |
| } |
| } |
| } |
| Relation<String, R3<Boolean, DtdType, String>> attributesToTypeElementUsed = |
| Relation.of( |
| new TreeMap<String, Set<R3<Boolean, DtdType, String>>>(), |
| LinkedHashSet.class); |
| |
| for (Entry<R2<DtdType, String>, Set<String>> s : theoryAttributes.keyValuesSet()) { |
| R2<DtdType, String> typeElement = s.getKey(); |
| Set<String> theoryAttributeSet = s.getValue(); |
| DtdType type = typeElement.get0(); |
| String element = typeElement.get1(); |
| if (element.equals("ANY") || element.equals("#PCDATA")) { |
| continue; |
| } |
| boolean deprecatedElement = |
| SUPPLEMENTAL_DATA_INFO.isDeprecated(type, element, "*", "*"); |
| String header = type + "\t" + element + "\t" + (deprecatedElement ? "X" : "") + "\t"; |
| Set<String> usedAttributes = foundAttributes.get(typeElement); |
| Set<String> unusedAttributes = new LinkedHashSet<>(theoryAttributeSet); |
| if (usedAttributes == null) { |
| logln( |
| header |
| + "<NOT-FOUND>\t\t" |
| + siftDeprecated( |
| type, |
| element, |
| unusedAttributes, |
| attributesToTypeElementUsed, |
| false)); |
| continue; |
| } |
| unusedAttributes.removeAll(usedAttributes); |
| logln( |
| header |
| + siftDeprecated( |
| type, |
| element, |
| usedAttributes, |
| attributesToTypeElementUsed, |
| true) |
| + "\t" |
| + siftDeprecated( |
| type, |
| element, |
| unusedAttributes, |
| attributesToTypeElementUsed, |
| false)); |
| } |
| |
| logln("Undeprecated Attributes\t"); |
| for (Entry<String, R3<Boolean, DtdType, String>> s : |
| attributesToTypeElementUsed.keyValueSet()) { |
| R3<Boolean, DtdType, String> typeElementUsed = s.getValue(); |
| logln( |
| s.getKey() |
| + "\t" |
| + typeElementUsed.get0() |
| + "\t" |
| + typeElementUsed.get1() |
| + "\t" |
| + typeElementUsed.get2()); |
| } |
| } |
| |
| private String siftDeprecated( |
| DtdType type, |
| String element, |
| Set<String> attributeSet, |
| Relation<String, R3<Boolean, DtdType, String>> attributesToTypeElementUsed, |
| boolean used) { |
| StringBuilder b = new StringBuilder(); |
| StringBuilder bdep = new StringBuilder(); |
| for (String attribute : attributeSet) { |
| String attributeName = |
| "«" |
| + attribute |
| + (!"NONE".equals(attribute) |
| && CLDRFile.isDistinguishing(type, element, attribute) |
| ? "*" |
| : "") |
| + "»"; |
| if (!"NONE".equals(attribute) |
| && SUPPLEMENTAL_DATA_INFO.isDeprecated(type, element, attribute, "*")) { |
| if (bdep.length() != 0) { |
| bdep.append(" "); |
| } |
| bdep.append(attributeName); |
| } else { |
| if (b.length() != 0) { |
| b.append(" "); |
| } |
| b.append(attributeName); |
| if (!"NONE".equals(attribute)) { |
| attributesToTypeElementUsed.put(attribute, Row.of(used, type, element)); |
| } |
| } |
| } |
| return b.toString() + "\t" + bdep.toString(); |
| } |
| |
| class MyErrorHandler implements ErrorHandler { |
| @Override |
| public void error(SAXParseException exception) throws SAXException { |
| errln("error: " + XMLFileReader.showSAX(exception)); |
| throw exception; |
| } |
| |
| @Override |
| public void fatalError(SAXParseException exception) throws SAXException { |
| errln("fatalError: " + XMLFileReader.showSAX(exception)); |
| throw exception; |
| } |
| |
| @Override |
| public void warning(SAXParseException exception) throws SAXException { |
| errln("warning: " + XMLFileReader.showSAX(exception)); |
| throw exception; |
| } |
| } |
| |
| private class TimingInfo { |
| File file; |
| long nanos; |
| } |
| |
| public TimingInfo check(File systemID) { |
| long start = System.nanoTime(); |
| try (InputStream fis = InputStreamFactory.createInputStream(systemID)) { |
| // FileInputStream fis = new FileInputStream(systemID); |
| XMLReader xmlReader = XMLFileReader.createXMLReader(true); |
| xmlReader.setErrorHandler(new MyErrorHandler()); |
| InputSource is = new InputSource(fis); |
| is.setSystemId(systemID.toString()); |
| DoctypeXmlStreamWrapper.wrap(is); |
| xmlReader.parse(is); |
| // fis.close(); |
| } catch (SAXException | IOException e) { |
| errln("\t" + "Can't read " + systemID + "\t" + e.getClass() + "\t" + e.getMessage()); |
| } |
| // catch (SAXParseException e) { |
| // errln("\t" + "Can't read " + systemID + "\t" + e.getClass() + "\t" + |
| // e.getMessage()); |
| // } catch (IOException e) { |
| // errln("\t" + "Can't read " + systemID + "\t" + e.getClass() + "\t" + |
| // e.getMessage()); |
| // } |
| TimingInfo timingInfo = new TimingInfo(); |
| timingInfo.nanos = System.nanoTime() - start; |
| timingInfo.file = systemID; |
| return timingInfo; |
| } |
| |
| public void TestCurrencyFallback() { |
| Factory cldrFactory = testInfo.getCldrFactory(); |
| Set<String> currencies = StandardCodes.make().getAvailableCodes("currency"); |
| |
| final UnicodeSet CHARACTERS_THAT_SHOULD_HAVE_FALLBACKS = |
| new UnicodeSet("[[:sc:]-[\\u0000-\\u00FF]]").freeze(); |
| |
| CharacterFallbacks fallbacks = CharacterFallbacks.make(); |
| |
| for (String locale : cldrFactory.getAvailable()) { |
| if (!StandardCodes.isLocaleAtLeastBasic(locale)) { |
| continue; |
| } |
| CLDRFile file = testInfo.getCLDRFile(locale, false); |
| if (file.isNonInheriting()) continue; |
| |
| final UnicodeSet OK_CURRENCY_FALLBACK = |
| new UnicodeSet("[\\u0000-\\u00FF]") |
| .addAll(safeExemplars(file, "")) |
| .addAll(safeExemplars(file, "auxiliary")) |
| .freeze(); |
| UnicodeSet badSoFar = new UnicodeSet(); |
| |
| for (Iterator<String> it = file.iterator(); it.hasNext(); ) { |
| String path = it.next(); |
| if (path.endsWith("/alias")) { |
| continue; |
| } |
| String value = file.getStringValue(path); |
| |
| // check for special characters |
| if (CHARACTERS_THAT_SHOULD_HAVE_FALLBACKS.containsSome(value)) { |
| XPathParts parts = XPathParts.getFrozenInstance(path); |
| if (!parts.getElement(-1).equals("symbol")) { |
| continue; |
| } |
| // We don't care about fallbacks for narrow currency symbols |
| if ("narrow".equals(parts.getAttributeValue(-1, "alt"))) { |
| continue; |
| } |
| String currencyType = parts.getAttributeValue(-2, "type"); |
| |
| UnicodeSet fishy = |
| new UnicodeSet() |
| .addAll(value) |
| .retainAll(CHARACTERS_THAT_SHOULD_HAVE_FALLBACKS) |
| .removeAll(badSoFar); |
| for (UnicodeSetIterator it2 = new UnicodeSetIterator(fishy); it2.next(); ) { |
| final int fishyCodepoint = it2.codepoint; |
| List<String> fallbackList = fallbacks.getSubstitutes(fishyCodepoint); |
| |
| String nfkc = Normalizer.normalize(fishyCodepoint, Normalizer.NFKC); |
| if (!nfkc.equals(UTF16.valueOf(fishyCodepoint))) { |
| if (fallbackList == null) { |
| fallbackList = new ArrayList<>(); |
| } else { |
| fallbackList = new ArrayList<>(fallbackList); // writable |
| } |
| fallbackList.add(nfkc); |
| } |
| // later test for all Latin-1 |
| if (fallbackList == null) { |
| if (locale.equals("nqo") |
| && logKnownIssue("CLDR-16987", "fishy fallback test")) { |
| continue; |
| } |
| errln( |
| "Locale:\t" |
| + locale |
| + ";\tCharacter with no fallback:\t" |
| + it2.getString() |
| + "\t" |
| + UCharacter.getName(fishyCodepoint)); |
| badSoFar.add(fishyCodepoint); |
| } else { |
| String fallback = null; |
| for (String fb : fallbackList) { |
| if (OK_CURRENCY_FALLBACK.containsAll(fb)) { |
| if (!fb.equals(currencyType) && currencies.contains(fb)) { |
| errln( |
| "Locale:\t" |
| + locale |
| + ";\tCurrency:\t" |
| + currencyType |
| + ";\tFallback converts to different code!:\t" |
| + fb |
| + "\t" |
| + it2.getString() |
| + "\t" |
| + UCharacter.getName(fishyCodepoint)); |
| } |
| if (fallback == null) { |
| fallback = fb; |
| } |
| } |
| } |
| if (fallback == null) { |
| errln( |
| "Locale:\t" |
| + locale |
| + ";\tCharacter with no good fallback (exemplars+Latin1):\t" |
| + it2.getString() |
| + "\t" |
| + UCharacter.getName(fishyCodepoint)); |
| badSoFar.add(fishyCodepoint); |
| } else { |
| logln( |
| "Locale:\t" |
| + locale |
| + ";\tCharacter with good fallback:\t" |
| + it2.getString() |
| + " " |
| + UCharacter.getName(fishyCodepoint) |
| + " => " |
| + fallback); |
| // badSoFar.add(fishyCodepoint); |
| } |
| } |
| } |
| } |
| } |
| } |
| } |
| |
| public void TestAbstractPaths() { |
| Factory cldrFactory = testInfo.getCldrFactory(); |
| CLDRFile english = testInfo.getEnglish(); |
| Map<String, Counter<Level>> abstactPaths = new TreeMap<>(); |
| RegexTransform abstractPathTransform = |
| new RegexTransform(RegexTransform.Processing.ONE_PASS) |
| .add("//ldml/", "") |
| .add("\\[@alt=\"[^\"]*\"\\]", "") |
| .add("=\"[^\"]*\"", "=\"*\"") |
| .add("([^]])\\[", "$1\t[") |
| .add("([^]])/", "$1\t/") |
| .add("/", "\t"); |
| |
| for (String locale : getInclusion() <= 5 ? eightPointLocales : cldrFactory.getAvailable()) { |
| CLDRFile file = testInfo.getCLDRFile(locale, resolved); |
| if (file.isNonInheriting()) continue; |
| logln(locale + "\t-\t" + english.getName(locale)); |
| |
| for (Iterator<String> it = file.iterator(); it.hasNext(); ) { |
| String path = it.next(); |
| if (path.endsWith("/alias")) { |
| continue; |
| } |
| // collect abstracted paths |
| String abstractPath = abstractPathTransform.transform(path); |
| Level level = SUPPLEMENTAL_DATA_INFO.getCoverageLevel(path, locale); |
| if (level == Level.OPTIONAL) { |
| level = Level.COMPREHENSIVE; |
| } |
| Counter<Level> row = abstactPaths.get(abstractPath); |
| if (row == null) { |
| abstactPaths.put(abstractPath, row = new Counter<>()); |
| } |
| row.add(level, 1); |
| } |
| } |
| logln(CldrUtility.LINE_SEPARATOR + "Abstract Paths"); |
| for (Entry<String, Counter<Level>> pathInfo : abstactPaths.entrySet()) { |
| String path = pathInfo.getKey(); |
| Counter<Level> counter = pathInfo.getValue(); |
| logln(counter.getTotal() + "\t" + getCoverage(counter) + "\t" + path); |
| } |
| } |
| |
| private CharSequence getCoverage(Counter<Level> counter) { |
| StringBuilder result = new StringBuilder(); |
| boolean first = true; |
| for (Level level : counter.getKeysetSortedByKey()) { |
| if (first) { |
| first = false; |
| } else { |
| result.append(' '); |
| } |
| result.append("L").append(level.ordinal()).append("=").append(counter.get(level)); |
| } |
| return result; |
| } |
| |
| // public void TestCLDRFileCache() { |
| // long start = System.nanoTime(); |
| // Factory cldrFactory = testInfo.getCldrFactory(); |
| // String unusualLocale = "hi"; |
| // CLDRFile file = cldrFactory.make(unusualLocale, true); |
| // long afterOne = System.nanoTime(); |
| // logln("First: " + (afterOne-start)); |
| // CLDRFile file2 = cldrFactory.make(unusualLocale, true); |
| // long afterTwo = System.nanoTime(); |
| // logln("Second: " + (afterTwo-afterOne)); |
| // } |
| // |
| public void TestPaths() { |
| Relation<String, String> distinguishing = |
| Relation.of(new TreeMap<String, Set<String>>(), TreeSet.class); |
| Relation<String, String> nonDistinguishing = |
| Relation.of(new TreeMap<String, Set<String>>(), TreeSet.class); |
| Factory cldrFactory = testInfo.getCldrFactory(); |
| CLDRFile english = testInfo.getEnglish(); |
| |
| Relation<String, String> pathToLocale = |
| Relation.of( |
| new TreeMap<String, Set<String>>(CLDRFile.getComparator(DtdType.ldml)), |
| TreeSet.class, |
| null); |
| Set<String> localesToTest = |
| getInclusion() <= 5 ? eightPointLocales : cldrFactory.getAvailable(); |
| for (String locale : localesToTest) { |
| CLDRFile file = testInfo.getCLDRFile(locale, resolved); |
| DtdType dtdType = null; |
| if (file.isNonInheriting()) continue; |
| DisplayAndInputProcessor displayAndInputProcessor = |
| new DisplayAndInputProcessor(file, false); |
| |
| logln(locale + "\t-\t" + english.getName(locale)); |
| |
| for (Iterator<String> it = file.iterator(); it.hasNext(); ) { |
| String path = it.next(); |
| if (dtdType == null) { |
| dtdType = DtdType.fromPath(path); |
| } |
| |
| if (path.endsWith("/alias")) { |
| continue; |
| } |
| String value = file.getStringValue(path); |
| if (value == null) { |
| throw new IllegalArgumentException( |
| locale + "\tError: in null value at " + path); |
| } |
| |
| String displayValue = displayAndInputProcessor.processForDisplay(path, value); |
| if (!displayValue.equals(value)) { |
| logln( |
| "\t" |
| + locale |
| + "\tdisplayAndInputProcessor changes display value <" |
| + value |
| + ">\t=>\t<" |
| + displayValue |
| + ">\t\t" |
| + path); |
| } |
| String inputValue = |
| displayAndInputProcessor.processInput( |
| path, displayValue, internalException); |
| if (internalException[0] != null) { |
| errln( |
| "\t" |
| + locale |
| + "\tdisplayAndInputProcessor internal error <" |
| + value |
| + ">\t=>\t<" |
| + inputValue |
| + ">\t\t" |
| + path); |
| internalException[0].printStackTrace(System.out); |
| } |
| if (isVerbose() && !inputValue.equals(value)) { |
| displayAndInputProcessor.processInput(path, value, internalException); // for |
| // debugging |
| logln( |
| "\t" |
| + locale |
| + "\tdisplayAndInputProcessor changes input value <" |
| + value |
| + ">\t=>\t<" |
| + inputValue |
| + ">\t\t" |
| + path); |
| } |
| |
| pathToLocale.put(path, locale); |
| |
| // also check for non-distinguishing attributes |
| if (path.contains("/identity")) continue; |
| |
| String fullPath = file.getFullXPath(path); |
| XPathParts parts = XPathParts.getFrozenInstance(fullPath); |
| for (int i = 0; i < parts.size(); ++i) { |
| if (parts.getAttributeCount(i) == 0) { |
| continue; |
| } |
| String element = parts.getElement(i); |
| for (String attribute : parts.getAttributeKeys(i)) { |
| if (skipAttributes.contains(attribute)) continue; |
| if (CLDRFile.isDistinguishing(dtdType, element, attribute)) { |
| distinguishing.put(element, attribute); |
| } else { |
| nonDistinguishing.put(element, attribute); |
| } |
| } |
| } |
| } |
| } |
| |
| if (isVerbose()) { |
| System.out.format( |
| "Distinguishing Elements: %s" + CldrUtility.LINE_SEPARATOR, distinguishing); |
| System.out.format( |
| "Nondistinguishing Elements: %s" + CldrUtility.LINE_SEPARATOR, |
| nonDistinguishing); |
| System.out.format("Skipped %s" + CldrUtility.LINE_SEPARATOR, skipAttributes); |
| } |
| } |
| |
| /** The verbose output shows the results of 1..3 \u00a4 signs. */ |
| public void checkCurrency() { |
| Map<String, Set<R2<String, Integer>>> results = |
| new TreeMap<>(Collator.getInstance(ULocale.ENGLISH)); |
| for (ULocale locale : ULocale.getAvailableLocales()) { |
| if (locale.getCountry().length() != 0) { |
| continue; |
| } |
| for (int i = 1; i < 4; ++i) { |
| NumberFormat format = getCurrencyInstance(locale, i); |
| for (Currency c : |
| new Currency[] { |
| Currency.getInstance("USD"), |
| Currency.getInstance("EUR"), |
| Currency.getInstance("INR") |
| }) { |
| format.setCurrency(c); |
| final String formatted = format.format(12345.67); |
| Set<R2<String, Integer>> set = results.get(formatted); |
| if (set == null) { |
| results.put(formatted, set = new TreeSet<>()); |
| } |
| set.add(Row.of(locale.toString(), i)); |
| } |
| } |
| } |
| for (String formatted : results.keySet()) { |
| logln(formatted + "\t" + results.get(formatted)); |
| } |
| } |
| |
| private static NumberFormat getCurrencyInstance(ULocale locale, int type) { |
| NumberFormat format = NumberFormat.getCurrencyInstance(locale); |
| if (type > 1) { |
| DecimalFormat format2 = (DecimalFormat) format; |
| String pattern = format2.toPattern(); |
| String replacement = "\u00a4\u00a4"; |
| for (int i = 2; i < type; ++i) { |
| replacement += "\u00a4"; |
| } |
| pattern = pattern.replace("\u00a4", replacement); |
| format2.applyPattern(pattern); |
| } |
| return format; |
| } |
| |
| private UnicodeSet safeExemplars(CLDRFile file, String string) { |
| final UnicodeSet result = file.getExemplarSet(string, WinningChoice.NORMAL); |
| return result != null ? result : new UnicodeSet(); |
| } |
| |
| public void TestAPath() { |
| // <month type="1">1</month> |
| String path = |
| "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/months/monthContext[@type=\"format\"]/monthWidth[@type=\"abbreviated\"]/month[@type=\"1\"]"; |
| CLDRFile root = testInfo.getRoot(); |
| logln("path: " + path); |
| String fullpath = root.getFullXPath(path); |
| logln("fullpath: " + fullpath); |
| String value = root.getStringValue(path); |
| logln("value: " + value); |
| Status status = new Status(); |
| String source = root.getSourceLocaleID(path, status); |
| logln("locale: " + source); |
| logln("status: " + status); |
| } |
| |
| public void TestDefaultContents() { |
| Set<String> defaultContents = Inheritance.defaultContents; |
| Multimap<String, String> parentToChildren = Inheritance.parentToChildren; |
| |
| // Put a list of locales that should be default content here. |
| final String expectDC[] = { |
| "os_GE" // see CLDR-14118 |
| }; |
| for (final String locale : expectDC) { |
| assertTrue( |
| "expect " + locale + " to be a default content locale", |
| defaultContents.contains(locale)); |
| } |
| |
| if (DEBUG) { |
| Inheritance.showChain("", "", "root"); |
| } |
| |
| for (String locale : defaultContents) { |
| CLDRFile cldrFile; |
| try { |
| cldrFile = testInfo.getCLDRFile(locale, false); |
| } catch (RuntimeException e) { |
| logln("Can't open default content file:\t" + locale); |
| continue; |
| } |
| // we check that the default content locale is always empty |
| for (Iterator<String> it = cldrFile.iterator(); it.hasNext(); ) { |
| String path = it.next(); |
| if (path.contains("/identity")) { |
| continue; |
| } |
| errln("Default content file not empty:\t" + locale); |
| showDifferences(locale); |
| break; |
| } |
| } |
| |
| // check that if a locale has any children, that exactly one of them is |
| // the default content. Ignore locales with variants |
| |
| for (Entry<String, Collection<String>> localeAndKids : |
| parentToChildren.asMap().entrySet()) { |
| String locale = localeAndKids.getKey(); |
| if (locale.equals("root")) { |
| continue; |
| } |
| |
| Collection<String> rawChildren = localeAndKids.getValue(); |
| |
| // remove variant children |
| Set<String> children = new LinkedHashSet<>(); |
| for (String child : rawChildren) { |
| if (new LocaleIDParser().set(child).getVariants().length == 0) { |
| children.add(child); |
| } |
| } |
| if (children.isEmpty()) { |
| continue; |
| } |
| |
| Set<String> defaultContentChildren = new LinkedHashSet<>(children); |
| defaultContentChildren.retainAll(defaultContents); |
| if (defaultContentChildren.size() == 1) { |
| continue; |
| // If we're already down to the region level then it's OK not to have |
| // default contents. |
| } else if (!new LocaleIDParser().set(locale).getRegion().isEmpty()) { |
| continue; |
| } else if (defaultContentChildren.isEmpty()) { |
| Object possible = highestShared(locale, children); |
| errln( |
| "Locale has children but is missing default contents locale: " |
| + locale |
| + ", children: " |
| + children |
| + "; possible fixes for children:\n" |
| + possible); |
| } else { |
| errln( |
| "Locale has too many defaultContent locales!!: " |
| + locale |
| + ", defaultContents: " |
| + defaultContentChildren); |
| } |
| } |
| |
| // check that each default content locale is likely-subtag equivalent to |
| // its parent. |
| |
| for (String locale : defaultContents) { |
| String maxLocale = LikelySubtags.maximize(locale, likelyData); |
| String localeParent = LocaleIDParser.getParent(locale); |
| String maxLocaleParent = LikelySubtags.maximize(localeParent, likelyData); |
| if (locale.equals("ar_001") || locale.equals("nb")) { |
| logln( |
| "Known exception to likelyMax(locale=" |
| + locale |
| + ")" |
| + " == " |
| + "likelyMax(defaultContent=" |
| + localeParent |
| + ")"); |
| continue; |
| } |
| assertEquals( |
| "likelyMax(locale=" |
| + locale |
| + ")" |
| + " == " |
| + "likelyMax(defaultContent=" |
| + localeParent |
| + ")", |
| maxLocaleParent, |
| maxLocale); |
| } |
| } |
| |
| private String highestShared(String parent, Set<String> children) { |
| M4<PathHeader, String, String, Boolean> data = |
| ChainedMap.of( |
| new TreeMap<PathHeader, Object>(), |
| new TreeMap<String, Object>(), |
| new TreeMap<String, Object>(), |
| Boolean.class); |
| CLDRFile parentFile = testInfo.getCLDRFile(parent, true); |
| PathHeader.Factory phf = PathHeader.getFactory(testInfo.getEnglish()); |
| for (String child : children) { |
| CLDRFile cldrFile = testInfo.getCLDRFile(child, false); |
| for (String path : cldrFile) { |
| if (path.contains("/identity")) { |
| continue; |
| } |
| if (path.contains("provisional") || path.contains("unconfirmed")) { |
| continue; |
| } |
| String value = cldrFile.getStringValue(path); |
| // double-check |
| String parentValue = parentFile.getStringValue(path); |
| if (value.equals(parentValue)) { |
| continue; |
| } |
| PathHeader ph = phf.fromPath(path); |
| data.put(ph, value, child, Boolean.TRUE); |
| data.put(ph, parentValue == null ? "∅∅∅" : parentValue, child, Boolean.TRUE); |
| } |
| } |
| StringBuilder result = new StringBuilder(); |
| for (Entry<PathHeader, Map<String, Map<String, Boolean>>> entry : data) { |
| for (Entry<String, Map<String, Boolean>> item : entry.getValue().entrySet()) { |
| result.append("\n") |
| .append(entry.getKey()) |
| .append("\t") |
| .append(item.getKey() + "\t" + item.getValue().keySet()); |
| } |
| } |
| return result.toString(); |
| } |
| |
| public static class Inheritance { |
| public static final Set<String> defaultContents = |
| SUPPLEMENTAL_DATA_INFO.getDefaultContentLocales(); |
| public static final Multimap<String, String> parentToChildren; |
| |
| static { |
| Multimap<String, String> _parentToChildren = TreeMultimap.create(); |
| for (String child : testInfo.getCldrFactory().getAvailable()) { |
| if (child.equals("root")) { |
| continue; |
| } |
| String localeParent = LocaleIDParser.getParent(child); |
| _parentToChildren.put(localeParent, child); |
| } |
| parentToChildren = ImmutableMultimap.copyOf(_parentToChildren); |
| } |
| |
| public static void showChain(String prefix, String gparent, String current) { |
| Collection<String> children = parentToChildren.get(current); |
| if (children == null) { |
| throw new IllegalArgumentException(); |
| } |
| prefix += |
| current |
| + (defaultContents.contains(current) ? "*" : "") |
| + (isLikelyEquivalent(gparent, current) ? "~" : "") |
| + "\t"; |
| |
| // find leaves |
| Set<String> parents = new LinkedHashSet<>(children); |
| parents.retainAll(parentToChildren.keySet()); |
| Set<String> leaves = new LinkedHashSet<>(children); |
| leaves.removeAll(parentToChildren.keySet()); |
| if (!leaves.isEmpty()) { |
| List<String> presentation = new ArrayList<>(); |
| boolean gotDc = false; |
| for (String s : leaves) { |
| String shown = s; |
| if (isLikelyEquivalent(current, s)) { |
| shown += "~"; |
| } |
| if (defaultContents.contains(s)) { |
| gotDc = true; |
| shown += "*"; |
| } |
| if (!shown.equals(s)) { |
| presentation.add(0, shown); |
| } else { |
| presentation.add(shown); |
| } |
| } |
| if (!gotDc) { |
| int debug = 0; |
| } |
| if (leaves.size() == 1) { |
| System.out.println(prefix + Joiner.on(" ").join(presentation)); |
| } else { |
| System.out.println(prefix + "{" + Joiner.on(" ").join(presentation) + "}"); |
| } |
| } |
| for (String parent : parents) { |
| showChain(prefix, current, parent); |
| } |
| } |
| |
| static boolean isLikelyEquivalent(String locale1, String locale2) { |
| if (locale1.equals(locale2)) { |
| return true; |
| } |
| try { |
| String maxLocale1 = LikelySubtags.maximize(locale1, likelyData); |
| String maxLocale2 = LikelySubtags.maximize(locale2, likelyData); |
| return maxLocale1 != null && Objects.equal(maxLocale1, maxLocale2); |
| } catch (Exception e) { |
| return false; |
| } |
| } |
| } |
| |
| static final Map<String, String> likelyData = SUPPLEMENTAL_DATA_INFO.getLikelySubtags(); |
| |
| private static final EnumSet<CldrVersion> badLdmlICUVersions = |
| EnumSet.of( |
| CldrVersion.v1_1_1, CldrVersion.v1_2, CldrVersion.v1_4_1, CldrVersion.v1_5_1); |
| |
| public void TestLikelySubtagsComplete() { |
| LanguageTagParser ltp = new LanguageTagParser(); |
| for (String locale : testInfo.getCldrFactory().getAvailable()) { |
| if (locale.equals("root")) { |
| continue; |
| } |
| String maxLocale = LikelySubtags.maximize(locale, likelyData); |
| if (maxLocale == null) { |
| errln("Locale missing likely subtag: " + locale); |
| continue; |
| } |
| ltp.set(maxLocale); |
| if (ltp.getLanguage().isEmpty() |
| || ltp.getScript().isEmpty() |
| || ltp.getRegion().isEmpty()) { |
| errln("Locale has defective likely subtag: " + locale + " => " + maxLocale); |
| } |
| } |
| } |
| |
| private void showDifferences(String locale) { |
| CLDRFile cldrFile = testInfo.getCLDRFile(locale, false); |
| final String localeParent = LocaleIDParser.getParent(locale); |
| CLDRFile parentFile = testInfo.getCLDRFile(localeParent, true); |
| int funnyCount = 0; |
| for (Iterator<String> it = cldrFile.iterator("", cldrFile.getComparator()); |
| it.hasNext(); ) { |
| String path = it.next(); |
| if (path.contains("/identity")) { |
| continue; |
| } |
| final String fullXPath = cldrFile.getFullXPath(path); |
| if (fullXPath.contains("[@draft=\"unconfirmed\"]") |
| || fullXPath.contains("[@draft=\"provisional\"]")) { |
| funnyCount++; |
| continue; |
| } |
| logln("\tpath:\t" + path); |
| logln("\t\t" + locale + " value:\t<" + cldrFile.getStringValue(path) + ">"); |
| final String parentFullPath = parentFile.getFullXPath(path); |
| logln("\t\t" + localeParent + " value:\t<" + parentFile.getStringValue(path) + ">"); |
| logln("\t\t" + locale + " fullpath:\t" + fullXPath); |
| logln("\t\t" + localeParent + " fullpath:\t" + parentFullPath); |
| } |
| logln("\tCount of non-approved:\t" + funnyCount); |
| } |
| |
| enum MissingType { |
| plurals, |
| main_exemplars, |
| no_main, |
| collation, |
| index_exemplars, |
| punct_exemplars |
| } |
| |
| public void TestCoreData() { |
| Set<String> availableLanguages = testInfo.getCldrFactory().getAvailableLanguages(); |
| PluralInfo rootRules = SUPPLEMENTAL_DATA_INFO.getPlurals(PluralType.cardinal, "root"); |
| Multimap<MissingType, Comparable> errors = TreeMultimap.create(); |
| errors.put(MissingType.collation, "?"); |
| |
| Multimap<MissingType, Comparable> warnings = TreeMultimap.create(); |
| warnings.put(MissingType.collation, "?"); |
| warnings.put(MissingType.index_exemplars, "?"); |
| warnings.put(MissingType.punct_exemplars, "?"); |
| |
| Set<String> collations = new HashSet<>(); |
| |
| // collect collation info |
| Factory collationFactory = |
| Factory.make(CLDRPaths.COLLATION_DIRECTORY, ".*", DraftStatus.contributed); |
| for (String localeID : collationFactory.getAvailable()) { |
| if (isTopLevel(localeID)) { |
| collations.add(localeID); |
| } |
| } |
| logln(collations.toString()); |
| |
| Set<String> allLanguages = |
| Builder.with(new TreeSet<String>()) |
| .addAll(collations) |
| .addAll(availableLanguages) |
| .freeze(); |
| |
| for (String localeID : allLanguages) { |
| if (localeID.equals("root")) { |
| continue; // skip script locales |
| } |
| if (!isTopLevel(localeID)) { |
| continue; |
| } |
| if (!StandardCodes.isLocaleAtLeastBasic(localeID)) { |
| continue; |
| } |
| errors.clear(); |
| warnings.clear(); |
| |
| String name = |
| "Locale:" + localeID + " (" + testInfo.getEnglish().getName(localeID) + ")"; |
| |
| if (!collations.contains(localeID)) { |
| warnings.put(MissingType.collation, "missing"); |
| logln(name + " is missing " + MissingType.collation.toString()); |
| } |
| |
| try { |
| CLDRFile cldrFile = |
| testInfo.getCldrFactory().make(localeID, false, DraftStatus.contributed); |
| |
| String wholeFileAlias = cldrFile.getStringValue("//ldml/alias"); |
| if (wholeFileAlias != null) { |
| logln("Whole-file alias:" + name); |
| continue; |
| } |
| |
| PluralInfo pluralInfo = |
| SUPPLEMENTAL_DATA_INFO.getPlurals(PluralType.cardinal, localeID); |
| if (pluralInfo == rootRules) { |
| logln(name + " is missing " + MissingType.plurals.toString()); |
| warnings.put(MissingType.plurals, "missing"); |
| } |
| UnicodeSet main = cldrFile.getExemplarSet("", WinningChoice.WINNING); |
| if (main == null || main.isEmpty()) { |
| errln(" " + name + " is missing " + MissingType.main_exemplars.toString()); |
| errors.put(MissingType.main_exemplars, "missing"); |
| } |
| UnicodeSet index = cldrFile.getExemplarSet("index", WinningChoice.WINNING); |
| if (index == null || index.isEmpty()) { |
| logln(name + " is missing " + MissingType.index_exemplars.toString()); |
| warnings.put(MissingType.index_exemplars, "missing"); |
| } |
| UnicodeSet punctuation = |
| cldrFile.getExemplarSet("punctuation", WinningChoice.WINNING); |
| if (punctuation == null || punctuation.isEmpty()) { |
| logln(name + " is missing " + MissingType.punct_exemplars.toString()); |
| warnings.put(MissingType.punct_exemplars, "missing"); |
| } |
| } catch (Exception e) { |
| StringWriter x = new StringWriter(); |
| PrintWriter pw = new PrintWriter(x); |
| e.printStackTrace(pw); |
| pw.flush(); |
| errln(" " + name + " is missing main locale data." + x); |
| errors.put(MissingType.no_main, x.toString()); |
| } |
| |
| // report errors |
| |
| if (errors.isEmpty() && warnings.isEmpty()) { |
| logln(name + ": No problems..."); |
| } |
| } |
| } |
| |
| private boolean isTopLevel(String localeID) { |
| return "root".equals(LocaleIDParser.getParent(localeID)); |
| } |
| |
| /** Tests that every dtd item is connected from root */ |
| public void TestDtdCompleteness() { |
| for (DtdType type : DtdType.values()) { |
| if (type.getStatus() != DtdType.DtdStatus.active) { |
| continue; |
| } |
| DtdData dtdData = DtdData.getInstance(type); |
| Set<Element> descendents = new LinkedHashSet<>(); |
| dtdData.getDescendents(dtdData.ROOT, descendents); |
| Set<Element> elements = dtdData.getElements(); |
| if (!elements.equals(descendents)) { |
| for (Element e : elements) { |
| if (!descendents.contains(e) |
| && !e.equals(dtdData.PCDATA) |
| && !e.equals(dtdData.ANY)) { |
| errln(type + ": Element " + e + " not contained in descendents of ROOT."); |
| } |
| } |
| for (Element e : descendents) { |
| if (!elements.contains(e)) { |
| errln(type + ": Element " + e + ", descendent of ROOT, not in elements."); |
| } |
| } |
| } |
| LinkedHashSet<Element> all = new LinkedHashSet<>(descendents); |
| all.addAll(elements); |
| Set<Attribute> attributes = dtdData.getAttributes(); |
| for (Attribute a : attributes) { |
| if (!elements.contains(a.element)) { |
| errln(type + ": Attribute " + a + " isn't for any element."); |
| } |
| } |
| } |
| } |
| |
| public void TestBasicDTDCompatibility() { |
| |
| if (!TestCLDRPaths.canUseArchiveDirectory()) { |
| return; |
| } |
| |
| final String oldCommon = CldrVersion.LAST_RELEASE_VERSION.getBaseDirectory() + "/common"; |
| |
| // set up exceptions |
| Set<String> changedToEmpty = |
| new HashSet<>( |
| Arrays.asList( |
| new String[] { |
| "version", |
| "languageCoverage", |
| "scriptCoverage", |
| "territoryCoverage", |
| "currencyCoverage", |
| "timezoneCoverage", |
| "skipDefaultLocale" |
| })); |
| Set<String> PCDATA = new HashSet<>(); |
| PCDATA.add("PCDATA"); |
| Set<String> EMPTY = new HashSet<>(); |
| EMPTY.add("EMPTY"); |
| Set<String> VERSION = new HashSet<>(); |
| VERSION.add("version"); |
| |
| // test all DTDs |
| for (DtdType dtd : DtdType.values()) { |
| if (dtd.getStatus() != DtdType.DtdStatus.active) { |
| continue; |
| } |
| if (dtd.firstVersion != null |
| && CldrVersion.LAST_RELEASE_VERSION.isOlderThan( |
| CldrVersion.from(dtd.firstVersion))) { |
| continue; // DTD didn't exist in last release |
| } |
| if (dtd == DtdType.ldmlICU) continue; |
| try { |
| ElementAttributeInfo oldDtd = ElementAttributeInfo.getInstance(oldCommon, dtd); |
| ElementAttributeInfo newDtd = ElementAttributeInfo.getInstance(dtd); |
| |
| if (oldDtd == newDtd) { |
| continue; |
| } |
| Relation<String, String> oldElement2Children = oldDtd.getElement2Children(); |
| Relation<String, String> newElement2Children = newDtd.getElement2Children(); |
| |
| Relation<String, String> oldElement2Attributes = oldDtd.getElement2Attributes(); |
| Relation<String, String> newElement2Attributes = newDtd.getElement2Attributes(); |
| |
| for (String element : oldElement2Children.keySet()) { |
| Set<String> oldChildren = oldElement2Children.getAll(element); |
| Set<String> newChildren = newElement2Children.getAll(element); |
| if (newChildren == null) { |
| if (!knownElementExceptions.contains(Pair.of(dtd.toString(), element))) { |
| errln("Old " + dtd + " contains element not in new: <" + element + ">"); |
| } |
| continue; |
| } |
| Set<String> funny = containsInOrder(newChildren, oldChildren); |
| if (funny != null) { |
| if (changedToEmpty.contains(element) |
| && oldChildren.equals(PCDATA) |
| && newChildren.equals(EMPTY)) { |
| // ok, skip |
| } else { |
| errln( |
| "Old " |
| + dtd |
| + " element <" |
| + element |
| + "> has children Missing/Misordered:\t" |
| + funny |
| + "\n\t\tOld:\t" |
| + oldChildren |
| + "\n\t\tNew:\t" |
| + newChildren); |
| } |
| } |
| |
| Set<String> oldAttributes = oldElement2Attributes.getAll(element); |
| if (oldAttributes == null) { |
| oldAttributes = Collections.emptySet(); |
| } |
| Set<String> newAttributes = newElement2Attributes.getAll(element); |
| if (newAttributes == null) { |
| newAttributes = Collections.emptySet(); |
| } |
| if (!newAttributes.containsAll(oldAttributes)) { |
| LinkedHashSet<String> missing = new LinkedHashSet<>(oldAttributes); |
| missing.removeAll(newAttributes); |
| if (element.equals(dtd.toString()) && missing.equals(VERSION)) { |
| // ok, skip |
| } else { |
| errln( |
| "Old " |
| + dtd |
| + " element <" |
| + element |
| + "> has attributes Missing:\t" |
| + missing |
| + "\n\t\tOld:\t" |
| + oldAttributes |
| + "\n\t\tNew:\t" |
| + newAttributes); |
| } |
| } |
| } |
| } catch (Exception e) { |
| e.printStackTrace(); |
| errln("Failure with " + dtd); |
| } |
| } |
| } |
| |
| private <T> Set<T> containsInOrder(Set<T> superset, Set<T> subset) { |
| if (!superset.containsAll(subset)) { |
| LinkedHashSet<T> missing = new LinkedHashSet<>(subset); |
| missing.removeAll(superset); |
| return missing; |
| } |
| // ok, we know that they are subsets, try order |
| Set<T> result = null; |
| DiscreteComparator<T> comp = |
| new DiscreteComparator.Builder<T>(Ordering.ARBITRARY).add(superset).get(); |
| T last = null; |
| for (T item : subset) { |
| if (last != null) { |
| int order = comp.compare(last, item); |
| if (order != -1) { |
| if (result == null) { |
| result = new HashSet<>(); |
| result.add(last); |
| result.add(item); |
| } |
| } |
| } |
| last = item; |
| } |
| return result; |
| } |
| |
| public void TestDtdCompatibility() { |
| |
| for (DtdType type : DtdType.values()) { |
| if (type.getStatus() != DtdType.DtdStatus.active) { |
| continue; |
| } |
| DtdData dtdData = DtdData.getInstance(type); |
| Map<String, Element> currentElementFromName = dtdData.getElementFromName(); |
| |
| // current has no orphan |
| Set<Element> orphans = new LinkedHashSet<>(dtdData.getElementFromName().values()); |
| orphans.remove(dtdData.ROOT); |
| orphans.remove(dtdData.PCDATA); |
| orphans.remove(dtdData.ANY); |
| Set<String> elementsWithoutAlt = new TreeSet<>(); |
| Set<String> elementsWithoutDraft = new TreeSet<>(); |
| Set<String> elementsWithoutAlias = new TreeSet<>(); |
| Set<String> elementsWithoutSpecial = new TreeSet<>(); |
| |
| for (Element element : dtdData.getElementFromName().values()) { |
| Set<Element> children = element.getChildren().keySet(); |
| orphans.removeAll(children); |
| if (type == DtdType.ldml |
| && !SUPPLEMENTAL_DATA_INFO.isDeprecated(type, element.name, "*", "*")) { |
| if (element.getType() == ElementType.PCDATA) { |
| if (element.getAttributeNamed("alt") == null) { |
| elementsWithoutAlt.add(element.name); |
| } |
| if (element.getAttributeNamed("draft") == null) { |
| elementsWithoutDraft.add(element.name); |
| } |
| } else { |
| if (children.size() != 0 && !"alias".equals(element.name)) { |
| if (element.getChildNamed("alias") == null) { |
| elementsWithoutAlias.add(element.name); |
| } |
| if (element.getChildNamed("special") == null) { |
| elementsWithoutSpecial.add(element.name); |
| } |
| } |
| } |
| } |
| } |
| assertEquals( |
| type + " DTD Must not have orphan elements", Collections.EMPTY_SET, orphans); |
| assertEquals( |
| type + " DTD elements with PCDATA must have 'alt' attributes", |
| Collections.EMPTY_SET, |
| elementsWithoutAlt); |
| assertEquals( |
| type + " DTD elements with PCDATA must have 'draft' attributes", |
| Collections.EMPTY_SET, |
| elementsWithoutDraft); |
| assertEquals( |
| type + " DTD elements with children must have 'alias' elements", |
| Collections.EMPTY_SET, |
| elementsWithoutAlias); |
| assertEquals( |
| type + " DTD elements with children must have 'special' elements", |
| Collections.EMPTY_SET, |
| elementsWithoutSpecial); |
| |
| if (!TestCLDRPaths.canUseArchiveDirectory()) { |
| return; |
| } |
| |
| for (CldrVersion version : CldrVersion.CLDR_VERSIONS_DESCENDING) { |
| if (version == CldrVersion.unknown || version == CldrVersion.baseline) { |
| continue; |
| } |
| if (type.getStatus() != DtdStatus.active) { |
| continue; // not active |
| } |
| if (type.firstVersion != null |
| && version.isOlderThan(CldrVersion.from(type.firstVersion))) { |
| continue; // didn't exist at that point |
| } |
| DtdData dtdDataOld; |
| try { |
| dtdDataOld = DtdData.getInstance(type, version.toString()); |
| } catch (IllegalArgumentException e) { |
| boolean tooOld = false; |
| switch (type) { |
| case ldmlICU: |
| tooOld = badLdmlICUVersions.contains(version); |
| break; |
| case ldmlBCP47: |
| case keyboard3: |
| if (type.firstVersion != null) { |
| tooOld = version.isOlderThan(CldrVersion.from(type.firstVersion)); |
| } |
| break; |
| default: |
| break; |
| } |
| if (tooOld) { |
| continue; |
| } else { |
| errln( |
| "v" |
| + version |
| + ": " |
| + e.getClass().getSimpleName() |
| + ", " |
| + e.getMessage()); |
| continue; |
| } |
| } |
| // verify that if E is in dtdDataOld, then it is in dtdData, and |
| // has at least the same children and attributes |
| for (Entry<String, Element> entry : dtdDataOld.getElementFromName().entrySet()) { |
| Element oldElement = entry.getValue(); |
| Element newElement = currentElementFromName.get(entry.getKey()); |
| if (knownElementExceptions.contains( |
| Pair.of(type.toString(), oldElement.getName()))) { |
| continue; |
| } |
| if (assertNotNull( |
| type |
| + " DTD for trunk must be superset of v" |
| + version |
| + ", and must contain «" |
| + oldElement.getName() |
| + "»", |
| newElement)) { |
| // TODO Check order also |
| for (Element oldChild : oldElement.getChildren().keySet()) { |
| if (oldChild == null) { |
| continue; |
| } |
| Element newChild = newElement.getChildNamed(oldChild.getName()); |
| // skip certain items |
| if (version.isOlderThan(CldrVersion.v1_6_1) |
| && newElement.getName().equals("zone") |
| && oldChild.getName().equals("usesMetazone")) { |
| if (logKnownIssue( |
| "CLDR-17054", |
| "Breakage with items older than 1.6.1: " |
| + newElement.getName() |
| + " / " |
| + oldChild.getName())) { |
| continue; |
| } |
| } |
| |
| if (knownChildExceptions.contains( |
| Pair.of(newElement.getName(), oldChild.getName()))) { |
| continue; |
| } |
| assertNotNull( |
| type |
| + " DTD - Trunk children of «" |
| + newElement.getName() |
| + "» must be superset of v" |
| + version |
| + ", and must contain «" |
| + oldChild.getName() |
| + "»", |
| newChild); |
| } |
| for (Attribute oldAttribute : oldElement.getAttributes().keySet()) { |
| Attribute newAttribute = |
| newElement.getAttributeNamed(oldAttribute.getName()); |
| |
| if (knownAttributeExceptions.contains( |
| Pair.of(newElement.getName(), oldAttribute.getName()))) { |
| continue; |
| } |
| assertNotNull( |
| type |
| + " DTD - Trunk attributes of «" |
| + newElement.getName() |
| + "» must be superset of v" |
| + version |
| + ", and must contain «" |
| + oldAttribute.getName() |
| + "»", |
| newAttribute); |
| } |
| } |
| } |
| } |
| } |
| } |
| |
| /** Compare each path to each other path for every single file in CLDR */ |
| public void TestDtdComparison() { |
| // try some simple paths for regression |
| |
| sortPaths( |
| DtdData.getInstance(DtdType.ldml).getDtdComparator(null), |
| "//ldml/dates/calendars/calendar[@type=\"generic\"]/dateTimeFormats/dateTimeFormatLength[@type=\"full\"]/dateTimeFormat[@type=\"standard\"]/pattern[@type=\"standard\"]", |
| "//ldml/dates/calendars/calendar[@type=\"generic\"]/dateTimeFormats"); |
| |
| sortPaths( |
| DtdData.getInstance(DtdType.supplementalData).getDtdComparator(null), |
| "//supplementalData/territoryContainment/group[@type=\"419\"][@contains=\"013 029 005\"][@grouping=\"true\"]", |
| "//supplementalData/territoryContainment/group[@type=\"003\"][@contains=\"021 013 029\"][@grouping=\"true\"]"); |
| } |
| |
| public void TestDtdComparisonsAll() { |
| if (getInclusion() <= 5) { // Only run this test in exhaustive mode. |
| return; |
| } |
| for (File file : CLDRConfig.getInstance().getAllCLDRFilesEndingWith(".xml")) { |
| if (file.getParentFile().getName().equals("import") |
| && file.getParentFile().getParentFile().getName().equals("keyboards")) { |
| return; // skip imports |
| } |
| checkDtdComparatorFor(file, null); |
| } |
| } |
| |
| public void checkDtdComparatorForResource(String fileToRead, DtdType overrideDtdType) { |
| MyHandler myHandler = new MyHandler(overrideDtdType); |
| XMLFileReader xfr = new XMLFileReader().setHandler(myHandler); |
| try { |
| myHandler.fileName = fileToRead; |
| xfr.read(myHandler.fileName, TestBasic.class, -1, true); |
| logln(myHandler.fileName); |
| } catch (Exception e) { |
| Throwable t = e; |
| StringBuilder b = new StringBuilder(); |
| String indent = ""; |
| while (t != null) { |
| b.append(indent).append(t.getMessage()); |
| indent = indent.isEmpty() ? "\n\t\t" : indent + "\t"; |
| t = t.getCause(); |
| } |
| errln(b.toString()); |
| return; |
| } |
| DtdData dtdData = DtdData.getInstance(myHandler.dtdType); |
| sortPaths(dtdData.getDtdComparator(null), myHandler.data); |
| } |
| |
| public void checkDtdComparatorFor(File fileToRead, DtdType overrideDtdType) { |
| MyHandler myHandler = new MyHandler(overrideDtdType); |
| XMLFileReader xfr = new XMLFileReader().setHandler(myHandler); |
| try { |
| myHandler.fileName = PathUtilities.getNormalizedPathString(fileToRead); |
| xfr.read(myHandler.fileName, -1, true); |
| logln(myHandler.fileName); |
| } catch (Exception e) { |
| e.printStackTrace(); |
| Throwable t = e; |
| StringBuilder b = new StringBuilder(); |
| String indent = ""; |
| while (t != null) { |
| b.append(indent).append(t.getMessage()); |
| indent = indent.isEmpty() ? "\n\t\t" : indent + "\t"; |
| t = t.getCause(); |
| } |
| errln(b.toString()); |
| return; |
| } |
| DtdData dtdData = DtdData.getInstance(myHandler.dtdType); |
| sortPaths(dtdData.getDtdComparator(null), myHandler.data); |
| } |
| |
| static class MyHandler extends XMLFileReader.SimpleHandler { |
| private String fileName; |
| private DtdType dtdType; |
| private final Set<String> data = new LinkedHashSet<>(); |
| |
| public MyHandler(DtdType overrideDtdType) { |
| dtdType = overrideDtdType; |
| } |
| |
| @Override |
| public void handlePathValue(String path, @SuppressWarnings("unused") String value) { |
| if (dtdType == null) { |
| try { |
| dtdType = DtdType.fromPath(path); |
| } catch (Exception e) { |
| throw new IllegalArgumentException("Can't read " + fileName, e); |
| } |
| } |
| data.add(path); |
| } |
| } |
| |
| public void sortPaths(Comparator<String> dc, Collection<String> paths) { |
| String[] array = paths.toArray(new String[paths.size()]); |
| sortPaths(dc, array); |
| } |
| |
| public void sortPaths(Comparator<String> dc, String... array) { |
| Arrays.sort(array, 0, array.length, dc); |
| } |
| // public void TestNewDtdData() moved to TestDtdData |
| } |