| /* |
| * Copyright (c) 2008, 2016, Oracle and/or its affiliates. All rights reserved. |
| * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
| * |
| * This code is free software; you can redistribute it and/or modify it |
| * under the terms of the GNU General Public License version 2 only, as |
| * published by the Free Software Foundation. |
| * |
| * This code is distributed in the hope that it will be useful, but WITHOUT |
| * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
| * version 2 for more details (a copy is included in the LICENSE file that |
| * accompanied this code). |
| * |
| * You should have received a copy of the GNU General Public License version |
| * 2 along with this work; if not, write to the Free Software Foundation, |
| * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. |
| * |
| * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
| * or visit www.oracle.com if you need additional information or have any |
| * questions. |
| */ |
| |
| /* |
| * @test |
| * @bug 6233345 6381699 6381702 6381705 6381706 |
| * @summary Encode many char sequences in many ways |
| * @library /lib/testlibrary/ |
| * @build jdk.testlibrary.* |
| * @run main/timeout=1200 FindEncoderBugs |
| * @author Martin Buchholz |
| * @key randomness |
| */ |
| |
| import java.util.*; |
| import java.util.regex.*; |
| import java.nio.*; |
| import java.nio.charset.*; |
| import jdk.testlibrary.RandomFactory; |
| |
| public class FindEncoderBugs { |
| |
| static boolean isBroken(String csn) { |
| if (csn.equals("x-COMPOUND_TEXT")) return true; |
| return false; |
| } |
| |
| static <T extends Comparable<? super T>> List<T> sort(Collection<T> c) { |
| List<T> list = new ArrayList<T>(c); |
| Collections.sort(list); |
| return list; |
| } |
| |
| static class TooManyFailures extends RuntimeException { |
| private static final long serialVersionUID = 0L; |
| } |
| |
| static String string(byte[] a) { |
| final StringBuilder sb = new StringBuilder(); |
| for (byte b : a) { |
| if (sb.length() != 0) sb.append(' '); |
| sb.append(String.format("%02x", b & 0xff)); |
| } |
| return sb.toString(); |
| } |
| |
| static String string(char[] a) { |
| final StringBuilder sb = new StringBuilder(); |
| for (char c : a) { |
| if (sb.length() != 0) sb.append(' '); |
| sb.append(String.format("\\u%04x", (int) c)); |
| } |
| return sb.toString(); |
| } |
| |
| static class Reporter { |
| // Some machinery to make sure only a small number of errors |
| // that are "too similar" are reported. |
| static class Counts extends HashMap<String, Long> { |
| private static final long serialVersionUID = -1; |
| long inc(String signature) { |
| Long count = get(signature); |
| if (count == null) count = 0L; |
| put(signature, count+1); |
| return count+1; |
| } |
| } |
| |
| final Counts failureCounts = new Counts(); |
| final static long maxFailures = 2; |
| |
| final static Pattern hideBytes = Pattern.compile("\"[0-9a-f ]+\""); |
| final static Pattern hideChars = Pattern.compile("\\\\u[0-9a-f]{4}"); |
| |
| boolean bug(String format, Object... args) { |
| String signature = String.format(format, args); |
| // signature = hideBytes.matcher(signature).replaceAll("\"??\""); |
| // signature = hideChars.matcher(signature).replaceAll("\\u????"); |
| failed++; |
| if (failureCounts.inc(signature) <= maxFailures) { |
| System.out.printf(format, args); |
| System.out.println(); |
| return true; |
| } |
| return false; |
| } |
| |
| void summarize() { |
| for (String key : sort(failureCounts.keySet())) |
| System.out.printf("-----%n%s%nfailures=%d%n", |
| key, failureCounts.get(key)); |
| } |
| } |
| |
| static final Reporter reporter = new Reporter(); |
| |
| static class Result { |
| final int limit; |
| final int ipos; |
| final boolean direct; |
| final char[] ia; |
| final byte[] oa; |
| final CoderResult cr; |
| |
| private static byte[] toByteArray(ByteBuffer bb) { |
| byte[] bytes = new byte[bb.position()]; |
| for (int i = 0; i < bytes.length; i++) |
| bytes[i] = bb.get(i); |
| return bytes; |
| } |
| |
| Result(CharBuffer ib, ByteBuffer ob, CoderResult cr) { |
| ipos = ib.position(); |
| ia = toArray(ib); |
| oa = toArray(ob); |
| direct = ib.isDirect(); |
| limit = ob.limit(); |
| this.cr = cr; |
| } |
| |
| static char[] toArray(CharBuffer b) { |
| int pos = b.position(); |
| char[] a = new char[b.limit()]; |
| b.position(0); |
| b.get(a); |
| b.position(pos); |
| return a; |
| } |
| |
| static byte[] toArray(ByteBuffer b) { |
| byte[] a = new byte[b.position()]; |
| b.position(0); |
| b.get(a); |
| return a; |
| } |
| |
| static boolean eq(Result x, Result y) { |
| return x == y || |
| (x != null && y != null && |
| (Arrays.equals(x.oa, y.oa) && |
| x.ipos == y.ipos && |
| x.cr == y.cr)); |
| } |
| |
| public String toString() { |
| return String.format("\"%s\"[%d/%d] => %s \"%s\"[%d/%d]%s", |
| string(ia), ipos, ia.length, |
| cr, string(oa), oa.length, limit, |
| (direct ? " (direct)" : "")); |
| } |
| } |
| |
| static class CharsetTester { |
| private final Charset cs; |
| private final boolean hasBom; |
| private static final int maxFailures = 5; |
| private int failures = 0; |
| // private static final long maxCharsetFailures = Long.MAX_VALUE; |
| private static final long maxCharsetFailures = 10000L; |
| private final long failed0 = failed; |
| |
| // legend: r=regular d=direct In=Input Ou=Output |
| static final int maxBufSize = 20; |
| static final CharBuffer[] rInBuffers = new CharBuffer[maxBufSize]; |
| static final CharBuffer[] dInBuffers = new CharBuffer[maxBufSize]; |
| |
| static final ByteBuffer[] rOuBuffers = new ByteBuffer[maxBufSize]; |
| static final ByteBuffer[] dOuBuffers = new ByteBuffer[maxBufSize]; |
| static { |
| for (int i = 0; i < maxBufSize; i++) { |
| rInBuffers[i] = CharBuffer.allocate(i); |
| dInBuffers[i] = ByteBuffer.allocateDirect(i*2).asCharBuffer(); |
| rOuBuffers[i] = ByteBuffer.allocate(i); |
| dOuBuffers[i] = ByteBuffer.allocateDirect(i); |
| } |
| } |
| |
| CharsetTester(Charset cs) { |
| this.cs = cs; |
| this.hasBom = |
| cs.name().matches(".*BOM.*") || |
| cs.name().equals("UTF-16"); |
| } |
| |
| static boolean bug(String format, Object... args) { |
| return reporter.bug(format, args); |
| } |
| |
| static boolean hasBom(byte[] a) { |
| switch (a.length) { |
| case 2: case 4: |
| int sum = 0; |
| for (byte x : a) |
| sum += x; |
| return sum == (byte) 0xfe + (byte) 0xff; |
| default: return false; |
| } |
| } |
| |
| void testSurrogates() { |
| int failures = 0; |
| for (int i = 0; i < 10; i++) { |
| Result r = test(new char[] { randomHighSurrogate() }); |
| if (r == null) break; |
| if (! (r.cr.isUnderflow() && |
| r.ipos == 0)) |
| bug("Lone high surrogate not UNDERFLOW: %s %s", |
| cs, r); |
| } |
| for (int i = 0; i < 10; i++) { |
| Result r = test(new char[] { randomLowSurrogate() }); |
| if (r == null) break; |
| if (! (r.cr.isMalformed() && r.cr.length() == 1)) |
| bug("Lone low surrogate not MALFORMED[1]: %s %s", |
| cs, r); |
| } |
| char[] chars = new char[2]; |
| for (int i = 0; i < 10; i++) { |
| chars[0] = randomLowSurrogate(); // Always illegal |
| chars[1] = randomChar(); |
| Result r = test(chars); |
| if (r == null) break; |
| if (! (r.cr.isMalformed() && |
| r.cr.length() == 1 && |
| (r.ipos == 0 || (hasBom && hasBom(r.oa))))) { |
| if (failures++ > 5) return; |
| bug("Unpaired low surrogate not MALFORMED[1]: %s %s", |
| cs, r); |
| } |
| } |
| for (int i = 0; i < 10; i++) { |
| chars[0] = randomHighSurrogate(); |
| do { |
| chars[1] = randomChar(); |
| } while (Character.isLowSurrogate(chars[1])); |
| Result r = test(chars); |
| if (r == null) break; |
| if (! (r.cr.isMalformed() && |
| r.cr.length() == 1 && |
| (r.ipos == 0 || (hasBom && hasBom(r.oa))))) { |
| if (failures++ > 5) return; |
| bug("Unpaired high surrogate not MALFORMED[1]: %s %s", |
| cs, r); |
| } |
| } |
| for (int i = 0; i < 1000; i++) { |
| chars[0] = randomHighSurrogate(); |
| chars[1] = randomLowSurrogate(); |
| Result r = test(chars); |
| if (r == null) break; |
| if (! ((r.cr.isUnmappable() && |
| r.cr.length() == 2 && |
| r.oa.length == 0) |
| || |
| (r.cr.isUnderflow() && |
| r.oa.length > 0 && |
| r.ipos == 2))) { |
| if (failures++ > 5) return; |
| bug("Legal supplementary character bug: %s %s", |
| cs, r); |
| } |
| } |
| } |
| |
| // if (! (r.cr.isMalformed() && |
| // r.cr.length() == 1 && |
| // (rob.position() == 0 || hasBom(rob)))) { |
| // if (failures++ > 5) return; |
| // bug("Unpaired surrogate not malformed: %s %s", |
| // cs, r); |
| // } |
| // } |
| |
| // dib.clear(); dib.put(chars); dib.flip(); |
| // rib.position(0); |
| // rob.clear(); rob.limit(lim); |
| // for (CharBuffer ib : new CharBuffer[] { rib, dib }) { |
| // Result r = recode(ib, rob); |
| // if (! (r.cr.isMalformed() && |
| // r.cr.length() == 1 && |
| // (rob.position() == 0 || hasBom(rob)))) { |
| // if (failures++ > 5) return; |
| // bug("Unpaired surrogate not malformed: %s %s", |
| // cs, r); |
| // } |
| // } |
| // //} |
| // for (int i = 0; i < 10000; i++) { |
| // chars[0] = randomHighSurrogate(); |
| // chars[1] = randomLowSurrogate(); |
| // dib.clear(); dib.put(chars); dib.flip(); |
| // rib.position(0); |
| // rob.clear(); rob.limit(lim); |
| // for (CharBuffer ib : new CharBuffer[] { rib, dib }) { |
| // Result r = recode(ib, rob); |
| // if (! ((r.cr.isUnmappable() && |
| // r.cr.length() == 2 && |
| // rob.position() == 0) |
| // || |
| // (r.cr.isUnderflow() && |
| // rob.position() > 0 && |
| // ib.position() == 2))) { |
| // if (failures++ > 5) return; |
| // bug("Legal supplementary character bug: %s %s", |
| // cs, r); |
| // } |
| // } |
| // } |
| // } |
| // } |
| |
| Result recode(CharBuffer ib, ByteBuffer ob) { |
| try { |
| byte canary = 22; |
| ib.clear(); // Prepare to read |
| ob.clear(); // Prepare to write |
| for (int i = 0; i < ob.limit(); i++) |
| ob.put(i, canary); |
| CharsetEncoder coder = cs.newEncoder(); |
| CoderResult cr = coder.encode(ib, ob, false); |
| equal(ib.limit(), ib.capacity()); |
| equal(ob.limit(), ob.capacity()); |
| Result r = new Result(ib, ob, cr); |
| if (cr.isError()) |
| check(cr.length() > 0); |
| if (cr.isOverflow() && ob.remaining() > 10) |
| bug("OVERFLOW, but there's lots of room: %s %s", |
| cs, r); |
| // if (cr.isOverflow() && ib.remaining() == 0 && ! hasBom) |
| // bug("OVERFLOW, yet remaining() == 0: %s %s", |
| // cs, r); |
| if (cr.isError() && ib.remaining() < cr.length()) |
| bug("remaining() < CoderResult.length(): %s %s", |
| cs, r); |
| // if (ib.position() == 0 |
| // && ob.position() > 0 |
| // && ! hasBom(r.oa)) |
| // bug("output only if input consumed: %s %s", |
| // cs, r); |
| CoderResult cr2 = coder.encode(ib, ob, false); |
| if (ib.position() != r.ipos || |
| ob.position() != r.oa.length || |
| cr != cr2) |
| bug("Coding operation not idempotent: %s%n %s%n %s", |
| cs, r, new Result(ib, ob, cr2)); |
| if (ob.position() < ob.limit() && |
| ob.get(ob.position()) != canary) |
| bug("Buffer overrun: %s %s %s", |
| cs, r, ob.get(ob.position())); |
| return r; |
| } catch (Throwable t) { |
| if (bug("Unexpected exception: %s %s %s", |
| cs, t.getClass().getSimpleName(), |
| new Result(ib, ob, null))) |
| t.printStackTrace(); |
| return null; |
| } |
| } |
| |
| Result recode2(char[] ia, int n) { |
| int len = ia.length; |
| CharBuffer rib = CharBuffer.wrap(ia); |
| CharBuffer dib = dInBuffers[len]; |
| dib.clear(); dib.put(ia); dib.clear(); |
| ByteBuffer rob = rOuBuffers[n]; |
| ByteBuffer dob = dOuBuffers[n]; |
| equal(rob.limit(), n); |
| equal(dob.limit(), n); |
| check(dib.isDirect()); |
| check(dob.isDirect()); |
| Result r1 = recode(rib, rob); |
| Result r2 = recode(dib, dob); |
| if (r1 != null && r2 != null && ! Result.eq(r1, r2)) |
| bug("Results differ for direct buffers: %s%n %s%n %s", |
| cs, r1, r2); |
| return r1; |
| } |
| |
| Result test(char[] ia) { |
| if (failed - failed0 >= maxCharsetFailures) |
| throw new TooManyFailures(); |
| |
| Result roomy = recode2(ia, maxBufSize - 1); |
| if (roomy == null) return roomy; |
| int olen = roomy.oa.length; |
| if (olen > 0) { |
| if (roomy.ipos == roomy.ia.length) { |
| Result perfectFit = recode2(ia, olen); |
| if (! Result.eq(roomy, perfectFit)) |
| bug("Results differ: %s%n %s%n %s", |
| cs, roomy, perfectFit); |
| } |
| for (int i = 0; i < olen; i++) { |
| Result claustrophobic = recode2(ia, i); |
| if (claustrophobic == null) return roomy; |
| if (roomy.cr.isUnderflow() && |
| ! claustrophobic.cr.isOverflow()) |
| bug("Expected OVERFLOW: %s%n %s%n %s", |
| cs, roomy, claustrophobic); |
| } |
| } |
| return roomy; |
| } |
| |
| void testExhaustively(char[] prefix, int n) { |
| int len = prefix.length; |
| char[] ia = Arrays.copyOf(prefix, len + 1); |
| for (int i = 0; i < 0x10000; i++) { |
| ia[len] = (char) i; |
| if (n == 1) |
| test(ia); |
| else |
| testExhaustively(ia, n - 1); |
| } |
| } |
| |
| void testRandomly(char[] prefix, int n) { |
| int len = prefix.length; |
| char[] ia = Arrays.copyOf(prefix, len + n); |
| for (int i = 0; i < 10000; i++) { |
| for (int j = 0; j < n; j++) |
| ia[len + j] = randomChar(); |
| test(ia); |
| } |
| } |
| |
| void testPrefix(char[] prefix) { |
| if (prefix.length > 0) |
| System.out.printf("Testing prefix %s%n", string(prefix)); |
| |
| test(prefix); |
| |
| testExhaustively(prefix, 1); |
| // Can you spare a year of CPU time? |
| //testExhaustively(prefix, 2); |
| |
| testRandomly(prefix, 2); |
| testRandomly(prefix, 3); |
| } |
| } |
| |
| private final static Random rnd = RandomFactory.getRandom(); |
| private static char randomChar() { |
| return (char) rnd.nextInt(Character.MAX_VALUE); |
| } |
| private static char randomHighSurrogate() { |
| return (char) (Character.MIN_HIGH_SURROGATE + rnd.nextInt(1024)); |
| } |
| private static char randomLowSurrogate() { |
| return (char) (Character.MIN_LOW_SURROGATE + rnd.nextInt(1024)); |
| } |
| |
| private static void testCharset(Charset cs) throws Throwable { |
| if (! cs.canEncode()) |
| return; |
| |
| final String csn = cs.name(); |
| |
| if (isBroken(csn)) { |
| System.out.printf("Skipping possibly broken charset %s%n", csn); |
| return; |
| } |
| System.out.println(csn); |
| |
| CharsetTester tester = new CharsetTester(cs); |
| |
| tester.testSurrogates(); |
| |
| tester.testPrefix(new char[] {}); |
| |
| if (csn.equals("x-ISCII91")) { |
| System.out.println("More ISCII testing..."); |
| new CharsetTester(cs).testPrefix(new char[]{'\u094d'}); // Halant |
| new CharsetTester(cs).testPrefix(new char[]{'\u093c'}); // Nukta |
| } |
| } |
| |
| private static void realMain(String[] args) { |
| for (Charset cs : sort(Charset.availableCharsets().values())) { |
| try { |
| testCharset(cs); |
| } catch (TooManyFailures e) { |
| System.out.printf("Too many failures for %s%n", cs); |
| } catch (Throwable t) { |
| unexpected(t); |
| } |
| } |
| reporter.summarize(); |
| } |
| |
| //--------------------- Infrastructure --------------------------- |
| static volatile long passed = 0, failed = 0; |
| static void pass() {passed++;} |
| static void fail() {failed++; Thread.dumpStack();} |
| static void fail(String format, Object... args) { |
| System.out.println(String.format(format, args)); failed++;} |
| static void fail(String msg) {System.out.println(msg); fail();} |
| static void unexpected(Throwable t) {failed++; t.printStackTrace();} |
| static void check(boolean cond) {if (cond) pass(); else fail();} |
| static void equal(Object x, Object y) { |
| if (x == null ? y == null : x.equals(y)) pass(); |
| else fail(x + " not equal to " + y);} |
| public static void main(String[] args) throws Throwable { |
| try {realMain(args);} catch (Throwable t) {unexpected(t);} |
| System.out.printf("%nPassed = %d, failed = %d%n%n", passed, failed); |
| if (failed > 0) throw new AssertionError("Some tests failed");} |
| } |
