blob: b1b81c511f41a4e3b4678de581229e72a9a48024 [file] [log] [blame]
/********************************************************************
* COPYRIGHT:
* Copyright (c) 1999-2009, International Business Machines Corporation and
* others. All Rights Reserved.
********************************************************************/
/************************************************************************
* Date Name Description
* 12/15/99 Madhu Creation.
* 01/12/2000 Madhu Updated for changed API and added new tests
************************************************************************/
#include "unicode/utypes.h"
#if !UCONFIG_NO_BREAK_ITERATION
#include "unicode/utypes.h"
#include "unicode/brkiter.h"
#include "unicode/rbbi.h"
#include "unicode/uchar.h"
#include "unicode/utf16.h"
#include "unicode/ucnv.h"
#include "unicode/schriter.h"
#include "unicode/uniset.h"
#include "unicode/regex.h" // TODO: make conditional on regexp being built.
#include "unicode/ustring.h"
#include "unicode/utext.h"
#include "intltest.h"
#include "rbbitst.h"
#include <string.h>
#include "uvector.h"
#include "uvectr32.h"
#include "triedict.h"
#include <string.h>
#include <stdio.h>
#include <stdlib.h>
#define TEST_ASSERT(x) {if (!(x)) { \
errln("Failure in file %s, line %d", __FILE__, __LINE__);}}
#define TEST_ASSERT_SUCCESS(errcode) { if (U_FAILURE(errcode)) { \
errcheckln(errcode, "Failure in file %s, line %d, status = \"%s\"", __FILE__, __LINE__, u_errorName(errcode));}}
//---------------------------------------------
// runIndexedTest
//---------------------------------------------
void RBBITest::runIndexedTest( int32_t index, UBool exec, const char* &name, char* params )
{
if (exec) logln("TestSuite RuleBasedBreakIterator: ");
switch (index) {
case 0: name = "TestBug4153072";
if(exec) TestBug4153072(); break;
case 1: name = "TestJapaneseLineBreak";
if(exec) TestJapaneseLineBreak(); break;
case 2: name = "TestStatusReturn";
if(exec) TestStatusReturn(); break;
case 3: name = "TestUnicodeFiles";
if(exec) TestUnicodeFiles(); break;
case 4: name = "TestEmptyString";
if(exec) TestEmptyString(); break;
case 5: name = "TestGetAvailableLocales";
if(exec) TestGetAvailableLocales(); break;
case 6: name = "TestGetDisplayName";
if(exec) TestGetDisplayName(); break;
case 7: name = "TestEndBehaviour";
if(exec) TestEndBehaviour(); break;
case 8: name = "TestMixedThaiLineBreak";
// BEGIN android-removed
// Disable all Thai breakiterator tests.
/* if(exec) TestMixedThaiLineBreak(); */ break;
// END android-removed
case 9: name = "TestThaiLineBreak";
// BEGIN android-removed
// Disable all Thai breakiterator tests.
/* if(exec) TestThaiLineBreak(); */ break;
// END android-removed
case 10: name = "TestMaiyamok";
// BEGIN android-removed
// Disable all Thai breakiterator tests.
/* if(exec) TestMaiyamok(); */ break;
// END android-removed
case 11: name = "TestWordBreaks";
if(exec) TestWordBreaks(); break;
case 12: name = "TestWordBoundary";
if(exec) TestWordBoundary(); break;
case 13: name = "TestLineBreaks";
if(exec) TestLineBreaks(); break;
case 14: name = "TestSentBreaks";
if(exec) TestSentBreaks(); break;
case 15: name = "TestExtended";
if(exec) TestExtended(); break;
case 16: name = "TestMonkey";
if(exec) {
#if !UCONFIG_NO_REGULAR_EXPRESSIONS
TestMonkey(params);
#else
logln("skipping TestMonkey (UCONFIG_NO_REGULAR_EXPRESSIONS)");
#endif
}
break;
case 17: name = "TestBug3818";
// BEGIN android-removed
// Disable all Thai breakiterator tests.
/* if(exec) TestBug3818(); */ break;
// END android-removed
case 18: name = "TestJapaneseWordBreak";
if(exec) TestJapaneseWordBreak(); break;
case 19: name = "TestDebug";
if(exec) TestDebug(); break;
case 20: name = "TestTrieDict";
if(exec) TestTrieDict(); break;
case 21: name = "TestBug5775";
if (exec) TestBug5775(); break;
case 22: name = "TestThaiBreaks";
// BEGIN android-removed
// Disable all Thai breakiterator tests.
/* if (exec) TestThaiBreaks(); */ break;
// END android-removed
case 23: name = "TestTailoredBreaks";
if (exec) TestTailoredBreaks(); break;
default: name = ""; break; //needed to end loop
}
}
//---------------------------------------------------------------------------
//
// class BITestData Holds a set of Break iterator test data and results
// Includes
// - the string data to be broken
// - a vector of the expected break positions.
// - a vector of source line numbers for the data,
// (to help see where errors occured.)
// - The expected break tag values.
// - Vectors of actual break positions and tag values.
// - Functions for comparing actual with expected and
// reporting errors.
//
//----------------------------------------------------------------------------
class BITestData {
public:
UnicodeString fDataToBreak;
UVector fExpectedBreakPositions;
UVector fExpectedTags;
UVector fLineNum;
UVector fActualBreakPositions; // Test Results.
UVector fActualTags;
BITestData(UErrorCode &status);
void addDataChunk(const char *data, int32_t tag, int32_t lineNum, UErrorCode status);
void checkResults(const char *heading, RBBITest *test);
void err(const char *heading, RBBITest *test, int32_t expectedIdx, int32_t actualIdx);
void clearResults();
};
//
// Constructor.
//
BITestData::BITestData(UErrorCode &status)
: fExpectedBreakPositions(status), fExpectedTags(status), fLineNum(status), fActualBreakPositions(status),
fActualTags(status)
{
}
//
// addDataChunk. Add a section (non-breaking) piece if data to the test data.
// The macro form collects the line number, which is helpful
// when tracking down failures.
//
// A null data item is inserted at the start of each test's data
// to put the starting zero into the data list. The position saved for
// each non-null item is its ending position.
//
#define ADD_DATACHUNK(td, data, tag, status) td.addDataChunk(data, tag, __LINE__, status);
void BITestData::addDataChunk(const char *data, int32_t tag, int32_t lineNum, UErrorCode status) {
if (U_FAILURE(status)) {return;}
if (data != NULL) {
fDataToBreak.append(CharsToUnicodeString(data));
}
fExpectedBreakPositions.addElement(fDataToBreak.length(), status);
fExpectedTags.addElement(tag, status);
fLineNum.addElement(lineNum, status);
}
//
// checkResults. Compare the actual and expected break positions, report any differences.
//
void BITestData::checkResults(const char *heading, RBBITest *test) {
int32_t expectedIndex = 0;
int32_t actualIndex = 0;
for (;;) {
// If we've run through both the expected and actual results vectors, we're done.
// break out of the loop.
if (expectedIndex >= fExpectedBreakPositions.size() &&
actualIndex >= fActualBreakPositions.size()) {
break;
}
if (expectedIndex >= fExpectedBreakPositions.size()) {
err(heading, test, expectedIndex-1, actualIndex);
actualIndex++;
continue;
}
if (actualIndex >= fActualBreakPositions.size()) {
err(heading, test, expectedIndex, actualIndex-1);
expectedIndex++;
continue;
}
if (fActualBreakPositions.elementAti(actualIndex) != fExpectedBreakPositions.elementAti(expectedIndex)) {
err(heading, test, expectedIndex, actualIndex);
// Try to resync the positions of the indices, to avoid a rash of spurious erros.
if (fActualBreakPositions.elementAti(actualIndex) < fExpectedBreakPositions.elementAti(expectedIndex)) {
actualIndex++;
} else {
expectedIndex++;
}
continue;
}
if (fActualTags.elementAti(actualIndex) != fExpectedTags.elementAti(expectedIndex)) {
test->errln("%s, tag mismatch. Test Line = %d, expected tag=%d, got %d",
heading, fLineNum.elementAt(expectedIndex),
fExpectedTags.elementAti(expectedIndex), fActualTags.elementAti(actualIndex));
}
actualIndex++;
expectedIndex++;
}
}
//
// err - An error was found. Report it, along with information about where the
// incorrectly broken test data appeared in the source file.
//
void BITestData::err(const char *heading, RBBITest *test, int32_t expectedIdx, int32_t actualIdx)
{
int32_t expected = fExpectedBreakPositions.elementAti(expectedIdx);
int32_t actual = fActualBreakPositions.elementAti(actualIdx);
int32_t o = 0;
int32_t line = fLineNum.elementAti(expectedIdx);
if (expectedIdx > 0) {
// The line numbers are off by one because a premature break occurs somewhere
// within the previous item, rather than at the start of the current (expected) item.
// We want to report the offset of the unexpected break from the start of
// this previous item.
o = actual - fExpectedBreakPositions.elementAti(expectedIdx-1);
}
if (actual < expected) {
test->errln("%s unexpected break at offset %d in test item from line %d. actual break: %d expected break: %d", heading, o, line, actual, expected);
} else {
test->errln("%s Failed to find break at end of item from line %d. actual break: %d expected break: %d", heading, line, actual, expected);
}
}
void BITestData::clearResults() {
fActualBreakPositions.removeAllElements();
fActualTags.removeAllElements();
}
//-----------------------------------------------------------------------------------
//
// Cannned Test Characters
//
//-----------------------------------------------------------------------------------
static const UChar cannedTestArray[] = {
0x0001, 0x0002, 0x0003, 0x0004, 0x0020, 0x0021, '\\', 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0028, 0x0029, 0x002b, 0x002d, 0x0030, 0x0031,
0x0032, 0x0033, 0x0034, 0x003c, 0x003d, 0x003e, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x005b, 0x005d, 0x005e, 0x005f, 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x007b,
0x007d, 0x007c, 0x002c, 0x00a0, 0x00a2,
0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, 0x00a8, 0x00a9, 0x00ab, 0x00ad, 0x00ae, 0x00af, 0x00b0, 0x00b2, 0x00b3,
0x00b4, 0x00b9, 0x00bb, 0x00bc, 0x00bd, 0x02b0, 0x02b1, 0x02b2, 0x02b3, 0x02b4, 0x0300, 0x0301, 0x0302, 0x0303,
0x0304, 0x05d0, 0x05d1, 0x05d2, 0x05d3, 0x05d4, 0x0903, 0x093e, 0x093f, 0x0940, 0x0949, 0x0f3a, 0x0f3b, 0x2000,
0x2001, 0x2002, 0x200c, 0x200d, 0x200e, 0x200f, 0x2010, 0x2011, 0x2012, 0x2028, 0x2029, 0x202a, 0x203e, 0x203f,
0x2040, 0x20dd, 0x20de, 0x20df, 0x20e0, 0x2160, 0x2161, 0x2162, 0x2163, 0x2164, 0x0000
};
static UnicodeString* cannedTestChars = 0;
#define halfNA "\\u0928\\u094d\\u200d"
#define halfSA "\\u0938\\u094d\\u200d"
#define halfCHA "\\u091a\\u094d\\u200d"
#define halfKA "\\u0915\\u094d\\u200d"
#define deadTA "\\u0924\\u094d"
//--------------------------------------------------------------------------------------
//
// RBBITest constructor and destructor
//
//--------------------------------------------------------------------------------------
RBBITest::RBBITest() {
UnicodeString temp(cannedTestArray);
cannedTestChars = new UnicodeString();
*cannedTestChars += (UChar)0x0000;
*cannedTestChars += temp;
}
RBBITest::~RBBITest() {
delete cannedTestChars;
}
static const int T_NUMBER = 100;
static const int T_LETTER = 200;
static const int T_H_OR_K = 300;
static const int T_IDEO = 400;
//--------------------------------------------------------------------
//Testing the BreakIterator for devanagari script
//--------------------------------------------------------------------
#define deadRA "\\u0930\\u094d" /*deadform RA = devanagari RA + virama*/
#define deadPHA "\\u092b\\u094d" /*deadform PHA = devanagari PHA + virama*/
#define deadTTHA "\\u0920\\u094d"
#define deadPA "\\u092a\\u094d"
#define deadSA "\\u0938\\u094d"
#define visarga "\\u0903" /*devanagari visarga looks like a english colon*/
//-----------------------------------------------------------------------------------
//
// Test for status {tag} return value from break rules.
// TODO: a more thorough test.
//
//-----------------------------------------------------------------------------------
void RBBITest::TestStatusReturn() {
UnicodeString rulesString1("$Letters = [:L:];\n"
"$Numbers = [:N:];\n"
"$Letters+{1};\n"
"$Numbers+{2};\n"
"Help\\ {4}/me\\!;\n"
"[^$Letters $Numbers];\n"
"!.*;\n", -1, US_INV);
UnicodeString testString1 = "abc123..abc Help me Help me!";
// 01234567890123456789012345678
int32_t bounds1[] = {0, 3, 6, 7, 8, 11, 12, 16, 17, 19, 20, 25, 27, 28, -1};
int32_t brkStatus[] = {0, 1, 2, 0, 0, 1, 0, 1, 0, 1, 0, 4, 1, 0, -1};
UErrorCode status=U_ZERO_ERROR;
UParseError parseError;
RuleBasedBreakIterator *bi = new RuleBasedBreakIterator(rulesString1, parseError, status);
if(U_FAILURE(status)) {
dataerrln("FAIL : in construction - %s", u_errorName(status));
} else {
int32_t pos;
int32_t i = 0;
bi->setText(testString1);
for (pos=bi->first(); pos!= BreakIterator::DONE; pos=bi->next()) {
if (pos != bounds1[i]) {
errln("FAIL: expected break at %d, got %d\n", bounds1[i], pos);
break;
}
int tag = bi->getRuleStatus();
if (tag != brkStatus[i]) {
errln("FAIL: break at %d, expected tag %d, got tag %d\n", pos, brkStatus[i], tag);
break;
}
i++;
}
}
delete bi;
}
static void printStringBreaks(UnicodeString ustr, int expected[],
int expectedcount)
{
UErrorCode status = U_ZERO_ERROR;
char name[100];
printf("code alpha extend alphanum type word sent line name\n");
int j;
for (j = 0; j < ustr.length(); j ++) {
if (expectedcount > 0) {
int k;
for (k = 0; k < expectedcount; k ++) {
if (j == expected[k]) {
printf("------------------------------------------------ %d\n",
j);
}
}
}
UChar32 c = ustr.char32At(j);
if (c > 0xffff) {
j ++;
}
u_charName(c, U_UNICODE_CHAR_NAME, name, 100, &status);
printf("%7x %5d %6d %8d %4s %4s %4s %4s %s\n", (int)c,
u_isUAlphabetic(c),
u_hasBinaryProperty(c, UCHAR_GRAPHEME_EXTEND),
u_isalnum(c),
u_getPropertyValueName(UCHAR_GENERAL_CATEGORY,
u_charType(c),
U_SHORT_PROPERTY_NAME),
u_getPropertyValueName(UCHAR_WORD_BREAK,
u_getIntPropertyValue(c,
UCHAR_WORD_BREAK),
U_SHORT_PROPERTY_NAME),
u_getPropertyValueName(UCHAR_SENTENCE_BREAK,
u_getIntPropertyValue(c,
UCHAR_SENTENCE_BREAK),
U_SHORT_PROPERTY_NAME),
u_getPropertyValueName(UCHAR_LINE_BREAK,
u_getIntPropertyValue(c,
UCHAR_LINE_BREAK),
U_SHORT_PROPERTY_NAME),
name);
}
}
void RBBITest::TestThaiLineBreak() {
UErrorCode status = U_ZERO_ERROR;
BITestData thaiLineSelection(status);
// \u0e2f-- the Thai paiyannoi character-- isn't a letter. It's a symbol that
// represents elided letters at the end of a long word. It should be bound to
// the end of the word and not treated as an independent punctuation mark.
ADD_DATACHUNK(thaiLineSelection, NULL, 0, status); // Break at start of data
ADD_DATACHUNK(thaiLineSelection, "\\u0e2a\\u0e16\\u0e32\\u0e19\\u0e35\\u0e2f", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e08\\u0e30", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e23\\u0e30\\u0e14\\u0e21", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e40\\u0e08\\u0e49\\u0e32", 0, status);
// ADD_DATACHUNK(thaiLineSelection, "\\u0e2b\\u0e19\\u0e49\\u0e32", 0, status);
// ADD_DATACHUNK(thaiLineSelection, "\\u0e17\\u0e35\\u0e48", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e2b\\u0e19\\u0e49\\u0e32\\u0e17\\u0e35\\u0e48", 0, status);
// the commented-out lines (I think) are the preferred result; this line is what our current dictionary is giving us
ADD_DATACHUNK(thaiLineSelection, "\\u0e2d\\u0e2d\\u0e01", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e21\\u0e32", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e40\\u0e23\\u0e48\\u0e07", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e23\\u0e30\\u0e1a\\u0e32\\u0e22", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e2d\\u0e22\\u0e48\\u0e32\\u0e07", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e40\\u0e15\\u0e47\\u0e21", 0, status);
// the one time where the paiyannoi occurs somewhere other than at the end
// of a word is in the Thai abbrevation for "etc.", which both begins and
// ends with a paiyannoi
ADD_DATACHUNK(thaiLineSelection, "\\u0e2f\\u0e25\\u0e2f", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e17\\u0e35\\u0e48", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e19\\u0e31\\u0e49\\u0e19", 0, status);
RuleBasedBreakIterator* e = (RuleBasedBreakIterator *)BreakIterator::createLineInstance(
Locale("th"), status);
if (U_FAILURE(status))
{
errcheckln(status, "Failed to create the BreakIterator for Thai locale in TestThaiLineBreak. - %s", u_errorName(status));
return;
}
generalIteratorTest(*e, thaiLineSelection);
delete e;
}
void RBBITest::TestMixedThaiLineBreak()
{
UErrorCode status = U_ZERO_ERROR;
BITestData thaiLineSelection(status);
ADD_DATACHUNK(thaiLineSelection, NULL, 0, status); // Break at start of data
// @suwit -- Test Arabic numerals, Thai numerals, Punctuation and English characters
// start
ADD_DATACHUNK(thaiLineSelection, "\\u0E1B\\u0E35", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0E1E\\u0E38\\u0E17\\u0E18\\u0E28\\u0E31\\u0E01\\u0E23\\u0E32\\u0E0A ", 0, status);
ADD_DATACHUNK(thaiLineSelection, "2545 ", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0E40\\u0E1B\\u0E47\\u0E19", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0E1B\\u0E35", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0E09\\u0E25\\u0E2D\\u0E07", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0E04\\u0E23\\u0E1A", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0E23\\u0E2D\\u0E1A ", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\"\\u0E52\\u0E52\\u0E50 ", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0E1b\\u0E35\" ", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0E02\\u0E2d\\u0E07", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0E01\\u0E23\\u0E38\\u0E07", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0E23\\u0E31\\u0E15\\u0E19\\u0E42\\u0E01\\u0E2A\\u0E34\\u0E19\\u0E17\\u0E23\\u0E4C ", 0, status);
ADD_DATACHUNK(thaiLineSelection, "(\\u0E01\\u0E23\\u0E38\\u0E07\\u0E40\\u0E17\\u0E1e\\u0E2F", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0E2B\\u0E23\\u0E37\\u0E2D ", 0, status);
ADD_DATACHUNK(thaiLineSelection, "Bangkok)", 0, status);
// @suwit - end of changes
RuleBasedBreakIterator* e = (RuleBasedBreakIterator *)BreakIterator::createLineInstance(Locale("th"), status);
if (U_FAILURE(status))
{
errcheckln(status, "Failed to create the BreakIterator for Thai locale in TestMixedThaiLineBreak. - %s", u_errorName(status));
return;
}
generalIteratorTest(*e, thaiLineSelection);
delete e;
}
void RBBITest::TestMaiyamok()
{
UErrorCode status = U_ZERO_ERROR;
BITestData thaiLineSelection(status);
ADD_DATACHUNK(thaiLineSelection, NULL, 0, status); // Break at start of data
// the Thai maiyamok character is a shorthand symbol that means "repeat the previous
// word". Instead of appearing as a word unto itself, however, it's kept together
// with the word before it
ADD_DATACHUNK(thaiLineSelection, "\\u0e44\\u0e1b\\u0e46", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e21\\u0e32\\u0e46", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e23\\u0e30\\u0e2b\\u0e27\\u0e48\\u0e32\\u0e07", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e01\\u0e23\\u0e38\\u0e07", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e40\\u0e17\\u0e1e", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e41\\u0e25\\u0e30", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e40\\u0e03\\u0e35", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e22\\u0e07", 0, status);
ADD_DATACHUNK(thaiLineSelection, "\\u0e43\\u0e2b\\u0e21\\u0e48", 0, status);
RuleBasedBreakIterator* e = (RuleBasedBreakIterator *)BreakIterator::createLineInstance(
Locale("th"), status);
if (U_FAILURE(status))
{
errcheckln(status, "Failed to create the BreakIterator for Thai locale in TestMaiyamok. - %s", u_errorName(status));
return;
}
generalIteratorTest(*e, thaiLineSelection);
delete e;
}
void RBBITest::TestBug3818() {
UErrorCode status = U_ZERO_ERROR;
// Four Thai words...
static const UChar thaiWordData[] = { 0x0E43,0x0E2B,0x0E0D,0x0E48, 0x0E43,0x0E2B,0x0E0D,0x0E48,
0x0E43,0x0E2B,0x0E0D,0x0E48, 0x0E43,0x0E2B,0x0E0D,0x0E48, 0 };
UnicodeString thaiStr(thaiWordData);
RuleBasedBreakIterator* bi =
(RuleBasedBreakIterator *)BreakIterator::createWordInstance(Locale("th"), status);
if (U_FAILURE(status) || bi == NULL) {
errcheckln(status, "Fail at file %s, line %d, status = %s", __FILE__, __LINE__, u_errorName(status));
return;
}
bi->setText(thaiStr);
int32_t startOfSecondWord = bi->following(1);
if (startOfSecondWord != 4) {
errln("Fail at file %s, line %d expected start of word at 4, got %d",
__FILE__, __LINE__, startOfSecondWord);
}
startOfSecondWord = bi->following(0);
if (startOfSecondWord != 4) {
errln("Fail at file %s, line %d expected start of word at 4, got %d",
__FILE__, __LINE__, startOfSecondWord);
}
delete bi;
}
void RBBITest::TestJapaneseWordBreak() {
UErrorCode status = U_ZERO_ERROR;
BITestData japaneseWordSelection(status);
ADD_DATACHUNK(japaneseWordSelection, NULL, 0, status); // Break at start of data
ADD_DATACHUNK(japaneseWordSelection, "\\u4ECA\\u65E5", 400, status); //2
ADD_DATACHUNK(japaneseWordSelection, "\\u306F\\u3044\\u3044", 300, status); //5
ADD_DATACHUNK(japaneseWordSelection, "\\u5929\\u6C17", 400, status); //7
ADD_DATACHUNK(japaneseWordSelection, "\\u3067\\u3059\\u306D", 300, status); //10
ADD_DATACHUNK(japaneseWordSelection, "\\u3002", 0, status); //11
ADD_DATACHUNK(japaneseWordSelection, "\\u000D\\u000A", 0, status); //12
RuleBasedBreakIterator* e = (RuleBasedBreakIterator *)BreakIterator::createWordInstance(
Locale("ja"), status);
if (U_FAILURE(status))
{
errcheckln(status, "Failed to create the BreakIterator for Japanese locale in TestJapaneseWordBreak.\n");
return;
}
generalIteratorTest(*e, japaneseWordSelection);
delete e;
}
void RBBITest::TestTrieDict() {
UErrorCode status = U_ZERO_ERROR;
//
// Open and read the test data file.
//
const char *testDataDirectory = IntlTest::getSourceTestData(status);
char testFileName[1000];
if (testDataDirectory == NULL || strlen(testDataDirectory) + strlen("riwords.txt") + 10 >= sizeof(testFileName)) {
errln("Can't open test data. Path too long.");
return;
}
strcpy(testFileName, testDataDirectory);
strcat(testFileName, "riwords.txt");
// Items needing deleting at the end
MutableTrieDictionary *mutableDict = NULL;
CompactTrieDictionary *compactDict = NULL;
UnicodeSet *breaks = NULL;
UChar *testFile = NULL;
StringEnumeration *enumer1 = NULL;
StringEnumeration *enumer2 = NULL;
MutableTrieDictionary *mutable2 = NULL;
StringEnumeration *cloneEnum = NULL;
CompactTrieDictionary *compact2 = NULL;
const UnicodeString *originalWord = NULL;
const UnicodeString *cloneWord = NULL;
UChar *current;
UChar *word;
UChar uc;
int32_t wordLen;
int32_t wordCount;
int32_t testCount;
int len;
testFile = ReadAndConvertFile(testFileName, len, NULL, status);
if (U_FAILURE(status)) {
goto cleanup; /* something went wrong, error already output */
}
mutableDict = new MutableTrieDictionary(0x0E1C, status);
if (U_FAILURE(status)) {
errln("Error creating MutableTrieDictionary: %s\n", u_errorName(status));
goto cleanup;
}
breaks = new UnicodeSet;
breaks->add(0x000A); // Line Feed
breaks->add(0x000D); // Carriage Return
breaks->add(0x2028); // Line Separator
breaks->add(0x2029); // Paragraph Separator
// Now add each non-comment line of the file as a word.
current = testFile;
word = current;
uc = *current++;
wordLen = 0;
wordCount = 0;
while (uc) {
if (uc == 0x0023) { // #comment line, skip
while (uc && !breaks->contains(uc)) {
uc = *current++;
}
}
else while (uc && !breaks->contains(uc)) {
++wordLen;
uc = *current++;
}
if (wordLen > 0) {
mutableDict->addWord(word, wordLen, status);
if (U_FAILURE(status)) {
errln("Could not add word to mutable dictionary; status %s\n", u_errorName(status));
goto cleanup;
}
wordCount += 1;
}
// Find beginning of next line
while (uc && breaks->contains(uc)) {
uc = *current++;
}
word = current-1;
wordLen = 0;
}
if (wordCount < 50) {
errln("Word count (%d) unreasonably small\n", wordCount);
goto cleanup;
}
enumer1 = mutableDict->openWords(status);
if (U_FAILURE(status)) {
errln("Could not open mutable dictionary enumerator: %s\n", u_errorName(status));
goto cleanup;
}
testCount = 0;
if (wordCount != (testCount = enumer1->count(status))) {
errln("MutableTrieDictionary word count (%d) differs from file word count (%d), with status %s\n",
testCount, wordCount, u_errorName(status));
goto cleanup;
}
// Now compact it
compactDict = new CompactTrieDictionary(*mutableDict, status);
if (U_FAILURE(status)) {
errln("Failed to create CompactTrieDictionary: %s\n", u_errorName(status));
goto cleanup;
}
enumer2 = compactDict->openWords(status);
if (U_FAILURE(status)) {
errln("Could not open compact trie dictionary enumerator: %s\n", u_errorName(status));
goto cleanup;
}
if (wordCount != (testCount = enumer2->count(status))) {
errln("CompactTrieDictionary word count (%d) differs from file word count (%d), with status %s\n",
testCount, wordCount, u_errorName(status));
goto cleanup;
}
if (enumer1->getDynamicClassID() == enumer2->getDynamicClassID()) {
errln("CompactTrieEnumeration and MutableTrieEnumeration ClassIDs are the same");
}
delete enumer1;
enumer1 = NULL;
delete enumer2;
enumer2 = NULL;
// Now un-compact it
mutable2 = compactDict->cloneMutable(status);
if (U_FAILURE(status)) {
errln("Could not clone CompactTrieDictionary to MutableTrieDictionary: %s\n", u_errorName(status));
goto cleanup;
}
cloneEnum = mutable2->openWords(status);
if (U_FAILURE(status)) {
errln("Could not create cloned mutable enumerator: %s\n", u_errorName(status));
goto cleanup;
}
if (wordCount != (testCount = cloneEnum->count(status))) {
errln("Cloned MutableTrieDictionary word count (%d) differs from file word count (%d), with status %s\n",
testCount, wordCount, u_errorName(status));
goto cleanup;
}
// Compact original dictionary to clone. Note that we can only compare the same kind of
// dictionary as the order of the enumerators is not guaranteed to be the same between
// different kinds
enumer1 = mutableDict->openWords(status);
if (U_FAILURE(status)) {
errln("Could not re-open mutable dictionary enumerator: %s\n", u_errorName(status));
goto cleanup;
}
originalWord = enumer1->snext(status);
cloneWord = cloneEnum->snext(status);
while (U_SUCCESS(status) && originalWord != NULL && cloneWord != NULL) {
if (*originalWord != *cloneWord) {
errln("Original and cloned MutableTrieDictionary word mismatch\n");
goto cleanup;
}
originalWord = enumer1->snext(status);
cloneWord = cloneEnum->snext(status);
}
if (U_FAILURE(status)) {
errln("Enumeration failed: %s\n", u_errorName(status));
goto cleanup;
}
if (originalWord != cloneWord) {
errln("Original and cloned MutableTrieDictionary ended enumeration at different points\n");
goto cleanup;
}
// Test the data copying constructor for CompactTrieDict, and the data access APIs.
compact2 = new CompactTrieDictionary(compactDict->data(), status);
if (U_FAILURE(status)) {
errln("CompactTrieDictionary(const void *,...) failed\n");
goto cleanup;
}
if (compact2->dataSize() == 0) {
errln("CompactTrieDictionary->dataSize() == 0\n");
goto cleanup;
}
// Now count the words via the second dictionary
delete enumer1;
enumer1 = compact2->openWords(status);
if (U_FAILURE(status)) {
errln("Could not open compact trie dictionary 2 enumerator: %s\n", u_errorName(status));
goto cleanup;
}
if (wordCount != (testCount = enumer1->count(status))) {
errln("CompactTrieDictionary 2 word count (%d) differs from file word count (%d), with status %s\n",
testCount, wordCount, u_errorName(status));
goto cleanup;
}
cleanup:
delete compactDict;
delete mutableDict;
delete breaks;
delete[] testFile;
delete enumer1;
delete mutable2;
delete cloneEnum;
delete compact2;
}
//----------------------------------------------------------------------------
//
// generalIteratorTest Given a break iterator and a set of test data,
// Run the tests and report the results.
//
//----------------------------------------------------------------------------
void RBBITest::generalIteratorTest(RuleBasedBreakIterator& bi, BITestData &td)
{
bi.setText(td.fDataToBreak);
testFirstAndNext(bi, td);
testLastAndPrevious(bi, td);
testFollowing(bi, td);
testPreceding(bi, td);
testIsBoundary(bi, td);
doMultipleSelectionTest(bi, td);
}
//
// testFirstAndNext. Run the iterator forwards in the obvious first(), next()
// kind of loop.
//
void RBBITest::testFirstAndNext(RuleBasedBreakIterator& bi, BITestData &td)
{
UErrorCode status = U_ZERO_ERROR;
int32_t p;
int32_t lastP = -1;
int32_t tag;
logln("Test first and next");
bi.setText(td.fDataToBreak);
td.clearResults();
for (p=bi.first(); p!=RuleBasedBreakIterator::DONE; p=bi.next()) {
td.fActualBreakPositions.addElement(p, status); // Save result.
tag = bi.getRuleStatus();
td.fActualTags.addElement(tag, status);
if (p <= lastP) {
// If the iterator is not making forward progress, stop.
// No need to raise an error here, it'll be detected in the normal check of results.
break;
}
lastP = p;
}
td.checkResults("testFirstAndNext", this);
}
//
// TestLastAndPrevious. Run the iterator backwards, starting with last().
//
void RBBITest::testLastAndPrevious(RuleBasedBreakIterator& bi, BITestData &td)
{
UErrorCode status = U_ZERO_ERROR;
int32_t p;
int32_t lastP = 0x7ffffffe;
int32_t tag;
logln("Test last and previous");
bi.setText(td.fDataToBreak);
td.clearResults();
for (p=bi.last(); p!=RuleBasedBreakIterator::DONE; p=bi.previous()) {
// Save break position. Insert it at start of vector of results, shoving
// already-saved results further towards the end.
td.fActualBreakPositions.insertElementAt(p, 0, status);
// bi.previous(); // TODO: Why does this fix things up????
// bi.next();
tag = bi.getRuleStatus();
td.fActualTags.insertElementAt(tag, 0, status);
if (p >= lastP) {
// If the iterator is not making progress, stop.
// No need to raise an error here, it'll be detected in the normal check of results.
break;
}
lastP = p;
}
td.checkResults("testLastAndPrevious", this);
}
void RBBITest::testFollowing(RuleBasedBreakIterator& bi, BITestData &td)
{
UErrorCode status = U_ZERO_ERROR;
int32_t p;
int32_t tag;
int32_t lastP = -2; // A value that will never be returned as a break position.
// cannot be -1; that is returned for DONE.
int i;
logln("testFollowing():");
bi.setText(td.fDataToBreak);
td.clearResults();
// Save the starting point, since we won't get that out of following.
p = bi.first();
td.fActualBreakPositions.addElement(p, status); // Save result.
tag = bi.getRuleStatus();
td.fActualTags.addElement(tag, status);
for (i = 0; i <= td.fDataToBreak.length()+1; i++) {
p = bi.following(i);
if (p != lastP) {
if (p == RuleBasedBreakIterator::DONE) {
break;
}
// We've reached a new break position. Save it.
td.fActualBreakPositions.addElement(p, status); // Save result.
tag = bi.getRuleStatus();
td.fActualTags.addElement(tag, status);
lastP = p;
}
}
// The loop normally exits by means of the break in the middle.
// Make sure that the index was at the correct position for the break iterator to have
// returned DONE.
if (i != td.fDataToBreak.length()) {
errln("testFollowing(): iterator returned DONE prematurely.");
}
// Full check of all results.
td.checkResults("testFollowing", this);
}
void RBBITest::testPreceding(RuleBasedBreakIterator& bi, BITestData &td) {
UErrorCode status = U_ZERO_ERROR;
int32_t p;
int32_t tag;
int32_t lastP = 0x7ffffffe;
int i;
logln("testPreceding():");
bi.setText(td.fDataToBreak);
td.clearResults();
p = bi.last();
td.fActualBreakPositions.addElement(p, status);
tag = bi.getRuleStatus();
td.fActualTags.addElement(tag, status);
for (i = td.fDataToBreak.length(); i>=-1; i--) {
p = bi.preceding(i);
if (p != lastP) {
if (p == RuleBasedBreakIterator::DONE) {
break;
}
// We've reached a new break position. Save it.
td.fActualBreakPositions.insertElementAt(p, 0, status);
lastP = p;
tag = bi.getRuleStatus();
td.fActualTags.insertElementAt(tag, 0, status);
}
}
// The loop normally exits by means of the break in the middle.
// Make sure that the index was at the correct position for the break iterator to have
// returned DONE.
if (i != 0) {
errln("testPreceding(): iterator returned DONE prematurely.");
}
// Full check of all results.
td.checkResults("testPreceding", this);
}
void RBBITest::testIsBoundary(RuleBasedBreakIterator& bi, BITestData &td) {
UErrorCode status = U_ZERO_ERROR;
int i;
int32_t tag;
logln("testIsBoundary():");
bi.setText(td.fDataToBreak);
td.clearResults();
for (i = 0; i <= td.fDataToBreak.length(); i++) {
if (bi.isBoundary(i)) {
td.fActualBreakPositions.addElement(i, status); // Save result.
tag = bi.getRuleStatus();
td.fActualTags.addElement(tag, status);
}
}
td.checkResults("testIsBoundary: ", this);
}
void RBBITest::doMultipleSelectionTest(RuleBasedBreakIterator& iterator, BITestData &td)
{
iterator.setText(td.fDataToBreak);
RuleBasedBreakIterator* testIterator =(RuleBasedBreakIterator*)iterator.clone();
int32_t offset = iterator.first();
int32_t testOffset;
int32_t count = 0;
logln("doMultipleSelectionTest text of length: %d", td.fDataToBreak.length());
if (*testIterator != iterator)
errln("clone() or operator!= failed: two clones compared unequal");
do {
testOffset = testIterator->first();
testOffset = testIterator->next(count);
if (offset != testOffset)
errln(UnicodeString("next(n) and next() not returning consistent results: for step ") + count + ", next(n) returned " + testOffset + " and next() had " + offset);
if (offset != RuleBasedBreakIterator::DONE) {
count++;
offset = iterator.next();
if (offset != RuleBasedBreakIterator::DONE && *testIterator == iterator) {
errln("operator== failed: Two unequal iterators compared equal. count=%d offset=%d", count, offset);
if (count > 10000 || offset == -1) {
errln("operator== failed too many times. Stopping test.");
if (offset == -1) {
errln("Does (RuleBasedBreakIterator::DONE == -1)?");
}
return;
}
}
}
} while (offset != RuleBasedBreakIterator::DONE);
// now do it backwards...
offset = iterator.last();
count = 0;
do {
testOffset = testIterator->last();
testOffset = testIterator->next(count); // next() with a negative arg is same as previous
if (offset != testOffset)
errln(UnicodeString("next(n) and next() not returning consistent results: for step ") + count + ", next(n) returned " + testOffset + " and next() had " + offset);
if (offset != RuleBasedBreakIterator::DONE) {
count--;
offset = iterator.previous();
}
} while (offset != RuleBasedBreakIterator::DONE);
delete testIterator;
}
//---------------------------------------------
//
// other tests
//
//---------------------------------------------
void RBBITest::TestEmptyString()
{
UnicodeString text = "";
UErrorCode status = U_ZERO_ERROR;
BITestData x(status);
ADD_DATACHUNK(x, "", 0, status); // Break at start of data
RuleBasedBreakIterator* bi = (RuleBasedBreakIterator *)BreakIterator::createLineInstance(Locale::getDefault(), status);
if (U_FAILURE(status))
{
errcheckln(status, "Failed to create the BreakIterator for default locale in TestEmptyString. - %s", u_errorName(status));
return;
}
generalIteratorTest(*bi, x);
delete bi;
}
void RBBITest::TestGetAvailableLocales()
{
int32_t locCount = 0;
const Locale* locList = BreakIterator::getAvailableLocales(locCount);
if (locCount == 0)
dataerrln("getAvailableLocales() returned an empty list!");
// Just make sure that it's returning good memory.
int32_t i;
for (i = 0; i < locCount; ++i) {
logln(locList[i].getName());
}
}
//Testing the BreakIterator::getDisplayName() function
void RBBITest::TestGetDisplayName()
{
UnicodeString result;
BreakIterator::getDisplayName(Locale::getUS(), result);
if (Locale::getDefault() == Locale::getUS() && result != "English (United States)")
dataerrln("BreakIterator::getDisplayName() failed: expected \"English (United States)\", got \""
+ result);
BreakIterator::getDisplayName(Locale::getFrance(), Locale::getUS(), result);
if (result != "French (France)")
dataerrln("BreakIterator::getDisplayName() failed: expected \"French (France)\", got \""
+ result);
}
/**
* Test End Behaviour
* @bug 4068137
*/
void RBBITest::TestEndBehaviour()
{
UErrorCode status = U_ZERO_ERROR;
UnicodeString testString("boo.");
BreakIterator *wb = BreakIterator::createWordInstance(Locale::getDefault(), status);
if (U_FAILURE(status))
{
errcheckln(status, "Failed to create the BreakIterator for default locale in TestEndBehaviour. - %s", u_errorName(status));
return;
}
wb->setText(testString);
if (wb->first() != 0)
errln("Didn't get break at beginning of string.");
if (wb->next() != 3)
errln("Didn't get break before period in \"boo.\"");
if (wb->current() != 4 && wb->next() != 4)
errln("Didn't get break at end of string.");
delete wb;
}
/*
* @bug 4153072
*/
void RBBITest::TestBug4153072() {
UErrorCode status = U_ZERO_ERROR;
BreakIterator *iter = BreakIterator::createWordInstance(Locale::getDefault(), status);
if (U_FAILURE(status))
{
errcheckln(status, "Failed to create the BreakIterator for default locale in TestBug4153072 - %s", u_errorName(status));
return;
}
UnicodeString str("...Hello, World!...");
int32_t begin = 3;
int32_t end = str.length() - 3;
UBool onBoundary;
StringCharacterIterator* textIterator = new StringCharacterIterator(str, begin, end, begin);
iter->adoptText(textIterator);
int index;
// Note: with the switch to UText, there is no way to restrict the
// iteration range to begin at an index other than zero.
// String character iterators created with a non-zero bound are
// treated by RBBI as being empty.
for (index = -1; index < begin + 1; ++index) {
onBoundary = iter->isBoundary(index);
if (index == 0? !onBoundary : onBoundary) {
errln((UnicodeString)"Didn't handle isBoundary correctly with offset = " + index +
" and begin index = " + begin);
}
}
delete iter;
}
//
// Test for problem reported by Ashok Matoria on 9 July 2007
// One.<kSoftHyphen><kSpace>Two.
//
// Sentence break at start (0) and then on calling next() it breaks at
// 'T' of "Two". Now, at this point if I do next() and
// then previous(), it breaks at <kSOftHyphen> instead of 'T' of "Two".
//
void RBBITest::TestBug5775() {
UErrorCode status = U_ZERO_ERROR;
BreakIterator *bi = BreakIterator::createSentenceInstance(Locale::getEnglish(), status);
TEST_ASSERT_SUCCESS(status);
if (U_FAILURE(status)) {
return;
}
// Check for status first for better handling of no data errors.
TEST_ASSERT(bi != NULL);
if (bi == NULL) {
return;
}
UnicodeString s("One.\\u00ad Two.", -1, US_INV);
// 01234 56789
s = s.unescape();
bi->setText(s);
int pos = bi->next();
TEST_ASSERT(pos == 6);
pos = bi->next();
TEST_ASSERT(pos == 10);
pos = bi->previous();
TEST_ASSERT(pos == 6);
delete bi;
}
/**
* Test Japanese Line Break
* @bug 4095322
*/
void RBBITest::TestJapaneseLineBreak()
{
#if 0
// Test needs updating some more... Dump it for now.
// Change for Unicode TR 14: Punctuation characters with categories Pi and Pf do not count
// as opening and closing punctuation for line breaking.
// Also, \u30fc and \u30fe are not counted as hyphens. Remove these chars
// from these tests. 6-13-2002
//
UErrorCode status = U_ZERO_ERROR;
UnicodeString testString = CharsToUnicodeString("\\u4e00x\\u4e8c");
UnicodeString precedingChars = CharsToUnicodeString(
//"([{\\u00ab$\\u00a5\\u00a3\\u00a4\\u2018\\u201a\\u201c\\u201e\\u201b\\u201f");
"([{$\\u00a5\\u00a3\\u00a4\\u201a\\u201e");
UnicodeString followingChars = CharsToUnicodeString(
// ")]}\\u00bb!%,.\\u3001\\u3002\\u3063\\u3083\\u3085\\u3087\\u30c3\\u30e3\\u30e5\\u30e7\\u30fc"
")]}!%,.\\u3001\\u3002\\u3063\\u3083\\u3085\\u3087\\u30c3\\u30e3\\u30e5\\u30e7"
// ":;\\u309b\\u309c\\u3005\\u309d\\u309e\\u30fd\\u30fe\\u2019\\u201d\\u00b0\\u2032\\u2033\\u2034"
":;\\u309b\\u309c\\u3005\\u309d\\u309e\\u30fd\\u00b0\\u2032\\u2033\\u2034"
"\\u2030\\u2031\\u2103\\u2109\\u00a2\\u0300\\u0301\\u0302");
BreakIterator *iter = BreakIterator::createLineInstance(Locale::getJapan(), status);
int32_t i;
if (U_FAILURE(status))
{
errln("Failed to create the BreakIterator for Japanese locale in TestJapaneseLineBreak.\n");
return;
}
for (i = 0; i < precedingChars.length(); i++) {
testString.setCharAt(1, precedingChars[i]);
iter->setText(testString);
int32_t j = iter->first();
if (j != 0)
errln("ja line break failure: failed to start at 0");
j = iter->next();
if (j != 1)
errln("ja line break failure: failed to stop before '" + UCharToUnicodeString(precedingChars[i])
+ "' (" + ((int)(precedingChars[i])) + ")");
j = iter->next();
if (j != 3)
errln("ja line break failure: failed to skip position after '" + UCharToUnicodeString(precedingChars[i])
+ "' (" + ((int)(precedingChars[i])) + ")");
}
for (i = 0; i < followingChars.length(); i++) {
testString.setCharAt(1, followingChars[i]);
iter->setText(testString);
int j = iter->first();
if (j != 0)
errln("ja line break failure: failed to start at 0");
j = iter->next();
if (j != 2)
errln("ja line break failure: failed to skip position before '" + UCharToUnicodeString(followingChars[i])
+ "' (" + ((int)(followingChars[i])) + ")");
j = iter->next();
if (j != 3)
errln("ja line break failure: failed to stop after '" + UCharToUnicodeString(followingChars[i])
+ "' (" + ((int)(followingChars[i])) + ")");
}
delete iter;
#endif
}
//------------------------------------------------------------------------------
//
// RBBITest::Extended Run RBBI Tests from an external test data file
//
//------------------------------------------------------------------------------
struct TestParams {
BreakIterator *bi;
UnicodeString dataToBreak;
UVector32 *expectedBreaks;
UVector32 *srcLine;
UVector32 *srcCol;
};
void RBBITest::executeTest(TestParams *t) {
int32_t bp;
int32_t prevBP;
int32_t i;
if (t->bi == NULL) {
return;
}
t->bi->setText(t->dataToBreak);
//
// Run the iterator forward
//
prevBP = -1;
for (bp = t->bi->first(); bp != BreakIterator::DONE; bp = t->bi->next()) {
if (prevBP == bp) {
// Fail for lack of forward progress.
errln("Forward Iteration, no forward progress. Break Pos=%4d File line,col=%4d,%4d",
bp, t->srcLine->elementAti(bp), t->srcCol->elementAti(bp));
break;
}
// Check that there were we didn't miss an expected break between the last one
// and this one.
for (i=prevBP+1; i<bp; i++) {
if (t->expectedBreaks->elementAti(i) != 0) {
int expected[] = {0, i};
printStringBreaks(t->dataToBreak, expected, 2);
errln("Forward Iteration, break expected, but not found. Pos=%4d File line,col= %4d,%4d",
i, t->srcLine->elementAti(i), t->srcCol->elementAti(i));
}
}
// Check that the break we did find was expected
if (t->expectedBreaks->elementAti(bp) == 0) {
int expected[] = {0, bp};
printStringBreaks(t->dataToBreak, expected, 2);
errln("Forward Iteration, break found, but not expected. Pos=%4d File line,col= %4d,%4d",
bp, t->srcLine->elementAti(bp), t->srcCol->elementAti(bp));
} else {
// The break was expected.
// Check that the {nnn} tag value is correct.
int32_t expectedTagVal = t->expectedBreaks->elementAti(bp);
if (expectedTagVal == -1) {
expectedTagVal = 0;
}
int32_t line = t->srcLine->elementAti(bp);
int32_t rs = ((RuleBasedBreakIterator *)t->bi)->getRuleStatus();
if (rs != expectedTagVal) {
errln("Incorrect status for forward break. Pos=%4d File line,col= %4d,%4d.\n"
" Actual, Expected status = %4d, %4d",
bp, line, t->srcCol->elementAti(bp), rs, expectedTagVal);
}
}
prevBP = bp;
}
// Verify that there were no missed expected breaks after the last one found
for (i=prevBP+1; i<t->expectedBreaks->size(); i++) {
if (t->expectedBreaks->elementAti(i) != 0) {
errln("Forward Iteration, break expected, but not found. Pos=%4d File line,col= %4d,%4d",
i, t->srcLine->elementAti(i), t->srcCol->elementAti(i));
}
}
//
// Run the iterator backwards, verify that the same breaks are found.
//
prevBP = t->dataToBreak.length()+2; // start with a phony value for the last break pos seen.
for (bp = t->bi->last(); bp != BreakIterator::DONE; bp = t->bi->previous()) {
if (prevBP == bp) {
// Fail for lack of progress.
errln("Reverse Iteration, no progress. Break Pos=%4d File line,col=%4d,%4d",
bp, t->srcLine->elementAti(bp), t->srcCol->elementAti(bp));
break;
}
// Check that there were we didn't miss an expected break between the last one
// and this one. (UVector returns zeros for index out of bounds.)
for (i=prevBP-1; i>bp; i--) {
if (t->expectedBreaks->elementAti(i) != 0) {
errln("Reverse Itertion, break expected, but not found. Pos=%4d File line,col= %4d,%4d",
i, t->srcLine->elementAti(i), t->srcCol->elementAti(i));
}
}
// Check that the break we did find was expected
if (t->expectedBreaks->elementAti(bp) == 0) {
errln("Reverse Itertion, break found, but not expected. Pos=%4d File line,col= %4d,%4d",
bp, t->srcLine->elementAti(bp), t->srcCol->elementAti(bp));
} else {
// The break was expected.
// Check that the {nnn} tag value is correct.
int32_t expectedTagVal = t->expectedBreaks->elementAti(bp);
if (expectedTagVal == -1) {
expectedTagVal = 0;
}
int line = t->srcLine->elementAti(bp);
int32_t rs = ((RuleBasedBreakIterator *)t->bi)->getRuleStatus();
if (rs != expectedTagVal) {
errln("Incorrect status for reverse break. Pos=%4d File line,col= %4d,%4d.\n"
" Actual, Expected status = %4d, %4d",
bp, line, t->srcCol->elementAti(bp), rs, expectedTagVal);
}
}
prevBP = bp;
}
// Verify that there were no missed breaks prior to the last one found
for (i=prevBP-1; i>=0; i--) {
if (t->expectedBreaks->elementAti(i) != 0) {
errln("Forward Itertion, break expected, but not found. Pos=%4d File line,col= %4d,%4d",
i, t->srcLine->elementAti(i), t->srcCol->elementAti(i));
}
}
}
void RBBITest::TestExtended() {
#if !UCONFIG_NO_REGULAR_EXPRESSIONS
UErrorCode status = U_ZERO_ERROR;
Locale locale("");
UnicodeString rules;
TestParams tp;
tp.bi = NULL;
tp.expectedBreaks = new UVector32(status);
tp.srcLine = new UVector32(status);
tp.srcCol = new UVector32(status);
RegexMatcher localeMatcher(UNICODE_STRING_SIMPLE("<locale *([\\p{L}\\p{Nd}_]*) *>"), 0, status);
if (U_FAILURE(status)) {
dataerrln("Failure in file %s, line %d, status = \"%s\"", __FILE__, __LINE__, u_errorName(status));
}
//
// Open and read the test data file.
//
const char *testDataDirectory = IntlTest::getSourceTestData(status);
char testFileName[1000];
if (testDataDirectory == NULL || strlen(testDataDirectory) >= sizeof(testFileName)) {
errln("Can't open test data. Path too long.");
return;
}
strcpy(testFileName, testDataDirectory);
strcat(testFileName, "rbbitst.txt");
int len;
UChar *testFile = ReadAndConvertFile(testFileName, len, "UTF-8", status);
if (U_FAILURE(status)) {
return; /* something went wrong, error already output */
}
//
// Put the test data into a UnicodeString
//
UnicodeString testString(FALSE, testFile, len);
enum EParseState{
PARSE_COMMENT,
PARSE_TAG,
PARSE_DATA,
PARSE_NUM
}
parseState = PARSE_TAG;
EParseState savedState = PARSE_TAG;
static const UChar CH_LF = 0x0a;
static const UChar CH_CR = 0x0d;
static const UChar CH_HASH = 0x23;
/*static const UChar CH_PERIOD = 0x2e;*/
static const UChar CH_LT = 0x3c;
static const UChar CH_GT = 0x3e;
static const UChar CH_BACKSLASH = 0x5c;
static const UChar CH_BULLET = 0x2022;
int32_t lineNum = 1;
int32_t colStart = 0;
int32_t column = 0;
int32_t charIdx = 0;
int32_t tagValue = 0; // The numeric value of a <nnn> tag.
for (charIdx = 0; charIdx < len; ) {
status = U_ZERO_ERROR;
UChar c = testString.charAt(charIdx);
charIdx++;
if (c == CH_CR && charIdx<len && testString.charAt(charIdx) == CH_LF) {
// treat CRLF as a unit
c = CH_LF;
charIdx++;
}
if (c == CH_LF || c == CH_CR) {
lineNum++;
colStart = charIdx;
}
column = charIdx - colStart + 1;
switch (parseState) {
case PARSE_COMMENT:
if (c == 0x0a || c == 0x0d) {
parseState = savedState;
}
break;
case PARSE_TAG:
{
if (c == CH_HASH) {
parseState = PARSE_COMMENT;
savedState = PARSE_TAG;
break;
}
if (u_isUWhiteSpace(c)) {
break;
}
if (testString.compare(charIdx-1, 6, "<word>") == 0) {
delete tp.bi;
tp.bi = BreakIterator::createWordInstance(locale, status);
charIdx += 5;
break;
}
if (testString.compare(charIdx-1, 6, "<char>") == 0) {
delete tp.bi;
tp.bi = BreakIterator::createCharacterInstance(locale, status);
charIdx += 5;
break;
}
if (testString.compare(charIdx-1, 6, "<line>") == 0) {
delete tp.bi;
tp.bi = BreakIterator::createLineInstance(locale, status);
charIdx += 5;
break;
}
if (testString.compare(charIdx-1, 6, "<sent>") == 0) {
delete tp.bi;
tp.bi = NULL;
tp.bi = BreakIterator::createSentenceInstance(locale, status);
charIdx += 5;
break;
}
if (testString.compare(charIdx-1, 7, "<title>") == 0) {
delete tp.bi;
tp.bi = BreakIterator::createTitleInstance(locale, status);
charIdx += 6;
break;
}
// <locale loc_name>
localeMatcher.reset(testString);
if (localeMatcher.lookingAt(charIdx-1, status)) {
UnicodeString localeName = localeMatcher.group(1, status);
char localeName8[100];
localeName.extract(0, localeName.length(), localeName8, sizeof(localeName8), 0);
locale = Locale::createFromName(localeName8);
charIdx += localeMatcher.group(0, status).length();
TEST_ASSERT_SUCCESS(status);
break;
}
if (testString.compare(charIdx-1, 6, "<data>") == 0) {
parseState = PARSE_DATA;
charIdx += 5;
tp.dataToBreak = "";
tp.expectedBreaks->removeAllElements();
tp.srcCol ->removeAllElements();
tp.srcLine->removeAllElements();
break;
}
errln("line %d: Tag expected in test file.", lineNum);
parseState = PARSE_COMMENT;
savedState = PARSE_DATA;
goto end_test; // Stop the test.
}
break;
case PARSE_DATA:
if (c == CH_BULLET) {
int32_t breakIdx = tp.dataToBreak.length();
tp.expectedBreaks->setSize(breakIdx+1);
tp.expectedBreaks->setElementAt(-1, breakIdx);
tp.srcLine->setSize(breakIdx+1);
tp.srcLine->setElementAt(lineNum, breakIdx);
tp.srcCol ->setSize(breakIdx+1);
tp.srcCol ->setElementAt(column, breakIdx);
break;
}
if (testString.compare(charIdx-1, 7, "</data>") == 0) {
// Add final entry to mappings from break location to source file position.
// Need one extra because last break position returned is after the
// last char in the data, not at the last char.
tp.srcLine->addElement(lineNum, status);
tp.srcCol ->addElement(column, status);
parseState = PARSE_TAG;
charIdx += 6;
// RUN THE TEST!
executeTest(&tp);
break;
}
if (testString.compare(charIdx-1, 3, UNICODE_STRING_SIMPLE("\\N{")) == 0) {
// Named character, e.g. \N{COMBINING GRAVE ACCENT}
// Get the code point from the name and insert it into the test data.
// (Damn, no API takes names in Unicode !!!
// we've got to take it back to char *)
int32_t nameEndIdx = testString.indexOf((UChar)0x7d/*'}'*/, charIdx);
int32_t nameLength = nameEndIdx - (charIdx+2);
char charNameBuf[200];
UChar32 theChar = -1;
if (nameEndIdx != -1) {
UErrorCode status = U_ZERO_ERROR;
testString.extract(charIdx+2, nameLength, charNameBuf, sizeof(charNameBuf));
charNameBuf[sizeof(charNameBuf)-1] = 0;
theChar = u_charFromName(U_UNICODE_CHAR_NAME, charNameBuf, &status);
if (U_FAILURE(status)) {
theChar = -1;
}
}
if (theChar == -1) {
errln("Error in named character in test file at line %d, col %d",
lineNum, column);
} else {
// Named code point was recognized. Insert it
// into the test data.
tp.dataToBreak.append(theChar);
while (tp.dataToBreak.length() > tp.srcLine->size()) {
tp.srcLine->addElement(lineNum, status);
tp.srcCol ->addElement(column, status);
}
}
if (nameEndIdx > charIdx) {
charIdx = nameEndIdx+1;
}
break;
}
if (testString.compare(charIdx-1, 2, "<>") == 0) {
charIdx++;
int32_t breakIdx = tp.dataToBreak.length();
tp.expectedBreaks->setSize(breakIdx+1);
tp.expectedBreaks->setElementAt(-1, breakIdx);
tp.srcLine->setSize(breakIdx+1);
tp.srcLine->setElementAt(lineNum, breakIdx);
tp.srcCol ->setSize(breakIdx+1);
tp.srcCol ->setElementAt(column, breakIdx);
break;
}
if (c == CH_LT) {
tagValue = 0;
parseState = PARSE_NUM;
break;
}
if (c == CH_HASH && column==3) { // TODO: why is column off so far?
parseState = PARSE_COMMENT;
savedState = PARSE_DATA;
break;
}
if (c == CH_BACKSLASH) {
// Check for \ at end of line, a line continuation.
// Advance over (discard) the newline
UChar32 cp = testString.char32At(charIdx);
if (cp == CH_CR && charIdx<len && testString.charAt(charIdx+1) == CH_LF) {
// We have a CR LF
// Need an extra increment of the input ptr to move over both of them
charIdx++;
}
if (cp == CH_LF || cp == CH_CR) {
lineNum++;
colStart = charIdx;
charIdx++;
break;
}
// Let unescape handle the back slash.
cp = testString.unescapeAt(charIdx);
if (cp != -1) {
// Escape sequence was recognized. Insert the char
// into the test data.
tp.dataToBreak.append(cp);
while (tp.dataToBreak.length() > tp.srcLine->size()) {
tp.srcLine->addElement(lineNum, status);
tp.srcCol ->addElement(column, status);
}
break;
}
// Not a recognized backslash escape sequence.
// Take the next char as a literal.
// TODO: Should this be an error?
c = testString.charAt(charIdx);
charIdx = testString.moveIndex32(charIdx, 1);
}
// Normal, non-escaped data char.
tp.dataToBreak.append(c);
// Save the mapping from offset in the data to line/column numbers in
// the original input file. Will be used for better error messages only.
// If there's an expected break before this char, the slot in the mapping
// vector will already be set for this char; don't overwrite it.
if (tp.dataToBreak.length() > tp.srcLine->size()) {
tp.srcLine->addElement(lineNum, status);
tp.srcCol ->addElement(column, status);
}
break;
case PARSE_NUM:
// We are parsing an expected numeric tag value, like <1234>,
// within a chunk of data.
if (u_isUWhiteSpace(c)) {
break;
}
if (c == CH_GT) {
// Finished the number. Add the info to the expected break data,
// and switch parse state back to doing plain data.
parseState = PARSE_DATA;
if (tagValue == 0) {
tagValue = -1;
}
int32_t breakIdx = tp.dataToBreak.length();
tp.expectedBreaks->setSize(breakIdx+1);
tp.expectedBreaks->setElementAt(tagValue, breakIdx);
tp.srcLine->setSize(breakIdx+1);
tp.srcLine->setElementAt(lineNum, breakIdx);
tp.srcCol ->setSize(breakIdx+1);
tp.srcCol ->setElementAt(column, breakIdx);
break;
}
if (u_isdigit(c)) {
tagValue = tagValue*10 + u_charDigitValue(c);
break;
}
errln("Syntax Error in test file at line %d, col %d",
lineNum, column);
parseState = PARSE_COMMENT;
goto end_test; // Stop the test
break;
}
if (U_FAILURE(status)) {
errln("ICU Error %s while parsing test file at line %d.",
u_errorName(status), lineNum);
status = U_ZERO_ERROR;
goto end_test; // Stop the test
}
}
end_test:
delete tp.bi;
delete tp.expectedBreaks;
delete tp.srcLine;
delete tp.srcCol;
delete [] testFile;
#endif
}
void RBBITest::TestThaiBreaks() {
UErrorCode status=U_ZERO_ERROR;
BreakIterator* b;
Locale locale = Locale("th");
int32_t p, index;
UChar c[]= {
0x0E01, 0x0E39, 0x0020, 0x0E01, 0x0E34, 0x0E19, 0x0E01, 0x0E38, 0x0E49, 0x0E07, 0x0020, 0x0E1B,
0x0E34, 0x0E49, 0x0E48, 0x0E07, 0x0E2D, 0x0E22, 0x0E39, 0x0E48, 0x0E43, 0x0E19,
0x0E16, 0x0E49, 0x0E33
};
int32_t expectedWordResult[] = {
2, 3, 6, 10, 11, 15, 17, 20, 22
};
int32_t expectedLineResult[] = {
3, 6, 11, 15, 17, 20, 22
};
int32_t size = sizeof(c)/sizeof(UChar);
UnicodeString text=UnicodeString(c);
b = BreakIterator::createWordInstance(locale, status);
if (U_FAILURE(status)) {
errcheckln(status, "Unable to create thai word break iterator. - %s", u_errorName(status));
return;
}
b->setText(text);
p = index = 0;
while ((p=b->next())!=BreakIterator::DONE && p < size) {
if (p != expectedWordResult[index++]) {
errln("Incorrect break given by thai word break iterator. Expected: %d Got: %d", expectedWordResult[index-1], p);
}
}
delete b;
b = BreakIterator::createLineInstance(locale, status);
if (U_FAILURE(status)) {
printf("Unable to create thai line break iterator.\n");
return;
}
b->setText(text);
p = index = 0;
while ((p=b->next())!=BreakIterator::DONE && p < size) {
if (p != expectedLineResult[index++]) {
errln("Incorrect break given by thai line break iterator. Expected: %d Got: %d", expectedLineResult[index-1], p);
}
}
delete b;
}
// UBreakIteratorType UBRK_WORD, Locale "en_US_POSIX"
// Words don't include colon or period (cldrbug #1969).
static const char posxWordText[] = "Can't have breaks in xx:yy or struct.field for CS-types.";
static const int32_t posxWordTOffsets[] = { 5, 6, 10, 11, 17, 18, 20, 21, 23, 24, 26, 27, 29, 30, 36, 37, 42, 43, 46, 47, 49, 50, 55, 56 };
static const int32_t posxWordROffsets[] = { 5, 6, 10, 11, 17, 18, 20, 21, 26, 27, 29, 30, 42, 43, 46, 47, 49, 50, 55, 56 };
// UBreakIteratorType UBRK_WORD, Locale "ja"
// Don't break in runs of hiragana or runs of ideograph, where the latter includes \u3005 \u3007 \u303B (cldrbug #2009).
static const char jaWordText[] = "\\u79C1\\u9054\\u306B\\u4E00\\u3007\\u3007\\u3007\\u306E\\u30B3\\u30F3\\u30D4\\u30E5\\u30FC\\u30BF"
"\\u304C\\u3042\\u308B\\u3002\\u5948\\u3005\\u306F\\u30EF\\u30FC\\u30C9\\u3067\\u3042\\u308B\\u3002";
static const int32_t jaWordTOffsets[] = { 2, 3, 7, 8, 14, 17, 18, 20, 21, 24, 27, 28 };
static const int32_t jaWordROffsets[] = { 1, 2, 3, 4, 5, 6, 7, 8, 14, 15, 16, 17, 18, 19, 20, 21, 24, 25, 26, 27, 28 };
// UBreakIteratorType UBRK_SENTENCE, Locale "el"
// Add break after Greek question mark (cldrbug #2069).
static const char elSentText[] = "\\u0391\\u03B2, \\u03B3\\u03B4; \\u0395 \\u03B6\\u03B7\\u037E \\u0398 \\u03B9\\u03BA. "
"\\u039B\\u03BC \\u03BD\\u03BE! \\u039F\\u03C0, \\u03A1\\u03C2? \\u03A3";
static const int32_t elSentTOffsets[] = { 8, 14, 20, 27, 35, 36 };
static const int32_t elSentROffsets[] = { 20, 27, 35, 36 };
// UBreakIteratorType UBRK_CHARACTER, Locale "th"
// Clusters should not include spacing Thai/Lao vowels (prefix or postfix), except for [SARA] AM (cldrbug #2161).
static const char thCharText[] = "\\u0E01\\u0E23\\u0E30\\u0E17\\u0E48\\u0E2D\\u0E21\\u0E23\\u0E08\\u0E19\\u0E32 "
"(\\u0E2A\\u0E38\\u0E0A\\u0E32\\u0E15\\u0E34-\\u0E08\\u0E38\\u0E11\\u0E32\\u0E21\\u0E32\\u0E28) "
"\\u0E40\\u0E14\\u0E47\\u0E01\\u0E21\\u0E35\\u0E1B\\u0E31\\u0E0D\\u0E2B\\u0E32 ";
static const int32_t thCharTOffsets[] = { 1, 2, 3, 5, 6, 7, 8, 9, 10, 11,
12, 13, 15, 16, 17, 19, 20, 22, 23, 24, 25, 26, 27, 28,
29, 30, 32, 33, 35, 37, 38, 39, 40, 41 };
static const int32_t thCharROffsets[] = { 1, 3, 5, 6, 7, 8, 9, 11,
12, 13, 15, 17, 19, 20, 22, 24, 26, 27, 28,
29, 32, 33, 35, 37, 38, 40, 41 };
typedef struct {
UBreakIteratorType type;
const char * locale;
const char * escapedText;
const int32_t * tailoredOffsets;
int32_t tailoredOffsetsCount;
const int32_t * rootOffsets;
int32_t rootOffsetsCount;
} TailoredBreakItem;
#define ARRAY_PTR_LEN(array) (array),(sizeof(array)/sizeof(array[0]))
static const TailoredBreakItem tbItems[] = {
{ UBRK_WORD, "en_US_POSIX", posxWordText, ARRAY_PTR_LEN(posxWordTOffsets), ARRAY_PTR_LEN(posxWordROffsets) },
{ UBRK_WORD, "ja", jaWordText, ARRAY_PTR_LEN(jaWordTOffsets), ARRAY_PTR_LEN(jaWordROffsets) },
{ UBRK_SENTENCE, "el", elSentText, ARRAY_PTR_LEN(elSentTOffsets), ARRAY_PTR_LEN(elSentROffsets) },
{ UBRK_CHARACTER, "th", thCharText, ARRAY_PTR_LEN(thCharTOffsets), ARRAY_PTR_LEN(thCharROffsets) },
{ UBRK_CHARACTER, NULL, NULL, NULL,0, NULL,0 } // terminator
};
static void formatOffsets(char* buffer, int32_t buflen, int32_t count, const int32_t* offsets) {
while (count-- > 0) {
int writeCount;
sprintf(buffer, /* buflen, */ " %d%n", *offsets++, &writeCount); /* wants to be snprintf */
buffer += writeCount;
buflen -= writeCount;
}
}
enum { kMaxOffsetCount = 128 };
void RBBITest::TBTest(BreakIterator* brkitr, int type, const char *locale, const char* escapedText, const int32_t *expectOffsets, int32_t expectOffsetsCount) {
brkitr->setText( CharsToUnicodeString(escapedText) );
int32_t foundOffsets[kMaxOffsetCount];
int32_t offset, foundOffsetsCount = 0;
// do forwards iteration test
while ( foundOffsetsCount < kMaxOffsetCount && (offset = brkitr->next()) != BreakIterator::DONE ) {
foundOffsets[foundOffsetsCount++] = offset;
}
if ( foundOffsetsCount != expectOffsetsCount || memcmp(expectOffsets, foundOffsets, foundOffsetsCount*sizeof(foundOffsets[0])) != 0 ) {
// log error for forwards test
char formatExpect[512], formatFound[512];
formatOffsets(formatExpect, sizeof(formatExpect), expectOffsetsCount, expectOffsets);
formatOffsets(formatFound, sizeof(formatFound), foundOffsetsCount, foundOffsets);
errln("For type %d %-5s, text \"%.16s\"...; expect %d offsets:%s; found %d offsets fwd:%s\n",
type, locale, escapedText, expectOffsetsCount, formatExpect, foundOffsetsCount, formatFound);
} else {
// do backwards iteration test
--foundOffsetsCount; // back off one from the end offset
while ( foundOffsetsCount > 0 ) {
offset = brkitr->previous();
if ( offset != foundOffsets[--foundOffsetsCount] ) {
// log error for backwards test
char formatExpect[512];
formatOffsets(formatExpect, sizeof(formatExpect), expectOffsetsCount, expectOffsets);
errln("For type %d %-5s, text \"%.16s\"...; expect %d offsets:%s; found rev offset %d where expect %d\n",
type, locale, escapedText, expectOffsetsCount, formatExpect, offset, foundOffsets[foundOffsetsCount]);
break;
}
}
}
}
void RBBITest::TestTailoredBreaks() {
const TailoredBreakItem * tbItemPtr;
Locale rootLocale = Locale("root");
for (tbItemPtr = tbItems; tbItemPtr->escapedText != NULL; ++tbItemPtr) {
Locale testLocale = Locale(tbItemPtr->locale);
BreakIterator * tailoredBrkiter;
BreakIterator * rootBrkiter;
UErrorCode status = U_ZERO_ERROR;
switch (tbItemPtr->type) {
case UBRK_CHARACTER:
tailoredBrkiter = BreakIterator::createCharacterInstance(testLocale, status);
rootBrkiter = BreakIterator::createCharacterInstance(rootLocale, status);
break;
case UBRK_WORD:
tailoredBrkiter = BreakIterator::createWordInstance(testLocale, status);
rootBrkiter = BreakIterator::createWordInstance(rootLocale, status);
break;
case UBRK_LINE:
tailoredBrkiter = BreakIterator::createLineInstance(testLocale, status);
rootBrkiter = BreakIterator::createLineInstance(rootLocale, status);
break;
case UBRK_SENTENCE:
tailoredBrkiter = BreakIterator::createSentenceInstance(testLocale, status);
rootBrkiter = BreakIterator::createSentenceInstance(rootLocale, status);
break;
default:
status = U_UNSUPPORTED_ERROR;
break;
}
if (U_FAILURE(status)) {
errcheckln(status, "BreakIterator create failed for type %d, locales root or %s - Error: %s", (int)(tbItemPtr->type), tbItemPtr->locale, u_errorName(status));
continue;
}
TBTest(tailoredBrkiter, (int)(tbItemPtr->type), tbItemPtr->locale, tbItemPtr->escapedText, tbItemPtr->tailoredOffsets, tbItemPtr->tailoredOffsetsCount);
TBTest(rootBrkiter, (int)(tbItemPtr->type), "root", tbItemPtr->escapedText, tbItemPtr->rootOffsets, tbItemPtr->rootOffsetsCount);
delete rootBrkiter;
delete tailoredBrkiter;
}
}
//-------------------------------------------------------------------------------
//
// ReadAndConvertFile Read a text data file, convert it to UChars, and
// return the datain one big UChar * buffer, which the caller must delete.
//
// parameters:
// fileName: the name of the file, with no directory part. The test data directory
// is assumed.
// ulen an out parameter, receives the actual length (in UChars) of the file data.
// encoding The file encoding. If the file contains a BOM, that will override the encoding
// specified here. The BOM, if it exists, will be stripped from the returned data.
// Pass NULL for the system default encoding.
// status
// returns:
// The file data, converted to UChar.
// The caller must delete this when done with
// delete [] theBuffer;
//
// TODO: This is a clone of RegexTest::ReadAndConvertFile.
// Move this function to some common place.
//
//--------------------------------------------------------------------------------
UChar *RBBITest::ReadAndConvertFile(const char *fileName, int &ulen, const char *encoding, UErrorCode &status) {
UChar *retPtr = NULL;
char *fileBuf = NULL;
UConverter* conv = NULL;
FILE *f = NULL;
ulen = 0;
if (U_FAILURE(status)) {
return retPtr;
}
//
// Open the file.
//
f = fopen(fileName, "rb");
if (f == 0) {
dataerrln("Error opening test data file %s\n", fileName);
status = U_FILE_ACCESS_ERROR;
return NULL;
}
//
// Read it in
//
int fileSize;
int amt_read;
fseek( f, 0, SEEK_END);
fileSize = ftell(f);
fileBuf = new char[fileSize];
fseek(f, 0, SEEK_SET);
amt_read = fread(fileBuf, 1, fileSize, f);
if (amt_read != fileSize || fileSize <= 0) {
errln("Error reading test data file.");
goto cleanUpAndReturn;
}
//
// Look for a Unicode Signature (BOM) on the data just read
//
int32_t signatureLength;
const char * fileBufC;
const char* bomEncoding;
fileBufC = fileBuf;
bomEncoding = ucnv_detectUnicodeSignature(
fileBuf, fileSize, &signatureLength, &status);
if(bomEncoding!=NULL ){
fileBufC += signatureLength;
fileSize -= signatureLength;
encoding = bomEncoding;
}
//
// Open a converter to take the rule file to UTF-16
//
conv = ucnv_open(encoding, &status);
if (U_FAILURE(status)) {
goto cleanUpAndReturn;
}
//
// Convert the rules to UChar.
// Preflight first to determine required buffer size.
//
ulen = ucnv_toUChars(conv,
NULL, // dest,
0, // destCapacity,
fileBufC,
fileSize,
&status);
if (status == U_BUFFER_OVERFLOW_ERROR) {
// Buffer Overflow is expected from the preflight operation.
status = U_ZERO_ERROR;
retPtr = new UChar[ulen+1];
ucnv_toUChars(conv,
retPtr, // dest,
ulen+1,
fileBufC,
fileSize,
&status);
}
cleanUpAndReturn:
fclose(f);
delete []fileBuf;
ucnv_close(conv);
if (U_FAILURE(status)) {
errln("ucnv_toUChars: ICU Error \"%s\"\n", u_errorName(status));
delete retPtr;
retPtr = 0;
ulen = 0;
};
return retPtr;
}
//--------------------------------------------------------------------------------------------
//
// Run tests from each of the boundary test data files distributed by the Unicode Consortium
//
//-------------------------------------------------------------------------------------------
void RBBITest::TestUnicodeFiles() {
RuleBasedBreakIterator *bi;
UErrorCode status = U_ZERO_ERROR;
bi = (RuleBasedBreakIterator *)BreakIterator::createCharacterInstance(Locale::getDefault(), status);
TEST_ASSERT_SUCCESS(status);
if (U_SUCCESS(status)) {
runUnicodeTestData("GraphemeBreakTest.txt", bi);
}
delete bi;
bi = (RuleBasedBreakIterator *)BreakIterator::createWordInstance(Locale::getDefault(), status);
TEST_ASSERT_SUCCESS(status);
if (U_SUCCESS(status)) {
runUnicodeTestData("WordBreakTest.txt", bi);
}
delete bi;
bi = (RuleBasedBreakIterator *)BreakIterator::createSentenceInstance(Locale::getDefault(), status);
TEST_ASSERT_SUCCESS(status);
if (U_SUCCESS(status)) {
runUnicodeTestData("SentenceBreakTest.txt", bi);
}
delete bi;
bi = (RuleBasedBreakIterator *)BreakIterator::createLineInstance(Locale::getDefault(), status);
TEST_ASSERT_SUCCESS(status);
if (U_SUCCESS(status)) {
runUnicodeTestData("LineBreakTest.txt", bi);
}
delete bi;
}
//--------------------------------------------------------------------------------------------
//
// Run tests from one of the boundary test data files distributed by the Unicode Consortium
//
//-------------------------------------------------------------------------------------------
void RBBITest::runUnicodeTestData(const char *fileName, RuleBasedBreakIterator *bi) {
#if !UCONFIG_NO_REGULAR_EXPRESSIONS
UErrorCode status = U_ZERO_ERROR;
//
// Open and read the test data file, put it into a UnicodeString.
//
const char *testDataDirectory = IntlTest::getSourceTestData(status);
char testFileName[1000];
if (testDataDirectory == NULL || strlen(testDataDirectory) >= sizeof(testFileName)) {
dataerrln("Can't open test data. Path too long.");
return;
}
strcpy(testFileName, testDataDirectory);
strcat(testFileName, fileName);
logln("Opening data file %s\n", fileName);
int len;
UChar *testFile = ReadAndConvertFile(testFileName, len, "UTF-8", status);
if (status != U_FILE_ACCESS_ERROR) {
TEST_ASSERT_SUCCESS(status);
TEST_ASSERT(testFile != NULL);
}
if (U_FAILURE(status) || testFile == NULL) {
return; /* something went wrong, error already output */
}
UnicodeString testFileAsString(TRUE, testFile, len);
//
// Parse the test data file using a regular expression.
// Each kind of token is recognized in its own capture group; what type of item was scanned
// is identified by which group had a match.
//
// Caputure Group # 1 2 3 4 5
// Parses this item: divide x hex digits comment \n unrecognized \n
//
UnicodeString tokenExpr("[ \t]*(?:(\\u00F7)|(\\u00D7)|([0-9a-fA-F]+)|((?:#.*?)?$.)|(.*?$.))", -1, US_INV);
RegexMatcher tokenMatcher(tokenExpr, testFileAsString, UREGEX_MULTILINE | UREGEX_DOTALL, status);
UnicodeString testString;
UVector32 breakPositions(status);
int lineNumber = 1;
TEST_ASSERT_SUCCESS(status);
if (U_FAILURE(status)) {
return;
}
//
// Scan through each test case, building up the string to be broken in testString,
// and the positions that should be boundaries in the breakPositions vector.
//
while (tokenMatcher.find()) {
if (tokenMatcher.start(1, status) >= 0) {
// Scanned a divide sign, indicating a break position in the test data.
if (testString.length()>0) {
breakPositions.addElement(testString.length(), status);
}
}
else if (tokenMatcher.start(2, status) >= 0) {
// Scanned an 'x', meaning no break at this position in the test data
// Nothing to be done here.
}
else if (tokenMatcher.start(3, status) >= 0) {
// Scanned Hex digits. Convert them to binary, append to the character data string.
const UnicodeString &hexNumber = tokenMatcher.group(3, status);
int length = hexNumber.length();
if (length<=8) {
char buf[10];
hexNumber.extract (0, length, buf, sizeof(buf), US_INV);
UChar32 c = (UChar32)strtol(buf, NULL, 16);
if (c<=0x10ffff) {
testString.append(c);
} else {
errln("Error: Unicode Character value out of range. \'%s\', line %d.\n",
fileName, lineNumber);
}
} else {
errln("Syntax Error: Hex Unicode Character value must have no more than 8 digits at \'%s\', line %d.\n",
fileName, lineNumber);
}
}
else if (tokenMatcher.start(4, status) >= 0) {
// Scanned to end of a line, possibly skipping over a comment in the process.
// If the line from the file contained test data, run the test now.
//
if (testString.length() > 0) {
checkUnicodeTestCase(fileName, lineNumber, testString, &breakPositions, bi);
}
// Clear out this test case.
// The string and breakPositions vector will be refilled as the next
// test case is parsed.
testString.remove();
breakPositions.removeAllElements();
lineNumber++;
} else {
// Scanner catchall. Something unrecognized appeared on the line.
char token[16];
UnicodeString uToken = tokenMatcher.group(0, status);
uToken.extract(0, uToken.length(), token, (uint32_t)sizeof(token));
token[sizeof(token)-1] = 0;
errln("Syntax error in test data file \'%s\', line %d. Scanning \"%s\"\n", fileName, lineNumber, token);
// Clean up, in preparation for continuing with the next line.
testString.remove();
breakPositions.removeAllElements();
lineNumber++;
}
TEST_ASSERT_SUCCESS(status);
if (U_FAILURE(status)) {
break;
}
}
delete [] testFile;
#endif // !UCONFIG_NO_REGULAR_EXPRESSIONS
}
//--------------------------------------------------------------------------------------------
//
// checkUnicodeTestCase() Run one test case from one of the Unicode Consortium
// test data files. Do only a simple, forward-only check -
// this test is mostly to check that ICU and the Unicode
// data agree with each other.
//
//--------------------------------------------------------------------------------------------
void RBBITest::checkUnicodeTestCase(const char *testFileName, int lineNumber,
const UnicodeString &testString, // Text data to be broken
UVector32 *breakPositions, // Positions where breaks should be found.
RuleBasedBreakIterator *bi) {
int32_t pos; // Break Position in the test string
int32_t expectedI = 0; // Index of expected break position in the vector of expected results.
int32_t expectedPos; // Expected break position (index into test string)
bi->setText(testString);
pos = bi->first();
pos = bi->next();
while (pos != BreakIterator::DONE) {
if (expectedI >= breakPositions->size()) {
errln("Test file \"%s\", line %d, unexpected break found at position %d",
testFileName, lineNumber, pos);
break;
}
expectedPos = breakPositions->elementAti(expectedI);
if (pos < expectedPos) {
errln("Test file \"%s\", line %d, unexpected break found at position %d",
testFileName, lineNumber, pos);
break;
}
if (pos > expectedPos) {
errln("Test file \"%s\", line %d, failed to find expected break at position %d",
testFileName, lineNumber, expectedPos);
break;
}
pos = bi->next();
expectedI++;
}
if (pos==BreakIterator::DONE && expectedI<breakPositions->size()) {
errln("Test file \"%s\", line %d, failed to find expected break at position %d",
testFileName, lineNumber, breakPositions->elementAti(expectedI));
}
}
#if !UCONFIG_NO_REGULAR_EXPRESSIONS
//---------------------------------------------------------------------------------------
//
// classs RBBIMonkeyKind
//
// Monkey Test for Break Iteration
// Abstract interface class. Concrete derived classes independently
// implement the break rules for different iterator types.
//
// The Monkey Test itself uses doesn't know which type of break iterator it is
// testing, but works purely in terms of the interface defined here.
//
//---------------------------------------------------------------------------------------
class RBBIMonkeyKind {
public:
// Return a UVector of UnicodeSets, representing the character classes used
// for this type of iterator.
virtual UVector *charClasses() = 0;
// Set the test text on which subsequent calls to next() will operate
virtual void setText(const UnicodeString &s) = 0;
// Find the next break postion, starting from the prev break position, or from zero.
// Return -1 after reaching end of string.
virtual int32_t next(int32_t i) = 0;
virtual ~RBBIMonkeyKind();
UErrorCode deferredStatus;
protected:
RBBIMonkeyKind();
private:
};
RBBIMonkeyKind::RBBIMonkeyKind() {
deferredStatus = U_ZERO_ERROR;
}
RBBIMonkeyKind::~RBBIMonkeyKind() {
}
//----------------------------------------------------------------------------------------
//
// Random Numbers. Similar to standard lib rand() and srand()
// Not using library to
// 1. Get same results on all platforms.
// 2. Get access to current seed, to more easily reproduce failures.
//
//---------------------------------------------------------------------------------------
static uint32_t m_seed = 1;
static uint32_t m_rand()
{
m_seed = m_seed * 1103515245 + 12345;
return (uint32_t)(m_seed/65536) % 32768;
}
//------------------------------------------------------------------------------------------
//
// class RBBICharMonkey Character (Grapheme Cluster) specific implementation
// of RBBIMonkeyKind.
//
//------------------------------------------------------------------------------------------
class RBBICharMonkey: public RBBIMonkeyKind {
public:
RBBICharMonkey();
virtual ~RBBICharMonkey();
virtual UVector *charClasses();
virtual void setText(const UnicodeString &s);
virtual int32_t next(int32_t i);
private:
UVector *fSets;
UnicodeSet *fCRLFSet;
UnicodeSet *fControlSet;
UnicodeSet *fExtendSet;
UnicodeSet *fPrependSet;
UnicodeSet *fSpacingSet;
UnicodeSet *fLSet;
UnicodeSet *fVSet;
UnicodeSet *fTSet;
UnicodeSet *fLVSet;
UnicodeSet *fLVTSet;
UnicodeSet *fHangulSet;
UnicodeSet *fAnySet;
const UnicodeString *fText;
};
RBBICharMonkey::RBBICharMonkey() {
UErrorCode status = U_ZERO_ERROR;
fText = NULL;
fCRLFSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\r\\n]"), status);
fControlSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Grapheme_Cluster_Break = Control}]"), status);
fExtendSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Grapheme_Cluster_Break = Extend}]"), status);
fPrependSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Grapheme_Cluster_Break = Prepend}]"), status);
fSpacingSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Grapheme_Cluster_Break = SpacingMark}]"), status);
fLSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Grapheme_Cluster_Break = L}]"), status);
fVSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Grapheme_Cluster_Break = V}]"), status);
fTSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Grapheme_Cluster_Break = T}]"), status);
fLVSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Grapheme_Cluster_Break = LV}]"), status);
fLVTSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Grapheme_Cluster_Break = LVT}]"), status);
fHangulSet = new UnicodeSet();
fHangulSet->addAll(*fLSet);
fHangulSet->addAll(*fVSet);
fHangulSet->addAll(*fTSet);
fHangulSet->addAll(*fLVSet);
fHangulSet->addAll(*fLVTSet);
fAnySet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\u0000-\\U0010ffff]"), status);
fSets = new UVector(status);
fSets->addElement(fCRLFSet, status);
fSets->addElement(fControlSet, status);
fSets->addElement(fExtendSet, status);
fSets->addElement(fPrependSet, status);
fSets->addElement(fSpacingSet, status);
fSets->addElement(fHangulSet, status);
fSets->addElement(fAnySet, status);
if (U_FAILURE(status)) {
deferredStatus = status;
}
}
void RBBICharMonkey::setText(const UnicodeString &s) {
fText = &s;
}
int32_t RBBICharMonkey::next(int32_t prevPos) {
int p0, p1, p2, p3; // Indices of the significant code points around the
// break position being tested. The candidate break
// location is before p2.
int breakPos = -1;
UChar32 c0, c1, c2, c3; // The code points at p0, p1, p2 & p3.
if (U_FAILURE(deferredStatus)) {
return -1;
}
// Previous break at end of string. return DONE.
if (prevPos >= fText->length()) {
return -1;
}
p0 = p1 = p2 = p3 = prevPos;
c3 = fText->char32At(prevPos);
c0 = c1 = c2 = 0;
// Loop runs once per "significant" character position in the input text.
for (;;) {
// Move all of the positions forward in the input string.
p0 = p1; c0 = c1;
p1 = p2; c1 = c2;
p2 = p3; c2 = c3;
// Advancd p3 by one codepoint
p3 = fText->moveIndex32(p3, 1);
c3 = fText->char32At(p3);
if (p1 == p2) {
// Still warming up the loop. (won't work with zero length strings, but we don't care)
continue;
}
if (p2 == fText->length()) {
// Reached end of string. Always a break position.
break;
}
// Rule GB3 CR x LF
// No Extend or Format characters may appear between the CR and LF,
// which requires the additional check for p2 immediately following p1.
//
if (c1==0x0D && c2==0x0A && p1==(p2-1)) {
continue;
}
// Rule (GB4). ( Control | CR | LF ) <break>
if (fControlSet->contains(c1) ||
c1 == 0x0D ||
c1 == 0x0A) {
break;
}
// Rule (GB5) <break> ( Control | CR | LF )
//
if (fControlSet->contains(c2) ||
c2 == 0x0D ||
c2 == 0x0A) {
break;
}
// Rule (GB6) L x ( L | V | LV | LVT )
if (fLSet->contains(c1) &&
(fLSet->contains(c2) ||
fVSet->contains(c2) ||
fLVSet->contains(c2) ||
fLVTSet->contains(c2))) {
continue;
}
// Rule (GB7) ( LV | V ) x ( V | T )
if ((fLVSet->contains(c1) || fVSet->contains(c1)) &&
(fVSet->contains(c2) || fTSet->contains(c2))) {
continue;
}
// Rule (GB8) ( LVT | T) x T
if ((fLVTSet->contains(c1) || fTSet->contains(c1)) &&
fTSet->contains(c2)) {
continue;
}
// Rule (GB9) Numeric x ALetter
if (fExtendSet->contains(c2)) {
continue;
}
// Rule (GB9a) x SpacingMark
if (fSpacingSet->contains(c2)) {
continue;
}
// Rule (GB9b) Prepend x
if (fPrependSet->contains(c1)) {
continue;