Collation.cpp

/*
 * PROGRAM: JRD Intl
 * MODULE: Collation.cpp
 * DESCRIPTION: International text support routines
 *
 * copyright (c) 1992, 1993 by Borland International
*/

/************* history ************
*
* COMPONENT: JRD MODULE: INTL.CPP
* generated by Marion V2.5 2/6/90
* from dev db on 4-JAN-1995
*****************************************************************
*
* PR 2002-06-02 Added ugly c hack in
* intl_back_compat_alloc_func_lookup.
* When someone has time we need to change the references to
* return (void*) function to something more C++ like
*
* 42 4711 3 11 17 tamlin 2001
* Added silly numbers before my name, and converted it to C++.
*
* 18850 daves 4-JAN-1995
* Fix gds__alloc usage
*
* 18837 deej 31-DEC-1994
* fixing up HARBOR_MERGE
*
* 18821 deej 27-DEC-1994
* HARBOR MERGE
*
* 18789 jdavid 19-DEC-1994
* Cast some functions
*
* 17508 jdavid 15-JUL-1994
* Bring it up to date
*
* 17500 daves 13-JUL-1994
* Bug 6645: Different calculation of partial keys
*
* 17202 katz 24-MAY-1994
* PC_PLATFORM requires the .dll extension
*
* 17191 katz 23-MAY-1994
* OS/2 requires the .dll extension
*
* 17180 katz 23-MAY-1994
* Define location of DLL on OS/2
*
* 17149 katz 20-MAY-1994
* In JRD, isc_arg_number arguments are SLONG's not int's
*
* 16633 daves 19-APR-1994
* Bug 6202: International licensing uses INTERNATIONAL product code
*
* 16555 katz 17-APR-1994
* The last argument of calls to ERR_post should be 0
*
* 16521 katz 14-APR-1994
* Borland C needs a decorated symbol to lookup
*
* 16403 daves 8-APR-1994
* Bug 6441: Emit an error whenever transliteration from ttype_binary attempted
*
* 16141 katz 28-MAR-1994
* Don't declare return value from ISC_lookup_entrypoint as API_ROUTINE
*
 * The contents of this file are subject to the Interbase Public
 * License Version 1.0 (the "License"); you may not use this file
 * except in compliance with the License. You may obtain a copy
 * of the License at http://www.Inprise.com/IPL.html
 *
 * Software distributed under the License is distributed on an
 * "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, either express
 * or implied. See the License for the specific language governing
 * rights and limitations under the License.
 *
 * The Original Code was created by Inprise Corporation
 * and its predecessors. Portions created by Inprise Corporation are
 * Copyright (C) Inprise Corporation.
 *
 * All Rights Reserved.
 * Contributor(s): ______________________________________.
 *
 * 2002.10.29 Sean Leyne - Removed obsolete "Netware" port
 *
 * 2002.10.30 Sean Leyne - Removed support for obsolete "PC_PLATFORM" define
 *
 * 2006.10.10 Adriano dos Santos Fernandes - refactored from intl.cpp
 *
*/

#include"firebird.h"
#include"iberror.h"
#include"../jrd/jrd.h"
#include"../jrd/err_proto.h"
#include"../jrd/evl_string.h"
#include"../jrd/intl_classes.h"
#include"../jrd/lck_proto.h"
#include"../jrd/intl_classes.h"
#include"../jrd/intl_proto.h"
#include"../jrd/Collation.h"
#include"../common/TextType.h"
#include"../common/SimilarToRegex.h"

usingnamespaceFirebird;
usingnamespaceJrd;


namespace {

classRe2SimilarMatcher : publicPatternMatcher
{
public:
Re2SimilarMatcher(thread_db* tdbb, MemoryPool& pool, TextType* textType,
const UCHAR* patternStr, SLONG patternLen, const UCHAR* escapeStr, SLONG escapeLen)
 : PatternMatcher(pool, textType),
converter(INTL_convert_lookup(tdbb, CS_UTF8, textType->getCharSet()->getId())),
 buffer(pool)
 {
 UCharBuffer patternBuffer, escapeBuffer;

constauto charSetId = textType->getCharSet()->getId();
unsigned flags = 0;

if (charSetId != CS_NONE && charSetId != CS_BINARY)
 {
if (charSetId != CS_UTF8)
 flags |= SimilarToFlag::WELLFORMED;

 flags |= (textType->getAttributes() & TEXTTYPE_ATTR_CASE_INSENSITIVE) ?
 SimilarToFlag::CASE_INSENSITIVE : 0;

 converter.convert(patternLen, patternStr, patternBuffer);

if (textType->getAttributes() & TEXTTYPE_ATTR_ACCENT_INSENSITIVE)
UnicodeUtil::utf8Normalize(patternBuffer);

 patternStr = patternBuffer.begin();
 patternLen = patternBuffer.getCount();

if (escapeStr)
 {
 converter.convert(escapeLen, escapeStr, escapeBuffer);

if (textType->getAttributes() & TEXTTYPE_ATTR_ACCENT_INSENSITIVE)
UnicodeUtil::utf8Normalize(escapeBuffer);

 escapeStr = escapeBuffer.begin();
 escapeLen = escapeBuffer.getCount();
 }
 }
else
 flags |= SimilarToFlag::LATIN;

 regex = FB_NEW_POOL(pool) SimilarToRegex(pool, flags,
 (constchar*) patternStr, patternLen,
 (escapeStr ? (constchar*) escapeStr : nullptr), escapeLen);
 }

public:
static Re2SimilarMatcher* create(thread_db* tdbb, MemoryPool& pool, TextType* textType,
const UCHAR* patternStr, SLONG patternLen, const UCHAR* escapeStr, SLONG escapeLen)
 {
returnFB_NEW_POOL(pool) Re2SimilarMatcher(tdbb, pool, textType, patternStr, patternLen, escapeStr, escapeLen);
 }

staticboolevaluate(thread_db* tdbb, MemoryPool& pool, TextType* textType, const UCHAR* str, SLONG strLen,
const UCHAR* patternStr, SLONG patternLen, const UCHAR* escapeStr, SLONG escapeLen)
 {
 Re2SimilarMatcher matcher(tdbb, pool, textType, patternStr, patternLen, escapeStr, escapeLen);
 matcher.process(str, strLen);
return matcher.result();
 }

public:
virtualvoidreset()
 {
 buffer.shrink(0);
 }

virtualboolprocess(const UCHAR* data, SLONG dataLen)
 {
const FB_SIZE_T pos = buffer.getCount();
memcpy(buffer.getBuffer(pos + dataLen) + pos, data, dataLen);
returntrue;
 }

virtualboolresult()
 {
 UCharBuffer utfBuffer;
constauto charSetId = textType->getCharSet()->getId();
 UCharBuffer* bufferPtr = &buffer;

if (charSetId != CS_NONE && charSetId != CS_BINARY && charSetId != CS_UTF8)
 {
 converter.convert(buffer.getCount(), buffer.begin(), utfBuffer);
 bufferPtr = &utfBuffer;
 }

if (textType->getAttributes() & TEXTTYPE_ATTR_ACCENT_INSENSITIVE)
UnicodeUtil::utf8Normalize(*bufferPtr);

return regex->matches((constchar*) bufferPtr->begin(), bufferPtr->getCount());
 }

private:
 CsConvert converter;
 AutoPtr<SimilarToRegex> regex;
 UCharBuffer buffer;
};

classRe2SubstringSimilarMatcher : publicBaseSubstringSimilarMatcher
{
public:
Re2SubstringSimilarMatcher(thread_db* tdbb, MemoryPool& pool, TextType* textType,
const UCHAR* patternStr, SLONG patternLen, const UCHAR* escapeStr, SLONG escapeLen)
 : BaseSubstringSimilarMatcher(pool, textType),
converter(INTL_convert_lookup(tdbb, CS_UTF8, textType->getCharSet()->getId())),
 buffer(pool),
 resultStart(0),
 resultLength(0)
 {
 UCharBuffer patternBuffer, escapeBuffer;

constauto charSetId = textType->getCharSet()->getId();
unsigned flags = 0;

if (charSetId != CS_NONE && charSetId != CS_BINARY)
 {
if (charSetId != CS_UTF8)
 flags |= SimilarToFlag::WELLFORMED;

 flags |= (textType->getAttributes() & TEXTTYPE_ATTR_CASE_INSENSITIVE) ?
 SimilarToFlag::CASE_INSENSITIVE : 0;

 converter.convert(patternLen, patternStr, patternBuffer);

if (textType->getAttributes() & TEXTTYPE_ATTR_ACCENT_INSENSITIVE)
UnicodeUtil::utf8Normalize(patternBuffer);

 patternStr = patternBuffer.begin();
 patternLen = patternBuffer.getCount();

if (escapeStr)
 {
 converter.convert(escapeLen, escapeStr, escapeBuffer);

if (textType->getAttributes() & TEXTTYPE_ATTR_ACCENT_INSENSITIVE)
UnicodeUtil::utf8Normalize(escapeBuffer);

 escapeStr = escapeBuffer.begin();
 escapeLen = escapeBuffer.getCount();
 }
 }
else
 flags |= SimilarToFlag::LATIN;

 regex = FB_NEW_POOL(pool) SubstringSimilarRegex(pool, flags,
 (constchar*) patternStr, patternLen,
 (escapeStr ? (constchar*) escapeStr : nullptr), escapeLen);
 }

virtual~Re2SubstringSimilarMatcher()
 {
 }

public:
static Re2SubstringSimilarMatcher* create(thread_db* tdbb, MemoryPool& pool, TextType* textType,
const UCHAR* patternStr, SLONG patternLen, const UCHAR* escapeStr, SLONG escapeLen)
 {
returnFB_NEW_POOL(pool) Re2SubstringSimilarMatcher(tdbb, pool, textType,
 patternStr, patternLen, escapeStr, escapeLen);
 }

staticboolevaluate(thread_db* tdbb, MemoryPool& pool, TextType* textType, const UCHAR* str, SLONG strLen,
const UCHAR* patternStr, SLONG patternLen, const UCHAR* escapeStr, SLONG escapeLen)
 {
 Re2SubstringSimilarMatcher matcher(tdbb, pool, textType, patternStr, patternLen, escapeStr, escapeLen);
 matcher.process(str, strLen);
return matcher.result();
 }

public:
virtualvoidreset()
 {
 buffer.shrink(0);
 resultStart = resultLength = 0;
 }

virtualboolprocess(const UCHAR* data, SLONG dataLen)
 {
const FB_SIZE_T pos = buffer.getCount();
memcpy(buffer.getBuffer(pos + dataLen) + pos, data, dataLen);
returntrue;
 }

virtualboolresult()
 {
 UCharBuffer utfBuffer;
constauto charSetId = textType->getCharSet()->getId();
 UCharBuffer* bufferPtr = &buffer;

if (charSetId != CS_NONE && charSetId != CS_BINARY && charSetId != CS_UTF8)
 {
 converter.convert(buffer.getCount(), buffer.begin(), utfBuffer);
 bufferPtr = &utfBuffer;
 }

if (textType->getAttributes() & TEXTTYPE_ATTR_ACCENT_INSENSITIVE)
UnicodeUtil::utf8Normalize(*bufferPtr);

if (!regex->matches((constchar*) bufferPtr->begin(), bufferPtr->getCount(), &resultStart, &resultLength))
returnfalse;

if (charSetId != CS_NONE && charSetId != CS_BINARY)
 {
// Get the character positions in the utf-8 string.
auto utf8CharSet = IntlUtil::getUtf8CharSet();
 resultLength = utf8CharSet->length(resultLength, bufferPtr->begin() + resultStart, true);
 resultStart = utf8CharSet->length(resultStart, bufferPtr->begin(), true);
 }

returntrue;
 }

virtualvoidgetResultInfo(unsigned* start, unsigned* length)
 {
 *start = resultStart;
 *length = resultLength;
 }

private:
 CsConvert converter;
 AutoPtr<SubstringSimilarRegex> regex;
 UCharBuffer buffer;
unsigned resultStart, resultLength;
};

// constants used in matches and sleuth
constint CHAR_GDML_MATCH_ONE = TextType::CHAR_QUESTION_MARK;
constint CHAR_GDML_MATCH_ANY = TextType::CHAR_ASTERISK;
constint CHAR_GDML_QUOTE = TextType::CHAR_AT;
constint CHAR_GDML_NOT = TextType::CHAR_TILDE;
constint CHAR_GDML_RANGE = TextType::CHAR_MINUS;
constint CHAR_GDML_CLASS_START = TextType::CHAR_OPEN_BRACKET;
constint CHAR_GDML_CLASS_END = TextType::CHAR_CLOSE_BRACKET;
constint CHAR_GDML_SUBSTITUTE = TextType::CHAR_EQUAL;
constint CHAR_GDML_FLAG_SET = TextType::CHAR_PLUS;
constint CHAR_GDML_FLAG_CLEAR = TextType::CHAR_MINUS;
constint CHAR_GDML_COMMA = TextType::CHAR_COMMA;
constint CHAR_GDML_LPAREN = TextType::CHAR_OPEN_PAREN;
constint CHAR_GDML_RPAREN = TextType::CHAR_CLOSE_PAREN;

staticconst UCHAR SLEUTH_SPECIAL[128] =
{
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, // $%*+- (dollar, percent, star, plus, minus)
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, // ? (question)
1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // @ (at-sign)
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, // [ (open square)
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0// ~ (tilde)
};

// Below are templates for functions used in Collation implementation

template <typename CharType, typename StrConverter = CanonicalConverter<> >
classLikeMatcher : publicPatternMatcher
{
public:
LikeMatcher(MemoryPool& pool, TextType* ttype, const CharType* str, SLONG str_len,
 CharType escape, bool use_escape, CharType sql_match_any, CharType sql_match_one)
 : PatternMatcher(pool, ttype),
evaluator(pool, str, str_len, escape, use_escape, sql_match_any, sql_match_one)
 {
 }

voidreset()
 {
 evaluator.reset();
 }

boolresult()
 {
return evaluator.getResult();
 }

boolprocess(const UCHAR* str, SLONG length)
 {
 StrConverter cvt(pool, textType, str, length);
fb_assert(length % sizeof(CharType) == 0);
return evaluator.processNextChunk(
reinterpret_cast<const CharType*>(str), length / sizeof(CharType));
 }

static LikeMatcher* create(MemoryPool& pool, TextType* ttype, const UCHAR* str,
 SLONG length, const UCHAR* escape, SLONG escape_length,
const UCHAR* sql_match_any, SLONG match_any_length,
const UCHAR* sql_match_one, SLONG match_one_length)
 {
 StrConverter cvt(pool, ttype, str, length),
cvt_escape(pool, ttype, escape, escape_length),
cvt_match_any(pool, ttype, sql_match_any, match_any_length),
cvt_match_one(pool, ttype, sql_match_one, match_one_length);

fb_assert(length % sizeof(CharType) == 0);
returnFB_NEW_POOL(pool) LikeMatcher(pool, ttype,
reinterpret_cast<const CharType*>(str), length / sizeof(CharType),
 (escape ? *reinterpret_cast<const CharType*>(escape) : 0), escape_length != 0,
 *reinterpret_cast<const CharType*>(sql_match_any),
 *reinterpret_cast<const CharType*>(sql_match_one));
 }

staticboolevaluate(MemoryPool& pool, TextType* ttype, const UCHAR* s, SLONG sl,
const UCHAR* p, SLONG pl, const UCHAR* escape, SLONG escape_length,
const UCHAR* sql_match_any, SLONG match_any_length,
const UCHAR* sql_match_one, SLONG match_one_length)
 {
 StrConverter cvt1(pool, ttype, p, pl),
cvt2(pool, ttype, s, sl),
cvt_escape(pool, ttype, escape, escape_length),
cvt_match_any(pool, ttype, sql_match_any, match_any_length),
cvt_match_one(pool, ttype, sql_match_one, match_one_length);

fb_assert(pl % sizeof(CharType) == 0);
fb_assert(sl % sizeof(CharType) == 0);
 Firebird::LikeEvaluator<CharType> evaluator(pool,
reinterpret_cast<const CharType*>(p), pl / sizeof(CharType),
 (escape ? *reinterpret_cast<const CharType*>(escape) : 0), escape_length != 0,
 *reinterpret_cast<const CharType*>(sql_match_any),
 *reinterpret_cast<const CharType*>(sql_match_one));
 evaluator.processNextChunk(reinterpret_cast<const CharType*>(s), sl / sizeof(CharType));
return evaluator.getResult();
 }

private:
 Firebird::LikeEvaluator<CharType> evaluator;
};

template <typename CharType, typename StrConverter>
classStartsMatcher : publicPatternMatcher
{
public:
StartsMatcher(MemoryPool& pool, TextType* ttype, const CharType* str, SLONG str_len, SLONG aByteLengthLimit)
 : PatternMatcher(pool, ttype),
evaluator(pool, str, str_len)
 {
auto charSet = ttype->getCharSet();

 byteLengthLimit = charSet->isMultiByte() ?
 aByteLengthLimit / charSet->minBytesPerChar() * charSet->maxBytesPerChar() :
 aByteLengthLimit;
 }

voidreset()
 {
 evaluator.reset();
 processedByteLength = 0;
 }

boolresult()
 {
return evaluator.getResult();
 }

boolprocess(const UCHAR* str, SLONG length)
 {
if (processedByteLength + length > byteLengthLimit)
 length = byteLengthLimit - processedByteLength;

 processedByteLength += length;

 StrConverter cvt(pool, textType, str, length);
fb_assert(length % sizeof(CharType) == 0);

return evaluator.processNextChunk(
reinterpret_cast<const CharType*>(str), length / sizeof(CharType));
 }

static StartsMatcher* create(MemoryPool& pool, TextType* ttype,
const UCHAR* str, SLONG length)
 {
constauto byteLengthLimit = length;

 StrConverter cvt(pool, ttype, str, length);
fb_assert(length % sizeof(CharType) == 0);

returnFB_NEW_POOL(pool) StartsMatcher(pool, ttype,
reinterpret_cast<const CharType*>(str), length / sizeof(CharType), byteLengthLimit);
 }

staticboolevaluate(MemoryPool& pool, TextType* ttype, const UCHAR* s, SLONG sl,
const UCHAR* p, SLONG pl)
 {
if (sl > pl)
 {
auto charSet = ttype->getCharSet();

 sl = charSet->isMultiByte() ?
MIN(sl, pl / charSet->minBytesPerChar() * charSet->maxBytesPerChar()) :
 pl;
 }

 StrConverter cvt1(pool, ttype, p, pl);
fb_assert(pl % sizeof(CharType) == 0);

 StrConverter cvt2(pool, ttype, s, sl);
fb_assert(sl % sizeof(CharType) == 0);

 Firebird::StartsEvaluator<CharType> evaluator(pool,
reinterpret_cast<const CharType*>(p), pl / sizeof(CharType));

 evaluator.processNextChunk(reinterpret_cast<const CharType*>(s), sl / sizeof(CharType));

return evaluator.getResult();
 }

private:
 Firebird::StartsEvaluator<CharType> evaluator;
 SLONG byteLengthLimit;
 SLONG processedByteLength = 0;
};

template <typename CharType, typename StrConverter = CanonicalConverter<UpcaseConverter<> > >
classContainsMatcher : publicPatternMatcher
{
public:
ContainsMatcher(MemoryPool& pool, TextType* ttype, const CharType* str, SLONG str_len)
 : PatternMatcher(pool, ttype),
evaluator(pool, str, str_len)
 {
 }

voidreset()
 {
 evaluator.reset();
 }

boolresult()
 {
return evaluator.getResult();
 }

boolprocess(const UCHAR* str, SLONG length)
 {
 StrConverter cvt(pool, textType, str, length);
fb_assert(length % sizeof(CharType) == 0);
return evaluator.processNextChunk(
reinterpret_cast<const CharType*>(str), length / sizeof(CharType));
 }

static ContainsMatcher* create(MemoryPool& pool, TextType* ttype, const UCHAR* str, SLONG length)
 {
 StrConverter cvt(pool, ttype, str, length);
fb_assert(length % sizeof(CharType) == 0);
returnFB_NEW_POOL(pool) ContainsMatcher(pool, ttype,
reinterpret_cast<const CharType*>(str), length / sizeof(CharType));
 }

staticboolevaluate(MemoryPool& pool, TextType* ttype, const UCHAR* s, SLONG sl,
const UCHAR* p, SLONG pl)
 {
 StrConverter cvt1(pool, ttype, p, pl);
 StrConverter cvt2(pool, ttype, s, sl);
fb_assert(pl % sizeof(CharType) == 0);
fb_assert(sl % sizeof(CharType) == 0);
 Firebird::ContainsEvaluator<CharType> evaluator(pool,
reinterpret_cast<const CharType*>(p), pl / sizeof(CharType));
 evaluator.processNextChunk(reinterpret_cast<const CharType*>(s), sl / sizeof(CharType));
return evaluator.getResult();
 }

private:
 Firebird::ContainsEvaluator<CharType> evaluator;
};

template <typename CharType, typename StrConverter = CanonicalConverter<> >
classMatchesMatcher
{
public:
staticboolevaluate(MemoryPool& pool, TextType* ttype, const UCHAR* s, SLONG sl,
const UCHAR* p, SLONG pl)
 {
 StrConverter cvt1(pool, ttype, p, pl);
 StrConverter cvt2(pool, ttype, s, sl);
fb_assert(pl % sizeof(CharType) == 0);
fb_assert(sl % sizeof(CharType) == 0);
returnmatches(pool, ttype, reinterpret_cast<const CharType*>(s), sl,
reinterpret_cast<const CharType*>(p), pl);
 }

private:
// Return true if a string (p1, l1) matches a given pattern (p2, l2).
// The character '?' in the pattern may match any single character
// in the the string, and the character '*' may match any sequence
// of characters.
//
// Wide SCHAR version operates on short-based buffer,
// instead of SCHAR-based.
//
// Matches is not a case-sensitive operation, thus it has no
// 8-bit international impact.
staticboolmatches(MemoryPool& pool, Jrd::TextType* obj, const CharType* p1,
 SLONG l1_bytes, const CharType* p2, SLONG l2_bytes)
 {
fb_assert(p1 != NULL);
fb_assert(p2 != NULL);
fb_assert((l1_bytes % sizeof(CharType)) == 0);
fb_assert((l2_bytes % sizeof(CharType)) == 0);
fb_assert((obj->getCanonicalWidth() == sizeof(CharType)));

 SLONG l1 = l1_bytes / sizeof(CharType);
 SLONG l2 = l2_bytes / sizeof(CharType);

while (l2-- > 0)
 {
const CharType c = *p2++;
if (c == *(CharType*) obj->getCanonicalChar(CHAR_GDML_MATCH_ANY))
 {
while ((l2 > 0) && (*p2 == *(CharType*) obj->getCanonicalChar(CHAR_GDML_MATCH_ANY)))
 {
 l2--;
 p2++;
 }
if (l2 == 0)
returntrue;
while (l1)
 {
if (matches(pool, obj, p1++, l1-- * sizeof(CharType), p2, l2 * sizeof(CharType)))
returntrue;
 }
returnfalse;
 }
if (l1-- == 0)
returnfalse;
if (c != *(CharType*) obj->getCanonicalChar(CHAR_GDML_MATCH_ONE) && c != *p1)
returnfalse;
 p1++;
 }

return !l1;
 }
};

template <typename CharType, typename StrConverter = CanonicalConverter<> >
classSleuthMatcher
{
public:
// Evaluate the "sleuth" search operator.
// Turn the (pointer, byte length) input parameters into
// (pointer, end_pointer) for use in aux function
staticboolcheck(MemoryPool& pool, TextType* ttype, USHORT flags,
const UCHAR* search, SLONG search_len, const UCHAR* match, SLONG match_len)
 {
 StrConverter cvt1(pool, ttype, search, search_len);//, cvt2(pool, ttype, match, match_len);

fb_assert((match_len % sizeof(CharType)) == 0);
fb_assert((search_len % sizeof(CharType)) == 0);
fb_assert(ttype->getCanonicalWidth() == sizeof(CharType));

const CharType* const end_match =
reinterpret_cast<const CharType*>(match) + (match_len / sizeof(CharType));
const CharType* const end_search =
reinterpret_cast<const CharType*>(search) + (search_len / sizeof(CharType));

returnaux(ttype, flags, reinterpret_cast<const CharType*>(search),
 end_search, reinterpret_cast<const CharType*>(match), end_match);
 }

static ULONG merge(MemoryPool& pool, TextType* ttype,
const UCHAR* match, SLONG match_bytes,
const UCHAR* control, SLONG control_bytes,
 UCHAR* combined) //, SLONG combined_bytes)
 {
 StrConverter cvt1(pool, ttype, match, match_bytes);
 StrConverter cvt2(pool, ttype, control, control_bytes);
fb_assert(match_bytes % sizeof(CharType) == 0);
fb_assert(control_bytes % sizeof(CharType) == 0);
returnactualMerge(/*pool,*/ ttype,
reinterpret_cast<const CharType*>(match), match_bytes,
reinterpret_cast<const CharType*>(control), control_bytes,
reinterpret_cast<CharType*>(combined)); //, combined_bytes);
 }

private:
// Evaluate the "sleuth" search operator.
staticboolaux(Jrd::TextType* obj, USHORT flags,
const CharType* search, const CharType* end_search,
const CharType* match, const CharType* end_match)
 {
fb_assert(search != NULL);
fb_assert(end_search != NULL);
fb_assert(match != NULL);
fb_assert(end_match != NULL);
fb_assert(search <= end_search);
fb_assert(match <= end_match);
fb_assert(obj->getCanonicalWidth() == sizeof(CharType));

while (match < end_match)
 {
 CharType c = *match++;
if ((c == *(CharType*) obj->getCanonicalChar(CHAR_GDML_QUOTE) && (c = *match++)) ||
 (size_t(c) < FB_NELEM(SLEUTH_SPECIAL) && !SLEUTH_SPECIAL[c]))
 {
if (match >= end_match || *match != *(CharType*) obj->getCanonicalChar(CHAR_GDML_MATCH_ANY))
 {
if (search >= end_search)
returnfalse;
const CharType d = *search++;
if (c != d)
returnfalse;
 }
else
 {
 ++match;
for (;;)
 {
if (aux(obj, flags, search, end_search, match, end_match))
returntrue;

if (search < end_search)
 {
const CharType d = *search++;
if (c != d)
returnfalse;
 }
else
returnfalse;
 }
 }
 }
elseif (c == *(CharType*) obj->getCanonicalChar(CHAR_GDML_MATCH_ONE))
 {
if (match >= end_match || *match != *(CharType*) obj->getCanonicalChar(CHAR_GDML_MATCH_ANY))
 {
if (search >= end_search)
returnfalse;

 search++;
 }
else
 {
if (++match >= end_match)
returntrue;

for (;;)
 {
if (aux(obj, flags, search, end_search, match, end_match))
returntrue;

if (++search >= end_search)
returnfalse;
 }
 }
 }
elseif (c == *(CharType*) obj->getCanonicalChar(CHAR_GDML_CLASS_START))
 {
const CharType* const char_class = match;
while (*match++ != *(CharType*) obj->getCanonicalChar(CHAR_GDML_CLASS_END))
 {
if (match >= end_match)
returnfalse;
 }
const CharType* const end_class = match - 1;
if (match >= end_match || *match != *(CharType*) obj->getCanonicalChar(CHAR_GDML_MATCH_ANY))
 {
if (!className(obj, /*flags,*/ char_class, end_class, *search++))
returnfalse;
 }
else
 {
 ++match;
for (;;)
 {
if (aux(obj, flags, search, end_search, match, end_match))
returntrue;

if (search < end_search)
 {
if (!className(obj, /*flags,*/ char_class, end_class, *search++))
returnfalse;
 }
else
returnfalse;
 }
 }
 }
elseif (c == *(CharType*) obj->getCanonicalChar(CHAR_GDML_FLAG_SET))
 {
 c = *match++;
if (c == *(CharType*) obj->getCanonicalChar(TextType::CHAR_LOWER_S) ||
 c == *(CharType*) obj->getCanonicalChar(TextType::CHAR_UPPER_S))
 {
 flags &= ~SLEUTH_INSENSITIVE;
 }
 }
elseif (c == *(CharType*) obj->getCanonicalChar(CHAR_GDML_FLAG_CLEAR))
 {
 c = *match++;
if (c == *(CharType*) obj->getCanonicalChar(TextType::CHAR_LOWER_S) ||
 c == *(CharType*) obj->getCanonicalChar(TextType::CHAR_UPPER_S))
 {
 flags |= SLEUTH_INSENSITIVE;
 }
 }
 }

if (search < end_search)
returnfalse;

returntrue;
 }

// See if a character is a member of a class.
// Japanese version operates on short-based buffer,
// instead of SCHAR-based.
staticboolclassName(Jrd::TextType* obj, // USHORT flags,
const CharType* char_class, const CharType* const end_class, CharType character)
 {
fb_assert(char_class != NULL);
fb_assert(end_class != NULL);
fb_assert(char_class <= end_class);
fb_assert(obj->getCanonicalWidth() == sizeof(CharType));

bool result = true;

if (*char_class == *(CharType*) obj->getCanonicalChar(CHAR_GDML_NOT))
 {
 ++char_class;
 result = false;
 }

while (char_class < end_class)
 {
const CharType c = *char_class++;
if (c == *(CharType*) obj->getCanonicalChar(CHAR_GDML_QUOTE))
 {
if (*char_class++ == character)
returntrue;
 }
elseif (*char_class == *(CharType*) obj->getCanonicalChar(CHAR_GDML_RANGE))
 {
 char_class += 2;
if (character >= c && character <= char_class[-1])
return result;
 }
elseif (character == c)
return result;
 }

return !result;
 }

// Merge the matching pattern and control strings to give a cannonical
// matching pattern. Return the length of the combined string.
//
// What this routine does is to take the language template, strip off
// the prefix and put it in the output string, then parse the definitions
// into an array of character pointers. The index array is the defined
// character. The routine then takes the actual match pattern and uses
// the characters in it to index into the definitions to produce an equivalent
// pattern in the cannonical language.
//
// The silly loop setting *v++ to zero initializes the array up to the
// highest character defined (also max_op). Believe it or not, that part
// is not a bug.
static ULONG actualMerge(/*MemoryPool& pool,*/ Jrd::TextType* obj,
const CharType* match, SLONG match_bytes,
const CharType* control, SLONG control_bytes,
 CharType* combined) //, SLONG combined_bytes)
 {
fb_assert(match != NULL);
fb_assert(control != NULL);
fb_assert(combined != NULL);

fb_assert((match_bytes % sizeof(CharType)) == 0);
fb_assert((control_bytes % sizeof(CharType)) == 0);
fb_assert(obj->getCanonicalWidth() == sizeof(CharType));

const CharType* const end_match = match + (match_bytes / sizeof(CharType));
const CharType* const end_control = control + (control_bytes / sizeof(CharType));

 CharType max_op = 0;
 CharType* comb = combined;
 CharType* vector[256];
 CharType** v = vector;
 CharType temp[256];
 CharType* t = temp;

// Parse control string into substitution strings and initializing string

while (control < end_control)
 {
 CharType c = *control++;
if (*control == *(CharType*) obj->getCanonicalChar(CHAR_GDML_SUBSTITUTE))
 {
// Note: don't allow substitution characters larger than vector
 CharType** const end_vector = vector + (((int) c < FB_NELEM(vector)) ? c : 0);
while (v <= end_vector)
 *v++ = 0;
 *end_vector = t;
 ++control;
while (control < end_control)
 {
 c = *control++;
if ((t > temp && t[-1] == *(CharType*) obj->getCanonicalChar(CHAR_GDML_QUOTE)) ||
 ((c != *(CharType*) obj->getCanonicalChar(CHAR_GDML_COMMA)) &&
 (c != *(CharType*) obj->getCanonicalChar(CHAR_GDML_RPAREN))))
 {
 *t++ = c;
 }
else
break;
 }
 *t++ = 0;
 }
elseif (c == *(CharType*) obj->getCanonicalChar(CHAR_GDML_QUOTE) && control < end_control)
 *comb++ = *control++;
elseif (c == *(CharType*) obj->getCanonicalChar(CHAR_GDML_RPAREN))
break;
elseif (c != *(CharType*) obj->getCanonicalChar(CHAR_GDML_LPAREN))
 *comb++ = c;
 }

 max_op = v - vector;

// Interpret matching string, substituting where appropriate

while (match < end_match)
 {
const CharType c = *match++;

// if we've got a defined character, slurp the definition

 CharType* p;
if (c <= max_op && (p = vector[c]))
 {
while (*p)
 *comb++ = *p++;

// if we've got the definition of a quote character,
// slurp the next character too

if (comb > combined &&
 comb[-1] == *(CharType*) obj->getCanonicalChar(CHAR_GDML_QUOTE) && *match)
 {
 *comb++ = *match++;
 }
 }
else
 {
// at this point we've got a non-match, but as it might be one of ours, quote it
if (size_t(c) < FB_NELEM(SLEUTH_SPECIAL) && SLEUTH_SPECIAL[c] &&
 comb > combined && comb[-1] != *(CharType*) obj->getCanonicalChar(CHAR_GDML_QUOTE))
 {
 *comb++ = *(CharType*) obj->getCanonicalChar(CHAR_GDML_QUOTE);
 }
 *comb++ = c;
 }
 }

// Put in trailing stuff

while (control < end_control)
 *comb++ = *control++;

// YYY - need to add code watching for overflow of combined

returnstatic_cast<ULONG>((comb - combined) * sizeof(CharType));
 }

private:
staticconstint SLEUTH_INSENSITIVE;
};

template <typename CharType, typename StrConverter>
constint SleuthMatcher<CharType, StrConverter>::SLEUTH_INSENSITIVE = 1;


template <
typename pStartsMatcher,
typename pContainsMatcher,
typename pLikeMatcher,
typename pMatchesMatcher,
typename pSleuthMatcher
>
classCollationImpl : publicCollation
{
public:
CollationImpl(TTYPE_ID a_type, texttype* a_tt, USHORT a_attributes, CharSet* a_cs)
 : Collation(a_type, a_tt, a_attributes, a_cs)
 {
 }

virtualboolmatches(MemoryPool& pool, const UCHAR* a, SLONG b, const UCHAR* c, SLONG d)
 {
returnpMatchesMatcher::evaluate(pool, this, a, b, c, d);