mi32/ucstring.h

Go to the documentation of this file.

/**
 * \file mi32/ucstring.h
 * \brief Definitions for Unicode string functions and related text conversion
 *
 * \if NODOC
 * $Id: ucstring.h_v 1.126 2003/10/06 17:05:12 mju Exp $
 *
 * $Log: ucstring.h_v $
 * Revision 1.126  2003/10/06 17:05:12  mju
 * Remove obsolete prototype.
 *
 * Revision 1.125  2003/10/03 20:05:23  linux32build!build
 * Doxygen
 *
 * Revision 1.124  2003/09/15 16:33:00  scowan
 * removed mac native code.
 *
 * Revision 1.123  2003/09/15 13:49:56  fileserver!dwilliss
 * Doxygen
 *
 * Revision 1.122  2003/07/09 14:27:09  mju
 * Don't define MISYSTEMclassexport at all as not needed.
 * Move libexport stuff to after all headers.
 *
 * Revision 1.121  2003/07/09 14:04:57  dwilliss
 * Don't leave LIBEXPORT defined after the end of the file
 *
 * Revision 1.120  2003/06/30 16:03:29  mju
 * Remove mttext fn to get from textid.
 *
 * Revision 1.119  2003/02/07 15:39:53  dwilliss
 * Added a prototype for a new function
 *
 * Revision 1.118  2002/09/11 17:01:23  dwilliss
 * Added KOI8 encoding
 *
 * Revision 1.117  2002/09/09 16:35:42  mju
 * Remove varargs fns.
 * \endif
**/

#ifndef INC_MI32_UCSTRING_H
#define INC_MI32_UCSTRING_H


#ifndef UCSTRING_H
#define UCSTRING_H

#ifndef INC_MI32_STDDEFNS_H
#include <mi32/stddefns.h>
#endif

#ifndef INC_MI32_TEXTID_H
#include <mi32/textid.h>
#endif

#include <stdio.h>

#if defined(SUN) && defined(BSD)

#include <varargs.h>
#include <sys/stdtypes.h>

#else

#include <stdarg.h>

#endif

#ifdef MISYSTEMDLL
   #define LIBEXPORT MI_DLLEXPORT
#else
   #define LIBEXPORT MI_DLLIMPORT
#endif

//! Structure used to keep track of state for calling
//! MucKeyEventToUnicode()
struct MUCEVENTSTATE {
   int  compose_mode;
   int  initialized;
   int  compose_byte1;
   int  compose_len;
   FILE  *debug;
   int  KeyboardCharsetLower128;
   int  KeyboardCharsetUpper128;
   int  KeyboardCharset2Byte;
   char compose_status[40];
   UINT8 compose_temp[80];
   };

#if defined(__cplusplus)
extern "C" {
#endif
#ifdef _XLIB_H_

//! Convert XKeyEvents to UNICODE
//!
//! Nothing uses this. It may not even work currently.
int MucKeyEventToUnicode (XEvent*, MUCEVENTSTATE*, UINT32);
#endif
#if defined(__cplusplus)
}
#endif

#define MUCVERSION   2
#ifndef GENERATING_DOXYGEN_OUTPUT
/* Structure of the header of unicode2.ref.  All numbers are stored
 * in LoHi byte order in unicode2.ref.  HiLo machines will have to
 * swap
 */
struct MUCFILEHDR {
   UINT32 FileHdrSize;     //!<  Size of this structure 
   UINT32 EncHdrSize;         //!<  sizeof(MUCENCODEHDR) 
   UINT32 Version;            //!<  MUCVERSION 
   UINT32 NumEncodings;    //!<  Number of MUCENCODEHDRs that follow 
   };

struct MUCENCODEHDR {
   UNICODE name[16]; 
   INT32 offset;           //!<  Offset into file of data for this encoding 
   INT32 numpairs;
   };
#endif // GENERATING_DOXYGEN_OUTPUT

/*
** Constants for MucCharsetToUnicode()/MucUnicodeToCharset()
*/
#define MASK_ALG                 0x7FFF0000
#define MASK_CHARSET             0x0000FFFF

#define CHARALG_SJIS             0x00010000
#define CHARALG_EUC              0x00020000
#define CHARALG_GL2GR            0x00040000  //!< Set high bits
#define CHARALG_GR2GL            0x00080000  //!< Strip high bits

#define CHARSET_Unicode          0x00000000
#define CHARSET_GB_2312          0x00000001
#define CHARSET_GB_12345         0x00000002
#define CHARSET_GB_7589          0x00000003
#define CHARSET_GB_7590          0x00000004
#define CHARSET_GB_Han           0x00000005
#define CHARSET_GB_8565          0x00000006
#define CHARSET_BigFive          0x00000007
#define CHARSET_CNS_11543_01     0x00000008
#define CHARSET_CNS_11543_02     0x00000009
#define CHARSET_CNS_11543_14     0x0000000A
#define CHARSET_JIS_X_0208_1990  0x0000000B
#define CHARSET_JIS_X_0212_1990  0x0000000C
#define CHARSET_KS_C_5601_1987   0x0000000D  //!< Wansung
#define CHARSET_KS_C_5657_1991   0x0000000E
#define CHARSET_ISO_8859_1       0x0000000F      //!<  ASCII 
#define CHARSET_ISO_8859_2       0x00000010     
#define CHARSET_ISO_8859_3       0x00000011    
#define CHARSET_ISO_8859_4       0x00000012  
#define CHARSET_ISO_8859_5       0x00000013
#define CHARSET_ISO_8859_6       0x00000014
#define CHARSET_ISO_8859_7       0x00000015
#define CHARSET_ISO_8859_8       0x00000016
#define CHARSET_ISO_8859_9       0x00000017
#define CHARSET_JIS_ROMAN        0x00000018
#define CHARSET_GB_ROMAN         0x00000019  //!<  From ISO 2022 
#define CHARSET_JIS_C_6226       0x0000001A  //!<  From ISO 2022 
#define CHARSET_HW_KATAKANA      0x0000001B  //!<  From ISO 2022 
#define CHARSET_WinCP_Turk       0x0000001C  //!<  Windows Codepage (cp1254)
#define CHARSET_WinCP_EE         0x0000001D  //!<  Windows Codepage (cp1250)
#define CHARSET_WinCP_Cyrl       0x0000001E  //!<  Windows Codepage (cp1251)
#define CHARSET_WinCP_Greek      0x0000001F  //!<  Windows Codepage (cp1253)
#define CHARSET_WinCP_Arab       0x00000020  //!<  Windows Codepage (cp1256)
#define CHARSET_WinCP_Hebr       0x00000021  //!<  Windows Codepage (cp1255)
#define CHARSET_DOSCP_437        0x00000022  //!<  DOS Codepage 
#define CHARSET_DOSCP_850        0x00000023  //!<  DOS Codepage 
#define CHARSET_DOSCP_852        0x00000024  //!<  DOS Codepage 
#define CHARSET_DOSCP_857        0x00000025  //!<  DOS Codepage 
#define CHARSET_DOSCP_861        0x00000026  //!<  DOS Codepage 
#define CHARSET_DOSCP_863        0x00000027  //!<  DOS Codepage 
#define CHARSET_DOSCP_865        0x00000028  //!<  DOS Codepage 
#define CHARSET_IBMCP_869        0x00000029  //!<  IBM Codepage 
#define CHARSET_IBMCP_855        0x0000002A  //!<  IBM Codepage 
#define CHARSET_IBMCP_864        0x0000002B  //!<  IBM Codepage 
#define CHARSET_IBMCP_1040       0x0000002C  //!<  IBM Codepage 
#define CHARSET_IBMCP_1041       0x0000002D  //!<  IBM Codepage 
#define CHARSET_IBMCP_1043       0x0000002E  //!<  IBM Codepage 
#define CHARSET_TIS620_2529      0x0000002F  //!<  Thai (BDF font order) 
#define CHARSET_WinCP_Thai       0x00000030  //!<  Thai Windows ordering 
#define CHARSET_MI_Thai          0x00000031  //!<  Thai Windows ordering 
#define CHARSET_WinCP_ANSI       0x00000032  //!<  Windows ANSI Code Page 
#define CHARSET_ArabicUC         0x00000033  //!<  The Arabic range of Unicode 
#define CHARSET_MacArabic        0x00000034
#define CHARSET_MacCroatian      0x00000035
#define CHARSET_MacCyrillic      0x00000036
#define CHARSET_MacDingbats      0x00000037
#define CHARSET_MacCentEurope    0x00000038
#define CHARSET_MacGreek         0x00000039
#define CHARSET_MacHebrew        0x0000003A
#define CHARSET_MacIcelandic     0x0000003B
#define CHARSET_MacJapanese      0x0000003C
#define CHARSET_MacRomanian      0x0000003D
#define CHARSET_MacRoman         0x0000003E
#define CHARSET_MacSymbol        0x0000003F
#define CHARSET_MacThai          0x00000040
#define CHARSET_MacTurkish       0x00000041
#define CHARSET_MacUkrainian     0x00000042
#define CHARSET_WinCP_Baltic     0x00000043
#define CHARSET_IBM_1046         0x00000044  //!<  IBM Arabic encoding 
#define CHARSET_KS_C_5601_Unif   0x00000045  //!< Unified Hangul (UHang)
#define CHARSET_KS_C_5601_1992   0x00000046  //!< Johab
#define CHARSET_ISCII_Devanagari 0x00000047  //!< Indic
#define CHARSET_ISCII_Bengali    0x00000048  //!< Indic
#define CHARSET_ISCII_Gurmukhi   0x00000049  //!< Indic
#define CHARSET_ISCII_Gujarati   0x0000004A  //!< Indic
#define CHARSET_ISCII_Oriya      0x0000004B  //!< Indic
#define CHARSET_ISCII_Tamil      0x0000004C  //!< Indic
#define CHARSET_ISCII_Telugu     0x0000004D  //!< Indic
#define CHARSET_ISCII_Kannada    0x0000004E  //!< Indic
#define CHARSET_ISCII_Malayalam  0x0000004F  //!< Indic
#define CHARSET_KOI8             0x00000050  //!< Russian/Ukranian/Etc.
#define CHARSET_MAX              0x00000050

/* Some aliases... */
#define CHARSET_ISO_Latin_1      CHARSET_ISO_8859_1   
#define CHARSET_ASCII            CHARSET_ISO_Latin_1
#define CHARSET_JIS_0208         CHARSET_JIS_X_0208_1990
#define CHARSET_JIS_0212         CHARSET_JIS_X_0212_1990
#define CHARSET_SHIFT_JIS        (CHARALG_SJIS | CHARSET_JIS_X_0208_1990)

#define UCCONV_NoByteOrderMark   0x00000001

//!
//! A character encoding exactly specifies both the set(s) of characters used
//! and how they are stored in memory (assuming native byte order).  Note that
//! this is *not* the same as a CHARSET.  Once an encoding value is assigned
//! it must never be changes as the value may be stored in a data file and is
//! also passed to the misystem DLL.
//!
//! These values should NOT be considered sequential!

#ifndef __cplusplus
typedef
#endif
enum CHAR_ENCODING {
   //! Base encodings
   CHAR_ENCODING_ASCII = 0,            //!< ASCII
   CHAR_ENCODING_Unicode,              //!< 2 byte encoding
   CHAR_ENCODING_UTF8,
   CHAR_ENCODING_XResFile,             //!< Variation on ISO-2022
   CHAR_ENCODING_Unicode_Decomposed,   //!< 2 byte encoding, but things like O with Umlat converted to 2 chars, and O and an Umlat
   CHAR_ENCODING_UTF8_Decomposed,      //!< Unicode_Decomposed converted to UTF8
   CHAR_ENCODING_WindowsMultiByte,     //!< WINDOWS ONLY - Uses MultiByteToWideChar/WideCharToMultiByte

   //! ISO Encodings
   CHAR_ENCODING_ISO_2022 = 100,
   CHAR_ENCODING_ISO_8859_1,
   CHAR_ENCODING_ISO_8859_2,
   CHAR_ENCODING_ISO_8859_3,
   CHAR_ENCODING_ISO_8859_4,
   CHAR_ENCODING_ISO_8859_5,
   CHAR_ENCODING_ISO_8859_6,
   CHAR_ENCODING_ISO_8859_7,
   CHAR_ENCODING_ISO_8859_8,
   CHAR_ENCODING_ISO_8859_9,

   //! Japanese encodings
   CHAR_ENCODING_JIS = 120,      //!< Japanese
   CHAR_ENCODING_EUC,            //!< Japeneseucstr
   CHAR_ENCODING_SJIS,           //!< Japanese
   CHAR_ENCODING_Japanese,       //!< Will gusss which of the 3 Japanese encodings

   //! Chinese encodings
   CHAR_ENCODING_Big5 = 130,     //!< Chinese
   CHAR_ENCODING_GB_2312,        //!< Chinese

   //! Korean encodings
   CHAR_ENCODING_KCS_5601 = 140,       //!< Korean

   //! ISCII (Indic) Encodings
   CHAR_ENCODING_ISCII_Devanagari = 150,
   CHAR_ENCODING_ISCII_Bengali,
   CHAR_ENCODING_ISCII_Gurmukhi,
   CHAR_ENCODING_ISCII_Gujarati,
   CHAR_ENCODING_ISCII_Oriya,
   CHAR_ENCODING_ISCII_Tamil,
   CHAR_ENCODING_ISCII_Telugu,
   CHAR_ENCODING_ISCII_Kannada,
   CHAR_ENCODING_ISCII_Malayalam,
   
   //! Windows Code Pages
   CHAR_ENCODING_WinCP_ANSI = 300,     //!< Most English and Western Europe
   CHAR_ENCODING_WinCP_Cyrl,           //!< Russian
   CHAR_ENCODING_WinCP_Greek,
   CHAR_ENCODING_WinCP_Arabic,
   CHAR_ENCODING_WinCP_Thai,
   CHAR_ENCODING_WinCP_EE,
   CHAR_ENCODING_WinCP_Turk,
   CHAR_ENCODING_WinCP_Hebr,

   //! IBM Code Pages
   CHAR_ENCODING_IBMCP_855 = 400,
   CHAR_ENCODING_IBMCP_869,

   //! DOS Code Pages
   CHAR_ENCODING_DOSCP_437 = 500,
   CHAR_ENCODING_DOSCP_850,
   CHAR_ENCODING_DOSCP_852,
   CHAR_ENCODING_DOSCP_857,
   CHAR_ENCODING_DOSCP_861,
   CHAR_ENCODING_DOSCP_863,
   CHAR_ENCODING_DOSCP_865,

   //! Macintosh Encodings
   CHAR_ENCODING_MacArabic = 600,
   CHAR_ENCODING_MacCroatian,
   CHAR_ENCODING_MacCyrillic,
   CHAR_ENCODING_MacDingbats,
   CHAR_ENCODING_MacCentEurope,
   CHAR_ENCODING_MacGreek,
   CHAR_ENCODING_MacHebrew,
   CHAR_ENCODING_MacIcelandic,
   CHAR_ENCODING_MacJapanese,
   CHAR_ENCODING_MacRomanian,
   CHAR_ENCODING_MacRoman,
   CHAR_ENCODING_MacSymbol,
   CHAR_ENCODING_MacThai,
   CHAR_ENCODING_MacTurkish,
   CHAR_ENCODING_MacUkrainian,

   //! Misc Code Pages
   CHAR_ENCODING_KOI8 = 700

   }
#ifndef __cplusplus
   CHAR_ENCODING
#endif
   ;

//! Enums which can be used when getting  the list of encodings
#ifndef __cplusplus
typedef
#endif
enum ENCODELIST_FLAGS {
   ENCODELIST_FLAG_Any        = 0x0000,
   ENCODELIST_FLAG_Importable = 0x0001,
   ENCODELIST_FLAG_Exportable = 0x0002,
   ENCODELIST_FLAG_Both       = 0x0003,      //!< Importable | Exportable
   ENCODELIST_FLAG_NoUnicode  = 0x0004
   }
#ifndef __cplusplus
   ENCODELIST_FLAGS
#endif
   ;

#ifdef __cplusplus
//! Following only defined for C++
//! Don't add a semicolon to the end of this!
DEFINE_ENUM_OPERATORS(ENCODELIST_FLAGS)
#endif

#ifndef __cplusplus
typedef
#endif
//! Script tags
//! These are used mostly for when multiple "scripts" (as in writing systems)
//! use the same character set but have different rules.  A good example of 
//! this is the Indic languages which mostly share the ISCII character set
//! but Devanagari and Bengali have different formatting rules.
//! The numbers below are not random gibberish as it may seem.  If you take
//! each pair of hex digits, they form an ASCII value so that together they
//! make up a 4 letter abbreviation.  This is how these tags are encoded in
//! TrueType fonts.  I would have #defined them as 'arab' instead of a big
//! number if I thought all compilers that we support would accept that.
enum SCRIPTTAG {

    SCRIPTTAG_arab = 0x61726162, //!< 'arab' *** Arabic
    SCRIPTTAG_armn = 0x61726D6E, //!< 'armn' *** Armenian: no MS definition ***
    SCRIPTTAG_beng = 0x62656E67, //!< 'beng' *** Bengali (Indic)
    SCRIPTTAG_bpmf = 0x62706D66, //!< 'bpmf' *** Bopomofo: no MS definition ***
    SCRIPTTAG_cyrl = 0x6379726C, //!< 'cyrl' *** Cyrillic
    SCRIPTTAG_deva = 0x64657661, //!< 'deva' *** Devanagari (Indic)
    SCRIPTTAG_grek = 0x6772656B, //!< 'grek' *** Greek
    SCRIPTTAG_grgn = 0x6772676E, //!< 'grgn' *** Georgian: no MS definition ***
    SCRIPTTAG_gujr = 0x67756A72, //!< 'gujr' *** Gujarati (Indic)
    SCRIPTTAG_hang = 0x68616E67, //!< 'hang' *** 
    SCRIPTTAG_hani = 0x68616E69, //!< 'hani' *** 
    SCRIPTTAG_hebr = 0x68656272, //!< 'hebr' *** Hebrew
    SCRIPTTAG_kana = 0x6B616E61, //!< 'kana' *** 
    SCRIPTTAG_knbn = 0x6B6E626E, //!< 'knbn' *** Kanbun: no MS definition ***
    SCRIPTTAG_knda = 0x6B6E6461, //!< 'knda' *** Kannada (Indic)
    SCRIPTTAG_laoS = 0x6C616F20, //!< 'lao ' *** Lao: no MS definition ***
    SCRIPTTAG_latn = 0x6C61746E, //!< 'latn' *** Latin
    SCRIPTTAG_mlym = 0x6D6C796D, //!< 'mlym' *** Malayalam (Indic)
    SCRIPTTAG_orya = 0x6F727961, //!< 'orya' *** Oriya (Indic)
    SCRIPTTAG_punj = 0x70756E6A, //!< 'punj' *** punjabi == gurmukhi
    SCRIPTTAG_taml = 0x74616D6C, //!< 'taml' *** Tamil (Indic)
    SCRIPTTAG_telu = 0x74656C75, //!< 'telu' *** Telugu (Indic)
    SCRIPTTAG_thai = 0x74686169, //!< 'thai' *** Thai
    SCRIPTTAG_tibt = 0x74696174, //!< 'tibt' *** Tibetan
      
    SCRIPTTAG_neut = 0x4E455554, //!< 'NEUT'
    SCRIPTTAG_puse = 0x50555345, //!< 'PUSE'
    SCRIPTTAG_spcl = 0x5350434C, //!< 'SPCL'
    SCRIPTTAG_surr = 0x53555252, //!< 'SURR'

    SCRIPTTAG_Default = 0x00000000 //!< ''
   }
#ifndef __cplusplus
   SCRIPTTAG
#endif
   ;



//!
//!   Some special glyph flags.  The method we use to generate GLYPHCONTEXTBITs
//!   requires that PartOfRtoLWord be 1 and that we don't use bits 0x02 and 0x04
//! here.  The PartOfRtoLWord flag will be shifted and OR'd with the bits of 
//! the glyphs on either side of it to create GLYPHCONTEXT bits.  
//! Yes, they're bytes, not longs.  Don't muck with `em!
//!
#define MucGLYPHFLAG_PartOfRtoLWord    0x01
#define MucGLYPHFLAG_NonJoining        0x80
#define MucGLYPHFLAG_Mark              0x40  

#define MucGLYPHDIRECTION_LtoR         0
#define MucGLYPHDIRECTION_RtoL         1
#define MucGLYPHDIRECTION_Weak         2


//! These are the general category codes from column 2 of the
//! UnicodeData.txt.  The numbers are arbitrarily assigned by
//! me.  UnicodeData.txt has the 2-letter codes.
//! You may be asking why I defined the values for all the
//! enums when the compiler would do that for me...See, this
//! way it would be a pain to insert something into the middle
//! and renumber them, wouldn't it?  Good!  don't do that.
//! ucdata.ref has the numbers and if you renumber them, you'll
//! break things.
enum UCCAT {
   UCCAT_Cn = 0,        //!< Other , Not assigned (norm)
   UCCAT_Lu = 1,        //!< Letter, Uppercase (norm)
   UCCAT_Ll = 2,        //!< Letter, Lowercase (norm)
   UCCAT_Lt = 3,        //!< Letter, Titlecase (norm)
   UCCAT_Lm = 4,        //!< Letter, Modifier
   UCCAT_Lo = 5,        //!< Letter, Other
   UCCAT_Mn = 6,        //!< Mark, non-spacing (norm)
   UCCAT_Mc = 7,        //!< Mark, spacing combining (norm)
   UCCAT_Me = 8,        //!< Mark, encolsing (norm)
   UCCAT_Nd = 9,        //!< Number Decimal digit (norm)
   UCCAT_Nl = 10,       //!< Number Letter (norm)
   UCCAT_No = 11,       //!< Number Other (norm)
   UCCAT_Zs = 12,       //!< Separator, space (norm)
   UCCAT_Zl = 13,       //!< Separator, line (norm)
   UCCAT_Zp = 14,       //!< Separator, paragraph (norm)
   UCCAT_Cc = 15,       //!< Other , control (norm)
   UCCAT_Cf = 16,       //!< Other , format (norm)
   UCCAT_Cs = 17,       //!< Other , surrogate (norm)
   UCCAT_Co = 18,       //!< Other , Private use (norm)
   UCCAT_Pc = 20,       //!< Punctuation, connector
   UCCAT_Pd = 21,       //!< Punctuation, Dash
   UCCAT_Ps = 22,       //!< Punctuation, Open
   UCCAT_Pe = 23,       //!< Punctuation, Close
   UCCAT_Pi = 24,       //!< Punctuation, Initial quote
   UCCAT_Pf = 25,       //!< Punctuation, final quote
   UCCAT_Po = 26,       //!< Punctuation, other
   UCCAT_Sm = 27,       //!< Symbol, math
   UCCAT_Sc = 28,       //!< Symbol, currency
   UCCAT_Sk = 29,       //!< Symbol, modifier
   UCCAT_So = 30        //!< Symbol, other
   };

//! Flags for the above which can be ORd together
enum UCCATFLAGS {
   UCCATFLAG_Cn = (1 << UCCAT_Cn),        //! Other , Not assigned (norm)
   UCCATFLAG_Lu = (1 << UCCAT_Lu),        //! Letter, Uppercase (norm)
   UCCATFLAG_Ll = (1 << UCCAT_Ll),        //! Letter, Lowercase (norm)
   UCCATFLAG_Lt = (1 << UCCAT_Lt),        //! Letter, Titlecase (norm)
   UCCATFLAG_Lm = (1 << UCCAT_Lm),        //! Letter, Modifier
   UCCATFLAG_Lo = (1 << UCCAT_Lo),        //! Letter, Other
   UCCATFLAG_Mn = (1 << UCCAT_Mn),        //! Mark, non-spacing (norm)
   UCCATFLAG_Mc = (1 << UCCAT_Mc),        //! Mark, spacing combining (norm)
   UCCATFLAG_Me = (1 << UCCAT_Me),        //! Mark, encolsing (norm)
   UCCATFLAG_Nd = (1 << UCCAT_Nd),        //! Number Decimal digit (norm)
   UCCATFLAG_Nl = (1 << UCCAT_Nl),        //! Number Letter (norm)
   UCCATFLAG_No = (1 << UCCAT_No),        //! Number Other (norm)
   UCCATFLAG_Zs = (1 << UCCAT_Zs),        //! Separator, space (norm)
   UCCATFLAG_Zl = (1 << UCCAT_Zl),        //! Separator, line (norm)
   UCCATFLAG_Zp = (1 << UCCAT_Zp),        //! Separator, paragraph (norm)
   UCCATFLAG_Cc = (1 << UCCAT_Cc),        //! Other , control (norm)
   UCCATFLAG_Cf = (1 << UCCAT_Cf),        //! Other , format (norm)
   UCCATFLAG_Cs = (1 << UCCAT_Cs),        //! Other , surrogate (norm)
   UCCATFLAG_Co = (1 << UCCAT_Co),        //! Other , Private use (norm)
   UCCATFLAG_Pc = (1 << UCCAT_Pc),        //! Punctuation, connector
   UCCATFLAG_Pd = (1 << UCCAT_Pd),        //! Punctuation, Dash
   UCCATFLAG_Ps = (1 << UCCAT_Ps),        //! Punctuation, Open
   UCCATFLAG_Pe = (1 << UCCAT_Pe),        //! Punctuation, Close
   UCCATFLAG_Pi = (1 << UCCAT_Pi),        //! Punctuation, Initial quote
   UCCATFLAG_Pf = (1 << UCCAT_Pf),        //! Punctuation, final quote
   UCCATFLAG_Po = (1 << UCCAT_Po),        //! Punctuation, other
   UCCATFLAG_Sm = (1 << UCCAT_Sm),        //! Symbol, math
   UCCATFLAG_Sc = (1 << UCCAT_Sc),        //! Symbol, currency
   UCCATFLAG_Sk = (1 << UCCAT_Sk),        //! Symbol, modifier
   UCCATFLAG_So = (1 << UCCAT_So),        //! other
   UCCATFLAG_Mi = (1 << 31)               //! Mirroring
   };
#ifdef __cplusplus
//! Following only defined for C++
//! Don't add a semicolon to the end of this!
DEFINE_ENUM_OPERATORS(UCCATFLAGS)
#endif
      
//! Unicode Bidirectional category from column 4 of UnicodeData.txt
//! See details on UCCATEGORY enum above.  These are summarized in
//! chapter 3 of the Unicode Standard
enum BIDITAG {
   BIDITAG_L   = 0,        //!< Left-to-right
   BIDITAG_LRE = 1,        //!< Left-to-right Embedding
   BIDITAG_LRO = 2,        //!< Left-to-right Override
   BIDITAG_R   = 3,        //!< Right-to-left
   BIDITAG_AL  = 4,        //!< Right-to-left arabic
   BIDITAG_RLE = 5,        //!< Right-to-left embedding
   BIDITAG_RLO = 6,        //!< Right-to-left override
   BIDITAG_PDF = 7,        //!< Pop Directional Format
   BIDITAG_EN  = 8,        //!< European Number
   BIDITAG_ES  = 9,        //!< European Number separator
   BIDITAG_ET  = 10,       //!< European Number Terminator
   BIDITAG_AN  = 11,       //!< Arabic Number 
   BIDITAG_CS  = 12,       //!< Common Number Separator
   BIDITAG_NSM = 13,       //!< Non Spacing Mark
   BIDITAG_BN  = 14,       //!< Boundary Neutral
   BIDITAG_B   = 15,       //!< Paragraph Separator
   BIDITAG_WS  = 16,       //!< White Space
   BIDITAG_ON  = 17        //!< Other Neutrals
   };

//! Flags for the above which can be ORd together
enum BIDIFLAGS {
   BIDIFLAG_L     = (1 << BIDITAG_L),        //! Left-to-right
   BIDIFLAG_LRE   = (1 << BIDITAG_LRE),      //! Left-to-right Embedding
   BIDIFLAG_LRO   = (1 << BIDITAG_LRO),      //! Left-to-right Override
   BIDIFLAG_R     = (1 << BIDITAG_R),        //! Right-to-left
   BIDIFLAG_AL    = (1 << BIDITAG_AL),       //! Right-to-left arabic
   BIDIFLAG_RLE   = (1 << BIDITAG_RLE),      //! Right-to-left embedding
   BIDIFLAG_RLO   = (1 << BIDITAG_RLO),      //! Right-to-left override
   BIDIFLAG_PDF   = (1 << BIDITAG_PDF),      //! Pop Directional Format
   BIDIFLAG_EN    = (1 << BIDITAG_EN),       //! European Number
   BIDIFLAG_ES    = (1 << BIDITAG_ES),       //! European Number separator
   BIDIFLAG_ET    = (1 << BIDITAG_ET),       //! European Number Terminator
   BIDIFLAG_AN    = (1 << BIDITAG_AN),       //! Arabic Number 
   BIDIFLAG_CS    = (1 << BIDITAG_CS),       //! Common Number Separator
   BIDIFLAG_NSM   = (1 << BIDITAG_NSM),      //! Non Spacing Mark
   BIDIFLAG_BN    = (1 << BIDITAG_BN),       //! Boundary Neutral
   BIDIFLAG_B     = (1 << BIDITAG_B),        //! Paragraph Separator
   BIDIFLAG_WS    = (1 << BIDITAG_WS),       //! White Space
   BIDIFLAG_ON    = (1 << BIDITAG_ON)        //! Other Neutrals
   };

#ifdef __cplusplus
//! Following only defined for C++
//! Don't add a semicolon to the end of this!
DEFINE_ENUM_OPERATORS(BIDIFLAGS)
#endif

//! Decompitition tag.  Column 5 of UnicodeData.txt sometimes has a tag
//! in angle brackets, (e.g. "<Isolated>")  This value represents that tag.
//! For example, 0xFE90 "ARABIC LETTER BEH FINAL FORM" has, in the 5th
//! column "<final> 0628" to which tells us that this glyph is the 
//! end-of-word form of the ARABIC LETTER BEH, which is at 0x0628
//! See Chapter 4 of the Unicode Standard.
enum UCDECOMP {
   UCDECOMP_NoTag       = 0,
   UCDECOMP_Initial     = 1,
   UCDECOMP_Medial      = 2,
   UCDECOMP_Final       = 3,
   UCDECOMP_Isolated    = 4,
   UCDECOMP_Compat      = 5,
   UCDECOMP_Wide        = 6,
   UCDECOMP_Narrow      = 7,
   UCDECOMP_Fraction    = 8,
   UCDECOMP_Subscript   = 9,
   UCDECOMP_Superscript = 10,
   UCDECOMP_Small       = 11, //!< A small veriant form (CNS compatibility)
   UCDECOMP_Square      = 12, //!< A CJK squared font variant
   UCDECOMP_Circle      = 13, //!< An encircled form
   UCDECOMP_Font        = 14, //!< A font variant (e.g. a blackletter form)
   UCDECOMP_NoBreak     = 15, //!< A non-breaking form of a space of hyphen
   UCDECOMP_Vertical    = 16  //!< A vertical layout presentation form
   };

#define UCDECOMPFLAG_Initial     (1 << UCDECOMP_Initial)
#define UCDECOMPFLAG_Medial      (1 << UCDECOMP_Medial)
#define UCDECOMPFLAG_Final       (1 << UCDECOMP_Final)
#define UCDECOMPFLAG_Isolated    (1 << UCDECOMP_Isolated)
#define UCDECOMPFLAG_PositionMask (UCDECOMPFLAG_Initial | UCDECOMPFLAG_Medial | UCDECOMPFLAG_Final | UCDECOMPFLAG_Isolated)

#ifdef __cplusplus

//! Structure representing "cooked" entry from UnicodeData.txt.
//! (see www.unicode.org).  Note, 3 of the values in this structure
//! could be enums, but I want them to take as little space as
//! possible, and I can't guarentee that enum will be a byte on
//! all platforms.  (have to be sure, because this struct is read
//! out of a file.
struct UCDATA {
   //! The actual Unicode value
   UNICODE  ucval;

   //! If cagegory == UCCAT_Lu, (Letter, Uppercase), this will
   //! be the Unicode value of the lowercase version of the same
   //! letter.  If category == UCCAT_Ll (Letter, Lowercase) this
   //! will be the uppercase version of the same letter
   UNICODE  altcase;

   //! Unicode value of the first composite character that starts
   //! with this Unicode value.  For example, Unicode value 0x0627
   //! (ARABIC LETTER ALEF) might have a firstcomp of 0x0626
   //! (ARABIC LETTER ALEF WITH HAMZA ABOVE).
   //! Due to the recursive nature of composition, this can't be
   //! done as a single linked list.  
   //! ARABIC LETTER ALEF WITH HAMZA ABOVE might have a nextcomp
   //! of ARABIC LETTER ALEF WITH HAMZA BELOW (next in the chain of
   //! things that started with ARABIC LETTER ALEF.  But it might
   //! have a firstcomp of ARABIC LETTER ALEF WITH HAMZA ABOVE FINAL 
   //! FORM (beginning a chain of things starting with ARABIC LETTER
   //! ALEF WITH HAMZA ABOVE)
   UNICODE firstcomp;

   //! Unicode value of the first composite character that starts
   //! with this Unicode value.  For example, Unicode value 0x00E0
   //! (ARABIC SMALL LETTER A WITH GRAVE) might have a nextcomp of 
   //! 0x00E1 (ARABIC SMALL LETTER A WITH ACUTE).  
   UNICODE nextcomp;

   //! General character category.  This is one of the UCCAT
   //! enums defined below.  The enum is based on the two letter
   //! code from UnicodeData.txt, so if UnicodeData.txt says a
   //! character is Lo (Letter, Other), the enum is UCCAT_Lo.
   UINT8    category;

   //! Bidirectional category tag.   Enums defined below and explained
   //! in Chapter 3 of the Unicode Standard
   UINT8    biditag;

   //! Decomposition tag
   UINT8    decomptag;

   //! Number of componants if this is a composite glyph or a
   //! variant of the glyph
   UINT8    numcomp;

   //! Return the CHARDATA for a given Unicode character.
   //! May return NULL if no data exists for a given Unicode char.
   static const UCDATA* GetCharData (
      UNICODE ucval
      );

   //! Get the list of componants for this character
   //! If numcomp > 0, the unicode character can be decomposed
   //! into that many componants.  This method will get the list
   //! of componants, assuming you got the componant by calling
   //! UCDATA::GetCharData()
   const UNICODE* GetComponants (
      ) const {
      return (reinterpret_cast<const UNICODE*>(this + 1));
      }
   };


//! Determine of a Unicode character has a given property.
//! The property flags are listed in ucstring.h.
//! (the name is derived from isprop() from ctype.h)
//! There are two sets of flags because there are so many
//! possibilites with Unicode.  You won't usually use this
//! function directly.  
LIBEXPORT bool ucisprop (
   UNICODE c,
   UINT32 uccatflags
   );
 
LIBEXPORT bool ucisdecompbidiprop(
   UNICODE c,
   UINT32 decompflags
   );
 
LIBEXPORT bool ucisbidiprop (
   UNICODE c,
   UINT32 bidiflags
   );

LIBEXPORT UNICODE uctolower (
   UNICODE c
   );

LIBEXPORT UNICODE uctoupper (
   UNICODE c
   );



//! Alpha
inline bool ucisalpha (UNICODE c) {
   return ucisprop(c, UCCATFLAG_Lu | UCCATFLAG_Ll | UCCATFLAG_Lm | UCCATFLAG_Lo | UCCATFLAG_Lt);
   }

//! Digit (any digit, not nust 0-9)
inline bool ucisdigit2 (UNICODE c) {
   return ucisprop(c, UCCATFLAG_Nd);
   }

//! Alpha or Digit
inline bool ucisalnum (UNICODE c) {
   return ucisprop(c, UCCATFLAG_Lu | UCCATFLAG_Ll | UCCATFLAG_Lm | UCCATFLAG_Lo | UCCATFLAG_Lt | UCCATFLAG_Nd);
   }

//! Control
inline bool uciscntrl (UNICODE c) {
   return ucisprop(c, UCCATFLAG_Cc | UCCATFLAG_Cf);
   }

//! Space
inline bool ucisspace (UNICODE c) {
   return ucisprop(c, UCCATFLAG_Zs);
   }

//! Space 
inline bool ucisblank (UNICODE c) {
   return ucisprop(c, UCCATFLAG_Zs);
   }

inline bool ucispunct (UNICODE c) {
   return ucisprop(c, UCCATFLAG_Pc | UCCATFLAG_Pd | UCCATFLAG_Ps | UCCATFLAG_Pe | UCCATFLAG_Po | UCCATFLAG_Pi | UCCATFLAG_Pf | UCCATFLAG_Po);
   }

inline bool ucisupper (UNICODE c) {
   return ucisprop(c, UCCATFLAG_Lu);
   }

inline bool ucislower (UNICODE c) {
   return ucisprop(c, UCCATFLAG_Ll);
   }

inline bool ucistitle (UNICODE c) {
   return ucisprop(c, UCCATFLAG_Lt);
   }

inline bool ucisisocntrl (UNICODE c) {
   return ucisprop(c, UCCATFLAG_Cc);
   }

inline bool ucisfmtcntrl (UNICODE c) {
   return ucisprop(c, UCCATFLAG_Cf);
   }

inline bool ucissymbol (UNICODE c) {
   return ucisprop(c, UCCATFLAG_Sm | UCCATFLAG_Sc | UCCATFLAG_So | UCCATFLAG_Sk);
   }

//! Number (digit, letter form, other)
inline bool ucisnumber (UNICODE c) {
   return ucisprop(c, UCCATFLAG_Nd | UCCATFLAG_No | UCCATFLAG_Nl);
   }

//! Mark, Non spacing
inline bool ucisnonspacing (UNICODE c) {
   return ucisprop(c, UCCATFLAG_Mn);
   }

//! Punctuation, Open
inline bool ucisopenpunct (UNICODE c) {
   return ucisprop(c, UCCATFLAG_Ps);
   }

//! Punctuation, Close
inline bool ucisclosepunct (UNICODE c) {
   return ucisprop(c, UCCATFLAG_Pe);
   }

//! Punctuation, Initial
inline bool ucisinitialpunct (UNICODE c) {
   return ucisprop(c, UCCATFLAG_Pi);
   }

//! Punctuation, Final
inline bool ucisfinalpunct (UNICODE c) {
   return ucisprop(c, UCCATFLAG_Pf);
   }

//! Strong Right-to-Left Directionality
inline bool ucisrtl (UNICODE c) {
   return ucisbidiprop(c, BIDIFLAG_R | BIDIFLAG_AL);
   }

//! Strong Left-to-Right Directionality
inline bool ucisltr (UNICODE c) {
   return ucisbidiprop(c, BIDIFLAG_L);
   }

//! Strong R-to-L or L-to-R Directionality
inline bool ucisstrong (UNICODE c) {
   return ucisbidiprop(c, BIDIFLAG_L | BIDIFLAG_R | BIDIFLAG_AL);
   }

//! Weak Directionality
inline bool ucisweak (UNICODE c) {
   return ucisbidiprop(c, BIDIFLAG_EN | BIDIFLAG_ES | BIDIFLAG_ET | BIDIFLAG_AN | BIDIFLAG_CS);
   }

//! Netutral Directionality
inline bool ucisneutral (UNICODE c) {
   return ucisbidiprop(c, BIDIFLAG_WS | BIDIFLAG_ON);
   }


//! Mark (any)
inline bool ucismark (UNICODE c) {
   return ucisprop(c, UCCATFLAG_Mn | UCCATFLAG_Mc | UCCATFLAG_Me);
   }

//! Letter, modifier
inline bool ucismodif (UNICODE c) {
   return ucisprop(c, UCCATFLAG_Lm);
   }

//! Punctuation, Connecting
inline bool ucisconnect (UNICODE c) {
   return ucisprop(c, UCCATFLAG_Pc);
   }

//! Punctuation, Dash
inline bool ucisdash (UNICODE c) {
   return ucisprop(c, UCCATFLAG_Pd);
   }

//! Symbol, math
inline bool ucismath (UNICODE c) {
   return ucisprop(c, UCCATFLAG_Sm);
   }

//! Symbol, Currency
inline bool uciscurrency (UNICODE c) {
   return ucisprop(c, UCCATFLAG_Sc);
   }

//! Symbol, modifier
inline bool ucismodifsymbol (UNICODE c) {
   return ucisprop(c, UCCATFLAG_Sk);
   }

//! Mark, non-spacing
inline bool ucisnsmark (UNICODE c) {
   return ucisprop(c, UCCATFLAG_Mn);
   }

//! Mark, spacing
inline bool ucisspmark (UNICODE c) {
   return ucisprop(c, UCCATFLAG_Mc);
   }

//! Mark, enclosing
inline bool ucisenclosing (UNICODE c) {
   return ucisprop(c, UCCATFLAG_Me);
   }

//! Seperator, Line
inline bool ucislsep (UNICODE c) {
   return ucisprop(c, UCCATFLAG_Zl);
   }

//! Seperator, Paragraph
inline bool ucispsep (UNICODE c) {
   return ucisprop(c, UCCATFLAG_Zp);
   }

//! Han glyph
inline bool ucishan (UNICODE c) {
   return ((c >= 0x4e00 && c <= 0x9fff) || (c >= 0xf900 && c <= 0xfaff));
   }

//! Hangul glyph
inline bool ucishangul (UNICODE c) {
   return (c >= 0xac00 && c <= 0xd7ff);
   }

#endif


/*----------------------------------------------------------------------------*/
/*    Flags for MtTextGetStringExtUC()                                        */
/*----------------------------------------------------------------------------*/

#define MTTEXT_NULLIfUnknown  0x00000001     //!<  Return null instead of "unknown message... 

/*----------------------------------------------------------------------------*/

#if defined(__cplusplus)
extern "C" {


//!:Associate with "MtText funtions"
//!\addtogroup MtText MtText Functions
//!@{
//!\deprecated Use TEXTID instead

#if !defined(DEPRECATE_GROUPKEY) || defined(MISYSTEMDLL)
//! Read a string from messages.txt
//!
//! \deprecated  Use MISTRING instead.
//!
//! Reads a string, which the caller should free when done using.
//! If the requested message can't be found, it returns a string
//! which says the message couldn't be found.  If you'd rather get
//! a NULL pointer in that case, use MtTextGetStringExtUC()
//!
//! @see MtTextGetStringExtUC()
LIBEXPORT UNICODE* MtTextGetStringUC (
   const char* group, 
   const char* key
   );
#endif

#if !defined(DEPRECATE_GROUPKEY) || defined(MISYSTEMDLL)
//! Read a string from messages.txt
//!
//! \deprecated  Use MISTRING instead.
//!
//! Reads a string, which the caller should free when done using.
//!
//! @param group
//!   Message group
//! @param key
//!   Message key
//! @param flags
//! \li <b>MTTEXT_NULLIfUnknown</b> Return NULL instead of 
//!   unknown message string
LIBEXPORT UNICODE* MtTextGetStringExtUC (
   const char* group, 
   const char* key, 
   UINT32 flags
   );
#endif

//!@} 

}  //! Extern "C"
#endif   //!< #if defined C++

//!:Associate with "Unicode Functions"
//!\addtogroup Unicode Unicode Functions
//!@{

#if defined(__cplusplus)
extern "C" {
#endif

//! Convert an ASCII string to Unicode (with length limit).
//!
//! Note, this function doesn't do any character set mapping.  It
//! simply casts all the source characters to (UNICODE)
LIBEXPORT UNICODE* strntouc (
   UNICODE *dest, 
   const char *source, 
   int len
   );

//! Convert an ASCII string to Unicode.
//!
//! Note, this function doesn't do any character set mapping.  It
//! simply casts all the source characters to (UNICODE)
LIBEXPORT UNICODE* strtouc (
   UNICODE *dest, 
   const char *source
   );

//! Convert an ASCII string to Unicode and append to existing string.
//!
//! Note, this function doesn't do any character set mapping.  It
//! simply casts all the source characters to (UNICODE)
LIBEXPORT UNICODE* strtouccat (
   UNICODE *dest, 
   const char * source
   );

//! UNICODE version of strdup().
//!
//! Note, this function doesn't do any character set mapping.  It
//! simply casts all the source characters to (UNICODE).
//! The resulting string should be MmFree()'d.
LIBEXPORT UNICODE* strtoucdup (
   const char*
   );

//! UNICODE version of isdigit().
//!
//!   It only detects the ASCII numerals 0-9.  
//!   Note, that some implementations of isdigit() will crash if given
//!   Unicode values > 255.
#ifdef __cplusplus
inline bool ucisdigit (UNICODE digit) {
   return (digit >= (UNICODE)'0' && digit <= (UNICODE)'9');
   }
#else
/* Inlines only work on C++ and some 3rd party libs don't
 * compile for C++.  The XmHTML lib is one such and has a module
 */
#define ucisdigit(x) ((x) >= (UNICODE)'0' && (x) <= (UNICODE)'9')
#endif

//! UNICODE version of strcat().
LIBEXPORT UNICODE* ucstrcat (
   UNICODE *dest, 
   const UNICODE *source
   );

//! UNICODE version of strchr().
//!
//! Find the first occurance of a given character in a Unicode string.
//! Returns a pointer to that character or NULL if not found.
LIBEXPORT UNICODE* ucstrchr (
   const UNICODE *p, 
   UNICODE value
   );

//! Compare two UNICODE strings.
//!
//! Returns -1 if (p1 < p2),
//!         1 if (p1 > p2)
//!         0 if (p1 == p2)
LIBEXPORT int ucstrcmp (
   const UNICODE *p1, 
   const UNICODE *p2
   );

//! Unicode version of strcpy().
LIBEXPORT UNICODE* ucstrcpy (
   UNICODE *dest, 
   const UNICODE *source
   );

//! Unicode version of strdup().
LIBEXPORT UNICODE* ucstrdup (
   const UNICODE *s
   );

//! Unicode version of strlen().
LIBEXPORT size_t ucstrlen (
   const UNICODE *p
   );

//! Compare two UNICODE strings (case insensitive).
//!
//! Returns -1 if (p1 < p2),
//!         1 if (p1 > p2)
//!         0 if (p1 == p2)
LIBEXPORT int ucstricmp (
   const UNICODE *p1, 
   const UNICODE *p2
   );

//! Convert UNICODE string to lowercase (in-place).
//!
//! Modifies the input string
LIBEXPORT UNICODE* ucstrlwr (UNICODE *p);

//! UNICODE version of strncat().
LIBEXPORT UNICODE* ucstrncat (
   UNICODE *p1, 
   const UNICODE *p2, 
   int len
   );

//! Compare two UNICODE strings (length limited).
//!
//! Returns -1 if (p1 < p2),
//!         1 if (p1 > p2)
//!         0 if (p1 == p2)
LIBEXPORT int ucstrncmp (
   const UNICODE *p1, 
   const UNICODE *p2, 
   int len
   );

//! UNICODE version of strncpy().
LIBEXPORT UNICODE* ucstrncpy (
   UNICODE *dest, 
   const UNICODE *source, 
   int len
   );

//! Compare two UNICODE strings (length limited and case insensitive).
//!
//! Returns -1 if (p1 < p2),
//!         1 if (p1 > p2)
//!         0 if (p1 == p2)
LIBEXPORT int ucstrnicmp (
   const UNICODE *p1, 
   const UNICODE *p2, 
   int len
   );

//! UNICODE version of strpbrk().
//!
//! Locate the first occurrence in the string s of any of the characters in the 
//! string accept.  Returns a pointer to it or NULL if not found.
LIBEXPORT UNICODE* ucstrpbrk (
   UNICODE *s, 
   UNICODE *accept
   );

//! UNICODE version of strrchr().
//!
//! Locates the last occurrence of the character value in the string s.
//! Returns a pointer to it or NULL if not found.
LIBEXPORT UNICODE* ucstrrchr (
   UNICODE *s, 
   UNICODE value
   );

//! UNICODE version of strspn().
//!
//! Returns the length of the initial segment of s which consists entirely of 
//! characters in accept.
LIBEXPORT size_t ucstrspn (
   const UNICODE *s, 
   const UNICODE *accept
   );

//! UNICODE version of strstr().
//!
//! Finds the first occurrence of the substring p in the string s.
//! Returns a pointer to it or NULL if not found.
LIBEXPORT UNICODE* ucstrstr (
   UNICODE *s, 
   const UNICODE *p
   );

//! Extract token from string
//!
//! A "token" is a nonempty string of characters not occurring in the string
//! delim, followed by \\0 or by a character occurring in delim.
//!
//! The ucstrtok() function can be used to parse the string s
//! into tokens.  The first call the ucstrtok() should have s
//! as its first argument.  Subsequent calls should have the
//! first argument set to NULL.  Each call returns a pointer to
//! next token, or