kdecore Library API Documentation

kcharsets.cpp

00001 /* This file is part of the KDE libraries
00002     Copyright (C) 1999 Lars Knoll (knoll@kde.org)
00003     $Id: kcharsets.cpp,v 1.134.2.3 2003/06/29 21:42:59 deller Exp $
00004 
00005     This library is free software; you can redistribute it and/or
00006     modify it under the terms of the GNU Library General Public
00007     License as published by the Free Software Foundation; either
00008     version 2 of the License, or (at your option) any later version.
00009 
00010     This library is distributed in the hope that it will be useful,
00011     but WITHOUT ANY WARRANTY; without even the implied warranty of
00012     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
00013     Library General Public License for more details.
00014 
00015     You should have received a copy of the GNU Library General Public License
00016     along with this library; see the file COPYING.LIB.  If not, write to
00017     the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
00018     Boston, MA 02111-1307, USA.
00019 */
00020 #include "kcharsets.h"
00021 
00022 #include "kentities.c"
00023 
00024 #include <kapplication.h>
00025 #include <kglobal.h>
00026 #include <klocale.h>
00027 #include <kconfig.h>
00028 
00029 #include <qfontinfo.h>
00030 #include <qstrlist.h>
00031 #include <qfontdatabase.h>
00032 #include <kdebug.h>
00033 
00034 #include <qtextcodec.h>
00035 #include <qmap.h>
00036 #include <qcstring.h>
00037 
00038 #include <assert.h>
00039 
00040 #define CHARSETS_COUNT 33
00041 
00042 static const char * const language_names[] = {
00043     I18N_NOOP( "Other" ),
00044     I18N_NOOP( "Arabic" ),
00045     I18N_NOOP( "Baltic" ),
00046     I18N_NOOP( "Central European" ),
00047     I18N_NOOP( "Chinese Simplified" ),
00048     I18N_NOOP( "Chinese Traditional" ),
00049     I18N_NOOP( "Cyrillic" ),
00050     I18N_NOOP( "Greek" ),
00051     I18N_NOOP( "Hebrew" ),
00052     I18N_NOOP( "Japanese" ),
00053     I18N_NOOP( "Korean" ),
00054     I18N_NOOP( "Thai" ),
00055     I18N_NOOP( "Turkish" ),
00056     I18N_NOOP( "Western European" ),
00057     I18N_NOOP( "Tamil" ),
00058     I18N_NOOP( "Unicode" ),
00059     I18N_NOOP( "Northern Saami" )
00060 };
00061 
00062 // this list gives the charsets that can be used to display a file given in a certain encoding.
00063 // the list should be in order of preference
00064 // left side is the name returned by the codec used, right side the name of the charset as
00065 // used in kcharsets.cpp
00066 // 'unicode' will always be chosen as last resort, so it only needs to be added to the list,
00067 // if it should get a higher priority
00068 // every line must end with 0
00069 
00070 static const char* const charsets_for_encoding[] = {
00071     "koi8-r",                "koi8-r","cp 1251","koi8-u","iso-8859-5", 0,
00072     "koi8-u",                "koi8-u","cp 1251","iso-8859-5","koi8-r", 0,
00073     "iso 8859-1",            "iso8859-1","iso8859-15", 0,
00074     "iso 8859-2",            "iso8859-2","unicode","iso8859-1", 0,
00075     "iso 8859-3",            "iso8859-3","unicode","iso8859-1", 0,
00076     "iso 8859-4",            "iso8859-4","unicode","iso8859-13", "iso8859-1", 0,
00077     "iso 8859-5",            "iso8859-5","koi8-u","koi8-r", 0,
00078     "iso 8859-6",            "unicode","iso8859-6", 0,
00079     "iso 8859-7",            "iso8859-7", 0,
00080     "iso 8859-8",            "iso8859-8", 0,
00081     "iso 8859-8-i",          "iso8859-8", 0,
00082     "iso 8859-9",            "iso8859-9","unicode","iso8859-1", 0,
00083     "iso 8859-11",           "iso8859-11", 0,
00084     "iso 8859-13",           "iso8859-13","unicode","iso8859-4", "iso8859-1", 0,
00085     "iso 8859-15",           "iso8859-15","unicode","iso8859-1", 0,
00086     "utf8",                  "unicode","iso8859-1", 0,
00087     "utf16",                 "unicode","iso8859-1", 0,
00088     "iso-10646-ucs-2",       "unicode","iso8859-1", 0,
00089     "cp 1250",               "iso8859-2", 0,
00090     "cp 1251",               "cp 1251","koi8-u","koi8-r","iso8859-5", 0,
00091     "cp 1252",               "iso8859-1", 0,
00092     "cp 1253",               "iso8859-7", 0,
00093     "cp 1254",               "iso8859-9", 0,
00094     "cp 1255",               "iso8859-8", 0,
00095     "cp 1256",               "unicode","iso8859-6", 0,
00096     "cp 1257",               "iso8859-13", "iso8859-4", 0,
00097     "ibm852",                "unicode","iso-8859-2", 0,
00098     "tis620",                "iso8859-11", 0,
00099     "eucjp",                 "eucjp","unicode","iso8859-1", 0,
00100     "sjis",                  "eucjp","unicode","iso8859-1", 0,
00101     "jis7",                  "eucjp","unicode","iso8859-1", 0,
00102     "big5",                  "big5","unicode","iso8859-1", 0,
00103     "gbk",                   "gb2312.1980-0","gbk-0","unicode","iso8859-1", 0,
00104     "gb18030",               "gb18030.2000-1", "gb18030.2000-0", "unicode", "gbk-0", "gb2313.1980-0", "iso8859-1", 0,
00105     "gb2312",                "gb2312.1980-0","unicode","iso8859-1", 0,
00106     "euckr",                 "euckr","unicode","iso8859-1", 0,
00107     "tscii",                 "tscii", 0,
00108     "pt 154",                "pt 154","cp 1251","koi8-u","koi8-r","iso8859-5", 0,
00109     "winsami2",              "winsami2", "cp1252", "unicode", 0,
00110     0 }; // extra 0 for end
00111 
00112 // 0 other
00113 // 1 Arabic
00114 // 2 Baltic
00115 // 3 Central European
00116 // 4 Chinese Simplified
00117 // 5 Chinese Traditional
00118 // 6 Cyrillic
00119 // 7 Greek
00120 // 8 Hebrew
00121 // 9 Japanese
00122 // 10 Korean
00123 // 11 Thai
00124 // 12 Turkish
00125 // 13 Western European
00126 // 14 Tamil
00127 // 15 Unicode
00128 // 16 Northern Sami
00129 static struct LanguageForEncoding
00130     {
00131     const char* index;
00132     int data;
00133     } const language_for_encoding[] = {
00134     { "iso 8859-1", 13 },
00135     { "iso 8859-15", 13 },
00136     { "cp 1252", 13 },
00137     { "iso 8859-2", 3 },
00138     { "iso 8859-3", 3 },
00139     { "iso 8859-4", 2 },
00140     { "iso 8859-13", 2 },
00141     { "cp 1250", 3 },
00142     { "cp 1254", 12 },
00143     { "cp 1257", 2 },
00144     { "ibm852", 3 },
00145     { "koi8-r", 6 },
00146     { "iso 8859-5", 6 },
00147     { "cp 1251", 6 },
00148     { "koi8-u", 6 },
00149     { "pt 154", 6 },
00150     { "big5", 5 },
00151     { "gb18030", 4 },
00152     { "gbk", 4 },
00153     { "gb2312", 4 },
00154     { "euckr", 10 },
00155     { "sjis", 9 },
00156     { "jis7", 9 },
00157     { "eucjp", 9 },
00158     { "iso 8859-7", 7 },
00159     { "cp 1253", 7 },
00160     { "iso 8859-6", 1 },
00161     { "cp 1256", 1 },
00162     { "iso 8859-8", 8 },
00163     { "iso 8859-8-i", 8 },
00164     { "cp 1255", 8 },
00165     { "iso 8859-9", 12 },
00166     { "tis620", 11 },
00167     { "iso 8859-11", 11 },
00168     { "utf8", 15 },
00169     { "utf16", 15 },
00170     { "utf7", 15 },
00171     { "ucs2", 15 },
00172     { "iso-10646-ucs-2", 15 },
00173     { "winsami2", 16},
00174     { 0, 0 } };
00175 
00176 // defines some different names for codecs that are built into Qt.
00177 static struct Builtin
00178     {
00179     const char* index;
00180     const char* data;
00181     } const builtin[] = {
00182     { "iso-ir-111", "koi8-r" },
00183     { "koi8-ru", "koi8-u" },
00184     { "koi8r", "koi8-r" },
00185     { "koi8u", "koi8-u" },
00186     { "koi unified", "koi8-r" },
00187     { "us-ascii", "iso 8859-1" },
00188     { "usascii", "iso 8859-1" },
00189     { "x-utf-8", "utf-8" },
00190     { "x-utf-7", "utf-7" },
00191     { "unicode-1-1-utf-7", "utf-7" },
00192     { "ucs2", "iso-10646-ucs-2" },
00193     { "iso10646-1", "iso-10646-ucs-2" },
00194     { "big5-0", "big5" },
00195     { "gb18030.2000-1", "gb18030" },
00196     { "gb18030.2000-0", "gb18030" },
00197     { "gbk-0", "gbk" },
00198     { "gb2312", "gbk" },
00199     { "gb2312.1980-0", "gbk" },
00200     { "euc-kr", "euckr" },
00201     { "x-euc-kr", "euckr" },
00202     { "euc-jp", "eucjp" },
00203     { "x-euc-jp", "eucjp" },
00204     { "jisx0201.1976-0", "eucjp" },
00205     { "jisx0208.1983-0", "eucjp" },
00206     { "jisx0208.1990-0", "eucjp" },
00207     { "jisx0208.1997-0", "eucjp" },
00208     { "jisx0212.1990-0", "eucjp" },
00209     { "jisx0213.2000-1", "eucjp" },
00210     { "jisx0213.2000-2", "eucjp" },
00211     { "shift_jis", "sjis" },
00212     { "shift-jis", "sjis" },
00213     { "x-sjis", "sjis" },
00214     { "iso-2022-jp", "jis7" },
00215     { "windows1250", "cp 1250" },
00216     { "windows1251", "cp 1251" },
00217     { "windows1252", "cp 1252" },
00218     { "windows1253", "cp 1253" },
00219     { "windows1254", "cp 1254" },
00220     { "windows1255", "cp 1255" },
00221     { "windows1256", "cp 1256" },
00222     { "windows1257", "cp 1257" },
00223     { "windows-1250", "cp 1250" },
00224     { "windows-1251", "cp 1251" },
00225     { "windows-1252", "cp 1252" },
00226     { "windows-1253", "cp 1253" },
00227     { "windows-1254", "cp 1254" },
00228     { "windows-1255", "cp 1255" },
00229     { "windows-1256", "cp 1256" },
00230     { "windows-1257", "cp 1257" },
00231     { "x-windows-1250", "cp 1250" },
00232     { "x-windows-1251", "cp 1251" },
00233     { "x-windows-1252", "cp 1252" },
00234     { "x-windows-1253", "cp 1253" },
00235     { "x-windows-1254", "cp 1254" },
00236     { "x-windows-1255", "cp 1255" },
00237     { "x-windows-1256", "cp 1256" },
00238     { "x-windows-1257", "cp 1257" },
00239     { "cp-1250", "cp 1250" },
00240     { "cp-1251", "cp 1251" },
00241     { "cp-1252", "cp 1252" },
00242     { "cp-1253", "cp 1253" },
00243     { "cp-1254", "cp 1254" },
00244     { "cp-1255", "cp 1255" },
00245     { "cp-1256", "cp 1256" },
00246     { "cp-1257", "cp 1257" },
00247     { "x-cp-1250", "cp 1250" },
00248     { "x-cp-1251", "cp 1251" },
00249     { "x-cp-1252", "cp 1252" },
00250     { "x-cp-1253", "cp 1253" },
00251     { "x-cp-1254", "cp 1254" },
00252     { "x-cp-1255", "cp 1255" },
00253     { "x-cp-1256", "cp 1256" },
00254     { "x-cp-1257", "cp 1257" },
00255     { "tis620", "iso 8859-11" },
00256     { "tis-620", "iso 8859-11" },
00257     { "thai-tis620", "iso 8859-11" },
00258     { "windows-874", "iso 8859-11" },
00259     { "windows874", "iso 8859-11" },
00260     { "x-windows-874", "iso 8859-11" },
00261     { "cp874", "iso 8859-11" },
00262     { "cp-874", "iso 8859-11" },
00263     { "x-cp-874", "iso 8859-11" },
00264     { "ksc5601.1987-0", "euckr" },
00265     { "ks_c_5601-1987", "euckr" },
00266     { "iso-8859-1", "iso 8859-1" },
00267     { "iso-8859-2", "iso 8859-2" },
00268     { "iso-8859-3", "iso 8859-3" },
00269     { "iso-8859-4", "iso 8859-4" },
00270     { "iso-8859-5", "iso 8859-5" },
00271     { "iso-8859-6", "iso 8859-6" },
00272     { "iso-8859-7", "iso 8859-7" },
00273     { "iso-8859-8", "iso 8859-8" },
00274     { "iso-8859-9", "iso 8859-9" },
00275     { "iso-8859-10", "iso 8859-10" },
00276     { "iso-8859-11", "iso 8859-11" },
00277     { "iso-8859-12", "iso 8859-12" },
00278     { "iso-8859-13", "iso 8859-13" },
00279     { "iso-8859-14", "iso 8859-14" },
00280     { "iso-8859-15", "iso 8859-15" },
00281     { "tscii", "tscii" },
00282     { "paratype-154", "pt 154" },
00283     { "pt-154", "pt 154" },
00284     { "x-winsami2", "winsami2" },
00285     { 0, 0 }};
00286 
00287 // some different names for the encodings defined in the charmaps files.
00288 // even though the charmap file names are all uppercase, the names are all lowercase here.
00289 static struct Aliases
00290     {
00291     const char* index;
00292     const char* data;
00293     } const aliases[] = {
00294     { "cp852", "ibm852" },
00295     { 0, 0 }};
00296 
00297 // some last resort hints in case the charmap file couldn't be found. This gives at least a partial conversion
00298 // and helps making things readable.
00299 // the name used as input here is already converted to the more canonical name as defined in the aliases array.
00300 static struct ConversionHints
00301     {
00302     const char* index;
00303     const char* data;
00304     } const conversion_hints[] = {
00305     { "cp1250", "iso-8859-2" },
00306     { "koi8-r", "iso-8859-5" },
00307     { "koi8-u", "koi8-r" },
00308     { "utf16", "iso-10646-ucs-2" },
00309     { 0, 0 }};
00310 
00311 
00312 // search an array of items index/data, index is const char*, data is T, find first matching index
00313 // and return data, or return 0
00314 template< typename T, typename Data >
00315 Data kcharsets_array_search( const T* start, const char* entry )
00316 {
00317     for( const T* pos = start;
00318          pos->index != 0;
00319          ++pos )
00320         if( qstrcmp( pos->index, entry ) == 0 )
00321             return pos->data;
00322     return 0;
00323 }
00324 
00325 
00326 class KCharsetsPrivate
00327 {
00328 public:
00329     KCharsetsPrivate(KCharsets* _kc)
00330         : codecForNameDict(43, false) // case insensitive
00331     {
00332         db = 0;
00333         kc = _kc;
00334     }
00335     ~KCharsetsPrivate()
00336     {
00337         delete db;
00338     }
00339     QFontDatabase *db;
00340     QAsciiDict<QTextCodec> codecForNameDict;
00341     KCharsets* kc;
00342 };
00343 
00344 // --------------------------------------------------------------------------
00345 
00346 KCharsets::KCharsets()
00347 {
00348     d = new KCharsetsPrivate(this);
00349 }
00350 
00351 KCharsets::~KCharsets()
00352 {
00353     delete d;
00354 }
00355 
00356 QChar KCharsets::fromEntity(const QString &str) 
00357 {
00358     QChar res = QChar::null;
00359 
00360     int pos = 0;
00361     if(str[pos] == '&') pos++;
00362 
00363     // Check for '&#000' or '&#x0000' sequence
00364     if (str[pos] == '#' && str.length()-pos > 1) {
00365         bool ok;
00366         pos++;
00367         if (str[pos] == 'x' || str[pos] == 'X') {
00368             pos++;
00369             // '&#x0000', hexadeciaml character reference
00370             QString tmp(str.unicode()+pos, str.length()-pos);
00371             res = tmp.toInt(&ok, 16);
00372         } else {
00373             //  '&#0000', deciaml character reference
00374             QString tmp(str.unicode()+pos, str.length()-pos);
00375             res = tmp.toInt(&ok, 10);
00376         }
00377         return res;
00378     }
00379 
00380     const entity *e = findEntity(str.ascii(), str.length());
00381 
00382     if(!e)
00383     {
00384         //kdDebug( 0 ) << "unknown entity " << str <<", len = " << str.length() << endl;
00385         return QChar::null;
00386     }
00387     //kdDebug() << "got entity " << str << " = " << e->code << endl;
00388 
00389     return QChar(e->code);
00390 }
00391 
00392 QChar KCharsets::fromEntity(const QString &str, int &len) 
00393 {
00394     // entities are never longer than 8 chars... we start from
00395     // that length and work backwards...
00396     len = 8;
00397     while(len > 0)
00398     {
00399         QString tmp = str.left(len);
00400         QChar res = fromEntity(tmp);
00401         if( res != QChar::null ) return res;
00402         len--;
00403     }
00404     return QChar::null;
00405 }
00406 
00407 
00408 QString KCharsets::toEntity(const QChar &ch) 
00409 {
00410     QString ent;
00411     ent.sprintf("&#0x%x;", ch.unicode());
00412     return ent;
00413 }
00414 
00415 QString KCharsets::resolveEntities( const QString &input )
00416 {
00417     QString text = input;
00418     const QChar *p = text.unicode();
00419     const QChar *end = p + text.length();
00420     const QChar *ampersand = 0;
00421     bool scanForSemicolon = false;
00422 
00423     for ( ; p < end; ++p ) {
00424         QChar ch = *p;
00425 
00426         if ( ch == '&' ) {
00427             ampersand = p;
00428             scanForSemicolon = true;
00429             continue;
00430         }
00431 
00432         if ( ch != ';' || scanForSemicolon == false )
00433             continue;
00434 
00435         assert( ampersand );
00436 
00437         scanForSemicolon = false;
00438 
00439         const QChar *entityBegin = ampersand + 1;
00440 
00441         uint entityLength = p - entityBegin;
00442         if ( entityLength == 0 )
00443             continue;
00444 
00445         QChar entityValue = KCharsets::fromEntity( QConstString( entityBegin, entityLength ).string() );
00446         if ( entityValue.isNull() )
00447             continue;
00448 
00449         uint ampersandPos = ampersand - text.unicode();
00450 
00451         text[ ampersandPos ] = entityValue;
00452         text.remove( ampersandPos + 1, entityLength + 1 );
00453         p = text.unicode() + ampersandPos;
00454         end = text.unicode() + text.length();
00455         ampersand = 0;
00456     }
00457 
00458     return text;
00459 }
00460 
00461 QStringList KCharsets::availableEncodingNames()
00462 {
00463     QStringList available;
00464 
00465     const char* const* pos = charsets_for_encoding;
00466     while( *pos != 0 ) {
00467         //kdDebug(0) << "key = " << *pos << endl;
00468 
00469 
00470         // iterate thorugh the list and find the first charset that is available
00471         for( const char* const* charsets = pos + 1;
00472              *charsets != 0;
00473              ++charsets ) {
00474             //kdDebug(0) << "checking for " << *charsets << endl;
00475 #ifdef __GNUC__
00476 #warning FIXME?
00477 #endif
00478             if( true ) {
00479                 //kdDebug(0) << *charsets << " available" << endl;
00480                 available.append( QString::fromLatin1( *pos ));
00481                 break;
00482             }
00483         }
00484         while( *pos != 0 ) // find end of line
00485             ++pos;
00486         ++pos; // move to the next line
00487     }
00488     return available;
00489 }
00490 
00491 QString KCharsets::languageForEncoding( const QString &encoding )
00492 {
00493     int lang = kcharsets_array_search< LanguageForEncoding, int >
00494         ( language_for_encoding, encoding.latin1());
00495     return i18n( language_names[lang] );
00496 }
00497 
00498 QString KCharsets::encodingForName( const QString &descriptiveName )
00499 {
00500     int left = descriptiveName.find( "( " );
00501     return descriptiveName.mid( left + 2, descriptiveName.find( " )" )
00502       - left - 2 );
00503 }
00504 
00505 QStringList KCharsets::descriptiveEncodingNames()
00506 {
00507   QStringList encodings = availableEncodingNames();
00508   QStringList::Iterator it;
00509   for( it = encodings.begin(); it != encodings.end(); ++it ) {
00510       QString lang = KGlobal::charsets()->languageForEncoding( *it );
00511       *it = lang + " ( " + *it + " )";
00512   }
00513   encodings.sort();
00514   return encodings;
00515 }
00516 
00517 QTextCodec *KCharsets::codecForName(const QString &n) const
00518 {
00519     bool b;
00520     return codecForName( n, b );
00521 }
00522 
00523 QTextCodec *KCharsets::codecForName(const QString &n, bool &ok) const
00524 {
00525     ok = true;
00526 
00527     QTextCodec* codec = 0;
00528     // dict lookup is case insensitive anyway
00529     if((codec = d->codecForNameDict[n.isEmpty() ? "->locale<-" : n.latin1()]))
00530         return codec; // cache hit, return
00531 
00532     if (n.isEmpty()) {
00533         codec = KGlobal::locale()->codecForEncoding();
00534         d->codecForNameDict.replace("->locale<-", codec);
00535         return codec;
00536     }
00537 
00538     QCString name = n.lower().latin1();
00539     QCString key = name;
00540     if (name.right(8) == "_charset")
00541        name.truncate(name.length()-8);
00542 
00543     if (name.isEmpty()) {
00544       ok = false;
00545       return QTextCodec::codecForName("iso8859-1");
00546     }
00547 
00548     codec = QTextCodec::codecForName(name);
00549 
00550     if(codec) {
00551         d->codecForNameDict.replace(key, codec);
00552         return codec;
00553     }
00554 
00555     // these codecs are built into Qt, but the name given for the codec is different,
00556     // so QTextCodec did not recognise it.
00557     QCString cname = kcharsets_array_search< Builtin, const char* >( builtin, name.data());
00558 
00559     if(!cname.isEmpty())
00560         codec = QTextCodec::codecForName(cname);
00561 
00562     if(codec)
00563     {
00564         d->codecForNameDict.replace(key, codec);
00565         return codec;
00566     }
00567 
00568     QString dir;
00569     {
00570     KConfigGroupSaver cfgsav( KGlobal::config(), "i18n" );
00571     dir = KGlobal::config()->readPathEntry("i18ndir", QString::fromLatin1("/usr/share/i18n/charmaps"));
00572     dir += "/";
00573     }
00574 
00575     // these are codecs not included in Qt. They can be build up if the corresponding charmap
00576     // is available in the charmap directory.
00577     cname = kcharsets_array_search< Aliases, const char* >( aliases, name.data());
00578 
00579     if(cname.isEmpty())
00580         cname = name;
00581     cname = cname.upper();
00582 
00583     codec = QTextCodec::loadCharmapFile((QString)(dir + cname.data()));
00584 
00585     if(codec) {
00586         d->codecForNameDict.replace(key, codec);
00587         return codec;
00588     }
00589 
00590     // this also failed, the last resort is now to take some compatibility charmap
00591 
00592     cname = cname.lower();
00593     cname = kcharsets_array_search< ConversionHints, const char* >( conversion_hints, (const char*)cname );
00594 
00595     if(!cname.isEmpty())
00596         codec = QTextCodec::codecForName(cname);
00597 
00598     if(codec) {
00599         d->codecForNameDict.replace(key, codec);
00600         return codec;
00601     }
00602 
00603     // could not assign a codec, let's return Latin1
00604     ok = false;
00605     return QTextCodec::codecForName("iso8859-1");
00606 }
KDE Logo
This file is part of the documentation for kdelibs Version 3.1.4.
Documentation copyright © 1996-2002 the KDE developers.
Generated on Sun Feb 27 22:14:45 2005 by doxygen 1.3.4 written by Dimitri van Heesch, © 1997-2001