khtml Library API Documentation
Overview Class Hierarchy Classes Classes (annotated) Members Source Files

decoder.cpp

00001 /*
00002     This file is part of the KDE libraries
00003 
00004     Copyright (C) 1999 Lars Knoll (knoll@mpi-hd.mpg.de)
00005 
00006     This library is free software; you can redistribute it and/or
00007     modify it under the terms of the GNU Library General Public
00008     License as published by the Free Software Foundation; either
00009     version 2 of the License, or (at your option) any later version.
00010 
00011     This library is distributed in the hope that it will be useful,
00012     but WITHOUT ANY WARRANTY; without even the implied warranty of
00013     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
00014     Library General Public License for more details.
00015 
00016     You should have received a copy of the GNU Library General Public License
00017     along with this library; see the file COPYING.LIB.  If not, write to
00018     the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
00019     Boston, MA 02111-1307, USA.
00020 */
00021 //----------------------------------------------------------------------------
00022 //
00023 // KDE HTML Widget -- decoder for input stream
00024 // $Id: decoder.cpp,v 1.59 2002/11/22 03:05:38 mueller Exp $
00025 
00026 #undef DECODE_DEBUG
00027 //#define DECODE_DEBUG
00028 
00029 #include <assert.h>
00030 
00031 #include "decoder.h"
00032 using namespace khtml;
00033 
00034 #include "htmlhashes.h"
00035 
00036 #include <qregexp.h>
00037 #include <qtextcodec.h>
00038 
00039 #include <kglobal.h>
00040 #include <kcharsets.h>
00041 
00042 #include <ctype.h>
00043 #include <kdebug.h>
00044 #include <klocale.h>
00045 
00046 class KanjiCode
00047 {
00048 public:
00049     enum Type {ASCII, JIS, EUC, SJIS, UNICODE, UTF8 };
00050     static enum Type judge(const char *str);
00051     static const int ESC;
00052     static const int _SS2_;
00053     static const unsigned char kanji_map_sjis[];
00054     static int ISkanji(int code)
00055     {
00056         if (code >= 0x100)
00057                     return 0;
00058         return (kanji_map_sjis[code & 0xff] & 1);
00059     }
00060 
00061     static int ISkana(int code)
00062     {
00063         if (code >= 0x100)
00064                     return 0;
00065         return (kanji_map_sjis[code & 0xff] & 2);
00066     }
00067 
00068 };
00069 
00070 const int KanjiCode::ESC = 0x1b;
00071 const int KanjiCode::_SS2_ = 0x8e;
00072 
00073 const unsigned char KanjiCode::kanji_map_sjis[] =
00074 {
00075     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
00076     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
00077     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
00078     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
00079     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
00080     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
00081     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
00082     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
00083     0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
00084     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
00085     0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
00086     2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
00087     2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
00088     2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
00089     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
00090     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0
00091 };
00092 
00093 /*
00094  * EUC-JP is
00095  *     [0xa1 - 0xfe][0xa1 - 0xfe]
00096  *     0x8e[0xa1 - 0xfe](SS2)
00097  *     0x8f[0xa1 - 0xfe][0xa1 - 0xfe](SS3)
00098  *
00099  * Shift_Jis is
00100  *     [0x81 - 0x9f, 0xe0 - 0xef(0xfe?)][0x40 - 0x7e, 0x80 - 0xfc]
00101  *
00102  * Shift_Jis Hankaku Kana is
00103  *     [0xa1 - 0xdf]
00104  */
00105 
00106 /*
00107  * KanjiCode::judge() is based on judge_jcode() from jvim
00108  *     http://hp.vector.co.jp/authors/VA003457/vim/
00109  *
00110  * Special Thanks to Kenichi Tsuchida
00111  */
00112 
00113 /*
00114  * Maybe we should use QTextCodec::heuristicContentMatch()
00115  * But it fails detection. It's not useful.
00116  */
00117 
00118 enum KanjiCode::Type KanjiCode::judge(const char *str)
00119 {
00120     enum Type code;
00121     int i;
00122     int bfr = FALSE;            /* Kana Moji */
00123     int bfk = 0;                /* EUC Kana */
00124     int sjis = 0;
00125     int euc = 0;
00126 
00127     const unsigned char *ptr = (const unsigned char *) str;
00128     int size = strlen(str);
00129 
00130     code = ASCII;
00131 
00132     i = 0;
00133     while (i < size) {
00134         if (ptr[i] == ESC && (size - i >= 3)) {
00135             if ((ptr[i + 1] == '$' && ptr[i + 2] == 'B')
00136             || (ptr[i + 1] == '(' && ptr[i + 2] == 'B')) {
00137                 code = JIS;
00138                 goto breakBreak;
00139             } else if ((ptr[i + 1] == '$' && ptr[i + 2] == '@')
00140                     || (ptr[i + 1] == '(' && ptr[i + 2] == 'J')) {
00141                 code = JIS;
00142                 goto breakBreak;
00143             } else if (ptr[i + 1] == '(' && ptr[i + 2] == 'I') {
00144                 code = JIS;
00145                 i += 3;
00146             } else if (ptr[i + 1] == ')' && ptr[i + 2] == 'I') {
00147                 code = JIS;
00148                 i += 3;
00149             } else {
00150                 i++;
00151             }
00152             bfr = FALSE;
00153             bfk = 0;
00154         } else {
00155             if (ptr[i] < 0x20) {
00156                 bfr = FALSE;
00157                 bfk = 0;
00158                 /* ?? check kudokuten ?? && ?? hiragana ?? */
00159                 if ((i >= 2) && (ptr[i - 2] == 0x81)
00160                         && (0x41 <= ptr[i - 1] && ptr[i - 1] <= 0x49)) {
00161                     code = SJIS;
00162                     sjis += 100;        /* kudokuten */
00163                 } else if ((i >= 2) && (ptr[i - 2] == 0xa1)
00164                         && (0xa2 <= ptr[i - 1] && ptr[i - 1] <= 0xaa)) {
00165                     code = EUC;
00166                     euc += 100;         /* kudokuten */
00167                 } else if ((i >= 2) && (ptr[i - 2] == 0x82) && (0xa0 <= ptr[i - 1])) {
00168                     sjis += 40;         /* hiragana */
00169                 } else if ((i >= 2) && (ptr[i - 2] == 0xa4) && (0xa0 <= ptr[i - 1])) {
00170                     euc += 40;  /* hiragana */
00171                 }
00172             } else {
00173                 /* ?? check hiragana or katana ?? */
00174                 if ((size - i > 1) && (ptr[i] == 0x82) && (0xa0 <= ptr[i + 1])) {
00175                     sjis++;     /* hiragana */
00176                 } else if ((size - i > 1) && (ptr[i] == 0x83)
00177                          && (0x40 <= ptr[i + 1] && ptr[i + 1] <= 0x9f)) {
00178                     sjis++;     /* katakana */
00179                 } else if ((size - i > 1) && (ptr[i] == 0xa4) && (0xa0 <= ptr[i + 1])) {
00180                     euc++;      /* hiragana */
00181                 } else if ((size - i > 1) && (ptr[i] == 0xa5) && (0xa0 <= ptr[i + 1])) {
00182                     euc++;      /* katakana */
00183                 }
00184                 if (bfr) {
00185                     if ((i >= 1) && (0x40 <= ptr[i] && ptr[i] <= 0xa0) && ISkanji(ptr[i - 1])) {
00186                         code = SJIS;
00187                         goto breakBreak;
00188                     } else if ((i >= 1) && (0x81 <= ptr[i - 1] && ptr[i - 1] <= 0x9f) && ((0x40 <= ptr[i] && ptr[i] < 0x7e) || (0x7e < ptr[i] && ptr[i] <= 0xfc))) {
00189                         code = SJIS;
00190                         goto breakBreak;
00191                     } else if ((i >= 1) && (0xfd <= ptr[i] && ptr[i] <= 0xfe) && (0xa1 <= ptr[i - 1] && ptr[i - 1] <= 0xfe)) {
00192                         code = EUC;
00193                         goto breakBreak;
00194                     } else if ((i >= 1) && (0xfd <= ptr[i - 1] && ptr[i - 1] <= 0xfe) && (0xa1 <= ptr[i] && ptr[i] <= 0xfe)) {
00195                         code = EUC;
00196                         goto breakBreak;
00197                     } else if ((i >= 1) && (ptr[i] < 0xa0 || 0xdf < ptr[i]) && (0x8e == ptr[i - 1])) {
00198                         code = SJIS;
00199                         goto breakBreak;
00200                     } else if (ptr[i] <= 0x7f) {
00201                         code = SJIS;
00202                         goto breakBreak;
00203                     } else {
00204                         if (0xa1 <= ptr[i] && ptr[i] <= 0xa6) {
00205                             euc++;      /* sjis hankaku kana kigo */
00206                         } else if (0xa1 <= ptr[i] && ptr[i] <= 0xdf) {
00207                             ;   /* sjis hankaku kana */
00208                         } else if (0xa1 <= ptr[i] && ptr[i] <= 0xfe) {
00209                             euc++;
00210                         } else if (0x8e == ptr[i]) {
00211                             euc++;
00212                         } else if (0x20 <= ptr[i] && ptr[i] <= 0x7f) {
00213                             sjis++;
00214                         }
00215                         bfr = FALSE;
00216                         bfk = 0;
00217                     }
00218                 } else if (0x8e == ptr[i]) {
00219                     if (size - i <= 1) {
00220                         ;
00221                     } else if (0xa1 <= ptr[i + 1] && ptr[i + 1] <= 0xdf) {
00222                         /* EUC KANA or SJIS KANJI */
00223                         if (bfk == 1) {
00224                             euc += 100;
00225                         }
00226                         bfk++;
00227                         i++;
00228                     } else {
00229                         /* SJIS only */
00230                         code = SJIS;
00231                         goto breakBreak;
00232                     }
00233                 } else if (0x81 <= ptr[i] && ptr[i] <= 0x9f) {
00234                     /* SJIS only */
00235                     code = SJIS;
00236                     if ((size - i >= 1)
00237                             && ((0x40 <= ptr[i + 1] && ptr[i + 1] <= 0x7e)
00238                             || (0x80 <= ptr[i + 1] && ptr[i + 1] <= 0xfc))) {
00239                         goto breakBreak;
00240                     }
00241                 } else if (0xfd <= ptr[i] && ptr[i] <= 0xfe) {
00242                     /* EUC only */
00243                     code = EUC;
00244                     if ((size - i >= 1)
00245                             && (0xa1 <= ptr[i + 1] && ptr[i + 1] <= 0xfe)) {
00246                         goto breakBreak;
00247                     }
00248                 } else if (ptr[i] <= 0x7f) {
00249                     ;
00250                 } else {
00251                     bfr = TRUE;
00252                     bfk = 0;
00253                 }
00254             }
00255             i++;
00256         }
00257     }
00258     if (code == ASCII) {
00259         if (sjis > euc) {
00260             code = SJIS;
00261         } else if (sjis < euc) {
00262             code = EUC;
00263         }
00264     }
00265 breakBreak:
00266     return (code);
00267 }
00268 
00269 Decoder::Decoder()
00270 {
00271     // latin1
00272     m_codec = QTextCodec::codecForMib(4);
00273     m_decoder = m_codec->makeDecoder();
00274     enc = 0;
00275     body = false;
00276     beginning = true;
00277     visualRTL = false;
00278     haveEncoding = false;
00279     m_automaticDetectionLanguage = SemiautomaticDetection;
00280 }
00281 Decoder::~Decoder()
00282 {
00283     delete m_decoder;
00284 }
00285 
00286 void Decoder::setEncoding(const char *_encoding, bool force)
00287 {
00288 #ifdef DECODE_DEBUG
00289     kdDebug(6005) << "setEncoding " << _encoding << " " << force << endl;
00290 #endif
00291     enc = _encoding;
00292 
00293     QTextCodec *old = m_codec;
00294 #ifdef DECODE_DEBUG
00295     kdDebug(6005) << "old encoding is:" << m_codec->name() << endl;
00296 #endif
00297     enc = enc.lower();
00298 #ifdef DECODE_DEBUG
00299     kdDebug(6005) << "requesting:" << enc << endl;
00300 #endif
00301     if(enc.isNull() || enc.isEmpty())
00302         return;
00303     if(enc == "visual") // hebrew visually ordered
00304         enc = "iso8859-8";
00305     bool b;
00306     m_codec = KGlobal::charsets()->codecForName(enc, b);
00307     if(m_codec->mibEnum() == 11)  {
00308         // iso8859-8 (visually ordered)
00309         m_codec = QTextCodec::codecForName("iso8859-8-i");
00310         visualRTL = true;
00311     }
00312     if( !b ) // in case the codec didn't exist, we keep the old one (fixes some sites specifying invalid codecs)
00313         m_codec = old;
00314     else
00315         haveEncoding = force;
00316     delete m_decoder;
00317     m_decoder = m_codec->makeDecoder();
00318     if (m_codec->mibEnum() == 1000) // utf 16
00319         haveEncoding = false; // force auto detection
00320 #ifdef DECODE_DEBUG
00321     kdDebug(6005) << "Decoder::encoding used is" << m_codec->name() << endl;
00322 #endif
00323 }
00324 
00325 const char *Decoder::encoding() const
00326 {
00327     return enc;
00328 }
00329 
00330 QString Decoder::decode(const char *data, int len)
00331 {
00332     // this is not completely efficient, since the function might go
00333     // through the html head several times...
00334 
00335     if(!haveEncoding && !body) {
00336 #ifdef DECODE_DEBUG
00337         kdDebug(6005) << "looking for charset definition" << endl;
00338 #endif
00339         // check for UTF-16
00340         uchar * uchars = (uchar *) data;
00341         if( uchars[0] == 0xfe && uchars[1] == 0xff ||
00342             uchars[0] == 0xff && uchars[1] == 0xfe ) {
00343             enc = "ISO-10646-UCS-2";
00344             haveEncoding = true;
00345             m_codec = QTextCodec::codecForMib(1000);
00346             delete m_decoder;
00347             m_decoder = m_codec->makeDecoder();
00348         } else {
00349 
00350             if(m_codec->mibEnum() != 1000) {  // utf16
00351                 // replace '\0' by spaces, for buggy pages
00352                 char *d = const_cast<char *>(data);
00353                 int i = len - 1;
00354                 while(i >= 0) {
00355                     if(d[i] == 0) d[i] = ' ';
00356                     i--;
00357                 }
00358             }
00359             buffer += QCString(data, len+1);
00360 
00361             // we still don't have an encoding, and are in the head
00362             // the following tags are allowed in <head>:
00363             // SCRIPT|STYLE|META|LINK|OBJECT|TITLE|BASE
00364 
00365             const char *ptr = buffer.data();
00366             while(*ptr != '\0')
00367             {
00368                 if(*ptr == '<') {
00369                     bool end = false;
00370                     ptr++;
00371                     if(*ptr == '/') ptr++, end=true;
00372                     char tmp[20];
00373                     int len = 0;
00374                     while (
00375                         ((*ptr >= 'a') && (*ptr <= 'z') ||
00376                          (*ptr >= 'A') && (*ptr <= 'Z') ||
00377                          (*ptr >= '0') && (*ptr <= '9'))
00378                         && len < 19 )
00379                     {
00380                         tmp[len] = tolower( *ptr );
00381                         ptr++;
00382                         len++;
00383                     }
00384                     tmp[len] = 0;
00385                     int id = khtml::getTagID(tmp, len);
00386                     if(end) id += ID_CLOSE_TAG;
00387 
00388                     switch( id ) {
00389                     case ID_META:
00390                     {
00391                         // found a meta tag...
00392                         //ptr += 5;
00393                         const char * end = ptr;
00394                         while(*end != '>' && *end != '\0') end++;
00395                         if ( *end == '\0' ) break;
00396                         QCString str( ptr, (end-ptr)+1);
00397                         str = str.lower();
00398                         int pos = 0;
00399                         //if( (pos = str.find("http-equiv", pos)) == -1) break;
00400                         //if( (pos = str.find("content-type", pos)) == -1) break;
00401                         while( pos < ( int ) str.length() ) {
00402                             if( (pos = str.find("charset", pos)) == -1) break;
00403                             pos += 7;
00404                             // skip whitespace..
00405                             while(  pos < (int)str.length() && str[pos] <= ' ' ) pos++;
00406                             if ( pos == ( int )str.length()) break;
00407                             if ( str[pos++] != '=' ) continue;
00408                             while ( pos < ( int )str.length() &&
00409                                     ( str[pos] <= ' ' ) || str[pos] == '=' || str[pos] == '"' || str[pos] == '\'')
00410                                 pos++;
00411 
00412                             // end ?
00413                             if ( pos == ( int )str.length() ) break;
00414                             uint endpos = pos;
00415                             while( endpos < str.length() &&
00416                                    (str[endpos] != ' ' && str[endpos] != '"' && str[endpos] != '\''
00417                                     && str[endpos] != ';' && str[endpos] != '>') )
00418                                 endpos++;
00419                             enc = str.mid(pos, endpos-pos);
00420 #ifdef DECODE_DEBUG
00421                             kdDebug( 6005 ) << "Decoder: found charset: " << enc.data() << endl;
00422 #endif
00423                             setEncoding(enc, true);
00424                             if( haveEncoding ) goto found;
00425 
00426                             if ( endpos >= str.length() || str[endpos] == '/' || str[endpos] == '>' ) break;
00427 
00428                             pos = endpos + 1;
00429                         }
00430                     }
00431                     case ID_SCRIPT:
00432                     case (ID_SCRIPT+ID_CLOSE_TAG):
00433                     case ID_NOSCRIPT:
00434                     case (ID_NOSCRIPT+ID_CLOSE_TAG):
00435                     case ID_STYLE:
00436                     case (ID_STYLE+ID_CLOSE_TAG):
00437                     case ID_LINK:
00438                     case (ID_LINK+ID_CLOSE_TAG):
00439                     case ID_OBJECT:
00440                     case (ID_OBJECT+ID_CLOSE_TAG):
00441                     case ID_TITLE:
00442                     case (ID_TITLE+ID_CLOSE_TAG):
00443                     case ID_BASE:
00444                     case (ID_BASE+ID_CLOSE_TAG):
00445                     case ID_HTML:
00446                     case ID_HEAD:
00447                     case 0:
00448                     case (0 + ID_CLOSE_TAG ):
00449                         break;
00450                     default:
00451                         body = true;
00452 #ifdef DECODE_DEBUG
00453                         kdDebug( 6005 ) << "Decoder: no charset found. Id=" << id << endl;
00454 #endif
00455                         goto found;
00456                     }
00457                 }
00458                 else
00459                     ptr++;
00460             }
00461             return QString::null;
00462         }
00463     }
00464 
00465  found:
00466     if ( !haveEncoding ) {
00467 #ifdef DECODE_DEBUG
00468         kdDebug( 6005 ) << "Decoder: use auto-detect (" << strlen(data) << ")" << endl;
00469 #endif
00470         if ( m_automaticDetectionLanguage == Decoder::Arabic ) {
00471             enc = automaticDetectionForArabic( data );
00472         }
00473         else if ( m_automaticDetectionLanguage == Decoder::Baltic ) {
00474             enc = automaticDetectionForBaltic( data );
00475         }
00476         else if ( m_automaticDetectionLanguage == Decoder::CentralEuropean ) {
00477             enc = automaticDetectionForCentralEuropean( data );
00478         }
00479         else if ( m_automaticDetectionLanguage == Decoder::Russian ) {
00480             enc = automaticDetectionForCyrillic( data, Decoder::Russian );
00481         }
00482         else if ( m_automaticDetectionLanguage == Decoder::Ukrainian ) {
00483             enc = automaticDetectionForCyrillic( data, Decoder::Ukrainian );
00484         }
00485         else if ( m_automaticDetectionLanguage == Decoder::Greek ) {
00486             enc = automaticDetectionForGreek( data );
00487         }
00488         else if ( m_automaticDetectionLanguage == Decoder::Hebrew ) {
00489             enc = automaticDetectionForHebrew( data );
00490         }
00491         else if ( m_automaticDetectionLanguage == Decoder::Japanese ) {
00492             switch ( KanjiCode::judge( data ) ) {
00493                 case KanjiCode::JIS:
00494                     enc = "jis7";
00495                     break;
00496                 case KanjiCode::EUC:
00497                     enc = "eucjp";
00498                     break;
00499                 case KanjiCode::SJIS:
00500                     enc = "sjis";
00501                     break;
00502                 default:
00503                     enc = NULL;
00504                     break;
00505             }
00506         }
00507         else if ( m_automaticDetectionLanguage == Decoder::Turkish ) {
00508             enc = automaticDetectionForTurkish( data );
00509         }
00510         else if ( m_automaticDetectionLanguage == Decoder::WesternEuropean ) {
00511             enc = automaticDetectionForWesternEuropean( data );
00512         }
00513 #ifdef DECODE_DEBUG
00514         kdDebug( 6005 ) << "Decoder: auto detect encoding is " << enc.data() << endl;
00515 #endif
00516         if ( !enc.isEmpty() )
00517             setEncoding( enc.data(), true );
00518     }
00519 
00520     // if we still haven't found an encoding latin1 will be used...
00521     // this is according to HTML4.0 specs
00522     if (!m_codec)
00523     {
00524         if(enc.isEmpty()) enc = "iso8859-1";
00525         m_codec = QTextCodec::codecForName(enc);
00526         // be sure not to crash
00527         if(!m_codec) {
00528             m_codec = QTextCodec::codecForMib(4);
00529             enc = "iso8859-1";
00530         }
00531         delete m_decoder;
00532         m_decoder = m_codec->makeDecoder();
00533     }
00534     QString out;
00535 
00536     if(!buffer.isEmpty() && enc != "ISO-10646-UCS-2") {
00537         out = m_decoder->toUnicode(buffer, buffer.length());
00538         buffer = "";
00539     } else {
00540         if(m_codec->mibEnum() != 1000) // utf16
00541         {
00542             // ### hack for a bug in QTextCodec. It cut's the input stream
00543             // in case there are \0 in it. ZDNET has them inside... :-(
00544             char *d = const_cast<char *>(data);
00545             int i = len - 1;
00546             while(i >= 0) {
00547                 if(*(d+i) == 0) *(d+i) = ' ';
00548                 i--;
00549             }
00550         }
00551         out = m_decoder->toUnicode(data, len);
00552     }
00553 
00554     // the hell knows, why the output does sometimes have a QChar::null at
00555     // the end...
00556     if(out[out.length()-1] == QChar::null)
00557         assert(0);
00558     return out;
00559 }
00560 
00561 QString Decoder::flush() const
00562 {
00563     return m_decoder->toUnicode(buffer, buffer.length());
00564 }
00565 
00566 QCString Decoder::automaticDetectionForArabic( const char* str )
00567 {
00568     const unsigned char *ptr = (const unsigned char*)str;
00569     int size = strlen( str );
00570 
00571     for ( int i = 0; i < size; ++i ) {
00572         if ( ( ptr[ i ] >= 0x80 && ptr[ i ] <= 0x9F ) || ptr[ i ] == 0xA1 || ptr[ i ] == 0xA2 || ptr[ i ] == 0xA3
00573              || ( ptr[ i ] >= 0xA5 && ptr[ i ] <= 0xAB ) || ( ptr[ i ] >= 0xAE && ptr[ i ] <= 0xBA )
00574              || ptr[ i ] == 0xBC || ptr[ i ] == 0xBD || ptr[ i ] == 0xBE || ptr[ i ] == 0xC0
00575              || ( ptr[ i ] >= 0xDB && ptr[ i ] <= 0xDF ) || ( ptr[ i ] >= 0xF3 && ptr[ i ] <= 0xFF ) ) {
00576             return "cp1256";
00577         }
00578     }
00579 
00580     return "iso-8859-6";
00581 }
00582 
00583 QCString Decoder::automaticDetectionForBaltic( const char* str )
00584 {
00585     const unsigned char *ptr = (const unsigned char*)str;
00586     int size = strlen( str );
00587 
00588     for ( int i = 0; i < size; ++i ) {
00589         if ( ( ptr[ i ] >= 0x80 && ptr[ i ] <= 0x9E ) )
00590              return "cp1257";
00591 
00592         if ( ptr[ i ] == 0xA1 || ptr[ i ] == 0xA5 )
00593             return "iso-8859-13";
00594     }
00595 
00596     return "iso-8859-13";
00597 }
00598 
00599 QCString Decoder::automaticDetectionForCentralEuropean( const char* str )
00600 {
00601     const unsigned char *ptr = (const unsigned char*)str;
00602     int size = strlen( str );
00603 
00604     QCString charset = QCString();
00605     for ( int i = 0; i < size; ++i ) {
00606         if ( ptr[ i ] >= 0x80 && ptr[ i ] <= 0x9F ) {
00607             if ( ptr[ i ] == 0x81 || ptr[ i ] == 0x83 || ptr[ i ] == 0x90 || ptr[ i ] == 0x98 )
00608                 return "ibm852";
00609 
00610             if ( i + 1 > size )
00611                 return "cp1250";
00612             else { // maybe ibm852 ?
00613                 charset = "cp1250";
00614                 continue;
00615             }
00616         }
00617         if ( ptr[ i ] == 0xA5 || ptr[ i ] == 0xAE || ptr[ i ] == 0xBE || ptr[ i ] == 0xC3 || ptr[ i ] == 0xD0 || ptr[ i ] == 0xE3 || ptr[ i ] == 0xF0 ) {
00618             if ( i + 1 > size )
00619                 return "iso-8859-2";
00620             else {  // maybe ibm852 ?
00621                 if ( charset.isNull() )
00622                     charset = "iso-8859-2";
00623                 continue;
00624             }
00625         }
00626     }
00627 
00628     if ( charset.isNull() )
00629         charset = "iso-8859-3";
00630 
00631     return charset.data();
00632 }
00633 
00634 QCString Decoder::automaticDetectionForCyrillic( const char* str, AutomaticDetectinonLanguage _language )
00635 {
00636     const unsigned char *ptr = (const unsigned char*)str;
00637     int size = strlen( str );
00638 
00639     QCString charset = QCString();
00640     for ( int i = 0; i < size; ++i ) {
00641         if ( ptr[ i ] >= 0x80 && ptr[ i ] <= 0x9F ) {
00642             if ( ptr[ i ] == 0x98 ) {
00643                 if ( _language == Russian )
00644                     return "koi8-r";
00645                 else if ( _language == Ukrainian )
00646                     return "koi8-u";
00647             }
00648 
00649             if ( i + 1 > size )
00650                 return "cp1251";
00651             else { // maybe koi8-r or koi8-u ?
00652                 charset = "cp1251";
00653                 continue;
00654             }
00655         }
00656         else {
00657             if ( i + 1 > size )
00658                 return "iso-8859-5";
00659             else {  // maybe koi8-r (koi8-u) or cp1251 ?
00660                 if ( charset.isNull() )
00661                     charset = "iso-8859-5";
00662                 continue;
00663             }
00664         }
00665     }
00666 
00667     if ( charset.isNull() )
00668         charset = "iso-8859-5";
00669 
00670     return charset.data();
00671 }
00672 
00673 QCString Decoder::automaticDetectionForGreek( const char* str )
00674 {
00675     const unsigned char *ptr = (const unsigned char*)str;
00676     int size = strlen( str );
00677 
00678     for ( int i = 0; i < size; ++i ) {
00679         if ( ptr[ i ] == 0x80 || ( ptr[ i ] >= 0x82 && ptr[ i ] <= 0x87 ) || ptr[ i ] == 0x89 || ptr[ i ] == 0x8B
00680              || ( ptr[ i ] >= 0x91 && ptr[ i ] <= 0x97 ) || ptr[ i ] == 0x99 || ptr[ i ] == 0x9B || ptr[ i ] == 0xA4
00681              || ptr[ i ] == 0xA5 || ptr[ i ] == 0xAE ) {
00682             return "cp1253";
00683         }
00684     }
00685 
00686     return "iso-8859-7";
00687 }
00688 
00689 QCString Decoder::automaticDetectionForHebrew( const char* str )
00690 {
00691     const unsigned char *ptr = (const unsigned char*)str;
00692     int size = strlen( str );
00693 
00694     for ( int i = 0; i < size; ++i ) {
00695         if ( ptr[ i ] == 0x80 || ( ptr[ i ] >= 0x82 && ptr[ i ] <= 0x89 ) || ptr[ i ] == 0x8B
00696              || ( ptr[ i ] >= 0x91 && ptr[ i ] <= 0x99 ) || ptr[ i ] == 0x9B || ptr[ i ] == 0xA1 || ( ptr[ i ] >= 0xBF && ptr[ i ] <= 0xC9 )
00697              || ( ptr[ i ] >= 0xCB && ptr[ i ] <= 0xD8 ) ) {
00698             return "cp1255";
00699         }
00700 
00701         if ( ptr[ i ] == 0xDF )
00702             return "iso-8859-8";;
00703     }
00704 
00705     return "iso-8859-8";
00706 }
00707 
00708 QCString Decoder::automaticDetectionForTurkish( const char* str )
00709 {
00710     const unsigned char *ptr = (const unsigned char*)str;
00711     int size = strlen( str );
00712 
00713     for ( int i = 0; i < size; ++i ) {
00714         if ( ptr[ i ] == 0x80 || ( ptr[ i ] >= 0x82 && ptr[ i ] <= 0x8C ) || ( ptr[ i ] >= 0x91 && ptr[ i ] <= 0x9C ) || ptr[ i ] == 0x9F ) {
00715             return "cp1254";
00716         }
00717     }
00718 
00719     return "iso-8859-9";
00720 }
00721 
00722 QCString Decoder::automaticDetectionForWesternEuropean( const char* str )
00723 {
00724     const unsigned char *ptr = (const unsigned char*)str;
00725     int size = strlen( str );
00726 
00727     for ( int i = 0; i < size; ++i ) {
00728         if ( ptr[ i ] >= 0x80 && ptr[ i ] <= 0x9F )
00729             return "cp1252";
00730     }
00731 
00732     return "iso-8859-1"; //"iso-8859-15"; Which better at default ?
00733 }
00734 
00735 
00736 // -----------------------------------------------------------------------------
00737 #undef DECODE_DEBUG
KDE Logo
This file is part of the documentation for kdelibs Version 3.1.5.
Documentation copyright © 1996-2002 the KDE developers.
Generated on Sun Mar 20 11:20:01 2005 by doxygen 1.3.3 written by Dimitri van Heesch, © 1997-2001