LCOV - code coverage report
Current view: directory - extensions/universalchardet/src/base - CharDistribution.h (source / functions) Found Hit Coverage
Test: app.info Lines: 49 0 0.0 %
Date: 2012-06-02 Functions: 11 0 0.0 %

       1                 : /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
       2                 : /* ***** BEGIN LICENSE BLOCK *****
       3                 :  * Version: MPL 1.1/GPL 2.0/LGPL 2.1
       4                 :  *
       5                 :  * The contents of this file are subject to the Mozilla Public License Version
       6                 :  * 1.1 (the "License"); you may not use this file except in compliance with
       7                 :  * the License. You may obtain a copy of the License at
       8                 :  * http://www.mozilla.org/MPL/
       9                 :  *
      10                 :  * Software distributed under the License is distributed on an "AS IS" basis,
      11                 :  * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
      12                 :  * for the specific language governing rights and limitations under the
      13                 :  * License.
      14                 :  *
      15                 :  * The Original Code is Mozilla Communicator client code.
      16                 :  *
      17                 :  * The Initial Developer of the Original Code is
      18                 :  * Netscape Communications Corporation.
      19                 :  * Portions created by the Initial Developer are Copyright (C) 1998
      20                 :  * the Initial Developer. All Rights Reserved.
      21                 :  *
      22                 :  * Contributor(s):
      23                 :  *
      24                 :  * Alternatively, the contents of this file may be used under the terms of
      25                 :  * either the GNU General Public License Version 2 or later (the "GPL"), or
      26                 :  * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
      27                 :  * in which case the provisions of the GPL or the LGPL are applicable instead
      28                 :  * of those above. If you wish to allow use of your version of this file only
      29                 :  * under the terms of either the GPL or the LGPL, and not to allow others to
      30                 :  * use your version of this file under the terms of the MPL, indicate your
      31                 :  * decision by deleting the provisions above and replace them with the notice
      32                 :  * and other provisions required by the GPL or the LGPL. If you do not delete
      33                 :  * the provisions above, a recipient may use your version of this file under
      34                 :  * the terms of any one of the MPL, the GPL or the LGPL.
      35                 :  *
      36                 :  * ***** END LICENSE BLOCK ***** */
      37                 : 
      38                 : #ifndef CharDistribution_h__
      39                 : #define CharDistribution_h__
      40                 : 
      41                 : #include "nscore.h"
      42                 : 
      43                 : #define ENOUGH_DATA_THRESHOLD 1024
      44                 :  
      45                 : #define MINIMUM_DATA_THRESHOLD  4
      46                 : 
      47                 : class CharDistributionAnalysis
      48                 : {
      49                 : public:
      50               0 :   CharDistributionAnalysis() {Reset(false);}
      51                 : 
      52                 :   //feed a block of data and do distribution analysis
      53                 :   void HandleData(const char* aBuf, PRUint32 aLen) {}
      54                 :   
      55                 :   //Feed a character with known length
      56               0 :   void HandleOneChar(const char* aStr, PRUint32 aCharLen)
      57                 :   {
      58                 :     PRInt32 order;
      59                 : 
      60                 :     //we only care about 2-bytes character in our distribution analysis
      61               0 :     order = (aCharLen == 2) ? GetOrder(aStr) : -1;
      62                 : 
      63               0 :     if (order >= 0)
      64                 :     {
      65               0 :       mTotalChars++;
      66                 :       //order is valid
      67               0 :       if ((PRUint32)order < mTableSize)
      68                 :       {
      69               0 :         if (512 > mCharToFreqOrder[order])
      70               0 :           mFreqChars++;
      71                 :       }
      72                 :     }
      73               0 :   }
      74                 : 
      75                 :   //return confidence base on existing data
      76                 :   float GetConfidence(void);
      77                 : 
      78                 :   //Reset analyser, clear any state 
      79               0 :   void      Reset(bool aIsPreferredLanguage) 
      80                 :   {
      81               0 :     mDone = false;
      82               0 :     mTotalChars = 0;
      83               0 :     mFreqChars = 0;
      84               0 :     mDataThreshold = aIsPreferredLanguage ? 0 : MINIMUM_DATA_THRESHOLD;
      85               0 :   }
      86                 : 
      87                 :   //This function is for future extension. Caller can use this function to control
      88                 :   //analyser's behavior
      89                 :   void      SetOpion(){}
      90                 : 
      91                 :   //It is not necessary to receive all data to draw conclusion. For charset detection,
      92                 :   // certain amount of data is enough
      93               0 :   bool GotEnoughData() {return mTotalChars > ENOUGH_DATA_THRESHOLD;}
      94                 : 
      95                 : protected:
      96                 :   //we do not handle character base on its original encoding string, but 
      97                 :   //convert this encoding string to a number, here called order.
      98                 :   //This allow multiple encoding of a language to share one frequency table 
      99               0 :   virtual PRInt32 GetOrder(const char* str) {return -1;}
     100                 :   
     101                 :   //If this flag is set to true, detection is done and conclusion has been made
     102                 :   bool     mDone;
     103                 : 
     104                 :   //The number of characters whose frequency order is less than 512
     105                 :   PRUint32 mFreqChars;
     106                 : 
     107                 :   //Total character encounted.
     108                 :   PRUint32 mTotalChars;
     109                 : 
     110                 :   //Number of hi-byte characters needed to trigger detection
     111                 :   PRUint32 mDataThreshold;
     112                 : 
     113                 :   //Mapping table to get frequency order from char order (get from GetOrder())
     114                 :   const PRInt16  *mCharToFreqOrder;
     115                 : 
     116                 :   //Size of above table
     117                 :   PRUint32 mTableSize;
     118                 : 
     119                 :   //This is a constant value varies from language to language, it is used in 
     120                 :   //calculating confidence. See my paper for further detail.
     121                 :   float    mTypicalDistributionRatio;
     122                 : };
     123                 : 
     124                 : 
     125                 : class EUCTWDistributionAnalysis: public CharDistributionAnalysis
     126                 : {
     127                 : public:
     128                 :   EUCTWDistributionAnalysis();
     129                 : protected:
     130                 : 
     131                 :   //for euc-TW encoding, we are interested 
     132                 :   //  first  byte range: 0xc4 -- 0xfe
     133                 :   //  second byte range: 0xa1 -- 0xfe
     134                 :   //no validation needed here. State machine has done that
     135               0 :   PRInt32 GetOrder(const char* str) 
     136               0 :   { if ((unsigned char)*str >= (unsigned char)0xc4)  
     137               0 :       return 94*((unsigned char)str[0]-(unsigned char)0xc4) + (unsigned char)str[1] - (unsigned char)0xa1;
     138                 :     else
     139               0 :       return -1;
     140                 :   }
     141                 : };
     142                 : 
     143                 : 
     144                 : class EUCKRDistributionAnalysis : public CharDistributionAnalysis
     145                 : {
     146                 : public:
     147                 :   EUCKRDistributionAnalysis();
     148                 : protected:
     149                 :   //for euc-KR encoding, we are interested 
     150                 :   //  first  byte range: 0xb0 -- 0xfe
     151                 :   //  second byte range: 0xa1 -- 0xfe
     152                 :   //no validation needed here. State machine has done that
     153               0 :   PRInt32 GetOrder(const char* str) 
     154               0 :   { if ((unsigned char)*str >= (unsigned char)0xb0)  
     155               0 :       return 94*((unsigned char)str[0]-(unsigned char)0xb0) + (unsigned char)str[1] - (unsigned char)0xa1;
     156                 :     else
     157               0 :       return -1;
     158                 :   }
     159                 : };
     160                 : 
     161                 : class GB2312DistributionAnalysis : public CharDistributionAnalysis
     162                 : {
     163                 : public:
     164                 :   GB2312DistributionAnalysis();
     165                 : protected:
     166                 :   //for GB2312 encoding, we are interested 
     167                 :   //  first  byte range: 0xb0 -- 0xfe
     168                 :   //  second byte range: 0xa1 -- 0xfe
     169                 :   //no validation needed here. State machine has done that
     170               0 :   PRInt32 GetOrder(const char* str) 
     171               0 :   { if ((unsigned char)*str >= (unsigned char)0xb0 && (unsigned char)str[1] >= (unsigned char)0xa1)  
     172               0 :       return 94*((unsigned char)str[0]-(unsigned char)0xb0) + (unsigned char)str[1] - (unsigned char)0xa1;
     173                 :     else
     174               0 :       return -1;
     175                 :   }
     176                 : };
     177                 : 
     178                 : 
     179                 : class Big5DistributionAnalysis : public CharDistributionAnalysis
     180                 : {
     181                 : public:
     182                 :   Big5DistributionAnalysis();
     183                 : protected:
     184                 :   //for big5 encoding, we are interested 
     185                 :   //  first  byte range: 0xa4 -- 0xfe
     186                 :   //  second byte range: 0x40 -- 0x7e , 0xa1 -- 0xfe
     187                 :   //no validation needed here. State machine has done that
     188               0 :   PRInt32 GetOrder(const char* str) 
     189               0 :   { if ((unsigned char)*str >= (unsigned char)0xa4)  
     190               0 :       if ((unsigned char)str[1] >= (unsigned char)0xa1)
     191               0 :         return 157*((unsigned char)str[0]-(unsigned char)0xa4) + (unsigned char)str[1] - (unsigned char)0xa1 +63;
     192                 :       else
     193               0 :         return 157*((unsigned char)str[0]-(unsigned char)0xa4) + (unsigned char)str[1] - (unsigned char)0x40;
     194                 :     else
     195               0 :       return -1;
     196                 :   }
     197                 : };
     198                 : 
     199                 : class SJISDistributionAnalysis : public CharDistributionAnalysis
     200                 : {
     201                 : public:
     202                 :   SJISDistributionAnalysis();
     203                 : protected:
     204                 :   //for sjis encoding, we are interested 
     205                 :   //  first  byte range: 0x81 -- 0x9f , 0xe0 -- 0xfe
     206                 :   //  second byte range: 0x40 -- 0x7e,  0x81 -- oxfe
     207                 :   //no validation needed here. State machine has done that
     208               0 :   PRInt32 GetOrder(const char* str) 
     209                 :   { 
     210                 :     PRInt32 order;
     211               0 :     if ((unsigned char)*str >= (unsigned char)0x81 && (unsigned char)*str <= (unsigned char)0x9f)  
     212               0 :       order = 188 * ((unsigned char)str[0]-(unsigned char)0x81);
     213               0 :     else if ((unsigned char)*str >= (unsigned char)0xe0 && (unsigned char)*str <= (unsigned char)0xef)  
     214               0 :       order = 188 * ((unsigned char)str[0]-(unsigned char)0xe0 + 31);
     215                 :     else
     216               0 :       return -1;
     217               0 :     order += (unsigned char)*(str+1) - 0x40;
     218               0 :     if ((unsigned char)str[1] > (unsigned char)0x7f)
     219               0 :       order--;
     220               0 :     return order;
     221                 :   }
     222                 : };
     223                 : 
     224                 : class EUCJPDistributionAnalysis : public CharDistributionAnalysis
     225                 : {
     226                 : public:
     227                 :   EUCJPDistributionAnalysis();
     228                 : protected:
     229                 :   //for euc-JP encoding, we are interested 
     230                 :   //  first  byte range: 0xa0 -- 0xfe
     231                 :   //  second byte range: 0xa1 -- 0xfe
     232                 :   //no validation needed here. State machine has done that
     233               0 :   PRInt32 GetOrder(const char* str) 
     234               0 :   { if ((unsigned char)*str >= (unsigned char)0xa0)  
     235               0 :       return 94*((unsigned char)str[0]-(unsigned char)0xa1) + (unsigned char)str[1] - (unsigned char)0xa1;
     236                 :     else
     237               0 :       return -1;
     238                 :   }
     239                 : };
     240                 : 
     241                 : #endif //CharDistribution_h__
     242                 : 

Generated by: LCOV version 1.7