1 : /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 : /* ***** BEGIN LICENSE BLOCK *****
3 : * Version: MPL 1.1/GPL 2.0/LGPL 2.1
4 : *
5 : * The contents of this file are subject to the Mozilla Public License Version
6 : * 1.1 (the "License"); you may not use this file except in compliance with
7 : * the License. You may obtain a copy of the License at
8 : * http://www.mozilla.org/MPL/
9 : *
10 : * Software distributed under the License is distributed on an "AS IS" basis,
11 : * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
12 : * for the specific language governing rights and limitations under the
13 : * License.
14 : *
15 : * The Original Code is Mozilla Communicator client code.
16 : *
17 : * The Initial Developer of the Original Code is
18 : * Netscape Communications Corporation.
19 : * Portions created by the Initial Developer are Copyright (C) 1998
20 : * the Initial Developer. All Rights Reserved.
21 : *
22 : * Contributor(s):
23 : *
24 : * Alternatively, the contents of this file may be used under the terms of
25 : * either the GNU General Public License Version 2 or later (the "GPL"), or
26 : * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
27 : * in which case the provisions of the GPL or the LGPL are applicable instead
28 : * of those above. If you wish to allow use of your version of this file only
29 : * under the terms of either the GPL or the LGPL, and not to allow others to
30 : * use your version of this file under the terms of the MPL, indicate your
31 : * decision by deleting the provisions above and replace them with the notice
32 : * and other provisions required by the GPL or the LGPL. If you do not delete
33 : * the provisions above, a recipient may use your version of this file under
34 : * the terms of any one of the MPL, the GPL or the LGPL.
35 : *
36 : * ***** END LICENSE BLOCK ***** */
37 :
38 : #ifndef CharDistribution_h__
39 : #define CharDistribution_h__
40 :
41 : #include "nscore.h"
42 :
43 : #define ENOUGH_DATA_THRESHOLD 1024
44 :
45 : #define MINIMUM_DATA_THRESHOLD 4
46 :
47 : class CharDistributionAnalysis
48 : {
49 : public:
50 0 : CharDistributionAnalysis() {Reset(false);}
51 :
52 : //feed a block of data and do distribution analysis
53 : void HandleData(const char* aBuf, PRUint32 aLen) {}
54 :
55 : //Feed a character with known length
56 0 : void HandleOneChar(const char* aStr, PRUint32 aCharLen)
57 : {
58 : PRInt32 order;
59 :
60 : //we only care about 2-bytes character in our distribution analysis
61 0 : order = (aCharLen == 2) ? GetOrder(aStr) : -1;
62 :
63 0 : if (order >= 0)
64 : {
65 0 : mTotalChars++;
66 : //order is valid
67 0 : if ((PRUint32)order < mTableSize)
68 : {
69 0 : if (512 > mCharToFreqOrder[order])
70 0 : mFreqChars++;
71 : }
72 : }
73 0 : }
74 :
75 : //return confidence base on existing data
76 : float GetConfidence(void);
77 :
78 : //Reset analyser, clear any state
79 0 : void Reset(bool aIsPreferredLanguage)
80 : {
81 0 : mDone = false;
82 0 : mTotalChars = 0;
83 0 : mFreqChars = 0;
84 0 : mDataThreshold = aIsPreferredLanguage ? 0 : MINIMUM_DATA_THRESHOLD;
85 0 : }
86 :
87 : //This function is for future extension. Caller can use this function to control
88 : //analyser's behavior
89 : void SetOpion(){}
90 :
91 : //It is not necessary to receive all data to draw conclusion. For charset detection,
92 : // certain amount of data is enough
93 0 : bool GotEnoughData() {return mTotalChars > ENOUGH_DATA_THRESHOLD;}
94 :
95 : protected:
96 : //we do not handle character base on its original encoding string, but
97 : //convert this encoding string to a number, here called order.
98 : //This allow multiple encoding of a language to share one frequency table
99 0 : virtual PRInt32 GetOrder(const char* str) {return -1;}
100 :
101 : //If this flag is set to true, detection is done and conclusion has been made
102 : bool mDone;
103 :
104 : //The number of characters whose frequency order is less than 512
105 : PRUint32 mFreqChars;
106 :
107 : //Total character encounted.
108 : PRUint32 mTotalChars;
109 :
110 : //Number of hi-byte characters needed to trigger detection
111 : PRUint32 mDataThreshold;
112 :
113 : //Mapping table to get frequency order from char order (get from GetOrder())
114 : const PRInt16 *mCharToFreqOrder;
115 :
116 : //Size of above table
117 : PRUint32 mTableSize;
118 :
119 : //This is a constant value varies from language to language, it is used in
120 : //calculating confidence. See my paper for further detail.
121 : float mTypicalDistributionRatio;
122 : };
123 :
124 :
125 : class EUCTWDistributionAnalysis: public CharDistributionAnalysis
126 : {
127 : public:
128 : EUCTWDistributionAnalysis();
129 : protected:
130 :
131 : //for euc-TW encoding, we are interested
132 : // first byte range: 0xc4 -- 0xfe
133 : // second byte range: 0xa1 -- 0xfe
134 : //no validation needed here. State machine has done that
135 0 : PRInt32 GetOrder(const char* str)
136 0 : { if ((unsigned char)*str >= (unsigned char)0xc4)
137 0 : return 94*((unsigned char)str[0]-(unsigned char)0xc4) + (unsigned char)str[1] - (unsigned char)0xa1;
138 : else
139 0 : return -1;
140 : }
141 : };
142 :
143 :
144 : class EUCKRDistributionAnalysis : public CharDistributionAnalysis
145 : {
146 : public:
147 : EUCKRDistributionAnalysis();
148 : protected:
149 : //for euc-KR encoding, we are interested
150 : // first byte range: 0xb0 -- 0xfe
151 : // second byte range: 0xa1 -- 0xfe
152 : //no validation needed here. State machine has done that
153 0 : PRInt32 GetOrder(const char* str)
154 0 : { if ((unsigned char)*str >= (unsigned char)0xb0)
155 0 : return 94*((unsigned char)str[0]-(unsigned char)0xb0) + (unsigned char)str[1] - (unsigned char)0xa1;
156 : else
157 0 : return -1;
158 : }
159 : };
160 :
161 : class GB2312DistributionAnalysis : public CharDistributionAnalysis
162 : {
163 : public:
164 : GB2312DistributionAnalysis();
165 : protected:
166 : //for GB2312 encoding, we are interested
167 : // first byte range: 0xb0 -- 0xfe
168 : // second byte range: 0xa1 -- 0xfe
169 : //no validation needed here. State machine has done that
170 0 : PRInt32 GetOrder(const char* str)
171 0 : { if ((unsigned char)*str >= (unsigned char)0xb0 && (unsigned char)str[1] >= (unsigned char)0xa1)
172 0 : return 94*((unsigned char)str[0]-(unsigned char)0xb0) + (unsigned char)str[1] - (unsigned char)0xa1;
173 : else
174 0 : return -1;
175 : }
176 : };
177 :
178 :
179 : class Big5DistributionAnalysis : public CharDistributionAnalysis
180 : {
181 : public:
182 : Big5DistributionAnalysis();
183 : protected:
184 : //for big5 encoding, we are interested
185 : // first byte range: 0xa4 -- 0xfe
186 : // second byte range: 0x40 -- 0x7e , 0xa1 -- 0xfe
187 : //no validation needed here. State machine has done that
188 0 : PRInt32 GetOrder(const char* str)
189 0 : { if ((unsigned char)*str >= (unsigned char)0xa4)
190 0 : if ((unsigned char)str[1] >= (unsigned char)0xa1)
191 0 : return 157*((unsigned char)str[0]-(unsigned char)0xa4) + (unsigned char)str[1] - (unsigned char)0xa1 +63;
192 : else
193 0 : return 157*((unsigned char)str[0]-(unsigned char)0xa4) + (unsigned char)str[1] - (unsigned char)0x40;
194 : else
195 0 : return -1;
196 : }
197 : };
198 :
199 : class SJISDistributionAnalysis : public CharDistributionAnalysis
200 : {
201 : public:
202 : SJISDistributionAnalysis();
203 : protected:
204 : //for sjis encoding, we are interested
205 : // first byte range: 0x81 -- 0x9f , 0xe0 -- 0xfe
206 : // second byte range: 0x40 -- 0x7e, 0x81 -- oxfe
207 : //no validation needed here. State machine has done that
208 0 : PRInt32 GetOrder(const char* str)
209 : {
210 : PRInt32 order;
211 0 : if ((unsigned char)*str >= (unsigned char)0x81 && (unsigned char)*str <= (unsigned char)0x9f)
212 0 : order = 188 * ((unsigned char)str[0]-(unsigned char)0x81);
213 0 : else if ((unsigned char)*str >= (unsigned char)0xe0 && (unsigned char)*str <= (unsigned char)0xef)
214 0 : order = 188 * ((unsigned char)str[0]-(unsigned char)0xe0 + 31);
215 : else
216 0 : return -1;
217 0 : order += (unsigned char)*(str+1) - 0x40;
218 0 : if ((unsigned char)str[1] > (unsigned char)0x7f)
219 0 : order--;
220 0 : return order;
221 : }
222 : };
223 :
224 : class EUCJPDistributionAnalysis : public CharDistributionAnalysis
225 : {
226 : public:
227 : EUCJPDistributionAnalysis();
228 : protected:
229 : //for euc-JP encoding, we are interested
230 : // first byte range: 0xa0 -- 0xfe
231 : // second byte range: 0xa1 -- 0xfe
232 : //no validation needed here. State machine has done that
233 0 : PRInt32 GetOrder(const char* str)
234 0 : { if ((unsigned char)*str >= (unsigned char)0xa0)
235 0 : return 94*((unsigned char)str[0]-(unsigned char)0xa1) + (unsigned char)str[1] - (unsigned char)0xa1;
236 : else
237 0 : return -1;
238 : }
239 : };
240 :
241 : #endif //CharDistribution_h__
242 :
|