1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
|
commit 4c8316f9cfda38d75fb015c0eb40e0eebb03d28f
Author: Jehan <jehan@girinstud.io>
Date: Sat Dec 5 21:04:20 2015 +0100
Nearly-ASCII text with NBSP is still not ASCII.
There is no "exception" in encoding. The non-breaking space 0xA0 is not
ASCII, and therefore returning "ASCII" will later create issues (for
instance trying to re-encode with iconv produces an error).
This was obviously an explicit decision in original code (according to
code comments), probably tied to specifity of the original program from
Mozilla. Now we want strict detection.
I will return "ISO-8859-1" for "nearly-ASCII texts with NBSP as only
exception" (note that I could have returned any ISO-8859 charsets since
they all have this character in common).
diff --git a/src/nsUniversalDetector.cpp b/src/nsUniversalDetector.cpp
index ab8bae0..ff06b9d 100644
--- a/src/nsUniversalDetector.cpp
+++ b/src/nsUniversalDetector.cpp
@@ -47,6 +47,7 @@
nsUniversalDetector::nsUniversalDetector(PRUint32 aLanguageFilter)
{
+ mNbspFound = PR_FALSE;
mDone = PR_FALSE;
mBestGuess = -1; //illegal value as signal
mInTag = PR_FALSE;
@@ -75,6 +76,7 @@ nsUniversalDetector::~nsUniversalDetector()
void
nsUniversalDetector::Reset()
{
+ mNbspFound = PR_FALSE;
mDone = PR_FALSE;
mBestGuess = -1; //illegal value as signal
mInTag = PR_FALSE;
@@ -162,9 +164,10 @@ nsresult nsUniversalDetector::HandleData(const char* aBuf, PRUint32 aLen)
PRUint32 i;
for (i = 0; i < aLen; i++)
{
- /* Other than 0xA0, if every other character is ASCII, the page is ASCII.
+ /* If every other character is ASCII or 0xA0, we don't run charset
+ * probers.
* 0xA0 (NBSP in a few charset) is apparently a rare exception
- * of non-ASCII character contained in ASCII text. */
+ * of non-ASCII character often contained in nearly-ASCII text. */
if (aBuf[i] & '\x80' && aBuf[i] != '\xA0')
{
/* We got a non-ASCII byte (high-byte) */
@@ -203,11 +206,19 @@ nsresult nsUniversalDetector::HandleData(const char* aBuf, PRUint32 aLen)
}
else
{
- //ok, just pure ascii so far
- if ( ePureAscii == mInputState &&
- (aBuf[i] == '\033' || (aBuf[i] == '{' && mLastChar == '~')) )
+ /* Just pure ASCII or NBSP so far. */
+ if (aBuf[i] == '\xA0')
{
- //found escape character or HZ "~{"
+ /* ASCII with the only exception of NBSP seems quite common.
+ * I doubt it is really necessary to train a model here, so let's
+ * just make an exception.
+ */
+ mNbspFound = PR_TRUE;
+ }
+ else if (mInputState == ePureAscii &&
+ (aBuf[i] == '\033' || (aBuf[i] == '{' && mLastChar == '~')))
+ {
+ /* We found an escape character or HZ "~{". */
mInputState = eEscAscii;
}
mLastChar = aBuf[i];
@@ -229,6 +240,10 @@ nsresult nsUniversalDetector::HandleData(const char* aBuf, PRUint32 aLen)
mDone = PR_TRUE;
mDetectedCharset = mEscCharSetProber->GetCharSetName();
}
+ else if (mNbspFound)
+ {
+ mDetectedCharset = "ISO-8859-1";
+ }
else
{
/* ASCII with the ESC character (or the sequence "~{") is still
@@ -253,8 +268,17 @@ nsresult nsUniversalDetector::HandleData(const char* aBuf, PRUint32 aLen)
break;
default:
- /* Pure ASCII */
- mDetectedCharset = "ASCII";
+ if (mNbspFound)
+ {
+ /* ISO-8859-1 is a good result candidate for ASCII + NBSP.
+ * (though it could have been any ISO-8859 encoding). */
+ mDetectedCharset = "ISO-8859-1";
+ }
+ else
+ {
+ /* Pure ASCII */
+ mDetectedCharset = "ASCII";
+ }
break;
}
return NS_OK;
diff --git a/src/nsUniversalDetector.h b/src/nsUniversalDetector.h
index 4d9b460..9f0a4b1 100644
--- a/src/nsUniversalDetector.h
+++ b/src/nsUniversalDetector.h
@@ -72,6 +72,7 @@ protected:
virtual void Report(const char* aCharset) = 0;
virtual void Reset();
nsInputState mInputState;
+ PRBool mNbspFound;
PRBool mDone;
PRBool mInTag;
PRBool mStart;
|
