1 : //* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 : /* ***** BEGIN LICENSE BLOCK *****
3 : * Version: MPL 1.1/GPL 2.0/LGPL 2.1
4 : *
5 : * The contents of this file are subject to the Mozilla Public License Version
6 : * 1.1 (the "License"); you may not use this file except in compliance with
7 : * the License. You may obtain a copy of the License at
8 : * http://www.mozilla.org/MPL/
9 : *
10 : * Software distributed under the License is distributed on an "AS IS" basis,
11 : * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
12 : * for the specific language governing rights and limitations under the
13 : * License.
14 : *
15 : * The Original Code is Url Classifier code
16 : *
17 : * The Initial Developer of the Original Code is
18 : * Google Inc.
19 : * Portions created by the Initial Developer are Copyright (C) 2006
20 : * the Initial Developer. All Rights Reserved.
21 : *
22 : * Contributor(s):
23 : * Tony Chang <tony@ponderer.org> (original author)
24 : * Brett Wilson <brettw@gmail.com>
25 : * Dave Camp <dcamp@mozilla.com>
26 : * David Dahl <ddahl@mozilla.com>
27 : * Gian-Carlo Pascutto <gpascutto@mozilla.com>
28 : *
29 : * Alternatively, the contents of this file may be used under the terms of
30 : * either the GNU General Public License Version 2 or later (the "GPL"), or
31 : * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
32 : * in which case the provisions of the GPL or the LGPL are applicable instead
33 : * of those above. If you wish to allow use of your version of this file only
34 : * under the terms of either the GPL or the LGPL, and not to allow others to
35 : * use your version of this file under the terms of the MPL, indicate your
36 : * decision by deleting the provisions above and replace them with the notice
37 : * and other provisions required by the GPL or the LGPL. If you do not delete
38 : * the provisions above, a recipient may use your version of this file under
39 : * the terms of any one of the MPL, the GPL or the LGPL.
40 : *
41 : * ***** END LICENSE BLOCK ***** */
42 :
43 : #include "nsAutoPtr.h"
44 : #include "nsCOMPtr.h"
45 : #include "mozIStorageService.h"
46 : #include "mozIStorageConnection.h"
47 : #include "mozIStorageStatement.h"
48 : #include "mozStorageHelper.h"
49 : #include "mozStorageCID.h"
50 : #include "nsAppDirectoryServiceDefs.h"
51 : #include "nsCRT.h"
52 : #include "nsDataHashtable.h"
53 : #include "nsICryptoHash.h"
54 : #include "nsICryptoHMAC.h"
55 : #include "nsIDirectoryService.h"
56 : #include "nsIKeyModule.h"
57 : #include "nsIObserverService.h"
58 : #include "nsIPermissionManager.h"
59 : #include "nsIPrefBranch.h"
60 : #include "nsIPrefService.h"
61 : #include "nsIProperties.h"
62 : #include "nsToolkitCompsCID.h"
63 : #include "nsIUrlClassifierUtils.h"
64 : #include "nsUrlClassifierDBService.h"
65 : #include "nsUrlClassifierUtils.h"
66 : #include "nsUrlClassifierProxies.h"
67 : #include "nsURILoader.h"
68 : #include "nsString.h"
69 : #include "nsReadableUtils.h"
70 : #include "nsTArray.h"
71 : #include "nsNetUtil.h"
72 : #include "nsNetCID.h"
73 : #include "nsThreadUtils.h"
74 : #include "nsXPCOMStrings.h"
75 : #include "mozilla/Mutex.h"
76 : #include "mozilla/Telemetry.h"
77 : #include "prlog.h"
78 : #include "prprf.h"
79 : #include "prnetdb.h"
80 : #include "zlib.h"
81 :
82 : // Needed to interpert mozIStorageConnection::GetLastError
83 : #include <sqlite3.h>
84 :
85 : using namespace mozilla;
86 :
87 : /**
88 : * The DBServices stores a set of Fragments. A fragment is one URL
89 : * fragment containing two or more domain components and some number
90 : * of path components.
91 : *
92 : * Fragment examples:
93 : * example.com/
94 : * www.example.com/foo/bar
95 : * www.mail.example.com/mail
96 : *
97 : * Fragments are described in "Simplified Regular Expression Lookup"
98 : * section of the protocol document at
99 : * http://code.google.com/p/google-safe-browsing/wiki/Protocolv2Spec
100 : *
101 : * A fragment is associated with a domain. The domain for a given
102 : * fragment is the three-host-component domain of the fragment (two
103 : * host components for URLs with only two components) with a trailing
104 : * slash. So for the fragments listed above, the domains are
105 : * example.com/, www.example.com/ and mail.example.com/.
106 : *
107 : * Fragments and domains are hashed in the database. The hash is described
108 : * in the protocol document, but it's basically a truncated SHA256 hash.
109 : *
110 : * A (table, chunk id, domain key, fragment) tuple is referred to as
111 : * an Entry.
112 : */
113 :
114 : // NSPR_LOG_MODULES=UrlClassifierDbService:5
115 : #if defined(PR_LOGGING)
116 : static const PRLogModuleInfo *gUrlClassifierDbServiceLog = nsnull;
117 : #define LOG(args) PR_LOG(gUrlClassifierDbServiceLog, PR_LOG_DEBUG, args)
118 : #define LOG_ENABLED() PR_LOG_TEST(gUrlClassifierDbServiceLog, 4)
119 : #else
120 : #define LOG(args)
121 : #define LOG_ENABLED() (false)
122 : #endif
123 :
124 : // Schema versioning: note that we don't bother to migrate between different
125 : // versions of the schema, we just start fetching the data freshly with each
126 : // migration.
127 :
128 : // The database filename is updated when there is an incompatible
129 : // schema change and we expect both implementations to continue
130 : // accessing the same database (such as between stable versions of the
131 : // platform).
132 : #define DATABASE_FILENAME "urlclassifier3.sqlite"
133 :
134 : // The implementation version is updated during development when we
135 : // want to change schema, or to recover from updating bugs. When an
136 : // implementation version change is detected, the database is scrapped
137 : // and we start over.
138 : #define IMPLEMENTATION_VERSION 7
139 :
140 : // Name of the persistent PrefixSet storage
141 : #define PREFIXSET_FILENAME "urlclassifier.pset"
142 :
143 : #define MAX_HOST_COMPONENTS 5
144 : #define MAX_PATH_COMPONENTS 4
145 :
146 : // Updates will fail if fed chunks larger than this
147 : #define MAX_CHUNK_SIZE (1024 * 1024)
148 :
149 : // Prefs for implementing nsIURIClassifier to block page loads
150 : #define CHECK_MALWARE_PREF "browser.safebrowsing.malware.enabled"
151 : #define CHECK_MALWARE_DEFAULT false
152 :
153 : #define CHECK_PHISHING_PREF "browser.safebrowsing.enabled"
154 : #define CHECK_PHISHING_DEFAULT false
155 :
156 : #define GETHASH_NOISE_PREF "urlclassifier.gethashnoise"
157 : #define GETHASH_NOISE_DEFAULT 4
158 :
159 : #define GETHASH_TABLES_PREF "urlclassifier.gethashtables"
160 :
161 : #define CONFIRM_AGE_PREF "urlclassifier.confirm-age"
162 : #define CONFIRM_AGE_DEFAULT_SEC (45 * 60)
163 :
164 : #define UPDATE_CACHE_SIZE_PREF "urlclassifier.updatecachemax"
165 : #define UPDATE_CACHE_SIZE_DEFAULT -1
166 :
167 : #define LOOKUP_CACHE_SIZE_PREF "urlclassifier.lookupcachemax"
168 : #define LOOKUP_CACHE_SIZE_DEFAULT -1
169 :
170 : // Amount of time to spend updating before committing and delaying, in
171 : // seconds. This is checked after each update stream, so the actual
172 : // time spent can be higher than this, depending on update stream size.
173 : #define UPDATE_WORKING_TIME "urlclassifier.workingtime"
174 : #define UPDATE_WORKING_TIME_DEFAULT 5
175 :
176 : // The amount of time to delay after hitting UPDATE_WORKING_TIME, in
177 : // seconds.
178 : #define UPDATE_DELAY_TIME "urlclassifier.updatetime"
179 : #define UPDATE_DELAY_TIME_DEFAULT 60
180 :
181 : class nsUrlClassifierDBServiceWorker;
182 :
183 : // Singleton instance.
184 : static nsUrlClassifierDBService* sUrlClassifierDBService;
185 :
186 : nsIThread* nsUrlClassifierDBService::gDbBackgroundThread = nsnull;
187 :
188 : // Once we've committed to shutting down, don't do work in the background
189 : // thread.
190 : static bool gShuttingDownThread = false;
191 :
192 : static PRInt32 gFreshnessGuarantee = CONFIRM_AGE_DEFAULT_SEC;
193 :
194 : static PRInt32 gUpdateCacheSize = UPDATE_CACHE_SIZE_DEFAULT;
195 : static PRInt32 gLookupCacheSize = LOOKUP_CACHE_SIZE_DEFAULT;
196 :
197 : static PRInt32 gWorkingTimeThreshold = UPDATE_WORKING_TIME_DEFAULT;
198 : static PRInt32 gDelayTime = UPDATE_DELAY_TIME_DEFAULT;
199 :
200 : static void
201 95 : SplitTables(const nsACString& str, nsTArray<nsCString>& tables)
202 : {
203 95 : tables.Clear();
204 :
205 95 : nsACString::const_iterator begin, iter, end;
206 95 : str.BeginReading(begin);
207 95 : str.EndReading(end);
208 379 : while (begin != end) {
209 189 : iter = begin;
210 189 : FindCharInReadable(',', iter, end);
211 189 : tables.AppendElement(Substring(begin, iter));
212 189 : begin = iter;
213 189 : if (begin != end)
214 94 : begin++;
215 : }
216 95 : }
217 :
218 : // -------------------------------------------------------------------------
219 : // Hash class implementation
220 :
221 : // A convenience wrapper around the potentially-truncated hash for a
222 : // domain or fragment.
223 :
224 : template <PRUint32 S>
225 : struct nsUrlClassifierHash
226 : {
227 : static const PRUint32 sHashSize = S;
228 : typedef nsUrlClassifierHash<S> self_type;
229 : PRUint8 buf[S];
230 :
231 916 : nsresult FromPlaintext(const nsACString& plainText, nsICryptoHash *hash) {
232 : // From the protocol doc:
233 : // Each entry in the chunk is composed of the 128 most significant bits
234 : // of the SHA 256 hash of a suffix/prefix expression.
235 :
236 916 : nsresult rv = hash->Init(nsICryptoHash::SHA256);
237 916 : NS_ENSURE_SUCCESS(rv, rv);
238 :
239 916 : rv = hash->Update
240 : (reinterpret_cast<const PRUint8*>(plainText.BeginReading()),
241 : plainText.Length());
242 916 : NS_ENSURE_SUCCESS(rv, rv);
243 :
244 1832 : nsCAutoString hashed;
245 916 : rv = hash->Finish(false, hashed);
246 916 : NS_ENSURE_SUCCESS(rv, rv);
247 :
248 916 : NS_ASSERTION(hashed.Length() >= sHashSize,
249 : "not enough characters in the hash");
250 :
251 916 : memcpy(buf, hashed.BeginReading(), sHashSize);
252 :
253 916 : return NS_OK;
254 : }
255 :
256 51 : void Assign(const nsACString& str) {
257 51 : NS_ASSERTION(str.Length() >= sHashSize,
258 : "string must be at least sHashSize characters long");
259 51 : memcpy(buf, str.BeginReading(), sHashSize);
260 51 : }
261 :
262 136 : void Clear() {
263 136 : memset(buf, 0, sizeof(buf));
264 136 : }
265 :
266 597 : const bool operator==(const self_type& hash) const {
267 597 : return (memcmp(buf, hash.buf, sizeof(buf)) == 0);
268 : }
269 0 : const bool operator!=(const self_type& hash) const {
270 0 : return !(*this == hash);
271 : }
272 180 : const bool operator<(const self_type& hash) const {
273 180 : return memcmp(buf, hash.buf, sizeof(self_type)) < 0;
274 : }
275 362 : const bool StartsWith(const nsUrlClassifierHash<PARTIAL_LENGTH>& hash) const {
276 : NS_ASSERTION(sHashSize >= PARTIAL_LENGTH, "nsUrlClassifierHash must be at least PARTIAL_LENGTH bytes long");
277 362 : return memcmp(buf, hash.buf, PARTIAL_LENGTH) == 0;
278 : }
279 84 : PRUint32 ToUint32() const {
280 84 : return *(reinterpret_cast<const PRUint32*>(buf));
281 : }
282 : };
283 :
284 : typedef nsUrlClassifierHash<DOMAIN_LENGTH> nsUrlClassifierDomainHash;
285 : typedef nsUrlClassifierHash<PARTIAL_LENGTH> nsUrlClassifierPartialHash;
286 : typedef nsUrlClassifierHash<COMPLETE_LENGTH> nsUrlClassifierCompleteHash;
287 :
288 :
289 : // -------------------------------------------------------------------------
290 : // Entry class implementation
291 :
292 : // This class represents one entry in the classifier database. It consists
293 : // of a table id, a chunk id, a domain hash, and a partial or complete hash.
294 : class nsUrlClassifierEntry
295 : {
296 : public:
297 524 : nsUrlClassifierEntry()
298 : : mId(-1)
299 : , mHavePartial(false)
300 : , mHaveComplete(false)
301 : , mTableId(0)
302 : , mChunkId(0)
303 524 : , mAddChunkId(0)
304 524 : {}
305 560 : ~nsUrlClassifierEntry() {}
306 :
307 : // Check that this entry could potentially match the complete hash.
308 : bool Match(const nsUrlClassifierCompleteHash &hash);
309 :
310 : // Check that the sub entry should apply to this entry.
311 : bool SubMatch(const nsUrlClassifierEntry& sub);
312 :
313 : // Clear out the entry structure
314 : void Clear();
315 :
316 : // Set the partial hash for this domain.
317 52 : void SetHash(const nsUrlClassifierPartialHash &partialHash) {
318 52 : mPartialHash = partialHash;
319 52 : mHavePartial = true;
320 52 : }
321 :
322 : // Set the complete hash for this domain.
323 173 : void SetHash(const nsUrlClassifierCompleteHash &completeHash) {
324 173 : mCompleteHash = completeHash;
325 173 : mHaveComplete = true;
326 173 : }
327 :
328 0 : bool operator== (const nsUrlClassifierEntry& entry) const {
329 : return ! (mTableId != entry.mTableId ||
330 : mChunkId != entry.mChunkId ||
331 : mHavePartial != entry.mHavePartial ||
332 0 : (mHavePartial && mPartialHash != entry.mPartialHash) ||
333 : mHaveComplete != entry.mHaveComplete ||
334 0 : (mHaveComplete && mCompleteHash != entry.mCompleteHash));
335 : }
336 :
337 65 : bool operator< (const nsUrlClassifierEntry& entry) const {
338 : return (mTableId < entry.mTableId ||
339 : mChunkId < entry.mChunkId ||
340 47 : (mHavePartial && !entry.mHavePartial) ||
341 47 : (mHavePartial && mPartialHash < entry.mPartialHash) ||
342 25 : (mHaveComplete && !entry.mHaveComplete) ||
343 184 : (mHaveComplete && mCompleteHash < entry.mCompleteHash));
344 : }
345 :
346 : PRInt64 mId;
347 :
348 : nsUrlClassifierDomainHash mKey;
349 :
350 : bool mHavePartial;
351 : nsUrlClassifierPartialHash mPartialHash;
352 :
353 : bool mHaveComplete;
354 : nsUrlClassifierCompleteHash mCompleteHash;
355 :
356 : PRUint32 mTableId;
357 : PRUint32 mChunkId;
358 : PRUint32 mAddChunkId;
359 : };
360 :
361 : bool
362 517 : nsUrlClassifierEntry::Match(const nsUrlClassifierCompleteHash &hash)
363 : {
364 517 : if (mHaveComplete)
365 268 : return mCompleteHash == hash;
366 :
367 249 : if (mHavePartial)
368 249 : return hash.StartsWith(mPartialHash);
369 :
370 0 : return false;
371 : }
372 :
373 : bool
374 19 : nsUrlClassifierEntry::SubMatch(const nsUrlClassifierEntry &subEntry)
375 : {
376 19 : if ((mTableId != subEntry.mTableId) || (mChunkId != subEntry.mAddChunkId))
377 0 : return false;
378 :
379 19 : if (subEntry.mHaveComplete)
380 18 : return mHaveComplete && mCompleteHash == subEntry.mCompleteHash;
381 :
382 1 : if (subEntry.mHavePartial)
383 1 : return mHavePartial && mPartialHash == subEntry.mPartialHash;
384 :
385 0 : return false;
386 : }
387 :
388 : void
389 0 : nsUrlClassifierEntry::Clear()
390 : {
391 0 : mId = -1;
392 0 : mHavePartial = false;
393 0 : mHaveComplete = false;
394 0 : }
395 :
396 : // -------------------------------------------------------------------------
397 : // Lookup result class implementation
398 :
399 : // This helper class wraps a nsUrlClassifierEntry found during a lookup.
400 : class nsUrlClassifierLookupResult
401 36 : {
402 : public:
403 136 : nsUrlClassifierLookupResult() : mConfirmed(false), mNoise(false) {
404 136 : mLookupFragment.Clear();
405 136 : }
406 172 : ~nsUrlClassifierLookupResult() {}
407 :
408 47 : bool operator==(const nsUrlClassifierLookupResult &result) const {
409 : // Don't need to compare table name, it's contained by id in the entry.
410 47 : return (mLookupFragment == result.mLookupFragment &&
411 : mConfirmed == result.mConfirmed &&
412 47 : mEntry == result.mEntry);
413 : }
414 :
415 108 : bool operator<(const nsUrlClassifierLookupResult &result) const {
416 : // Don't need to compare table name, it's contained by id in the entry.
417 108 : return (mLookupFragment < result.mLookupFragment ||
418 : mConfirmed < result.mConfirmed ||
419 108 : mEntry < result.mEntry);
420 : }
421 :
422 : // The hash that matched this entry.
423 : nsUrlClassifierCompleteHash mLookupFragment;
424 :
425 : // The entry that was found during the lookup.
426 : nsUrlClassifierEntry mEntry;
427 :
428 : // TRUE if the lookup matched a complete hash (not just a partial
429 : // one).
430 : bool mConfirmed;
431 :
432 : // TRUE if this lookup is gethash noise. Does not represent an actual
433 : // result.
434 : bool mNoise;
435 :
436 : // The table name associated with mEntry.mTableId.
437 : nsCString mTableName;
438 : };
439 :
440 : // -------------------------------------------------------------------------
441 : // Store class implementation
442 :
443 : // This class mediates access to the classifier and chunk entry tables.
444 : class nsUrlClassifierStore
445 : {
446 : public:
447 16 : nsUrlClassifierStore() {}
448 32 : virtual ~nsUrlClassifierStore() {}
449 :
450 : // Initialize the statements for the store.
451 : nsresult Init(nsUrlClassifierDBServiceWorker *worker,
452 : mozIStorageConnection *connection,
453 : const nsACString& entriesTableName);
454 : // Shut down the store.
455 : void Close();
456 :
457 : // Read an entry from a database statement
458 : virtual bool ReadStatement(mozIStorageStatement* statement,
459 : nsUrlClassifierEntry& entry);
460 :
461 : // Prepare a statement to write this entry to the database
462 : virtual nsresult BindStatement(const nsUrlClassifierEntry& entry,
463 : mozIStorageStatement* statement);
464 :
465 : // Read the entry with a given ID from the database
466 : nsresult ReadEntry(PRInt64 id, nsUrlClassifierEntry& entry, bool *exists);
467 :
468 : // Remove an entry from the database
469 : nsresult DeleteEntry(nsUrlClassifierEntry& entry);
470 :
471 : // Write an entry to the database
472 : nsresult WriteEntry(nsUrlClassifierEntry& entry);
473 :
474 : // Update an entry in the database. The entry must already exist in the
475 : // database or this method will fail.
476 : nsresult UpdateEntry(nsUrlClassifierEntry& entry);
477 :
478 : // Remove all entries for a given table/chunk pair from the database.
479 : nsresult Expire(PRUint32 tableId,
480 : PRUint32 chunkNum);
481 :
482 : // Read a certain number of rows adjacent to the requested rowid that
483 : // don't have complete hash data.
484 : nsresult ReadNoiseEntries(PRInt64 rowID,
485 : PRUint32 numRequested,
486 : bool before,
487 : nsTArray<nsUrlClassifierEntry> &entries);
488 :
489 : // Ask the db for a random number. This is temporary, and should be
490 : // replaced with nsIRandomGenerator when 419739 is fixed.
491 : nsresult RandomNumber(PRInt64 *randomNum);
492 : // Return an array with all Prefixes known
493 : nsresult ReadPrefixes(FallibleTArray<PRUint32>& array, PRUint32 aKey);
494 :
495 :
496 : protected:
497 : nsresult ReadEntries(mozIStorageStatement *statement,
498 : nsTArray<nsUrlClassifierEntry>& entries);
499 : nsUrlClassifierDBServiceWorker *mWorker;
500 : nsCOMPtr<mozIStorageConnection> mConnection;
501 :
502 : nsCOMPtr<mozIStorageStatement> mLookupWithIDStatement;
503 :
504 : nsCOMPtr<mozIStorageStatement> mInsertStatement;
505 : nsCOMPtr<mozIStorageStatement> mUpdateStatement;
506 : nsCOMPtr<mozIStorageStatement> mDeleteStatement;
507 : nsCOMPtr<mozIStorageStatement> mExpireStatement;
508 :
509 : nsCOMPtr<mozIStorageStatement> mPartialEntriesStatement;
510 : nsCOMPtr<mozIStorageStatement> mPartialEntriesAfterStatement;
511 : nsCOMPtr<mozIStorageStatement> mLastPartialEntriesStatement;
512 : nsCOMPtr<mozIStorageStatement> mPartialEntriesBeforeStatement;
513 :
514 : nsCOMPtr<mozIStorageStatement> mRandomStatement;
515 : nsCOMPtr<mozIStorageStatement> mAllPrefixGetStatement;
516 : nsCOMPtr<mozIStorageStatement> mAllPrefixCountStatement;
517 : };
518 :
519 : nsresult
520 282 : nsUrlClassifierStore::Init(nsUrlClassifierDBServiceWorker *worker,
521 : mozIStorageConnection *connection,
522 : const nsACString& entriesName)
523 : {
524 282 : mWorker = worker;
525 282 : mConnection = connection;
526 :
527 282 : nsresult rv = mConnection->CreateStatement
528 564 : (NS_LITERAL_CSTRING("SELECT * FROM ") + entriesName +
529 846 : NS_LITERAL_CSTRING(" WHERE id=?1"),
530 846 : getter_AddRefs(mLookupWithIDStatement));
531 282 : NS_ENSURE_SUCCESS(rv, rv);
532 :
533 282 : rv = mConnection->CreateStatement
534 564 : (NS_LITERAL_CSTRING("DELETE FROM ") + entriesName +
535 846 : NS_LITERAL_CSTRING(" WHERE id=?1"),
536 846 : getter_AddRefs(mDeleteStatement));
537 282 : NS_ENSURE_SUCCESS(rv, rv);
538 :
539 282 : rv = mConnection->CreateStatement
540 564 : (NS_LITERAL_CSTRING("DELETE FROM ") + entriesName +
541 846 : NS_LITERAL_CSTRING(" WHERE table_id=?1 AND chunk_id=?2"),
542 846 : getter_AddRefs(mExpireStatement));
543 282 : NS_ENSURE_SUCCESS(rv, rv);
544 :
545 282 : rv = mConnection->CreateStatement
546 564 : (NS_LITERAL_CSTRING("SELECT * FROM ") + entriesName +
547 846 : NS_LITERAL_CSTRING(" WHERE complete_data ISNULL"
548 : " LIMIT ?1"),
549 846 : getter_AddRefs(mPartialEntriesStatement));
550 282 : NS_ENSURE_SUCCESS(rv, rv);
551 :
552 282 : rv = mConnection->CreateStatement
553 564 : (NS_LITERAL_CSTRING("SELECT * FROM ") + entriesName +
554 846 : NS_LITERAL_CSTRING(" WHERE id > ?1 AND complete_data ISNULL"
555 : " LIMIT ?2"),
556 846 : getter_AddRefs(mPartialEntriesAfterStatement));
557 282 : NS_ENSURE_SUCCESS(rv, rv);
558 :
559 282 : rv = mConnection->CreateStatement
560 564 : (NS_LITERAL_CSTRING("SELECT * FROM ") + entriesName +
561 846 : NS_LITERAL_CSTRING(" WHERE complete_data ISNULL"
562 : " ORDER BY id DESC LIMIT ?1"),
563 846 : getter_AddRefs(mLastPartialEntriesStatement));
564 282 : NS_ENSURE_SUCCESS(rv, rv);
565 :
566 282 : rv = mConnection->CreateStatement
567 564 : (NS_LITERAL_CSTRING("SELECT * FROM ") + entriesName +
568 846 : NS_LITERAL_CSTRING(" WHERE id < ?1 AND complete_data ISNULL"
569 : " ORDER BY id DESC LIMIT ?2"),
570 846 : getter_AddRefs(mPartialEntriesBeforeStatement));
571 282 : NS_ENSURE_SUCCESS(rv, rv);
572 :
573 282 : rv = mConnection->CreateStatement
574 282 : (NS_LITERAL_CSTRING("SELECT abs(random())"),
575 564 : getter_AddRefs(mRandomStatement));
576 282 : NS_ENSURE_SUCCESS(rv, rv);
577 :
578 564 : rv = mConnection->CreateStatement(NS_LITERAL_CSTRING("SELECT domain, partial_data, complete_data FROM ")
579 282 : + entriesName,
580 846 : getter_AddRefs(mAllPrefixGetStatement));
581 282 : NS_ENSURE_SUCCESS(rv, rv);
582 :
583 564 : rv = mConnection->CreateStatement(NS_LITERAL_CSTRING("SELECT COUNT(1) FROM ")
584 282 : + entriesName,
585 846 : getter_AddRefs(mAllPrefixCountStatement));
586 282 : NS_ENSURE_SUCCESS(rv, rv);
587 :
588 282 : return NS_OK;
589 : }
590 :
591 : void
592 282 : nsUrlClassifierStore::Close()
593 : {
594 282 : mLookupWithIDStatement = nsnull;
595 :
596 282 : mInsertStatement = nsnull;
597 282 : mUpdateStatement = nsnull;
598 282 : mDeleteStatement = nsnull;
599 282 : mExpireStatement = nsnull;
600 :
601 282 : mPartialEntriesStatement = nsnull;
602 282 : mPartialEntriesAfterStatement = nsnull;
603 282 : mPartialEntriesBeforeStatement = nsnull;
604 282 : mLastPartialEntriesStatement = nsnull;
605 282 : mRandomStatement = nsnull;
606 :
607 282 : mAllPrefixGetStatement = nsnull;
608 282 : mAllPrefixCountStatement = nsnull;
609 :
610 282 : mConnection = nsnull;
611 282 : }
612 :
613 :
614 : bool
615 204 : nsUrlClassifierStore::ReadStatement(mozIStorageStatement* statement,
616 : nsUrlClassifierEntry& entry)
617 : {
618 204 : entry.mId = statement->AsInt64(0);
619 :
620 : PRUint32 size;
621 204 : const PRUint8* blob = statement->AsSharedBlob(1, &size);
622 204 : if (!blob || (size != DOMAIN_LENGTH))
623 0 : return false;
624 204 : memcpy(entry.mKey.buf, blob, DOMAIN_LENGTH);
625 :
626 204 : blob = statement->AsSharedBlob(2, &size);
627 204 : if (!blob || size == 0) {
628 147 : entry.mHavePartial = false;
629 : } else {
630 57 : if (size != PARTIAL_LENGTH)
631 0 : return false;
632 57 : entry.mHavePartial = true;
633 57 : memcpy(entry.mPartialHash.buf, blob, PARTIAL_LENGTH);
634 : }
635 :
636 204 : blob = statement->AsSharedBlob(3, &size);
637 204 : if (!blob || size == 0) {
638 59 : entry.mHaveComplete = false;
639 : } else {
640 145 : if (size != COMPLETE_LENGTH)
641 0 : return false;
642 145 : entry.mHaveComplete = true;
643 145 : memcpy(entry.mCompleteHash.buf, blob, COMPLETE_LENGTH);
644 : }
645 :
646 : // If we only have a partial entry, and that partial entry matches the
647 : // domain, we don't save the extra copy to the database.
648 204 : if (!(entry.mHavePartial || entry.mHaveComplete)) {
649 5 : entry.SetHash(entry.mKey);
650 : }
651 :
652 204 : entry.mChunkId = statement->AsInt32(4);
653 204 : entry.mTableId = statement->AsInt32(5);
654 :
655 204 : return true;
656 : }
657 :
658 : nsresult
659 199 : nsUrlClassifierStore::BindStatement(const nsUrlClassifierEntry &entry,
660 : mozIStorageStatement* statement)
661 : {
662 : nsresult rv;
663 :
664 199 : if (entry.mId == -1)
665 163 : rv = statement->BindNullByIndex(0);
666 : else
667 36 : rv = statement->BindInt64ByIndex(0, entry.mId);
668 199 : NS_ENSURE_SUCCESS(rv, rv);
669 :
670 199 : rv = statement->BindBlobByIndex(1, entry.mKey.buf, DOMAIN_LENGTH);
671 199 : NS_ENSURE_SUCCESS(rv, rv);
672 :
673 199 : if (entry.mHavePartial) {
674 : // If we only have a partial entry and that entry matches the domain,
675 : // we'll save some space by only storing the domain hash.
676 82 : if (!entry.mHaveComplete && entry.mKey == entry.mPartialHash) {
677 5 : rv = statement->BindNullByIndex(2);
678 : } else {
679 : rv = statement->BindBlobByIndex(2, entry.mPartialHash.buf,
680 77 : PARTIAL_LENGTH);
681 : }
682 : } else {
683 117 : rv = statement->BindNullByIndex(2);
684 : }
685 199 : NS_ENSURE_SUCCESS(rv, rv);
686 :
687 199 : if (entry.mHaveComplete) {
688 153 : rv = statement->BindBlobByIndex(3, entry.mCompleteHash.buf, COMPLETE_LENGTH);
689 : } else {
690 46 : rv = statement->BindNullByIndex(3);
691 : }
692 199 : NS_ENSURE_SUCCESS(rv, rv);
693 :
694 199 : rv = statement->BindInt32ByIndex(4, entry.mChunkId);
695 199 : NS_ENSURE_SUCCESS(rv, rv);
696 :
697 199 : rv = statement->BindInt32ByIndex(5, entry.mTableId);
698 199 : NS_ENSURE_SUCCESS(rv, rv);
699 :
700 199 : return true;
701 : }
702 :
703 : nsresult
704 242 : nsUrlClassifierStore::ReadEntries(mozIStorageStatement *statement,
705 : nsTArray<nsUrlClassifierEntry>& entries)
706 : {
707 : bool exists;
708 242 : nsresult rv = statement->ExecuteStep(&exists);
709 242 : NS_ENSURE_SUCCESS(rv, rv);
710 :
711 688 : while (exists) {
712 204 : nsUrlClassifierEntry *entry = entries.AppendElement();
713 204 : if (!entry) {
714 0 : return NS_ERROR_OUT_OF_MEMORY;
715 : }
716 :
717 204 : if (!ReadStatement(statement, *entry))
718 0 : return NS_ERROR_FAILURE;
719 :
720 204 : statement->ExecuteStep(&exists);
721 : }
722 :
723 242 : return NS_OK;
724 : }
725 :
726 : nsresult
727 0 : nsUrlClassifierStore::ReadEntry(PRInt64 id,
728 : nsUrlClassifierEntry& entry,
729 : bool *exists)
730 : {
731 0 : entry.Clear();
732 :
733 0 : mozStorageStatementScoper scoper(mLookupWithIDStatement);
734 :
735 0 : nsresult rv = mLookupWithIDStatement->BindInt64ByIndex(0, id);
736 0 : NS_ENSURE_SUCCESS(rv, rv);
737 :
738 0 : rv = mLookupWithIDStatement->ExecuteStep(exists);
739 0 : NS_ENSURE_SUCCESS(rv, rv);
740 :
741 0 : if (*exists) {
742 0 : if (ReadStatement(mLookupWithIDStatement, entry))
743 0 : return NS_ERROR_FAILURE;
744 : }
745 :
746 0 : return NS_OK;
747 : }
748 :
749 : nsresult
750 0 : nsUrlClassifierStore::ReadNoiseEntries(PRInt64 rowID,
751 : PRUint32 numRequested,
752 : bool before,
753 : nsTArray<nsUrlClassifierEntry> &entries)
754 : {
755 0 : if (numRequested == 0) {
756 0 : return NS_OK;
757 : }
758 :
759 : mozIStorageStatement *statement =
760 0 : before ? mPartialEntriesBeforeStatement : mPartialEntriesAfterStatement;
761 0 : mozStorageStatementScoper scoper(statement);
762 :
763 0 : nsresult rv = statement->BindInt64ByIndex(0, rowID);
764 0 : NS_ENSURE_SUCCESS(rv, rv);
765 :
766 0 : statement->BindInt32ByIndex(1, numRequested);
767 0 : NS_ENSURE_SUCCESS(rv, rv);
768 :
769 0 : PRUint32 length = entries.Length();
770 0 : rv = ReadEntries(statement, entries);
771 0 : NS_ENSURE_SUCCESS(rv, rv);
772 :
773 0 : PRUint32 numRead = entries.Length() - length;
774 :
775 0 : if (numRead >= numRequested)
776 0 : return NS_OK;
777 :
778 : // If we didn't get enough entries, we need the search to wrap around from
779 : // beginning to end (or vice-versa)
780 :
781 : mozIStorageStatement *wraparoundStatement =
782 0 : before ? mPartialEntriesStatement : mLastPartialEntriesStatement;
783 0 : mozStorageStatementScoper wraparoundScoper(wraparoundStatement);
784 :
785 0 : rv = wraparoundStatement->BindInt32ByIndex(0, numRequested - numRead);
786 0 : NS_ENSURE_SUCCESS(rv, rv);
787 :
788 0 : return ReadEntries(wraparoundStatement, entries);
789 : }
790 :
791 : nsresult
792 0 : nsUrlClassifierStore::RandomNumber(PRInt64 *randomNum)
793 : {
794 0 : mozStorageStatementScoper randScoper(mRandomStatement);
795 : bool exists;
796 0 : nsresult rv = mRandomStatement->ExecuteStep(&exists);
797 0 : NS_ENSURE_SUCCESS(rv, rv);
798 0 : if (!exists)
799 0 : return NS_ERROR_NOT_AVAILABLE;
800 :
801 0 : *randomNum = mRandomStatement->AsInt64(0);
802 :
803 0 : return NS_OK;
804 : }
805 :
806 : // -------------------------------------------------------------------------
807 : // nsUrlClassifierAddStore class implementation
808 :
809 : // This class accesses the moz_classifier table.
810 : class nsUrlClassifierAddStore: public nsUrlClassifierStore
811 : {
812 : public:
813 8 : nsUrlClassifierAddStore() {};
814 16 : virtual ~nsUrlClassifierAddStore() {};
815 :
816 : nsresult Init(nsUrlClassifierDBServiceWorker *worker,
817 : mozIStorageConnection *connection,
818 : const nsACString& entriesTableName);
819 :
820 : void Close();
821 :
822 : // Read the entries for a given key/table/chunk from the database
823 : nsresult ReadAddEntries(const nsUrlClassifierDomainHash& key,
824 : PRUint32 tableId,
825 : PRUint32 chunkId,
826 : nsTArray<nsUrlClassifierEntry>& entry);
827 :
828 : // Read the entries for a given host key from the database.
829 : nsresult ReadAddEntries(const nsUrlClassifierDomainHash& key,
830 : nsTArray<nsUrlClassifierEntry>& entry);
831 :
832 : protected:
833 : nsCOMPtr<mozIStorageStatement> mLookupStatement;
834 : nsCOMPtr<mozIStorageStatement> mLookupWithChunkStatement;
835 : };
836 :
837 : nsresult
838 141 : nsUrlClassifierAddStore::Init(nsUrlClassifierDBServiceWorker *worker,
839 : mozIStorageConnection *connection,
840 : const nsACString &entriesTableName)
841 : {
842 : nsresult rv = nsUrlClassifierStore::Init(worker, connection,
843 141 : entriesTableName);
844 141 : NS_ENSURE_SUCCESS(rv, rv);
845 :
846 141 : rv = mConnection->CreateStatement
847 282 : (NS_LITERAL_CSTRING("INSERT OR REPLACE INTO ") + entriesTableName +
848 423 : NS_LITERAL_CSTRING(" VALUES (?1, ?2, ?3, ?4, ?5, ?6)"),
849 423 : getter_AddRefs(mInsertStatement));
850 141 : NS_ENSURE_SUCCESS(rv, rv);
851 :
852 141 : rv = mConnection->CreateStatement
853 282 : (NS_LITERAL_CSTRING("UPDATE ") + entriesTableName +
854 423 : NS_LITERAL_CSTRING(" SET domain=?2, partial_data=?3, "
855 : " complete_data=?4, chunk_id=?5, table_id=?6"
856 : " WHERE id=?1"),
857 423 : getter_AddRefs(mUpdateStatement));
858 141 : NS_ENSURE_SUCCESS(rv, rv);
859 :
860 141 : rv = mConnection->CreateStatement
861 282 : (NS_LITERAL_CSTRING("SELECT * FROM ") + entriesTableName +
862 423 : NS_LITERAL_CSTRING(" WHERE domain=?1"),
863 423 : getter_AddRefs(mLookupStatement));
864 141 : NS_ENSURE_SUCCESS(rv, rv);
865 :
866 141 : rv = mConnection->CreateStatement
867 282 : (NS_LITERAL_CSTRING("SELECT * FROM ") + entriesTableName +
868 423 : NS_LITERAL_CSTRING(" WHERE domain=?1 AND table_id=?2 AND chunk_id=?3"),
869 423 : getter_AddRefs(mLookupWithChunkStatement));
870 141 : NS_ENSURE_SUCCESS(rv, rv);
871 :
872 141 : return NS_OK;
873 : }
874 :
875 : void
876 141 : nsUrlClassifierAddStore::Close()
877 : {
878 141 : nsUrlClassifierStore::Close();
879 :
880 141 : mLookupStatement = nsnull;
881 141 : mLookupWithChunkStatement = nsnull;
882 141 : }
883 :
884 : nsresult
885 12 : nsUrlClassifierAddStore::ReadAddEntries(const nsUrlClassifierDomainHash& hash,
886 : PRUint32 tableId,
887 : PRUint32 chunkId,
888 : nsTArray<nsUrlClassifierEntry>& entries)
889 : {
890 24 : mozStorageStatementScoper scoper(mLookupWithChunkStatement);
891 :
892 12 : nsresult rv = mLookupWithChunkStatement->BindBlobByIndex
893 12 : (0, hash.buf, DOMAIN_LENGTH);
894 12 : NS_ENSURE_SUCCESS(rv, rv);
895 :
896 12 : rv = mLookupWithChunkStatement->BindInt32ByIndex(1, tableId);
897 12 : NS_ENSURE_SUCCESS(rv, rv);
898 12 : rv = mLookupWithChunkStatement->BindInt32ByIndex(2, chunkId);
899 12 : NS_ENSURE_SUCCESS(rv, rv);
900 :
901 12 : return ReadEntries(mLookupWithChunkStatement, entries);
902 : }
903 :
904 : nsresult
905 149 : nsUrlClassifierAddStore::ReadAddEntries(const nsUrlClassifierDomainHash& hash,
906 : nsTArray<nsUrlClassifierEntry>& entries)
907 : {
908 298 : mozStorageStatementScoper scoper(mLookupStatement);
909 :
910 149 : nsresult rv = mLookupStatement->BindBlobByIndex
911 149 : (0, hash.buf, DOMAIN_LENGTH);
912 149 : NS_ENSURE_SUCCESS(rv, rv);
913 :
914 149 : return ReadEntries(mLookupStatement, entries);
915 : }
916 :
917 : // -------------------------------------------------------------------------
918 : // nsUrlClassifierSubStore class implementation
919 :
920 : // This class accesses the moz_subs table.
921 : class nsUrlClassifierSubStore : public nsUrlClassifierStore
922 : {
923 : public:
924 8 : nsUrlClassifierSubStore() {};
925 16 : virtual ~nsUrlClassifierSubStore() {};
926 :
927 : nsresult Init(nsUrlClassifierDBServiceWorker *worker,
928 : mozIStorageConnection *connection,
929 : const nsACString& entriesTableName);
930 :
931 : void Close();
932 :
933 : // Read an entry from a database statement
934 : virtual bool ReadStatement(mozIStorageStatement* statement,
935 : nsUrlClassifierEntry& entry);
936 :
937 : // Prepare a statement to write this entry to the database
938 : virtual nsresult BindStatement(const nsUrlClassifierEntry& entry,
939 : mozIStorageStatement* statement);
940 :
941 : // Read sub entries for a given add chunk
942 : nsresult ReadSubEntries(PRUint32 tableId, PRUint32 chunkId,
943 : nsTArray<nsUrlClassifierEntry> &subEntry);
944 :
945 : // Expire sub entries for a given add chunk
946 : nsresult ExpireAddChunk(PRUint32 tableId, PRUint32 chunkId);
947 :
948 : protected:
949 : nsCOMPtr<mozIStorageStatement> mLookupWithAddChunkStatement;
950 : nsCOMPtr<mozIStorageStatement> mExpireAddChunkStatement;
951 : };
952 :
953 : nsresult
954 141 : nsUrlClassifierSubStore::Init(nsUrlClassifierDBServiceWorker *worker,
955 : mozIStorageConnection *connection,
956 : const nsACString &entriesTableName)
957 : {
958 : nsresult rv = nsUrlClassifierStore::Init(worker, connection,
959 141 : entriesTableName);
960 141 : NS_ENSURE_SUCCESS(rv, rv);
961 :
962 141 : rv = mConnection->CreateStatement
963 282 : (NS_LITERAL_CSTRING("INSERT OR REPLACE INTO ") + entriesTableName +
964 423 : NS_LITERAL_CSTRING(" VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)"),
965 423 : getter_AddRefs(mInsertStatement));
966 141 : NS_ENSURE_SUCCESS(rv, rv);
967 :
968 141 : rv = mConnection->CreateStatement
969 282 : (NS_LITERAL_CSTRING("UPDATE ") + entriesTableName +
970 423 : NS_LITERAL_CSTRING(" SET domain=?2, partial_data=?3, complete_data=?4,"
971 : " chunk_id=?5, table_id=?6, add_chunk_id=?7"
972 : " WHERE id=?1"),
973 423 : getter_AddRefs(mUpdateStatement));
974 141 : NS_ENSURE_SUCCESS(rv, rv);
975 :
976 141 : rv = mConnection->CreateStatement
977 282 : (NS_LITERAL_CSTRING("SELECT * FROM ") + entriesTableName +
978 423 : NS_LITERAL_CSTRING(" WHERE table_id=?1 AND add_chunk_id=?2"),
979 423 : getter_AddRefs(mLookupWithAddChunkStatement));
980 141 : NS_ENSURE_SUCCESS(rv, rv);
981 :
982 141 : rv = mConnection->CreateStatement
983 282 : (NS_LITERAL_CSTRING("DELETE FROM ") + entriesTableName +
984 423 : NS_LITERAL_CSTRING(" WHERE table_id=?1 AND add_chunk_id=?2"),
985 423 : getter_AddRefs(mExpireAddChunkStatement));
986 141 : NS_ENSURE_SUCCESS(rv, rv);
987 :
988 141 : return NS_OK;
989 : }
990 :
991 : bool
992 8 : nsUrlClassifierSubStore::ReadStatement(mozIStorageStatement* statement,
993 : nsUrlClassifierEntry& entry)
994 : {
995 8 : if (!nsUrlClassifierStore::ReadStatement(statement, entry))
996 0 : return false;
997 :
998 8 : entry.mAddChunkId = statement->AsInt32(6);
999 8 : return true;
1000 : }
1001 :
1002 : nsresult
1003 15 : nsUrlClassifierSubStore::BindStatement(const nsUrlClassifierEntry& entry,
1004 : mozIStorageStatement* statement)
1005 : {
1006 15 : nsresult rv = nsUrlClassifierStore::BindStatement(entry, statement);
1007 15 : NS_ENSURE_SUCCESS(rv, rv);
1008 :
1009 15 : return statement->BindInt32ByIndex(6, entry.mAddChunkId);
1010 : }
1011 :
1012 : nsresult
1013 81 : nsUrlClassifierSubStore::ReadSubEntries(PRUint32 tableId, PRUint32 addChunkId,
1014 : nsTArray<nsUrlClassifierEntry>& entries)
1015 : {
1016 162 : mozStorageStatementScoper scoper(mLookupWithAddChunkStatement);
1017 :
1018 81 : nsresult rv = mLookupWithAddChunkStatement->BindInt32ByIndex(0, tableId);
1019 81 : NS_ENSURE_SUCCESS(rv, rv);
1020 81 : rv = mLookupWithAddChunkStatement->BindInt32ByIndex(1, addChunkId);
1021 81 : NS_ENSURE_SUCCESS(rv, rv);
1022 :
1023 81 : return ReadEntries(mLookupWithAddChunkStatement, entries);
1024 : }
1025 :
1026 : nsresult
1027 81 : nsUrlClassifierSubStore::ExpireAddChunk(PRUint32 tableId, PRUint32 addChunkId)
1028 : {
1029 162 : mozStorageStatementScoper scoper(mExpireAddChunkStatement);
1030 :
1031 81 : nsresult rv = mExpireAddChunkStatement->BindInt32ByIndex(0, tableId);
1032 81 : NS_ENSURE_SUCCESS(rv, rv);
1033 81 : rv = mExpireAddChunkStatement->BindInt32ByIndex(1, addChunkId);
1034 81 : NS_ENSURE_SUCCESS(rv, rv);
1035 :
1036 81 : return mExpireAddChunkStatement->Execute();
1037 : }
1038 :
1039 : void
1040 141 : nsUrlClassifierSubStore::Close()
1041 : {
1042 141 : nsUrlClassifierStore::Close();
1043 141 : mLookupWithAddChunkStatement = nsnull;
1044 141 : mExpireAddChunkStatement = nsnull;
1045 141 : }
1046 :
1047 : // Similar to GetKey(), but if the domain contains three or more components,
1048 : // two keys will be returned:
1049 : // hostname.com/foo/bar -> [hostname.com]
1050 : // mail.hostname.com/foo/bar -> [hostname.com, mail.hostname.com]
1051 : // www.mail.hostname.com/foo/bar -> [hostname.com, mail.hostname.com]
1052 : static nsresult GetHostKeys(const nsACString &spec,
1053 : nsTArray<nsCString> &hostKeys);
1054 :
1055 : // take a lookup string (www.hostname.com/path/to/resource.html) and
1056 : // expand it into the set of fragments that should be searched for in an
1057 : // entry
1058 : static nsresult GetLookupFragments(const nsCSubstring& spec,
1059 : nsTArray<nsCString>& fragments);
1060 :
1061 : // Check for a canonicalized IP address.
1062 : static bool IsCanonicalizedIP(const nsACString& host);
1063 :
1064 : // Get the database key for a given URI. This is the top three
1065 : // domain components if they exist, otherwise the top two.
1066 : // hostname.com/foo/bar -> hostname.com
1067 : // mail.hostname.com/foo/bar -> mail.hostname.com
1068 : // www.mail.hostname.com/foo/bar -> mail.hostname.com
1069 : static nsresult GetKey(const nsACString& spec, nsUrlClassifierDomainHash& hash,
1070 : nsICryptoHash * aCryptoHash);
1071 :
1072 : // We have both a prefix and a domain. Drop the domain, but
1073 : // hash the domain, the prefix and a random value together,
1074 : // ensuring any collisions happens at a different points for
1075 : // different users.
1076 : static nsresult KeyedHash(PRUint32 aPref, PRUint32 aDomain,
1077 : PRUint32 aKey, PRUint32 *aOut);
1078 :
1079 :
1080 : // -------------------------------------------------------------------------
1081 : // Actual worker implemenatation
1082 : class nsUrlClassifierDBServiceWorker : public nsIUrlClassifierDBServiceWorker
1083 : {
1084 : public:
1085 : nsUrlClassifierDBServiceWorker();
1086 :
1087 : NS_DECL_ISUPPORTS
1088 : NS_DECL_NSIURLCLASSIFIERDBSERVICE
1089 : NS_DECL_NSIURLCLASSIFIERDBSERVICEWORKER
1090 :
1091 : // Initialize, called in the main thread
1092 : nsresult Init(PRInt32 gethashNoise,
1093 : nsRefPtr<nsUrlClassifierPrefixSet> & prefSet);
1094 :
1095 : // Queue a lookup for the worker to perform, called in the main thread.
1096 : nsresult QueueLookup(const nsACString& lookupKey,
1097 : nsIUrlClassifierLookupCallback* callback);
1098 :
1099 : // Handle any queued-up lookups. We call this function during long-running
1100 : // update operations to prevent lookups from blocking for too long.
1101 : nsresult HandlePendingLookups();
1102 :
1103 : // Blocks the PrefixSet from being updated while the main thread is doing
1104 : // its lookups. LockPrefixSet will return whether the PrefixSet is in a
1105 : // usable state. If not, we should fall through to SQLite lookups.
1106 : bool LockPrefixSet();
1107 : void UnlockPrefixSet();
1108 :
1109 : private:
1110 : // No subclassing
1111 : ~nsUrlClassifierDBServiceWorker();
1112 :
1113 : // Disallow copy constructor
1114 : nsUrlClassifierDBServiceWorker(nsUrlClassifierDBServiceWorker&);
1115 :
1116 : // Try to open the db, DATABASE_FILENAME.
1117 : nsresult OpenDb();
1118 :
1119 : // Create table in the db if they don't exist.
1120 : nsresult MaybeCreateTables(mozIStorageConnection* connection);
1121 :
1122 : nsresult GetTableName(PRUint32 tableId, nsACString& table);
1123 : nsresult GetTableId(const nsACString& table, PRUint32* tableId);
1124 :
1125 : // Decompress a zlib'ed chunk (used for -exp tables)
1126 : nsresult InflateChunk(nsACString& chunk);
1127 :
1128 : // Expand shavar chunk into its individual entries
1129 : nsresult GetShaEntries(PRUint32 tableId,
1130 : PRUint32 chunkType,
1131 : PRUint32 chunkNum,
1132 : PRUint32 domainSize,
1133 : PRUint32 hashSize,
1134 : nsACString& chunk,
1135 : nsTArray<nsUrlClassifierEntry>& entries);
1136 :
1137 : // Expand a chunk into its individual entries
1138 : nsresult GetChunkEntries(const nsACString& table,
1139 : PRUint32 tableId,
1140 : PRUint32 chunkType,
1141 : PRUint32 chunkNum,
1142 : PRUint32 hashSize,
1143 : nsACString& chunk,
1144 : nsTArray<nsUrlClassifierEntry>& entries);
1145 :
1146 : // Parse one stringified range of chunks of the form "n" or "n-m" from a
1147 : // comma-separated list of chunks. Upon return, 'begin' will point to the
1148 : // next range of chunks in the list of chunks.
1149 : bool ParseChunkRange(nsACString::const_iterator &begin,
1150 : const nsACString::const_iterator &end,
1151 : PRUint32 *first, PRUint32 *last);
1152 :
1153 : // Expand a stringified chunk list into an array of ints.
1154 : nsresult ParseChunkList(const nsACString& chunkStr,
1155 : nsTArray<PRUint32>& chunks);
1156 :
1157 : // Join an array of ints into a stringified chunk list.
1158 : nsresult JoinChunkList(nsTArray<PRUint32>& chunks, nsCString& chunkStr);
1159 :
1160 : // List the add/subtract chunks that have been applied to a table
1161 : nsresult GetChunkLists(PRUint32 tableId,
1162 : nsACString& addChunks,
1163 : nsACString& subChunks);
1164 :
1165 : // Set the list of add/subtract chunks that have been applied to a table
1166 : nsresult SetChunkLists(PRUint32 tableId,
1167 : const nsACString& addChunks,
1168 : const nsACString& subChunks);
1169 :
1170 : // Cache the list of add/subtract chunks applied to the table, optionally
1171 : // parsing the add or sub lists. These lists are cached while updating
1172 : // tables to avoid excessive database reads/writes and parsing.
1173 : nsresult CacheChunkLists(PRUint32 tableId,
1174 : bool parseAdds,
1175 : bool parseSubs);
1176 :
1177 : // Clear the cached list of add/subtract chunks.
1178 : void ClearCachedChunkLists();
1179 :
1180 : // Flush the cached add/subtract lists to the database.
1181 : nsresult FlushChunkLists();
1182 :
1183 : // Inserts a chunk id into the list, sorted. Returns TRUE if the
1184 : // number was successfully added, FALSE if the chunk already exists.
1185 : bool InsertChunkId(nsTArray<PRUint32>& chunks, PRUint32 chunkNum);
1186 :
1187 : // Add a list of entries to the database, merging with
1188 : // existing entries as necessary
1189 : nsresult AddChunk(PRUint32 tableId, PRUint32 chunkNum,
1190 : nsTArray<nsUrlClassifierEntry>& entries);
1191 :
1192 : // Expire an add chunk
1193 : nsresult ExpireAdd(PRUint32 tableId, PRUint32 chunkNum);
1194 :
1195 : // Subtract a list of entries from the database
1196 : nsresult SubChunk(PRUint32 tableId, PRUint32 chunkNum,
1197 : nsTArray<nsUrlClassifierEntry>& entries);
1198 :
1199 : // Expire a subtract chunk
1200 : nsresult ExpireSub(PRUint32 tableId, PRUint32 chunkNum);
1201 :
1202 : // Handle line-oriented control information from a stream update
1203 : nsresult ProcessResponseLines(bool* done);
1204 : // Handle chunk data from a stream update
1205 : nsresult ProcessChunk(bool* done);
1206 :
1207 : // Sets up a transaction and begins counting update time.
1208 : nsresult SetupUpdate();
1209 :
1210 : // Applies the current transaction and resets the update/working times.
1211 : nsresult ApplyUpdate();
1212 :
1213 : // Reset the in-progress update stream
1214 : void ResetStream();
1215 :
1216 : // Reset the in-progress update
1217 : void ResetUpdate();
1218 :
1219 : // Look for a given lookup string (www.hostname.com/path/to/resource.html)
1220 : // Returns a list of entries that match.
1221 : nsresult Check(const nsCSubstring& spec,
1222 : nsTArray<nsUrlClassifierLookupResult>& results);
1223 :
1224 : // Perform a classifier lookup for a given url.
1225 : nsresult DoLookup(const nsACString& spec, nsIUrlClassifierLookupCallback* c);
1226 :
1227 : // Add entries to the results.
1228 : nsresult AddNoise(PRInt64 nearID,
1229 : PRInt32 count,
1230 : nsTArray<nsUrlClassifierLookupResult>& results);
1231 :
1232 : // Construct a Prefix Set with known prefixes
1233 : nsresult LoadPrefixSet(nsCOMPtr<nsIFile> & aFile);
1234 : nsresult ConstructPrefixSet();
1235 :
1236 : // Set the SQLite cache size
1237 : nsresult SetCacheSize(mozIStorageConnection * aConnection,
1238 : PRInt32 aCacheSize);
1239 :
1240 : nsCOMPtr<nsIFile> mDBFile;
1241 : nsCOMPtr<nsIFile> mPSFile;
1242 :
1243 : nsCOMPtr<nsICryptoHash> mCryptoHash;
1244 :
1245 : // Holds a connection to the Db. We lazily initialize this because it has
1246 : // to be created in the background thread (currently mozStorageConnection
1247 : // isn't thread safe).
1248 : nsCOMPtr<mozIStorageConnection> mConnection;
1249 :
1250 : // The main collection of entries. This is the store that will be checked
1251 : // when classifying a URL.
1252 : nsUrlClassifierAddStore mMainStore;
1253 :
1254 : // The collection of subs waiting for their accompanying add.
1255 : nsUrlClassifierSubStore mPendingSubStore;
1256 :
1257 : nsCOMPtr<mozIStorageStatement> mGetChunkListsStatement;
1258 : nsCOMPtr<mozIStorageStatement> mSetChunkListsStatement;
1259 :
1260 : nsCOMPtr<mozIStorageStatement> mGetTablesStatement;
1261 : nsCOMPtr<mozIStorageStatement> mGetTableIdStatement;
1262 : nsCOMPtr<mozIStorageStatement> mGetTableNameStatement;
1263 : nsCOMPtr<mozIStorageStatement> mInsertTableIdStatement;
1264 : nsCOMPtr<mozIStorageStatement> mGetPageSizeStatement;
1265 :
1266 : // Stores the last time a given table was updated.
1267 : nsDataHashtable<nsCStringHashKey, PRInt64> mTableFreshness;
1268 :
1269 : // We receive data in small chunks that may be broken in the middle of
1270 : // a line. So we save the last partial line here.
1271 : nsCString mPendingStreamUpdate;
1272 :
1273 : PRInt32 mUpdateWait;
1274 :
1275 : bool mResetRequested;
1276 : bool mGrewCache;
1277 :
1278 : enum {
1279 : STATE_LINE,
1280 : STATE_CHUNK
1281 : } mState;
1282 :
1283 : enum {
1284 : CHUNK_ADD,
1285 : CHUNK_SUB
1286 : } mChunkType;
1287 :
1288 : PRUint32 mChunkNum;
1289 : PRUint32 mHashSize;
1290 : PRUint32 mChunkLen;
1291 :
1292 : // List of tables included in this update.
1293 : nsTArray<nsCString> mUpdateTables;
1294 :
1295 : nsCString mUpdateTable;
1296 : PRUint32 mUpdateTableId;
1297 :
1298 : nsresult mUpdateStatus;
1299 :
1300 : nsCOMPtr<nsIUrlClassifierUpdateObserver> mUpdateObserver;
1301 : bool mInStream;
1302 : bool mPrimaryStream;
1303 :
1304 : bool mHaveCachedLists;
1305 : PRUint32 mCachedListsTable;
1306 : nsCAutoString mCachedSubsStr;
1307 : nsCAutoString mCachedAddsStr;
1308 :
1309 : bool mHaveCachedAddChunks;
1310 : nsTArray<PRUint32> mCachedAddChunks;
1311 :
1312 : bool mHaveCachedSubChunks;
1313 : nsTArray<PRUint32> mCachedSubChunks;
1314 :
1315 : // The client key with which the data from the server will be MAC'ed.
1316 : nsCString mUpdateClientKey;
1317 :
1318 : // The MAC stated by the server.
1319 : nsCString mServerMAC;
1320 :
1321 : // Start time of the current update interval. This will be reset
1322 : // every time we apply the update.
1323 : PRIntervalTime mUpdateStartTime;
1324 :
1325 : nsCOMPtr<nsICryptoHMAC> mHMAC;
1326 : // The number of noise entries to add to the set of lookup results.
1327 : PRInt32 mGethashNoise;
1328 :
1329 : // Set of prefixes known to be in the database
1330 : nsRefPtr<nsUrlClassifierPrefixSet> mPrefixSet;
1331 : // Can we use the PrefixSet (low memory conditions)
1332 : bool mPrefixSetEnabled;
1333 : Mutex mPrefixSetEnabledLock;
1334 :
1335 : // Pending lookups are stored in a queue for processing. The queue
1336 : // is protected by mPendingLookupLock.
1337 : Mutex mPendingLookupLock;
1338 :
1339 564 : class PendingLookup {
1340 : public:
1341 : nsCString mKey;
1342 : nsCOMPtr<nsIUrlClassifierLookupCallback> mCallback;
1343 : };
1344 :
1345 : // list of pending lookups
1346 : nsTArray<PendingLookup> mPendingLookups;
1347 : };
1348 :
1349 3631 : NS_IMPL_THREADSAFE_ISUPPORTS2(nsUrlClassifierDBServiceWorker,
1350 : nsIUrlClassifierDBServiceWorker,
1351 : nsIUrlClassifierDBService)
1352 :
1353 8 : nsUrlClassifierDBServiceWorker::nsUrlClassifierDBServiceWorker()
1354 : : mUpdateWait(0)
1355 : , mResetRequested(false)
1356 : , mGrewCache(false)
1357 : , mState(STATE_LINE)
1358 : , mChunkType(CHUNK_ADD)
1359 : , mChunkNum(0)
1360 : , mHashSize(0)
1361 : , mChunkLen(0)
1362 : , mUpdateTableId(0)
1363 : , mUpdateStatus(NS_OK)
1364 : , mInStream(false)
1365 : , mPrimaryStream(false)
1366 : , mHaveCachedLists(false)
1367 : , mCachedListsTable(PR_UINT32_MAX)
1368 : , mHaveCachedAddChunks(false)
1369 : , mHaveCachedSubChunks(false)
1370 : , mUpdateStartTime(0)
1371 : , mGethashNoise(0)
1372 : , mPrefixSet(0)
1373 : , mPrefixSetEnabled(true)
1374 : , mPrefixSetEnabledLock("mPrefixSetEnabledLock")
1375 8 : , mPendingLookupLock("nsUrlClassifierDBServerWorker.mPendingLookupLock")
1376 : {
1377 8 : }
1378 :
1379 16 : nsUrlClassifierDBServiceWorker::~nsUrlClassifierDBServiceWorker()
1380 : {
1381 8 : NS_ASSERTION(!mConnection,
1382 : "Db connection not closed, leaking memory! Call CloseDb "
1383 : "to close the connection.");
1384 8 : }
1385 :
1386 : nsresult
1387 8 : nsUrlClassifierDBServiceWorker::Init(PRInt32 gethashNoise,
1388 : nsRefPtr<nsUrlClassifierPrefixSet> & prefSet)
1389 : {
1390 8 : mGethashNoise = gethashNoise;
1391 8 : mPrefixSet = prefSet;
1392 :
1393 : // Compute database filename
1394 :
1395 : // Because we dump raw integers into the database, this database isn't
1396 : // portable between machine types, so store it in the local profile dir.
1397 : nsresult rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_LOCAL_50_DIR,
1398 8 : getter_AddRefs(mDBFile));
1399 :
1400 8 : if (NS_FAILED(rv)) {
1401 : rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR,
1402 0 : getter_AddRefs(mDBFile));
1403 : }
1404 :
1405 8 : if (NS_FAILED(rv)) return NS_ERROR_NOT_AVAILABLE;
1406 :
1407 8 : rv = mDBFile->Clone(getter_AddRefs(mPSFile));
1408 8 : NS_ENSURE_SUCCESS(rv, rv);
1409 :
1410 8 : rv = mDBFile->Append(NS_LITERAL_STRING(DATABASE_FILENAME));
1411 8 : NS_ENSURE_SUCCESS(rv, rv);
1412 :
1413 8 : rv = mPSFile->Append(NS_LITERAL_STRING(PREFIXSET_FILENAME));
1414 8 : NS_ENSURE_SUCCESS(rv, rv);
1415 :
1416 8 : ResetUpdate();
1417 :
1418 8 : mTableFreshness.Init();
1419 :
1420 8 : return NS_OK;
1421 : }
1422 :
1423 : nsresult
1424 141 : nsUrlClassifierDBServiceWorker::QueueLookup(const nsACString& spec,
1425 : nsIUrlClassifierLookupCallback* callback)
1426 : {
1427 282 : MutexAutoLock lock(mPendingLookupLock);
1428 :
1429 141 : PendingLookup* lookup = mPendingLookups.AppendElement();
1430 141 : if (!lookup) return NS_ERROR_OUT_OF_MEMORY;
1431 :
1432 141 : lookup->mKey = spec;
1433 141 : lookup->mCallback = callback;
1434 :
1435 141 : return NS_OK;
1436 : }
1437 :
1438 : nsresult
1439 21 : nsUrlClassifierDBService::CheckClean(const nsACString &spec,
1440 : bool *clean)
1441 : {
1442 42 : Telemetry::AutoTimer<Telemetry::URLCLASSIFIER_PS_LOOKUP_TIME> timer;
1443 :
1444 : // Is the PrefixSet usable?
1445 21 : bool usePrefixSet = mWorker->LockPrefixSet();
1446 :
1447 : // No, bail out and pretend the URL is not clean. We will do
1448 : // a database lookup and get the correct result.
1449 21 : if (!usePrefixSet) {
1450 0 : mWorker->UnlockPrefixSet();
1451 0 : *clean = false;
1452 0 : return NS_OK;
1453 : }
1454 :
1455 : // Get the set of fragments to look up.
1456 42 : nsTArray<nsCString> fragments;
1457 21 : nsresult rv = GetLookupFragments(spec, fragments);
1458 21 : if (NS_FAILED(rv)) {
1459 0 : goto error_checkclean;
1460 : }
1461 :
1462 : PRUint32 prefixkey;
1463 21 : rv = mPrefixSet->GetKey(&prefixkey);
1464 21 : if (NS_FAILED(rv)) {
1465 0 : goto error_checkclean;
1466 : }
1467 :
1468 21 : *clean = true;
1469 :
1470 63 : for (PRUint32 i = 0; i < fragments.Length(); i++) {
1471 : nsUrlClassifierDomainHash fragmentKeyHash;
1472 42 : fragmentKeyHash.FromPlaintext(fragments[i], mHash);
1473 :
1474 : // Find the corresponding host key
1475 : nsUrlClassifierDomainHash hostkey;
1476 42 : rv = GetKey(fragments[i], hostkey, mHash);
1477 42 : if (NS_FAILED(rv)) {
1478 : /* This happens for hosts on the local network,
1479 : can't check these against the DB */
1480 0 : continue;
1481 : }
1482 :
1483 42 : PRUint32 hostprefix = hostkey.ToUint32();
1484 42 : PRUint32 fragkey = fragmentKeyHash.ToUint32();
1485 : PRUint32 codedkey;
1486 42 : rv = KeyedHash(fragkey, hostprefix, prefixkey, &codedkey);
1487 42 : if (NS_FAILED(rv)) {
1488 0 : goto error_checkclean;
1489 : }
1490 :
1491 42 : bool found = false;
1492 42 : bool ready = false; /* opportunistic probe */
1493 42 : rv = mPrefixSet->Probe(codedkey, prefixkey, &ready, &found);
1494 42 : if (NS_FAILED(rv)) {
1495 0 : goto error_checkclean;
1496 : }
1497 42 : LOG(("CheckClean Probed %X ready: %d found: %d ",
1498 : codedkey, ready, found));
1499 42 : if (found || !ready) {
1500 1 : *clean = false;
1501 : }
1502 : }
1503 :
1504 21 : mWorker->UnlockPrefixSet();
1505 21 : return NS_OK;
1506 :
1507 : error_checkclean:
1508 0 : mWorker->UnlockPrefixSet();
1509 0 : return rv;
1510 : }
1511 :
1512 141 : static nsresult GetHostKeys(const nsACString &spec,
1513 : nsTArray<nsCString> &hostKeys)
1514 : {
1515 141 : nsACString::const_iterator begin, end, iter;
1516 141 : spec.BeginReading(begin);
1517 141 : spec.EndReading(end);
1518 :
1519 141 : iter = begin;
1520 141 : if (!FindCharInReadable('/', iter, end)) {
1521 0 : return NS_OK;
1522 : }
1523 :
1524 282 : const nsCSubstring& host = Substring(begin, iter);
1525 :
1526 141 : if (IsCanonicalizedIP(host)) {
1527 2 : nsCString *key = hostKeys.AppendElement();
1528 2 : if (!key)
1529 0 : return NS_ERROR_OUT_OF_MEMORY;
1530 :
1531 2 : key->Assign(host);
1532 2 : key->Append("/");
1533 2 : return NS_OK;
1534 : }
1535 :
1536 278 : nsTArray<nsCString> hostComponents;
1537 139 : ParseString(PromiseFlatCString(host), '.', hostComponents);
1538 :
1539 139 : if (hostComponents.Length() < 2) {
1540 : // no host or toplevel host, this won't match anything in the db
1541 0 : return NS_OK;
1542 : }
1543 :
1544 : // First check with two domain components
1545 139 : PRInt32 last = PRInt32(hostComponents.Length()) - 1;
1546 139 : nsCString *lookupHost = hostKeys.AppendElement();
1547 139 : if (!lookupHost)
1548 0 : return NS_ERROR_OUT_OF_MEMORY;
1549 :
1550 139 : lookupHost->Assign(hostComponents[last - 1]);
1551 139 : lookupHost->Append(".");
1552 139 : lookupHost->Append(hostComponents[last]);
1553 139 : lookupHost->Append("/");
1554 :
1555 : // Now check with three domain components
1556 139 : if (hostComponents.Length() > 2) {
1557 8 : nsCString *lookupHost2 = hostKeys.AppendElement();
1558 8 : if (!lookupHost2)
1559 0 : return NS_ERROR_OUT_OF_MEMORY;
1560 8 : lookupHost2->Assign(hostComponents[last - 2]);
1561 8 : lookupHost2->Append(".");
1562 8 : lookupHost2->Append(*lookupHost);
1563 : }
1564 :
1565 139 : return NS_OK;
1566 : }
1567 :
1568 : nsresult
1569 162 : GetLookupFragments(const nsACString& spec,
1570 : nsTArray<nsCString>& fragments)
1571 : {
1572 162 : fragments.Clear();
1573 :
1574 162 : nsACString::const_iterator begin, end, iter;
1575 162 : spec.BeginReading(begin);
1576 162 : spec.EndReading(end);
1577 :
1578 162 : iter = begin;
1579 162 : if (!FindCharInReadable('/', iter, end)) {
1580 0 : return NS_OK;
1581 : }
1582 :
1583 324 : const nsCSubstring& host = Substring(begin, iter++);
1584 324 : nsCAutoString path;
1585 162 : path.Assign(Substring(iter, end));
1586 :
1587 : /**
1588 : * From the protocol doc:
1589 : * For the hostname, the client will try at most 5 different strings. They
1590 : * are:
1591 : * a) The exact hostname of the url
1592 : * b) The 4 hostnames formed by starting with the last 5 components and
1593 : * successivly removing the leading component. The top-level component
1594 : * can be skipped. This is not done if the hostname is a numerical IP.
1595 : */
1596 324 : nsTArray<nsCString> hosts;
1597 162 : hosts.AppendElement(host);
1598 :
1599 162 : if (!IsCanonicalizedIP(host)) {
1600 160 : host.BeginReading(begin);
1601 160 : host.EndReading(end);
1602 160 : int numHostComponents = 0;
1603 510 : while (RFindInReadable(NS_LITERAL_CSTRING("."), begin, end) &&
1604 : numHostComponents < MAX_HOST_COMPONENTS) {
1605 : // don't bother checking toplevel domains
1606 190 : if (++numHostComponents >= 2) {
1607 30 : host.EndReading(iter);
1608 30 : hosts.AppendElement(Substring(end, iter));
1609 : }
1610 190 : end = begin;
1611 190 : host.BeginReading(begin);
1612 : }
1613 : }
1614 :
1615 : /**
1616 : * From the protocol doc:
1617 : * For the path, the client will also try at most 6 different strings.
1618 : * They are:
1619 : * a) the exact path of the url, including query parameters
1620 : * b) the exact path of the url, without query parameters
1621 : * c) the 4 paths formed by starting at the root (/) and
1622 : * successively appending path components, including a trailing
1623 : * slash. This behavior should only extend up to the next-to-last
1624 : * path component, that is, a trailing slash should never be
1625 : * appended that was not present in the original url.
1626 : */
1627 324 : nsTArray<nsCString> paths;
1628 324 : nsCAutoString pathToAdd;
1629 :
1630 162 : path.BeginReading(begin);
1631 162 : path.EndReading(end);
1632 162 : iter = begin;
1633 162 : if (FindCharInReadable('?', iter, end)) {
1634 3 : pathToAdd = Substring(begin, iter);
1635 3 : paths.AppendElement(pathToAdd);
1636 3 : end = iter;
1637 : }
1638 :
1639 162 : int numPathComponents = 1;
1640 162 : iter = begin;
1641 332 : while (FindCharInReadable('/', iter, end) &&
1642 : numPathComponents < MAX_PATH_COMPONENTS) {
1643 8 : iter++;
1644 8 : pathToAdd.Assign(Substring(begin, iter));
1645 8 : paths.AppendElement(pathToAdd);
1646 8 : numPathComponents++;
1647 : }
1648 :
1649 : // If we haven't already done so, add the full path
1650 162 : if (!pathToAdd.Equals(path)) {
1651 137 : paths.AppendElement(path);
1652 : }
1653 : // Check an empty path (for whole-domain blacklist entries)
1654 162 : paths.AppendElement(EmptyCString());
1655 :
1656 354 : for (PRUint32 hostIndex = 0; hostIndex < hosts.Length(); hostIndex++) {
1657 549 : for (PRUint32 pathIndex = 0; pathIndex < paths.Length(); pathIndex++) {
1658 714 : nsCString key;
1659 357 : key.Assign(hosts[hostIndex]);
1660 357 : key.Append('/');
1661 357 : key.Append(paths[pathIndex]);
1662 357 : LOG(("Chking %s", key.get()));
1663 :
1664 357 : fragments.AppendElement(key);
1665 : }
1666 : }
1667 :
1668 162 : return NS_OK;
1669 : }
1670 :
1671 : nsresult
1672 141 : nsUrlClassifierDBServiceWorker::Check(const nsACString& spec,
1673 : nsTArray<nsUrlClassifierLookupResult>& results)
1674 : {
1675 141 : PRInt64 now = (PR_Now() / PR_USEC_PER_SEC);
1676 :
1677 : // Get list of host keys to look up
1678 282 : nsAutoTArray<nsCString, 2> lookupHosts;
1679 141 : nsresult rv = GetHostKeys(spec, lookupHosts);
1680 :
1681 282 : nsTArray<nsUrlClassifierEntry> mCachedEntries;
1682 :
1683 : // Gather host's prefixes
1684 290 : for (PRUint32 i = 0; i < lookupHosts.Length(); i++) {
1685 : // Find the corresponding host key
1686 : nsUrlClassifierDomainHash hostKey;
1687 149 : nsresult rv = GetKey(lookupHosts[i], hostKey, mCryptoHash);
1688 149 : NS_ENSURE_SUCCESS(rv, rv);
1689 : // Read the entries for this fragments host from SQLite
1690 149 : mMainStore.ReadAddEntries(hostKey, mCachedEntries);
1691 : }
1692 :
1693 : // Now get the set of fragments to look up.
1694 282 : nsTArray<nsCString> fragments;
1695 141 : rv = GetLookupFragments(spec, fragments);
1696 141 : NS_ENSURE_SUCCESS(rv, rv);
1697 :
1698 : // Now check each lookup fragment against the entries in the DB.
1699 456 : for (PRUint32 i = 0; i < fragments.Length(); i++) {
1700 : nsUrlClassifierCompleteHash lookupHash;
1701 315 : lookupHash.FromPlaintext(fragments[i], mCryptoHash);
1702 :
1703 832 : for (PRUint32 j = 0; j < mCachedEntries.Length(); j++) {
1704 517 : nsUrlClassifierEntry &entry = mCachedEntries[j];
1705 517 : if (entry.Match(lookupHash)) {
1706 : // If the entry doesn't contain a complete hash, we need to
1707 : // save it here so that it can be compared against the
1708 : // complete hash. However, we don't set entry.mHaveComplete
1709 : // because it isn't a verified part of the entry yet.
1710 136 : nsUrlClassifierLookupResult *result = results.AppendElement();
1711 136 : if (!result)
1712 0 : return NS_ERROR_OUT_OF_MEMORY;
1713 :
1714 136 : result->mLookupFragment = lookupHash;
1715 136 : result->mEntry = entry;
1716 :
1717 : // Fill in the table name.
1718 136 : GetTableName(entry.mTableId, result->mTableName);
1719 :
1720 : bool fresh;
1721 : PRInt64 tableUpdateTime;
1722 136 : if (mTableFreshness.Get(result->mTableName, &tableUpdateTime)) {
1723 112 : LOG(("tableUpdateTime: %lld, now: %lld, freshnessGuarantee: %d\n",
1724 : tableUpdateTime, now, gFreshnessGuarantee));
1725 112 : fresh = ((now - tableUpdateTime) <= gFreshnessGuarantee);
1726 : } else {
1727 24 : LOG(("No expiration time for this table.\n"));
1728 24 : fresh = false;
1729 : }
1730 :
1731 : // This is a confirmed result if we match a complete fragment in
1732 : // an up-to-date table.
1733 136 : result->mConfirmed = entry.mHaveComplete && fresh;
1734 :
1735 136 : LOG(("Found a result. complete=%d, fresh=%d",
1736 : entry.mHaveComplete, fresh));
1737 : }
1738 : }
1739 : }
1740 :
1741 141 : return NS_OK;
1742 : }
1743 :
1744 : /**
1745 : * Lookup up a key in the database is a two step process:
1746 : *
1747 : * a) First we look for any Entries in the database that might apply to this
1748 : * url. For each URL there are one or two possible domain names to check:
1749 : * the two-part domain name (example.com) and the three-part name
1750 : * (www.example.com). We check the database for both of these.
1751 : * b) If we find any entries, we check the list of fragments for that entry
1752 : * against the possible subfragments of the URL as described in the
1753 : * "Simplified Regular Expression Lookup" section of the protocol doc.
1754 : */
1755 : nsresult
1756 141 : nsUrlClassifierDBServiceWorker::DoLookup(const nsACString& spec,
1757 : nsIUrlClassifierLookupCallback* c)
1758 : {
1759 141 : if (gShuttingDownThread) {
1760 0 : c->LookupComplete(nsnull);
1761 0 : return NS_ERROR_NOT_INITIALIZED;
1762 : }
1763 :
1764 141 : nsresult rv = OpenDb();
1765 141 : if (NS_FAILED(rv)) {
1766 0 : c->LookupComplete(nsnull);
1767 0 : return NS_ERROR_FAILURE;
1768 : }
1769 :
1770 : #if defined(PR_LOGGING)
1771 141 : PRIntervalTime clockStart = 0;
1772 141 : if (LOG_ENABLED()) {
1773 0 : clockStart = PR_IntervalNow();
1774 : }
1775 : #endif
1776 :
1777 282 : nsAutoPtr<nsTArray<nsUrlClassifierLookupResult> > results;
1778 141 : results = new nsTArray<nsUrlClassifierLookupResult>();
1779 141 : if (!results) {
1780 0 : c->LookupComplete(nsnull);
1781 0 : return NS_ERROR_OUT_OF_MEMORY;
1782 : }
1783 :
1784 : // we ignore failures from Check because we'd rather return the
1785 : // results that were found than fail.
1786 141 : Check(spec, *results);
1787 :
1788 : #if defined(PR_LOGGING)
1789 141 : if (LOG_ENABLED()) {
1790 0 : PRIntervalTime clockEnd = PR_IntervalNow();
1791 0 : LOG(("query took %dms\n",
1792 : PR_IntervalToMilliseconds(clockEnd - clockStart)));
1793 : }
1794 : #endif
1795 :
1796 200 : for (PRUint32 i = 0; i < results->Length(); i++) {
1797 111 : if (!results->ElementAt(i).mConfirmed) {
1798 : // We're going to be doing a gethash request, add some extra entries.
1799 52 : AddNoise(results->ElementAt(i).mEntry.mId, mGethashNoise, *results);
1800 52 : break;
1801 : }
1802 : }
1803 :
1804 : // At this point ownership of 'results' is handed to the callback.
1805 141 : c->LookupComplete(results.forget());
1806 :
1807 141 : return NS_OK;
1808 : }
1809 :
1810 : nsresult
1811 595 : nsUrlClassifierDBServiceWorker::HandlePendingLookups()
1812 : {
1813 1190 : MutexAutoLock lock(mPendingLookupLock);
1814 1331 : while (mPendingLookups.Length() > 0) {
1815 282 : PendingLookup lookup = mPendingLookups[0];
1816 141 : mPendingLookups.RemoveElementAt(0);
1817 : {
1818 282 : MutexAutoUnlock unlock(mPendingLookupLock);
1819 141 : DoLookup(lookup.mKey, lookup.mCallback);
1820 : }
1821 : }
1822 :
1823 595 : return NS_OK;
1824 : }
1825 :
1826 : nsresult
1827 52 : nsUrlClassifierDBServiceWorker::AddNoise(PRInt64 nearID,
1828 : PRInt32 count,
1829 : nsTArray<nsUrlClassifierLookupResult>& results)
1830 : {
1831 52 : if (count < 1) {
1832 52 : return NS_OK;
1833 : }
1834 :
1835 : PRInt64 randomNum;
1836 0 : nsresult rv = mMainStore.RandomNumber(&randomNum);
1837 0 : NS_ENSURE_SUCCESS(rv, rv);
1838 :
1839 0 : PRInt32 numBefore = randomNum % count;
1840 :
1841 0 : nsTArray<nsUrlClassifierEntry> noiseEntries;
1842 0 : rv = mMainStore.ReadNoiseEntries(nearID, numBefore, true, noiseEntries);
1843 0 : NS_ENSURE_SUCCESS(rv, rv);
1844 :
1845 0 : rv = mMainStore.ReadNoiseEntries(nearID, count - numBefore, false, noiseEntries);
1846 0 : NS_ENSURE_SUCCESS(rv, rv);
1847 :
1848 0 : for (PRUint32 i = 0; i < noiseEntries.Length(); i++) {
1849 0 : nsUrlClassifierLookupResult *result = results.AppendElement();
1850 0 : if (!result)
1851 0 : return NS_ERROR_OUT_OF_MEMORY;
1852 :
1853 0 : result->mEntry = noiseEntries[i];
1854 0 : result->mConfirmed = false;
1855 0 : result->mNoise = true;
1856 :
1857 : // Fill in the table name.
1858 0 : GetTableName(noiseEntries[i].mTableId, result->mTableName);
1859 : }
1860 :
1861 0 : return NS_OK;
1862 : }
1863 :
1864 :
1865 : // Lookup a key in the db.
1866 : NS_IMETHODIMP
1867 141 : nsUrlClassifierDBServiceWorker::Lookup(const nsACString& spec,
1868 : nsIUrlClassifierCallback* c)
1869 : {
1870 141 : return HandlePendingLookups();
1871 : }
1872 :
1873 : NS_IMETHODIMP
1874 53 : nsUrlClassifierDBServiceWorker::GetTables(nsIUrlClassifierCallback* c)
1875 : {
1876 53 : if (gShuttingDownThread)
1877 0 : return NS_ERROR_NOT_INITIALIZED;
1878 :
1879 53 : nsresult rv = OpenDb();
1880 53 : if (NS_FAILED(rv)) {
1881 0 : NS_ERROR("Unable to open database");
1882 0 : return NS_ERROR_FAILURE;
1883 : }
1884 :
1885 106 : mozStorageStatementScoper scoper(mGetTablesStatement);
1886 :
1887 106 : nsCAutoString response;
1888 : bool hasMore;
1889 163 : while (NS_SUCCEEDED(rv = mGetTablesStatement->ExecuteStep(&hasMore)) &&
1890 : hasMore) {
1891 114 : nsCAutoString val;
1892 57 : mGetTablesStatement->GetUTF8String(0, val);
1893 :
1894 57 : if (val.IsEmpty()) {
1895 0 : continue;
1896 : }
1897 :
1898 57 : response.Append(val);
1899 57 : response.Append(';');
1900 :
1901 57 : mGetTablesStatement->GetUTF8String(1, val);
1902 :
1903 57 : bool haveAdds = false;
1904 57 : if (!val.IsEmpty()) {
1905 56 : response.Append("a:");
1906 56 : response.Append(val);
1907 56 : haveAdds = true;
1908 : }
1909 :
1910 57 : mGetTablesStatement->GetUTF8String(2, val);
1911 57 : if (!val.IsEmpty()) {
1912 12 : if (haveAdds)
1913 12 : response.Append(":");
1914 :
1915 12 : response.Append("s:");
1916 12 : response.Append(val);
1917 : }
1918 :
1919 57 : response.Append('\n');
1920 : }
1921 :
1922 53 : if (NS_FAILED(rv)) {
1923 0 : response.Truncate();
1924 : }
1925 :
1926 53 : c->HandleEvent(response);
1927 :
1928 53 : return rv;
1929 : }
1930 :
1931 : nsresult
1932 102 : nsUrlClassifierDBServiceWorker::GetTableId(const nsACString& table,
1933 : PRUint32* tableId)
1934 : {
1935 204 : mozStorageStatementScoper findScoper(mGetTableIdStatement);
1936 :
1937 102 : nsresult rv = mGetTableIdStatement->BindUTF8StringByIndex(0, table);
1938 102 : NS_ENSURE_SUCCESS(rv, rv);
1939 :
1940 : bool exists;
1941 102 : rv = mGetTableIdStatement->ExecuteStep(&exists);
1942 102 : NS_ENSURE_SUCCESS(rv, rv);
1943 102 : if (exists) {
1944 43 : *tableId = mGetTableIdStatement->AsInt32(0);
1945 43 : return NS_OK;
1946 : }
1947 :
1948 118 : mozStorageStatementScoper insertScoper(mInsertTableIdStatement);
1949 59 : rv = mInsertTableIdStatement->BindUTF8StringByIndex(0, table);
1950 59 : NS_ENSURE_SUCCESS(rv, rv);
1951 :
1952 59 : rv = mInsertTableIdStatement->Execute();
1953 59 : NS_ENSURE_SUCCESS(rv, rv);
1954 :
1955 : PRInt64 rowId;
1956 59 : rv = mConnection->GetLastInsertRowID(&rowId);
1957 59 : NS_ENSURE_SUCCESS(rv, rv);
1958 :
1959 59 : if (rowId > PR_UINT32_MAX)
1960 0 : return NS_ERROR_FAILURE;
1961 :
1962 59 : *tableId = rowId;
1963 :
1964 59 : return NS_OK;
1965 : }
1966 :
1967 : nsresult
1968 136 : nsUrlClassifierDBServiceWorker::GetTableName(PRUint32 tableId,
1969 : nsACString& tableName)
1970 : {
1971 272 : mozStorageStatementScoper findScoper(mGetTableNameStatement);
1972 136 : nsresult rv = mGetTableNameStatement->BindInt32ByIndex(0, tableId);
1973 136 : NS_ENSURE_SUCCESS(rv, rv);
1974 : bool exists;
1975 136 : rv = mGetTableNameStatement->ExecuteStep(&exists);
1976 136 : NS_ENSURE_SUCCESS(rv, rv);
1977 136 : if (!exists) return NS_ERROR_FAILURE;
1978 :
1979 136 : return mGetTableNameStatement->GetUTF8String(0, tableName);
1980 : }
1981 :
1982 : nsresult
1983 0 : nsUrlClassifierDBServiceWorker::InflateChunk(nsACString& chunk)
1984 : {
1985 0 : nsCAutoString inflated;
1986 : char buf[4096];
1987 :
1988 0 : const nsPromiseFlatCString& flat = PromiseFlatCString(chunk);
1989 :
1990 : z_stream stream;
1991 0 : memset(&stream, 0, sizeof(stream));
1992 0 : stream.next_in = (Bytef*)flat.get();
1993 0 : stream.avail_in = flat.Length();
1994 :
1995 0 : if (inflateInit(&stream) != Z_OK) {
1996 0 : return NS_ERROR_FAILURE;
1997 : }
1998 :
1999 : int code;
2000 0 : do {
2001 0 : stream.next_out = (Bytef*)buf;
2002 0 : stream.avail_out = sizeof(buf);
2003 :
2004 0 : code = inflate(&stream, Z_NO_FLUSH);
2005 0 : PRUint32 numRead = sizeof(buf) - stream.avail_out;
2006 :
2007 0 : if (code == Z_OK || code == Z_STREAM_END) {
2008 0 : inflated.Append(buf, numRead);
2009 : }
2010 : } while (code == Z_OK);
2011 :
2012 0 : inflateEnd(&stream);
2013 :
2014 0 : if (code != Z_STREAM_END) {
2015 0 : return NS_ERROR_FAILURE;
2016 : }
2017 :
2018 0 : chunk = inflated;
2019 :
2020 0 : return NS_OK;
2021 : }
2022 :
2023 : nsresult
2024 8 : nsUrlClassifierStore::DeleteEntry(nsUrlClassifierEntry& entry)
2025 : {
2026 8 : if (entry.mId == -1) {
2027 0 : return NS_OK;
2028 : }
2029 :
2030 16 : mozStorageStatementScoper scoper(mDeleteStatement);
2031 8 : mDeleteStatement->BindInt64ByIndex(0, entry.mId);
2032 8 : nsresult rv = mDeleteStatement->Execute();
2033 8 : NS_ENSURE_SUCCESS(rv, rv);
2034 :
2035 8 : entry.mId = -1;
2036 :
2037 8 : return NS_OK;
2038 : }
2039 :
2040 : nsresult
2041 163 : nsUrlClassifierStore::WriteEntry(nsUrlClassifierEntry& entry)
2042 : {
2043 163 : if (entry.mId != -1) {
2044 : // existing entry, just ignore it
2045 0 : return NS_OK;
2046 : }
2047 :
2048 326 : mozStorageStatementScoper scoper(mInsertStatement);
2049 :
2050 163 : nsresult rv = BindStatement(entry, mInsertStatement);
2051 163 : NS_ENSURE_SUCCESS(rv, rv);
2052 :
2053 163 : rv = mInsertStatement->Execute();
2054 163 : NS_ENSURE_SUCCESS(rv, rv);
2055 :
2056 : PRInt64 rowId;
2057 163 : rv = mConnection->GetLastInsertRowID(&rowId);
2058 163 : NS_ENSURE_SUCCESS(rv, rv);
2059 :
2060 163 : if (rowId > PR_UINT32_MAX) {
2061 0 : return NS_ERROR_FAILURE;
2062 : }
2063 :
2064 163 : entry.mId = rowId;
2065 :
2066 163 : return NS_OK;
2067 : }
2068 :
2069 : nsresult
2070 36 : nsUrlClassifierStore::UpdateEntry(nsUrlClassifierEntry& entry)
2071 : {
2072 72 : mozStorageStatementScoper scoper(mUpdateStatement);
2073 :
2074 36 : NS_ENSURE_ARG(entry.mId != -1);
2075 :
2076 36 : nsresult rv = BindStatement(entry, mUpdateStatement);
2077 36 : NS_ENSURE_SUCCESS(rv, rv);
2078 :
2079 36 : rv = mUpdateStatement->Execute();
2080 36 : NS_ENSURE_SUCCESS(rv, rv);
2081 :
2082 36 : return NS_OK;
2083 : }
2084 :
2085 : static bool
2086 678 : IsCanonicalizedIP(const nsACString& host)
2087 : {
2088 : // The canonicalization process will have left IP addresses in dotted
2089 : // decimal with no surprises.
2090 : PRUint32 i1, i2, i3, i4;
2091 : char c;
2092 678 : if (PR_sscanf(PromiseFlatCString(host).get(), "%u.%u.%u.%u%c",
2093 678 : &i1, &i2, &i3, &i4, &c) == 4) {
2094 8 : return (i1 <= 0xFF && i2 <= 0xFF && i3 <= 0xFF && i4 <= 0xFF);
2095 : }
2096 :
2097 670 : return false;
2098 : }
2099 :
2100 : static nsresult
2101 375 : GetKey(const nsACString& spec,
2102 : nsUrlClassifierDomainHash& hash,
2103 : nsICryptoHash * aCryptoHash)
2104 : {
2105 375 : nsACString::const_iterator begin, end, iter;
2106 375 : spec.BeginReading(begin);
2107 375 : spec.EndReading(end);
2108 :
2109 375 : iter = begin;
2110 375 : if (!FindCharInReadable('/', iter, end)) {
2111 0 : return NS_OK;
2112 : }
2113 :
2114 750 : const nsCSubstring& host = Substring(begin, iter);
2115 :
2116 375 : if (IsCanonicalizedIP(host)) {
2117 8 : nsCAutoString key;
2118 4 : key.Assign(host);
2119 4 : key.Append("/");
2120 4 : return hash.FromPlaintext(key, aCryptoHash);
2121 : }
2122 :
2123 742 : nsTArray<nsCString> hostComponents;
2124 371 : ParseString(PromiseFlatCString(host), '.', hostComponents);
2125 :
2126 371 : if (hostComponents.Length() < 2)
2127 0 : return NS_ERROR_FAILURE;
2128 :
2129 371 : PRInt32 last = PRInt32(hostComponents.Length()) - 1;
2130 742 : nsCAutoString lookupHost;
2131 :
2132 371 : if (hostComponents.Length() > 2) {
2133 43 : lookupHost.Append(hostComponents[last - 2]);
2134 43 : lookupHost.Append(".");
2135 : }
2136 :
2137 371 : lookupHost.Append(hostComponents[last - 1]);
2138 371 : lookupHost.Append(".");
2139 371 : lookupHost.Append(hostComponents[last]);
2140 371 : lookupHost.Append("/");
2141 :
2142 371 : return hash.FromPlaintext(lookupHost, aCryptoHash);
2143 : }
2144 :
2145 : nsresult
2146 0 : nsUrlClassifierDBServiceWorker::GetShaEntries(PRUint32 tableId,
2147 : PRUint32 chunkType,
2148 : PRUint32 chunkNum,
2149 : PRUint32 domainSize,
2150 : PRUint32 fragmentSize,
2151 : nsACString& chunk,
2152 : nsTArray<nsUrlClassifierEntry>& entries)
2153 : {
2154 0 : PRUint32 start = 0;
2155 0 : while (start + domainSize + 1 <= chunk.Length()) {
2156 : nsUrlClassifierDomainHash domain;
2157 0 : domain.Assign(Substring(chunk, start, DOMAIN_LENGTH));
2158 0 : start += domainSize;
2159 :
2160 : // then there is a one-byte count of fragments
2161 0 : PRUint8 numEntries = static_cast<PRUint8>(chunk[start]);
2162 0 : start++;
2163 :
2164 0 : if (numEntries == 0) {
2165 : // if there are no fragments, the domain itself is treated as a
2166 : // fragment. This will only work if domainHashSize == hashSize
2167 0 : if (domainSize != fragmentSize) {
2168 0 : NS_WARNING("Received 0-fragment entry where domainSize != fragmentSize");
2169 0 : return NS_ERROR_FAILURE;
2170 : }
2171 :
2172 0 : nsUrlClassifierEntry* entry = entries.AppendElement();
2173 0 : if (!entry) return NS_ERROR_OUT_OF_MEMORY;
2174 :
2175 0 : entry->mKey = domain;
2176 0 : entry->mTableId = tableId;
2177 0 : entry->mChunkId = chunkNum;
2178 0 : entry->SetHash(domain);
2179 :
2180 0 : if (chunkType == CHUNK_SUB) {
2181 0 : if (start + 4 > chunk.Length()) {
2182 : // there isn't as much data as there should be.
2183 0 : NS_WARNING("Received a zero-entry sub chunk without an associated add.");
2184 0 : return NS_ERROR_FAILURE;
2185 : }
2186 0 : const nsCSubstring& str = Substring(chunk, start, 4);
2187 : PRUint32 p;
2188 0 : memcpy(&p, str.BeginReading(), 4);
2189 0 : entry->mAddChunkId = PR_ntohl(p);
2190 0 : if (entry->mAddChunkId == 0) {
2191 0 : NS_WARNING("Received invalid chunk number.");
2192 0 : return NS_ERROR_FAILURE;
2193 : }
2194 0 : start += 4;
2195 : }
2196 : } else {
2197 0 : PRUint32 entrySize = fragmentSize;
2198 0 : if (chunkType == CHUNK_SUB) {
2199 0 : entrySize += 4;
2200 : }
2201 0 : if (start + (numEntries * entrySize) > chunk.Length()) {
2202 : // there isn't as much data as they said there would be.
2203 0 : NS_WARNING("Received a chunk without enough data");
2204 0 : return NS_ERROR_FAILURE;
2205 : }
2206 :
2207 0 : for (PRUint8 i = 0; i < numEntries; i++) {
2208 0 : nsUrlClassifierEntry* entry = entries.AppendElement();
2209 0 : if (!entry) return NS_ERROR_OUT_OF_MEMORY;
2210 :
2211 0 : entry->mKey = domain;
2212 0 : entry->mTableId = tableId;
2213 0 : entry->mChunkId = chunkNum;
2214 :
2215 0 : if (chunkType == CHUNK_SUB) {
2216 0 : const nsCSubstring& str = Substring(chunk, start, 4);
2217 : PRUint32 p;
2218 0 : memcpy(&p, str.BeginReading(), 4);
2219 0 : entry->mAddChunkId = PR_ntohl(p);
2220 0 : if (entry->mAddChunkId == 0) {
2221 0 : NS_WARNING("Received invalid chunk number.");
2222 0 : return NS_ERROR_FAILURE;
2223 : }
2224 0 : start += 4;
2225 : }
2226 :
2227 0 : if (fragmentSize == PARTIAL_LENGTH) {
2228 : nsUrlClassifierPartialHash hash;
2229 0 : hash.Assign(Substring(chunk, start, PARTIAL_LENGTH));
2230 0 : entry->SetHash(hash);
2231 0 : } else if (fragmentSize == COMPLETE_LENGTH) {
2232 : nsUrlClassifierCompleteHash hash;
2233 0 : hash.Assign(Substring(chunk, start, COMPLETE_LENGTH));
2234 0 : entry->SetHash(hash);
2235 : } else {
2236 0 : NS_ASSERTION(false, "Invalid fragment size!");
2237 0 : return NS_ERROR_FAILURE;
2238 : }
2239 :
2240 0 : start += fragmentSize;
2241 : }
2242 : }
2243 : }
2244 :
2245 0 : return NS_OK;
2246 : }
2247 :
2248 : nsresult
2249 103 : nsUrlClassifierDBServiceWorker::GetChunkEntries(const nsACString& table,
2250 : PRUint32 tableId,
2251 : PRUint32 chunkType,
2252 : PRUint32 chunkNum,
2253 : PRUint32 hashSize,
2254 : nsACString& chunk,
2255 : nsTArray<nsUrlClassifierEntry>& entries)
2256 : {
2257 : nsresult rv;
2258 103 : if (StringEndsWith(table, NS_LITERAL_CSTRING("-exp"))) {
2259 : // regexp tables need to be ungzipped
2260 0 : rv = InflateChunk(chunk);
2261 0 : NS_ENSURE_SUCCESS(rv, rv);
2262 : }
2263 :
2264 103 : if (StringEndsWith(table, NS_LITERAL_CSTRING("-shavar"))) {
2265 : rv = GetShaEntries(tableId, chunkType, chunkNum, DOMAIN_LENGTH, hashSize,
2266 0 : chunk, entries);
2267 0 : NS_ENSURE_SUCCESS(rv, rv);
2268 : } else {
2269 206 : nsTArray<nsCString> lines;
2270 103 : ParseString(PromiseFlatCString(chunk), '\n', lines);
2271 :
2272 : // non-hashed tables need to be hashed
2273 287 : for (PRInt32 i = 0; i < PRInt32(lines.Length()); i++) {
2274 184 : nsUrlClassifierEntry *entry = entries.AppendElement();
2275 184 : if (!entry)
2276 0 : return NS_ERROR_OUT_OF_MEMORY;
2277 :
2278 368 : nsCAutoString entryStr;
2279 184 : if (chunkType == CHUNK_SUB) {
2280 27 : nsCString::const_iterator begin, iter, end;
2281 27 : lines[i].BeginReading(begin);
2282 27 : lines[i].EndReading(end);
2283 27 : iter = begin;
2284 54 : if (!FindCharInReadable(':', iter, end) ||
2285 27 : PR_sscanf(lines[i].get(), "%d:", &entry->mAddChunkId) != 1) {
2286 0 : NS_WARNING("Received sub chunk without associated add chunk.");
2287 0 : return NS_ERROR_FAILURE;
2288 : }
2289 27 : iter++;
2290 27 : entryStr = Substring(iter, end);
2291 : } else {
2292 157 : entryStr = lines[i];
2293 : }
2294 :
2295 184 : rv = GetKey(entryStr, entry->mKey, mCryptoHash);
2296 184 : NS_ENSURE_SUCCESS(rv, rv);
2297 :
2298 184 : entry->mTableId = tableId;
2299 184 : entry->mChunkId = chunkNum;
2300 184 : if (hashSize == PARTIAL_LENGTH) {
2301 : nsUrlClassifierPartialHash hash;
2302 47 : hash.FromPlaintext(entryStr, mCryptoHash);
2303 47 : entry->SetHash(hash);
2304 137 : } else if (hashSize == COMPLETE_LENGTH) {
2305 : nsUrlClassifierCompleteHash hash;
2306 137 : hash.FromPlaintext(entryStr, mCryptoHash);
2307 137 : entry->SetHash(hash);
2308 : } else {
2309 0 : NS_ASSERTION(false, "Invalid fragment size!");
2310 0 : return NS_ERROR_FAILURE;
2311 : }
2312 : }
2313 : }
2314 :
2315 103 : return NS_OK;
2316 : }
2317 :
2318 : bool
2319 39 : nsUrlClassifierDBServiceWorker::ParseChunkRange(nsACString::const_iterator &begin,
2320 : const nsACString::const_iterator &end,
2321 : PRUint32 *first,
2322 : PRUint32 *last)
2323 : {
2324 39 : nsACString::const_iterator iter = begin;
2325 39 : FindCharInReadable(',', iter, end);
2326 :
2327 78 : nsCAutoString element(Substring(begin, iter));
2328 39 : begin = iter;
2329 39 : if (begin != end)
2330 6 : begin++;
2331 :
2332 39 : PRUint32 numRead = PR_sscanf(element.get(), "%u-%u", first, last);
2333 39 : if (numRead == 2) {
2334 11 : if (*first > *last) {
2335 0 : PRUint32 tmp = *first;
2336 0 : *first = *last;
2337 0 : *last = tmp;
2338 : }
2339 11 : return true;
2340 : }
2341 :
2342 28 : if (numRead == 1) {
2343 28 : *last = *first;
2344 28 : return true;
2345 : }
2346 :
2347 0 : return false;
2348 : }
2349 :
2350 : nsresult
2351 104 : nsUrlClassifierDBServiceWorker::ParseChunkList(const nsACString& chunkStr,
2352 : nsTArray<PRUint32>& chunks)
2353 : {
2354 104 : LOG(("Parsing %s", PromiseFlatCString(chunkStr).get()));
2355 :
2356 104 : nsACString::const_iterator begin, end;
2357 104 : chunkStr.BeginReading(begin);
2358 104 : chunkStr.EndReading(end);
2359 236 : while (begin != end) {
2360 : PRUint32 first, last;
2361 28 : if (ParseChunkRange(begin, end, &first, &last)) {
2362 68 : for (PRUint32 num = first; num <= last; num++) {
2363 40 : chunks.AppendElement(num);
2364 : }
2365 : }
2366 : }
2367 :
2368 104 : LOG(("Got %d elements.", chunks.Length()));
2369 :
2370 104 : return NS_OK;
2371 : }
2372 :
2373 : nsresult
2374 104 : nsUrlClassifierDBServiceWorker::JoinChunkList(nsTArray<PRUint32>& chunks,
2375 : nsCString& chunkStr)
2376 : {
2377 104 : chunkStr.Truncate();
2378 104 : chunks.Sort();
2379 :
2380 104 : PRUint32 i = 0;
2381 305 : while (i < chunks.Length()) {
2382 97 : if (i != 0) {
2383 4 : chunkStr.Append(',');
2384 : }
2385 97 : chunkStr.AppendInt(chunks[i]);
2386 :
2387 97 : PRUint32 first = i;
2388 97 : PRUint32 last = first;
2389 97 : i++;
2390 221 : while (i < chunks.Length() && (chunks[i] == chunks[i - 1] + 1 || chunks[i] == chunks[i - 1])) {
2391 27 : last = i++;
2392 : }
2393 :
2394 97 : if (last != first) {
2395 21 : chunkStr.Append('-');
2396 21 : chunkStr.AppendInt(chunks[last]);
2397 : }
2398 : }
2399 :
2400 104 : return NS_OK;
2401 : }
2402 :
2403 :
2404 : nsresult
2405 86 : nsUrlClassifierDBServiceWorker::GetChunkLists(PRUint32 tableId,
2406 : nsACString& addChunks,
2407 : nsACString& subChunks)
2408 : {
2409 86 : addChunks.Truncate();
2410 86 : subChunks.Truncate();
2411 :
2412 172 : mozStorageStatementScoper scoper(mGetChunkListsStatement);
2413 :
2414 86 : nsresult rv = mGetChunkListsStatement->BindInt32ByIndex(0, tableId);
2415 86 : NS_ENSURE_SUCCESS(rv, rv);
2416 :
2417 86 : bool hasMore = false;
2418 86 : rv = mGetChunkListsStatement->ExecuteStep(&hasMore);
2419 86 : NS_ENSURE_SUCCESS(rv, rv);
2420 :
2421 86 : if (!hasMore) {
2422 0 : LOG(("Getting chunks for %d, found nothing", tableId));
2423 0 : return NS_OK;
2424 : }
2425 :
2426 86 : rv = mGetChunkListsStatement->GetUTF8String(0, addChunks);
2427 86 : NS_ENSURE_SUCCESS(rv, rv);
2428 :
2429 86 : rv = mGetChunkListsStatement->GetUTF8String(1, subChunks);
2430 86 : NS_ENSURE_SUCCESS(rv, rv);
2431 :
2432 86 : LOG(("Getting chunks for %d, got %s/%s",
2433 : tableId,
2434 : PromiseFlatCString(addChunks).get(),
2435 : PromiseFlatCString(subChunks).get()));
2436 :
2437 86 : return NS_OK;
2438 : }
2439 :
2440 : nsresult
2441 86 : nsUrlClassifierDBServiceWorker::SetChunkLists(PRUint32 tableId,
2442 : const nsACString& addChunks,
2443 : const nsACString& subChunks)
2444 : {
2445 172 : mozStorageStatementScoper scoper(mSetChunkListsStatement);
2446 :
2447 86 : mSetChunkListsStatement->BindUTF8StringByIndex(0, addChunks);
2448 86 : mSetChunkListsStatement->BindUTF8StringByIndex(1, subChunks);
2449 86 : mSetChunkListsStatement->BindInt32ByIndex(2, tableId);
2450 86 : nsresult rv = mSetChunkListsStatement->Execute();
2451 86 : NS_ENSURE_SUCCESS(rv, rv);
2452 :
2453 86 : return NS_OK;
2454 : }
2455 :
2456 : nsresult
2457 120 : nsUrlClassifierDBServiceWorker::CacheChunkLists(PRUint32 tableId,
2458 : bool parseAdds,
2459 : bool parseSubs)
2460 : {
2461 : nsresult rv;
2462 :
2463 120 : if (mHaveCachedLists && mCachedListsTable != tableId) {
2464 3 : rv = FlushChunkLists();
2465 3 : NS_ENSURE_SUCCESS(rv, rv);
2466 : }
2467 :
2468 120 : if (!mHaveCachedLists) {
2469 86 : rv = GetChunkLists(tableId, mCachedAddsStr, mCachedSubsStr);
2470 86 : NS_ENSURE_SUCCESS(rv, rv);
2471 :
2472 86 : mHaveCachedLists = true;
2473 86 : mCachedListsTable = tableId;
2474 : }
2475 :
2476 120 : if (parseAdds && !mHaveCachedAddChunks) {
2477 84 : ParseChunkList(mCachedAddsStr, mCachedAddChunks);
2478 84 : mHaveCachedAddChunks = true;
2479 : }
2480 :
2481 120 : if (parseSubs && !mHaveCachedSubChunks) {
2482 20 : ParseChunkList(mCachedSubsStr, mCachedSubChunks);
2483 20 : mHaveCachedSubChunks = true;
2484 : }
2485 :
2486 120 : return NS_OK;
2487 : }
2488 :
2489 : nsresult
2490 87 : nsUrlClassifierDBServiceWorker::FlushChunkLists()
2491 : {
2492 87 : if (!mHaveCachedLists) {
2493 1 : return NS_OK;
2494 : }
2495 :
2496 86 : if (mHaveCachedAddChunks) {
2497 84 : JoinChunkList(mCachedAddChunks, mCachedAddsStr);
2498 : }
2499 :
2500 86 : if (mHaveCachedSubChunks) {
2501 20 : JoinChunkList(mCachedSubChunks, mCachedSubsStr);
2502 : }
2503 :
2504 : nsresult rv = SetChunkLists(mCachedListsTable,
2505 86 : mCachedAddsStr, mCachedSubsStr);
2506 :
2507 : // clear out the cache before checking/returning the error here.
2508 86 : ClearCachedChunkLists();
2509 :
2510 86 : return rv;
2511 : }
2512 :
2513 : void
2514 143 : nsUrlClassifierDBServiceWorker::ClearCachedChunkLists()
2515 : {
2516 143 : mCachedAddsStr.Truncate();
2517 143 : mCachedSubsStr.Truncate();
2518 143 : mCachedListsTable = PR_UINT32_MAX;
2519 143 : mHaveCachedLists = false;
2520 :
2521 143 : mCachedAddChunks.Clear();
2522 143 : mHaveCachedAddChunks = false;
2523 :
2524 143 : mCachedSubChunks.Clear();
2525 143 : mHaveCachedSubChunks = false;
2526 143 : }
2527 :
2528 : bool
2529 103 : nsUrlClassifierDBServiceWorker::InsertChunkId(nsTArray<PRUint32> &chunks,
2530 : PRUint32 chunkNum)
2531 : {
2532 103 : PRUint32 low = 0, high = chunks.Length();
2533 235 : while (high > low) {
2534 31 : PRUint32 mid = (high + low) >> 1;
2535 31 : if (chunks[mid] == chunkNum)
2536 2 : return false;
2537 29 : if (chunks[mid] < chunkNum)
2538 29 : low = mid + 1;
2539 : else
2540 0 : high = mid;
2541 : }
2542 :
2543 101 : PRUint32 *item = chunks.InsertElementAt(low, chunkNum);
2544 101 : return (item != nsnull);
2545 : }
2546 :
2547 : nsresult
2548 83 : nsUrlClassifierDBServiceWorker::AddChunk(PRUint32 tableId,
2549 : PRUint32 chunkNum,
2550 : nsTArray<nsUrlClassifierEntry>& entries)
2551 : {
2552 : #if defined(PR_LOGGING)
2553 83 : PRIntervalTime clockStart = 0;
2554 83 : if (LOG_ENABLED()) {
2555 0 : clockStart = PR_IntervalNow();
2556 : }
2557 : #endif
2558 :
2559 83 : nsresult rv = CacheChunkLists(tableId, true, false);
2560 83 : NS_ENSURE_SUCCESS(rv, rv);
2561 :
2562 83 : if (!InsertChunkId(mCachedAddChunks, chunkNum)) {
2563 2 : LOG(("Ignoring duplicate add chunk %d in table %d", chunkNum, tableId));
2564 2 : return NS_OK;
2565 : }
2566 :
2567 81 : LOG(("Adding %d entries to chunk %d in table %d", entries.Length(), chunkNum, tableId));
2568 :
2569 162 : nsTArray<PRUint32> entryIDs;
2570 :
2571 162 : nsAutoTArray<nsUrlClassifierEntry, 5> subEntries;
2572 81 : rv = mPendingSubStore.ReadSubEntries(tableId, chunkNum, subEntries);
2573 81 : NS_ENSURE_SUCCESS(rv, rv);
2574 :
2575 236 : for (PRUint32 i = 0; i < entries.Length(); i++) {
2576 155 : nsUrlClassifierEntry& thisEntry = entries[i];
2577 :
2578 155 : HandlePendingLookups();
2579 :
2580 155 : bool writeEntry = true;
2581 156 : for (PRUint32 j = 0; j < subEntries.Length(); j++) {
2582 8 : if (thisEntry.SubMatch(subEntries[j])) {
2583 7 : subEntries.RemoveElementAt(j);
2584 :
2585 7 : writeEntry = false;
2586 7 : break;
2587 : }
2588 : }
2589 :
2590 155 : HandlePendingLookups();
2591 :
2592 155 : if (writeEntry) {
2593 148 : rv = mMainStore.WriteEntry(thisEntry);
2594 148 : NS_ENSURE_SUCCESS(rv, rv);
2595 : }
2596 : }
2597 :
2598 81 : rv = mPendingSubStore.ExpireAddChunk(tableId, chunkNum);
2599 81 : NS_ENSURE_SUCCESS(rv, rv);
2600 :
2601 : #if defined(PR_LOGGING)
2602 81 : if (LOG_ENABLED()) {
2603 0 : PRIntervalTime clockEnd = PR_IntervalNow();
2604 0 : LOG(("adding chunk %d took %dms\n", chunkNum,
2605 : PR_IntervalToMilliseconds(clockEnd - clockStart)));
2606 : }
2607 : #endif
2608 :
2609 81 : return rv;
2610 : }
2611 :
2612 : nsresult
2613 17 : nsUrlClassifierStore::Expire(PRUint32 tableId, PRUint32 chunkNum)
2614 : {
2615 17 : LOG(("Expiring chunk %d\n", chunkNum));
2616 :
2617 34 : mozStorageStatementScoper expireScoper(mExpireStatement);
2618 :
2619 17 : nsresult rv = mExpireStatement->BindInt32ByIndex(0, tableId);
2620 17 : NS_ENSURE_SUCCESS(rv, rv);
2621 17 : rv = mExpireStatement->BindInt32ByIndex(1, chunkNum);
2622 17 : NS_ENSURE_SUCCESS(rv, rv);
2623 :
2624 17 : mWorker->HandlePendingLookups();
2625 :
2626 17 : rv = mExpireStatement->Execute();
2627 17 : NS_ENSURE_SUCCESS(rv, rv);
2628 :
2629 17 : return NS_OK;
2630 : }
2631 :
2632 : nsresult
2633 11 : nsUrlClassifierDBServiceWorker::ExpireAdd(PRUint32 tableId,
2634 : PRUint32 chunkNum)
2635 : {
2636 11 : nsresult rv = CacheChunkLists(tableId, true, false);
2637 11 : NS_ENSURE_SUCCESS(rv, rv);
2638 11 : mCachedAddChunks.RemoveElement(chunkNum);
2639 :
2640 11 : return mMainStore.Expire(tableId, chunkNum);
2641 : }
2642 :
2643 : nsresult
2644 20 : nsUrlClassifierDBServiceWorker::SubChunk(PRUint32 tableId,
2645 : PRUint32 chunkNum,
2646 : nsTArray<nsUrlClassifierEntry>& entries)
2647 : {
2648 20 : nsresult rv = CacheChunkLists(tableId, true, true);
2649 :
2650 20 : if (!InsertChunkId(mCachedSubChunks, chunkNum)) {
2651 0 : LOG(("Ignoring duplicate sub chunk %d in table %d", chunkNum, tableId));
2652 0 : return NS_OK;
2653 : }
2654 :
2655 20 : LOG(("Subbing %d entries in chunk %d in table %d", entries.Length(), chunkNum, tableId));
2656 :
2657 47 : for (PRUint32 i = 0; i < entries.Length(); i++) {
2658 54 : nsAutoTArray<nsUrlClassifierEntry, 5> existingEntries;
2659 27 : nsUrlClassifierEntry& thisEntry = entries[i];
2660 :
2661 27 : HandlePendingLookups();
2662 :
2663 : // Check if we have the add chunk associated with the sub.
2664 27 : bool haveAdds = (mCachedAddChunks.BinaryIndexOf(thisEntry.mAddChunkId) !=
2665 27 : mCachedAddChunks.NoIndex);
2666 :
2667 27 : if (haveAdds) {
2668 : rv = mMainStore.ReadAddEntries(thisEntry.mKey, thisEntry.mTableId,
2669 12 : thisEntry.mAddChunkId, existingEntries);
2670 12 : NS_ENSURE_SUCCESS(rv, rv);
2671 : }
2672 :
2673 30 : for (PRUint32 j = 0; j < existingEntries.Length(); j++) {
2674 11 : if (existingEntries[j].SubMatch(thisEntry)) {
2675 8 : rv = mMainStore.DeleteEntry(existingEntries[j]);
2676 8 : NS_ENSURE_SUCCESS(rv, rv);
2677 8 : existingEntries.RemoveElementAt(j);
2678 8 : break;
2679 : }
2680 : }
2681 :
2682 27 : if (!haveAdds) {
2683 : // Save this entry in the pending subtraction store.
2684 15 : rv = mPendingSubStore.WriteEntry(thisEntry);
2685 15 : NS_ENSURE_SUCCESS(rv, rv);
2686 : }
2687 : }
2688 :
2689 20 : return NS_OK;
2690 : }
2691 :
2692 : nsresult
2693 6 : nsUrlClassifierDBServiceWorker::ExpireSub(PRUint32 tableId, PRUint32 chunkNum)
2694 : {
2695 6 : nsresult rv = CacheChunkLists(tableId, false, true);
2696 6 : NS_ENSURE_SUCCESS(rv, rv);
2697 6 : mCachedSubChunks.RemoveElement(chunkNum);
2698 :
2699 6 : return mPendingSubStore.Expire(tableId, chunkNum);
2700 : }
2701 :
2702 : nsresult
2703 103 : nsUrlClassifierDBServiceWorker::ProcessChunk(bool* done)
2704 : {
2705 : // wait until the chunk has been read
2706 103 : if (mPendingStreamUpdate.Length() < static_cast<PRUint32>(mChunkLen)) {
2707 0 : *done = true;
2708 0 : return NS_OK;
2709 : }
2710 :
2711 206 : nsCAutoString chunk;
2712 103 : chunk.Assign(Substring(mPendingStreamUpdate, 0, mChunkLen));
2713 103 : mPendingStreamUpdate = Substring(mPendingStreamUpdate, mChunkLen);
2714 :
2715 103 : LOG(("Handling a chunk sized %d", chunk.Length()));
2716 :
2717 206 : nsTArray<nsUrlClassifierEntry> entries;
2718 : nsresult rv = GetChunkEntries(mUpdateTable, mUpdateTableId, mChunkType,
2719 103 : mChunkNum, mHashSize, chunk, entries);
2720 103 : NS_ENSURE_SUCCESS(rv, rv);
2721 :
2722 103 : if (mChunkType == CHUNK_ADD) {
2723 83 : rv = AddChunk(mUpdateTableId, mChunkNum, entries);
2724 : } else {
2725 20 : rv = SubChunk(mUpdateTableId, mChunkNum, entries);
2726 : }
2727 :
2728 103 : mState = STATE_LINE;
2729 103 : *done = false;
2730 :
2731 103 : return rv;
2732 : }
2733 :
2734 : nsresult
2735 203 : nsUrlClassifierDBServiceWorker::ProcessResponseLines(bool* done)
2736 : {
2737 203 : PRUint32 cur = 0;
2738 : PRInt32 next;
2739 :
2740 : nsresult rv;
2741 : // We will run to completion unless we find a chunk line
2742 203 : *done = true;
2743 :
2744 203 : nsACString& updateString = mPendingStreamUpdate;
2745 :
2746 726 : while(cur < updateString.Length() &&
2747 : (next = updateString.FindChar('\n', cur)) != kNotFound) {
2748 848 : const nsCSubstring& line = Substring(updateString, cur, next - cur);
2749 424 : cur = next + 1;
2750 :
2751 424 : LOG(("Processing %s\n", PromiseFlatCString(line).get()));
2752 :
2753 424 : if (mHMAC && mServerMAC.IsEmpty()) {
2754 : // If we did not receive a server MAC during BeginStream(), we
2755 : // require the first line of the update to be either a MAC or
2756 : // a request to rekey.
2757 2 : if (StringBeginsWith(line, NS_LITERAL_CSTRING("m:"))) {
2758 2 : mServerMAC = Substring(line, 2);
2759 2 : nsUrlClassifierUtils::UnUrlsafeBase64(mServerMAC);
2760 :
2761 : // The remainder of the pending update needs to be digested.
2762 4 : const nsCSubstring &toDigest = Substring(updateString, cur);
2763 4 : rv = mHMAC->Update(reinterpret_cast<const PRUint8*>(toDigest.BeginReading()),
2764 4 : toDigest.Length());
2765 2 : NS_ENSURE_SUCCESS(rv, rv);
2766 0 : } else if (line.EqualsLiteral("e:pleaserekey")) {
2767 0 : mUpdateObserver->RekeyRequested();
2768 : } else {
2769 0 : LOG(("No MAC specified!"));
2770 0 : return NS_ERROR_FAILURE;
2771 : }
2772 422 : } else if (StringBeginsWith(line, NS_LITERAL_CSTRING("n:"))) {
2773 101 : if (PR_sscanf(PromiseFlatCString(line).get(), "n:%d",
2774 101 : &mUpdateWait) != 1) {
2775 0 : LOG(("Error parsing n: field: %s", PromiseFlatCString(line).get()));
2776 0 : mUpdateWait = 0;
2777 : }
2778 321 : } else if (line.EqualsLiteral("r:pleasereset")) {
2779 1 : mResetRequested = true;
2780 320 : } else if (line.EqualsLiteral("e:pleaserekey")) {
2781 0 : mUpdateObserver->RekeyRequested();
2782 320 : } else if (StringBeginsWith(line, NS_LITERAL_CSTRING("i:"))) {
2783 87 : mUpdateTable.Assign(Substring(line, 2));
2784 87 : GetTableId(mUpdateTable, &mUpdateTableId);
2785 87 : LOG(("update table: '%s' (%d)", mUpdateTable.get(), mUpdateTableId));
2786 233 : } else if (StringBeginsWith(line, NS_LITERAL_CSTRING("u:"))) {
2787 17 : if (!mPrimaryStream) {
2788 0 : LOG(("Forwarded update tried to add its own forwarded update."));
2789 0 : return NS_ERROR_FAILURE;
2790 : }
2791 :
2792 34 : const nsCSubstring& data = Substring(line, 2);
2793 17 : if (mHMAC) {
2794 : // We're expecting MACs alongside any url forwards.
2795 3 : nsCSubstring::const_iterator begin, end, sepBegin, sepEnd;
2796 3 : data.BeginReading(begin);
2797 3 : sepBegin = begin;
2798 :
2799 3 : data.EndReading(end);
2800 3 : sepEnd = end;
2801 :
2802 3 : if (!RFindInReadable(NS_LITERAL_CSTRING(","), sepBegin, sepEnd)) {
2803 0 : NS_WARNING("No MAC specified for a redirect in a request that expects a MAC");
2804 0 : return NS_ERROR_FAILURE;
2805 : }
2806 :
2807 6 : nsCString serverMAC(Substring(sepEnd, end));
2808 3 : nsUrlClassifierUtils::UnUrlsafeBase64(serverMAC);
2809 6 : mUpdateObserver->UpdateUrlRequested(Substring(begin, sepBegin),
2810 : mUpdateTable,
2811 3 : serverMAC);
2812 : } else {
2813 : // We didn't ask for a MAC, none should have been specified.
2814 14 : mUpdateObserver->UpdateUrlRequested(data, mUpdateTable,
2815 14 : NS_LITERAL_CSTRING(""));
2816 : }
2817 696 : } else if (StringBeginsWith(line, NS_LITERAL_CSTRING("a:")) ||
2818 480 : StringBeginsWith(line, NS_LITERAL_CSTRING("s:"))) {
2819 104 : mState = STATE_CHUNK;
2820 : char command;
2821 104 : if (PR_sscanf(PromiseFlatCString(line).get(),
2822 104 : "%c:%d:%d:%d", &command, &mChunkNum, &mHashSize, &mChunkLen) != 4) {
2823 0 : return NS_ERROR_FAILURE;
2824 : }
2825 :
2826 104 : if (mChunkLen > MAX_CHUNK_SIZE) {
2827 0 : return NS_ERROR_FAILURE;
2828 : }
2829 :
2830 104 : if (!(mHashSize == PARTIAL_LENGTH || mHashSize == COMPLETE_LENGTH)) {
2831 1 : NS_WARNING("Invalid hash size specified in update.");
2832 1 : return NS_ERROR_FAILURE;
2833 : }
2834 :
2835 103 : mChunkType = (command == 'a') ? CHUNK_ADD : CHUNK_SUB;
2836 :
2837 : // Done parsing lines, move to chunk state now
2838 103 : *done = false;
2839 : break;
2840 112 : } else if (StringBeginsWith(line, NS_LITERAL_CSTRING("ad:"))) {
2841 12 : const nsCSubstring &list = Substring(line, 3);
2842 6 : nsACString::const_iterator begin, end;
2843 6 : list.BeginReading(begin);
2844 6 : list.EndReading(end);
2845 6 : while (begin != end) {
2846 : PRUint32 first, last;
2847 7 : if (ParseChunkRange(begin, end, &first, &last)) {
2848 18 : for (PRUint32 num = first; num <= last; num++) {
2849 11 : rv = ExpireAdd(mUpdateTableId, num);
2850 11 : NS_ENSURE_SUCCESS(rv, rv);
2851 : }
2852 : } else {
2853 0 : return NS_ERROR_FAILURE;
2854 : }
2855 : }
2856 106 : } else if (StringBeginsWith(line, NS_LITERAL_CSTRING("sd:"))) {
2857 6 : const nsCSubstring &list = Substring(line, 3);
2858 3 : nsACString::const_iterator begin, end;
2859 3 : list.BeginReading(begin);
2860 3 : list.EndReading(end);
2861 3 : while (begin != end) {
2862 : PRUint32 first, last;
2863 4 : if (ParseChunkRange(begin, end, &first, &last)) {
2864 10 : for (PRUint32 num = first; num <= last; num++) {
2865 6 : rv = ExpireSub(mUpdateTableId, num);
2866 6 : NS_ENSURE_SUCCESS(rv, rv);
2867 : }
2868 : } else {
2869 0 : return NS_ERROR_FAILURE;
2870 : }
2871 : }
2872 : } else {
2873 103 : LOG(("ignoring unknown line: '%s'", PromiseFlatCString(line).get()));
2874 : }
2875 : }
2876 :
2877 202 : mPendingStreamUpdate = Substring(updateString, cur);
2878 :
2879 202 : return NS_OK;
2880 : }
2881 :
2882 : void
2883 102 : nsUrlClassifierDBServiceWorker::ResetStream()
2884 : {
2885 102 : mState = STATE_LINE;
2886 102 : mChunkNum = 0;
2887 102 : mHashSize = 0;
2888 102 : mChunkLen = 0;
2889 102 : mInStream = false;
2890 102 : mPrimaryStream = false;
2891 102 : mUpdateTable.Truncate();
2892 102 : mPendingStreamUpdate.Truncate();
2893 102 : mServerMAC.Truncate();
2894 102 : mHMAC = nsnull;
2895 102 : }
2896 :
2897 : void
2898 95 : nsUrlClassifierDBServiceWorker::ResetUpdate()
2899 : {
2900 95 : mUpdateWait = 0;
2901 95 : mUpdateStatus = NS_OK;
2902 95 : mUpdateObserver = nsnull;
2903 95 : mUpdateClientKey.Truncate();
2904 95 : mResetRequested = false;
2905 95 : mUpdateTables.Clear();
2906 95 : }
2907 :
2908 : NS_IMETHODIMP
2909 0 : nsUrlClassifierDBServiceWorker::SetHashCompleter(const nsACString &tableName,
2910 : nsIUrlClassifierHashCompleter *completer)
2911 : {
2912 0 : return NS_ERROR_NOT_IMPLEMENTED;
2913 : }
2914 :
2915 : NS_IMETHODIMP
2916 87 : nsUrlClassifierDBServiceWorker::BeginUpdate(nsIUrlClassifierUpdateObserver *observer,
2917 : const nsACString &tables,
2918 : const nsACString &clientKey)
2919 : {
2920 87 : if (gShuttingDownThread)
2921 0 : return NS_ERROR_NOT_INITIALIZED;
2922 :
2923 87 : NS_ENSURE_STATE(!mUpdateObserver);
2924 :
2925 87 : nsresult rv = OpenDb();
2926 87 : if (NS_FAILED(rv)) {
2927 0 : NS_ERROR("Unable to open database");
2928 0 : return NS_ERROR_FAILURE;
2929 : }
2930 :
2931 : bool transaction;
2932 87 : rv = mConnection->GetTransactionInProgress(&transaction);
2933 87 : if (NS_FAILED(rv)) {
2934 0 : mUpdateStatus = rv;
2935 0 : return rv;
2936 : }
2937 :
2938 87 : if (transaction) {
2939 0 : NS_WARNING("Transaction already in progress in nsUrlClassifierDBServiceWorker::BeginUpdate. Cancelling update.");
2940 0 : mUpdateStatus = NS_ERROR_FAILURE;
2941 0 : return rv;
2942 : }
2943 :
2944 87 : rv = SetupUpdate();
2945 87 : if (NS_FAILED(rv)) {
2946 0 : mUpdateStatus = rv;
2947 0 : return rv;
2948 : }
2949 :
2950 87 : mUpdateObserver = observer;
2951 :
2952 87 : if (!clientKey.IsEmpty()) {
2953 2 : rv = nsUrlClassifierUtils::DecodeClientKey(clientKey, mUpdateClientKey);
2954 2 : NS_ENSURE_SUCCESS(rv, rv);
2955 : }
2956 :
2957 : // The first stream in an update is the only stream that may request
2958 : // forwarded updates.
2959 87 : mPrimaryStream = true;
2960 :
2961 87 : SplitTables(tables, mUpdateTables);
2962 :
2963 87 : return NS_OK;
2964 : }
2965 :
2966 : NS_IMETHODIMP
2967 102 : nsUrlClassifierDBServiceWorker::BeginStream(const nsACString &table,
2968 : const nsACString &serverMAC)
2969 : {
2970 102 : if (gShuttingDownThread)
2971 0 : return NS_ERROR_NOT_INITIALIZED;
2972 :
2973 102 : NS_ENSURE_STATE(mUpdateObserver);
2974 102 : NS_ENSURE_STATE(!mInStream);
2975 :
2976 : // We may have committed the update in FinishStream, if so set it up
2977 : // again here.
2978 102 : nsresult rv = SetupUpdate();
2979 102 : if (NS_FAILED(rv)) {
2980 0 : mUpdateStatus = rv;
2981 0 : return rv;
2982 : }
2983 :
2984 102 : mInStream = true;
2985 :
2986 : // If we're expecting a MAC, create the nsICryptoHMAC component now.
2987 102 : if (!mUpdateClientKey.IsEmpty()) {
2988 : nsCOMPtr<nsIKeyObjectFactory> keyObjectFactory(do_GetService(
2989 10 : "@mozilla.org/security/keyobjectfactory;1", &rv));
2990 5 : if (NS_FAILED(rv)) {
2991 0 : NS_WARNING("Failed to get nsIKeyObjectFactory service");
2992 0 : mUpdateStatus = rv;
2993 0 : return mUpdateStatus;
2994 : }
2995 :
2996 10 : nsCOMPtr<nsIKeyObject> keyObject;
2997 5 : rv = keyObjectFactory->KeyFromString(nsIKeyObject::HMAC, mUpdateClientKey,
2998 5 : getter_AddRefs(keyObject));
2999 5 : if (NS_FAILED(rv)) {
3000 0 : NS_WARNING("Failed to create key object, maybe not FIPS compliant?");
3001 0 : mUpdateStatus = rv;
3002 0 : return mUpdateStatus;
3003 : }
3004 :
3005 5 : mHMAC = do_CreateInstance(NS_CRYPTO_HMAC_CONTRACTID, &rv);
3006 5 : if (NS_FAILED(rv)) {
3007 0 : NS_WARNING("Failed to create nsICryptoHMAC instance");
3008 0 : mUpdateStatus = rv;
3009 0 : return mUpdateStatus;
3010 : }
3011 :
3012 5 : rv = mHMAC->Init(nsICryptoHMAC::SHA1, keyObject);
3013 5 : if (NS_FAILED(rv)) {
3014 0 : NS_WARNING("Failed to initialize nsICryptoHMAC instance");
3015 0 : mUpdateStatus = rv;
3016 0 : return mUpdateStatus;
3017 : }
3018 : }
3019 :
3020 102 : mServerMAC = serverMAC;
3021 :
3022 102 : if (!table.IsEmpty()) {
3023 15 : mUpdateTable = table;
3024 15 : GetTableId(mUpdateTable, &mUpdateTableId);
3025 15 : LOG(("update table: '%s' (%d)", mUpdateTable.get(), mUpdateTableId));
3026 : }
3027 :
3028 102 : return NS_OK;
3029 : }
3030 :
3031 : /**
3032 : * Updating the database:
3033 : *
3034 : * The Update() method takes a series of chunks separated with control data,
3035 : * as described in
3036 : * http://code.google.com/p/google-safe-browsing/wiki/Protocolv2Spec
3037 : *
3038 : * It will iterate through the control data until it reaches a chunk. By
3039 : * the time it reaches a chunk, it should have received
3040 : * a) the table to which this chunk applies
3041 : * b) the type of chunk (add, delete, expire add, expire delete).
3042 : * c) the chunk ID
3043 : * d) the length of the chunk.
3044 : *
3045 : * For add and subtract chunks, it needs to read the chunk data (expires
3046 : * don't have any data). Chunk data is a list of URI fragments whose
3047 : * encoding depends on the type of table (which is indicated by the end
3048 : * of the table name):
3049 : * a) tables ending with -exp are a zlib-compressed list of URI fragments
3050 : * separated by newlines.
3051 : * b) tables ending with -sha128 have the form
3052 : * [domain][N][frag0]...[fragN]
3053 : * 16 1 16 16
3054 : * If N is 0, the domain is reused as a fragment.
3055 : * c) any other tables are assumed to be a plaintext list of URI fragments
3056 : * separated by newlines.
3057 : *
3058 : * Update() can be fed partial data; It will accumulate data until there is
3059 : * enough to act on. Finish() should be called when there will be no more
3060 : * data.
3061 : */
3062 : NS_IMETHODIMP
3063 100 : nsUrlClassifierDBServiceWorker::UpdateStream(const nsACString& chunk)
3064 : {
3065 100 : if (gShuttingDownThread)
3066 0 : return NS_ERROR_NOT_INITIALIZED;
3067 :
3068 100 : NS_ENSURE_STATE(mInStream);
3069 :
3070 100 : HandlePendingLookups();
3071 :
3072 100 : LOG(("Update from Stream."));
3073 100 : nsresult rv = OpenDb();
3074 100 : if (NS_FAILED(rv)) {
3075 0 : NS_ERROR("Unable to open database");
3076 0 : return NS_ERROR_FAILURE;
3077 : }
3078 :
3079 : // if something has gone wrong during this update, just throw it away
3080 100 : if (NS_FAILED(mUpdateStatus)) {
3081 0 : return mUpdateStatus;
3082 : }
3083 :
3084 100 : if (mHMAC && !mServerMAC.IsEmpty()) {
3085 6 : rv = mHMAC->Update(reinterpret_cast<const PRUint8*>(chunk.BeginReading()),
3086 6 : chunk.Length());
3087 3 : if (NS_FAILED(rv)) {
3088 0 : mUpdateStatus = rv;
3089 0 : return mUpdateStatus;
3090 : }
3091 : }
3092 :
3093 100 : LOG(("Got %s\n", PromiseFlatCString(chunk).get()));
3094 :
3095 100 : mPendingStreamUpdate.Append(chunk);
3096 :
3097 100 : bool done = false;
3098 505 : while (!done) {
3099 306 : if (mState == STATE_CHUNK) {
3100 103 : rv = ProcessChunk(&done);
3101 : } else {
3102 203 : rv = ProcessResponseLines(&done);
3103 : }
3104 306 : if (NS_FAILED(rv)) {
3105 1 : mUpdateStatus = rv;
3106 1 : return rv;
3107 : }
3108 : }
3109 :
3110 99 : return NS_OK;
3111 : }
3112 :
3113 : NS_IMETHODIMP
3114 100 : nsUrlClassifierDBServiceWorker::FinishStream()
3115 : {
3116 100 : if (gShuttingDownThread)
3117 0 : return NS_ERROR_NOT_INITIALIZED;
3118 :
3119 100 : NS_ENSURE_STATE(mInStream);
3120 100 : NS_ENSURE_STATE(mUpdateObserver);
3121 :
3122 100 : PRInt32 nextStreamDelay = 0;
3123 :
3124 100 : if (NS_SUCCEEDED(mUpdateStatus) && mHMAC) {
3125 10 : nsCAutoString clientMAC;
3126 5 : mHMAC->Finish(true, clientMAC);
3127 :
3128 5 : if (clientMAC != mServerMAC) {
3129 0 : NS_WARNING("Invalid update MAC!");
3130 0 : LOG(("Invalid update MAC: expected %s, got %s",
3131 : mServerMAC.get(), clientMAC.get()));
3132 0 : mUpdateStatus = NS_ERROR_FAILURE;
3133 : }
3134 5 : PRIntervalTime updateTime = PR_IntervalNow() - mUpdateStartTime;
3135 5 : if (PR_IntervalToSeconds(updateTime) >=
3136 : static_cast<PRUint32>(gWorkingTimeThreshold)) {
3137 : // We've spent long enough working that we should commit what we
3138 : // have and hold off for a bit.
3139 0 : nsresult rv = ApplyUpdate();
3140 0 : if (NS_FAILED(rv)) {
3141 0 : if (rv == NS_ERROR_FILE_CORRUPTED) {
3142 0 : ResetDatabase();
3143 : }
3144 0 : return rv;
3145 : }
3146 0 : nextStreamDelay = gDelayTime * 1000;
3147 : }
3148 : }
3149 :
3150 100 : mUpdateObserver->StreamFinished(mUpdateStatus,
3151 100 : static_cast<PRUint32>(nextStreamDelay));
3152 :
3153 100 : ResetStream();
3154 :
3155 100 : return NS_OK;
3156 : }
3157 :
3158 : nsresult
3159 228 : nsUrlClassifierDBServiceWorker::SetCacheSize(
3160 : mozIStorageConnection * aConnection, PRInt32 aCacheSize)
3161 : {
3162 456 : mozStorageStatementScoper scoper(mGetPageSizeStatement);
3163 : bool hasResult;
3164 228 : nsresult rv = mGetPageSizeStatement->ExecuteStep(&hasResult);
3165 228 : NS_ENSURE_SUCCESS(rv, rv);
3166 :
3167 228 : NS_ASSERTION(hasResult, "Should always be able to get page size from sqlite");
3168 228 : PRUint32 pageSize = mGetPageSizeStatement->AsInt32(0);
3169 228 : PRUint32 cachePages = aCacheSize / pageSize;
3170 : nsCAutoString cacheSizePragma(MOZ_STORAGE_UNIQUIFY_QUERY_STR
3171 456 : "PRAGMA cache_size=");
3172 228 : cacheSizePragma.AppendInt(cachePages);
3173 228 : rv = aConnection->ExecuteSimpleSQL(cacheSizePragma);
3174 228 : NS_ENSURE_SUCCESS(rv, rv);
3175 :
3176 228 : return NS_OK;
3177 : }
3178 :
3179 : nsresult
3180 189 : nsUrlClassifierDBServiceWorker::SetupUpdate()
3181 : {
3182 189 : LOG(("nsUrlClassifierDBServiceWorker::SetupUpdate"));
3183 : bool inProgress;
3184 189 : nsresult rv = mConnection->GetTransactionInProgress(&inProgress);
3185 189 : if (inProgress) {
3186 102 : return NS_OK;
3187 : }
3188 :
3189 87 : mUpdateStartTime = PR_IntervalNow();
3190 :
3191 87 : rv = mConnection->BeginTransaction();
3192 87 : NS_ENSURE_SUCCESS(rv, rv);
3193 :
3194 87 : if (gUpdateCacheSize > 0) {
3195 87 : rv = SetCacheSize(mConnection, gUpdateCacheSize);
3196 87 : NS_ENSURE_SUCCESS(rv, rv);
3197 87 : if (gUpdateCacheSize != gLookupCacheSize) {
3198 87 : mGrewCache = true;
3199 : }
3200 : }
3201 :
3202 87 : return NS_OK;
3203 : }
3204 :
3205 : nsresult
3206 85 : nsUrlClassifierDBServiceWorker::ApplyUpdate()
3207 : {
3208 85 : LOG(("nsUrlClassifierDBServiceWorker::ApplyUpdate"));
3209 :
3210 85 : if (mConnection) {
3211 85 : if (NS_FAILED(mUpdateStatus)) {
3212 1 : mConnection->RollbackTransaction();
3213 : } else {
3214 84 : mUpdateStatus = FlushChunkLists();
3215 84 : if (NS_SUCCEEDED(mUpdateStatus)) {
3216 84 : mUpdateStatus = mConnection->CommitTransaction();
3217 : }
3218 : }
3219 : }
3220 :
3221 85 : if (NS_SUCCEEDED(mUpdateStatus)) {
3222 : // Reconstruct the prefix tree from the DB
3223 84 : nsresult rv = ConstructPrefixSet();
3224 84 : NS_ENSURE_SUCCESS(rv, rv);
3225 : }
3226 :
3227 85 : if (mGrewCache) {
3228 : // During the update we increased the page cache to bigger than we
3229 : // want to keep around. At the moment, the only reliable way to make
3230 : // sure that the page cache is freed is to reopen the connection.
3231 85 : LOG(("GrewCache true, reopening DB"));
3232 85 : mGrewCache = false;
3233 85 : CloseDb();
3234 85 : OpenDb();
3235 : }
3236 :
3237 85 : mUpdateStartTime = 0;
3238 :
3239 85 : return NS_OK;
3240 : }
3241 :
3242 : NS_IMETHODIMP
3243 85 : nsUrlClassifierDBServiceWorker::FinishUpdate()
3244 : {
3245 85 : LOG(("nsUrlClassifierDBServiceWorker::FinishUpdate()"));
3246 85 : if (gShuttingDownThread)
3247 0 : return NS_ERROR_NOT_INITIALIZED;
3248 :
3249 85 : NS_ENSURE_STATE(!mInStream);
3250 85 : NS_ENSURE_STATE(mUpdateObserver);
3251 :
3252 : // We need to get the error code before ApplyUpdate, because it might
3253 : // close/open the connection.
3254 85 : PRInt32 errcode = SQLITE_OK;
3255 85 : if (mConnection)
3256 85 : mConnection->GetLastError(&errcode);
3257 :
3258 85 : nsresult rv = ApplyUpdate();
3259 85 : if (NS_FAILED(rv)) {
3260 0 : if (rv == NS_ERROR_FILE_CORRUPTED) {
3261 0 : ResetDatabase();
3262 : }
3263 0 : return rv;
3264 : }
3265 :
3266 85 : if (NS_SUCCEEDED(mUpdateStatus)) {
3267 84 : mUpdateObserver->UpdateSuccess(mUpdateWait);
3268 : } else {
3269 1 : mUpdateObserver->UpdateError(mUpdateStatus);
3270 : }
3271 :
3272 : // It's important that we only reset the database on an update
3273 : // command if the update was successful, otherwise unauthenticated
3274 : // updates could cause a database reset.
3275 85 : bool resetDB = (NS_SUCCEEDED(mUpdateStatus) && mResetRequested) ||
3276 85 : errcode == SQLITE_CORRUPT;
3277 :
3278 85 : if (!resetDB) {
3279 84 : if (NS_SUCCEEDED(mUpdateStatus)) {
3280 83 : PRInt64 now = (PR_Now() / PR_USEC_PER_SEC);
3281 249 : for (PRUint32 i = 0; i < mUpdateTables.Length(); i++) {
3282 166 : LOG(("Successfully updated %s", mUpdateTables[i].get()));
3283 166 : mTableFreshness.Put(mUpdateTables[i], now);
3284 : }
3285 : } else {
3286 3 : for (PRUint32 i = 0; i < mUpdateTables.Length(); i++) {
3287 2 : LOG(("Failed updating %s", mUpdateTables[i].get()));
3288 2 : mTableFreshness.Remove(mUpdateTables[i]);
3289 : }
3290 : }
3291 : }
3292 :
3293 85 : ResetUpdate();
3294 :
3295 85 : if (resetDB) {
3296 1 : ResetDatabase();
3297 : }
3298 :
3299 85 : return NS_OK;
3300 : }
3301 :
3302 : NS_IMETHODIMP
3303 55 : nsUrlClassifierDBServiceWorker::ResetDatabase()
3304 : {
3305 55 : LOG(("nsUrlClassifierDBServiceWorker::ResetDatabase [%p]", this));
3306 55 : ClearCachedChunkLists();
3307 :
3308 55 : mTableFreshness.Clear();
3309 :
3310 55 : nsresult rv = CloseDb();
3311 55 : NS_ENSURE_SUCCESS(rv, rv);
3312 :
3313 55 : rv = mPrefixSet->SetPrefixes(nsnull, 0);
3314 55 : NS_ENSURE_SUCCESS(rv, rv);
3315 :
3316 55 : mDBFile->Remove(false);
3317 55 : mPSFile->Remove(false);
3318 :
3319 55 : return NS_OK;
3320 : }
3321 :
3322 : NS_IMETHODIMP
3323 10 : nsUrlClassifierDBServiceWorker::CancelUpdate()
3324 : {
3325 10 : LOG(("CancelUpdate"));
3326 :
3327 10 : if (mUpdateObserver) {
3328 2 : mUpdateStatus = NS_BINDING_ABORTED;
3329 :
3330 2 : ClearCachedChunkLists();
3331 2 : mConnection->RollbackTransaction();
3332 2 : mUpdateObserver->UpdateError(mUpdateStatus);
3333 :
3334 5 : for (PRUint32 i = 0; i < mUpdateTables.Length(); i++) {
3335 3 : LOG(("Failed updating %s", mUpdateTables[i].get()));
3336 3 : mTableFreshness.Remove(mUpdateTables[i]);
3337 : }
3338 :
3339 2 : ResetStream();
3340 2 : ResetUpdate();
3341 : }
3342 :
3343 10 : return NS_OK;
3344 : }
3345 :
3346 : // Allows the main thread to delete the connection which may be in
3347 : // a background thread.
3348 : // XXX This could be turned into a single shutdown event so the logic
3349 : // is simpler in nsUrlClassifierDBService::Shutdown.
3350 : NS_IMETHODIMP
3351 148 : nsUrlClassifierDBServiceWorker::CloseDb()
3352 : {
3353 148 : if (mConnection) {
3354 141 : mMainStore.Close();
3355 141 : mPendingSubStore.Close();
3356 :
3357 141 : mGetChunkListsStatement = nsnull;
3358 141 : mSetChunkListsStatement = nsnull;
3359 :
3360 141 : mGetTablesStatement = nsnull;
3361 141 : mGetTableIdStatement = nsnull;
3362 141 : mGetTableNameStatement = nsnull;
3363 141 : mInsertTableIdStatement = nsnull;
3364 141 : mGetPageSizeStatement = nsnull;
3365 :
3366 141 : mConnection = nsnull;
3367 141 : LOG(("urlclassifier db closed\n"));
3368 : }
3369 :
3370 148 : mCryptoHash = nsnull;
3371 :
3372 148 : return NS_OK;
3373 : }
3374 :
3375 : NS_IMETHODIMP
3376 20 : nsUrlClassifierDBServiceWorker::CacheCompletions(nsTArray<nsUrlClassifierLookupResult> *results)
3377 : {
3378 20 : LOG(("nsUrlClassifierDBServiceWorker::CacheCompletions [%p]", this));
3379 :
3380 40 : nsAutoPtr<nsTArray<nsUrlClassifierLookupResult> > resultsPtr(results);
3381 :
3382 : // Start a new transaction. If a transaction is open for an update
3383 : // this will be a noop, and this cache will be included in the
3384 : // update's transaction.
3385 40 : mozStorageTransaction trans(mConnection, true);
3386 :
3387 56 : for (PRUint32 i = 0; i < results->Length(); i++) {
3388 36 : nsUrlClassifierLookupResult& result = results->ElementAt(i);
3389 : // Failing to update here shouldn't be fatal (and might be common,
3390 : // if we're updating entries that were removed since they were
3391 : // returned after a lookup).
3392 36 : mMainStore.UpdateEntry(result.mEntry);
3393 : }
3394 :
3395 20 : return NS_OK;
3396 : }
3397 :
3398 : nsresult
3399 466 : nsUrlClassifierDBServiceWorker::OpenDb()
3400 : {
3401 : // Connection already open, don't do anything.
3402 466 : if (mConnection) {
3403 325 : return NS_OK;
3404 : }
3405 :
3406 141 : LOG(("Opening db\n"));
3407 :
3408 : nsresult rv;
3409 : // open the connection
3410 : nsCOMPtr<mozIStorageService> storageService =
3411 282 : do_GetService(MOZ_STORAGE_SERVICE_CONTRACTID, &rv);
3412 141 : NS_ENSURE_SUCCESS(rv, rv);
3413 :
3414 : bool exists;
3415 141 : rv = mDBFile->Exists(&exists);
3416 141 : NS_ENSURE_SUCCESS(rv, rv);
3417 141 : bool newDB = !exists;
3418 :
3419 282 : nsCOMPtr<mozIStorageConnection> connection;
3420 141 : rv = storageService->OpenDatabase(mDBFile, getter_AddRefs(connection));
3421 141 : if (rv == NS_ERROR_FILE_CORRUPTED) {
3422 : // delete the db and try opening again
3423 0 : rv = mDBFile->Remove(false);
3424 0 : NS_ENSURE_SUCCESS(rv, rv);
3425 :
3426 0 : newDB = true;
3427 :
3428 0 : rv = storageService->OpenDatabase(mDBFile, getter_AddRefs(connection));
3429 : }
3430 141 : NS_ENSURE_SUCCESS(rv, rv);
3431 :
3432 141 : if (!newDB) {
3433 : PRInt32 databaseVersion;
3434 85 : rv = connection->GetSchemaVersion(&databaseVersion);
3435 85 : NS_ENSURE_SUCCESS(rv, rv);
3436 :
3437 85 : if (databaseVersion != IMPLEMENTATION_VERSION) {
3438 0 : LOG(("Incompatible database, removing."));
3439 :
3440 0 : rv = connection->Close();
3441 0 : NS_ENSURE_SUCCESS(rv, rv);
3442 :
3443 0 : rv = mDBFile->Remove(false);
3444 0 : NS_ENSURE_SUCCESS(rv, rv);
3445 :
3446 0 : newDB = true;
3447 :
3448 0 : rv = storageService->OpenDatabase(mDBFile, getter_AddRefs(connection));
3449 0 : NS_ENSURE_SUCCESS(rv, rv);
3450 : }
3451 : }
3452 :
3453 141 : connection->SetGrowthIncrement(5 * 1024 * 1024, EmptyCString());
3454 141 : rv = connection->ExecuteSimpleSQL(NS_LITERAL_CSTRING("PRAGMA synchronous=OFF"));
3455 141 : NS_ENSURE_SUCCESS(rv, rv);
3456 :
3457 141 : rv = connection->CreateStatement
3458 141 : (NS_LITERAL_CSTRING(MOZ_STORAGE_UNIQUIFY_QUERY_STR "PRAGMA page_size"),
3459 282 : getter_AddRefs(mGetPageSizeStatement));
3460 141 : NS_ENSURE_SUCCESS(rv, rv);
3461 :
3462 141 : rv = SetCacheSize(connection, gLookupCacheSize);
3463 141 : NS_ENSURE_SUCCESS(rv, rv);
3464 :
3465 141 : if (newDB) {
3466 56 : rv = connection->SetSchemaVersion(IMPLEMENTATION_VERSION);
3467 56 : NS_ENSURE_SUCCESS(rv, rv);
3468 : }
3469 :
3470 : // Create the table
3471 141 : rv = MaybeCreateTables(connection);
3472 141 : NS_ENSURE_SUCCESS(rv, rv);
3473 :
3474 : rv = mMainStore.Init(this, connection,
3475 141 : NS_LITERAL_CSTRING("moz_classifier"));
3476 141 : NS_ENSURE_SUCCESS(rv, rv);
3477 :
3478 : rv = mPendingSubStore.Init(this, connection,
3479 141 : NS_LITERAL_CSTRING("moz_subs"));
3480 141 : NS_ENSURE_SUCCESS(rv, rv);
3481 :
3482 141 : rv = connection->CreateStatement
3483 141 : (NS_LITERAL_CSTRING("SELECT add_chunks, sub_chunks FROM moz_tables"
3484 : " WHERE id=?1"),
3485 282 : getter_AddRefs(mGetChunkListsStatement));
3486 141 : NS_ENSURE_SUCCESS(rv, rv);
3487 :
3488 141 : rv = connection->CreateStatement
3489 141 : (NS_LITERAL_CSTRING("UPDATE moz_tables"
3490 : " SET add_chunks=?1, sub_chunks=?2"
3491 : " WHERE id=?3"),
3492 282 : getter_AddRefs(mSetChunkListsStatement));
3493 141 : NS_ENSURE_SUCCESS(rv, rv);
3494 :
3495 141 : rv = connection->CreateStatement
3496 141 : (NS_LITERAL_CSTRING("SELECT name, add_chunks, sub_chunks"
3497 : " FROM moz_tables"),
3498 282 : getter_AddRefs(mGetTablesStatement));
3499 141 : NS_ENSURE_SUCCESS(rv, rv);
3500 :
3501 141 : rv = connection->CreateStatement
3502 141 : (NS_LITERAL_CSTRING("SELECT id FROM moz_tables"
3503 : " WHERE name = ?1"),
3504 282 : getter_AddRefs(mGetTableIdStatement));
3505 141 : NS_ENSURE_SUCCESS(rv, rv);
3506 :
3507 141 : rv = connection->CreateStatement
3508 141 : (NS_LITERAL_CSTRING("SELECT name FROM moz_tables"
3509 : " WHERE id = ?1"),
3510 282 : getter_AddRefs(mGetTableNameStatement));
3511 141 : NS_ENSURE_SUCCESS(rv, rv);
3512 :
3513 141 : rv = connection->CreateStatement
3514 141 : (NS_LITERAL_CSTRING("INSERT INTO moz_tables(id, name, add_chunks, sub_chunks)"
3515 : " VALUES (null, ?1, null, null)"),
3516 282 : getter_AddRefs(mInsertTableIdStatement));
3517 141 : NS_ENSURE_SUCCESS(rv, rv);
3518 :
3519 141 : mConnection = connection;
3520 :
3521 141 : mCryptoHash = do_CreateInstance(NS_CRYPTO_HASH_CONTRACTID, &rv);
3522 141 : NS_ENSURE_SUCCESS(rv, rv);
3523 :
3524 141 : LOG(("loading Prefix Set\n"));
3525 141 : rv = LoadPrefixSet(mPSFile);
3526 141 : if (NS_FAILED(rv)) {
3527 0 : if (rv == NS_ERROR_FILE_CORRUPTED) {
3528 0 : ResetDatabase();
3529 : }
3530 0 : return rv;
3531 : }
3532 :
3533 141 : return NS_OK;
3534 : }
3535 :
3536 : // We have both a prefix and a domain. Drop the domain, but
3537 : // hash the domain, the prefix and a random value together,
3538 : // ensuring any collisions happens at a different points for
3539 : // different users.
3540 : // We need to calculate +- 500k hashes each update.
3541 : // The extensive initialization and finalization of normal
3542 : // cryptographic hashes, as well as fairly low speed, causes them
3543 : // to be prohibitively slow here, hence we can't use them.
3544 : // We use MurmurHash3 instead because it's reasonably well
3545 : // researched, trusted inside some other big projects, extremely
3546 : // fast and with a specific a 32-bit output version, and fairly
3547 : // compact. Upon testing with the actual prefix data, it does
3548 : // not appear to increase the number of collisions by any
3549 : // meaningful amount.
3550 224 : static nsresult KeyedHash(PRUint32 aPref, PRUint32 aDomain,
3551 : PRUint32 aKey, PRUint32 *aOut)
3552 : {
3553 : // This is a reimplementation of MurmurHash3 32-bit
3554 : // based on the public domain C++ sources.
3555 : // http://code.google.com/p/smhasher/source/browse/trunk/MurmurHash3.cpp
3556 : // for nblocks = 2
3557 224 : PRUint32 c1 = 0xCC9E2D51;
3558 224 : PRUint32 c2 = 0x1B873593;
3559 224 : PRUint32 c3 = 0xE6546B64;
3560 224 : PRUint32 c4 = 0x85EBCA6B;
3561 224 : PRUint32 c5 = 0xC2B2AE35;
3562 224 : PRUint32 h1 = aPref; // seed
3563 : PRUint32 k1;
3564 : PRUint32 karr[2];
3565 :
3566 224 : karr[0] = aDomain;
3567 224 : karr[1] = aKey;
3568 :
3569 672 : for (PRUint32 i = 0; i < 2; i++) {
3570 448 : k1 = karr[i];
3571 448 : k1 *= c1;
3572 448 : k1 = (k1 << 15) | (k1 >> (32-15));
3573 448 : k1 *= c2;
3574 :
3575 448 : h1 ^= k1;
3576 448 : h1 = (h1 << 13) | (h1 >> (32-13));
3577 448 : h1 *= 5;
3578 448 : h1 += c3;
3579 : }
3580 :
3581 224 : h1 ^= 2; // len
3582 : // fmix
3583 224 : h1 ^= h1 >> 16;
3584 224 : h1 *= c4;
3585 224 : h1 ^= h1 >> 13;
3586 224 : h1 *= c5;
3587 224 : h1 ^= h1 >> 16;
3588 :
3589 224 : *aOut = h1;
3590 :
3591 224 : return NS_OK;
3592 : }
3593 :
3594 140 : nsresult nsUrlClassifierStore::ReadPrefixes(FallibleTArray<PRUint32>& array,
3595 : PRUint32 aKey)
3596 : {
3597 280 : mozStorageStatementScoper scoper(mAllPrefixGetStatement);
3598 280 : mozStorageStatementScoper scoperToo(mAllPrefixCountStatement);
3599 : bool hasMoreData;
3600 140 : PRUint32 pcnt = 0;
3601 140 : PRUint32 fcnt = 0;
3602 :
3603 : #if defined(PR_LOGGING)
3604 140 : PRIntervalTime clockStart = 0;
3605 140 : if (LOG_ENABLED()) {
3606 0 : clockStart = PR_IntervalNow();
3607 : }
3608 : #endif
3609 :
3610 : // Make sure we allocate no more than we really need, so first
3611 : // check how much entries there are
3612 140 : if (NS_SUCCEEDED(mAllPrefixCountStatement->ExecuteStep(&hasMoreData)) && hasMoreData) {
3613 140 : PRUint32 count = mAllPrefixCountStatement->AsInt32(0);
3614 140 : if (!array.SetCapacity(count)) {
3615 0 : return NS_ERROR_OUT_OF_MEMORY;
3616 : }
3617 : } else {
3618 0 : return NS_ERROR_FILE_CORRUPTED;
3619 : }
3620 :
3621 462 : while (NS_SUCCEEDED(mAllPrefixGetStatement->ExecuteStep(&hasMoreData)) && hasMoreData) {
3622 : PRUint32 prefixval;
3623 : PRUint32 domainval;
3624 : PRUint32 size;
3625 :
3626 182 : const PRUint8 *blobdomain = mAllPrefixGetStatement->AsSharedBlob(0, &size);
3627 182 : if (!blobdomain || (size != DOMAIN_LENGTH))
3628 0 : return false;
3629 :
3630 182 : domainval = *(reinterpret_cast<const PRUint32*>(blobdomain));
3631 :
3632 182 : const PRUint8 *blobprefix = mAllPrefixGetStatement->AsSharedBlob(1, &size);
3633 182 : if (!blobprefix || (size != PARTIAL_LENGTH)) {
3634 139 : const PRUint8 *blobfull = mAllPrefixGetStatement->AsSharedBlob(2, &size);
3635 139 : if (!blobfull || (size != COMPLETE_LENGTH)) {
3636 5 : prefixval = domainval;
3637 5 : fcnt++;
3638 : } else {
3639 134 : prefixval = *(reinterpret_cast<const PRUint32*>(blobfull));
3640 139 : }
3641 : } else {
3642 43 : prefixval = *(reinterpret_cast<const PRUint32*>(blobprefix));
3643 : }
3644 :
3645 : PRUint32 keyedVal;
3646 182 : nsresult rv = KeyedHash(prefixval, domainval, aKey, &keyedVal);
3647 182 : NS_ENSURE_SUCCESS(rv, rv);
3648 :
3649 182 : PRUint32 *res = array.AppendElement(keyedVal);
3650 182 : MOZ_ASSERT(res != nsnull);
3651 182 : pcnt++;
3652 : // Normal DB size is about 500k entries. If we are getting 10x
3653 : // as much, the database must be corrupted.
3654 182 : if (pcnt > 5000000) {
3655 0 : return NS_ERROR_FILE_CORRUPTED;
3656 : }
3657 : }
3658 :
3659 140 : LOG(("SB prefixes: %d fulldomain: %d\n", pcnt, fcnt));
3660 :
3661 : #if defined(PR_LOGGING)
3662 140 : if (LOG_ENABLED()) {
3663 0 : PRIntervalTime clockEnd = PR_IntervalNow();
3664 0 : LOG(("Gathering took %dms\n",
3665 : PR_IntervalToMilliseconds(clockEnd - clockStart)));
3666 : }
3667 : #endif
3668 :
3669 140 : return NS_OK;
3670 : }
3671 :
3672 21 : bool nsUrlClassifierDBServiceWorker::LockPrefixSet()
3673 : {
3674 21 : mPrefixSetEnabledLock.Lock();
3675 21 : return mPrefixSetEnabled;
3676 : }
3677 :
3678 21 : void nsUrlClassifierDBServiceWorker::UnlockPrefixSet()
3679 : {
3680 21 : mPrefixSetEnabledLock.Unlock();
3681 21 : }
3682 :
3683 : nsresult
3684 140 : nsUrlClassifierDBServiceWorker::ConstructPrefixSet()
3685 : {
3686 280 : Telemetry::AutoTimer<Telemetry::URLCLASSIFIER_PS_CONSTRUCT_TIME> timer;
3687 :
3688 : PRUint32 key;
3689 140 : nsresult rv = mPrefixSet->GetKey(&key);
3690 140 : NS_ENSURE_SUCCESS(rv, rv);
3691 :
3692 280 : FallibleTArray<PRUint32> array;
3693 140 : rv = mMainStore.ReadPrefixes(array, key);
3694 140 : if (NS_FAILED(rv)) {
3695 0 : goto error_bailout;
3696 : }
3697 :
3698 : #ifdef HASHFUNCTION_COLLISION_TEST
3699 : array.Sort();
3700 : PRUint32 collisions = 0;
3701 : for (int i = 1; i < array.Length(); i++) {
3702 : if (array[i - 1] == array[i]) {
3703 : collisions++;
3704 : }
3705 : }
3706 : LOG(("%d collisions in the set", collisions));
3707 : #endif
3708 :
3709 140 : if (array.IsEmpty()) {
3710 : // DB is empty, put a sentinel to show that we loaded it
3711 69 : if (!array.AppendElement(0)) {
3712 0 : goto error_bailout;
3713 : }
3714 : }
3715 : // SetPrefixes requires sorted arrays
3716 140 : array.Sort();
3717 :
3718 : // construct new prefixset
3719 140 : rv = mPrefixSet->SetPrefixes(array.Elements(), array.Length());
3720 140 : if (NS_FAILED(rv)) {
3721 0 : goto error_bailout;
3722 : }
3723 :
3724 : // store the new tree to disk
3725 140 : rv = mPrefixSet->StoreToFile(mPSFile);
3726 140 : NS_WARN_IF_FALSE(NS_SUCCEEDED(rv), "failed to store the prefixset");
3727 :
3728 : // re-enable prefixset usage if disabled earlier
3729 140 : mPrefixSetEnabled = true;
3730 :
3731 140 : return NS_OK;
3732 :
3733 : error_bailout:
3734 : // disable prefixset usage
3735 0 : MutexAutoLock lock(mPrefixSetEnabledLock);
3736 0 : mPrefixSetEnabled = false;
3737 : // load an empty prefixset
3738 0 : nsAutoTArray<PRUint32, 1> sentinel;
3739 0 : sentinel.Clear();
3740 0 : sentinel.AppendElement(0);
3741 0 : mPrefixSet->SetPrefixes(sentinel.Elements(), sentinel.Length());
3742 0 : if (rv == NS_ERROR_OUT_OF_MEMORY) {
3743 0 : Telemetry::Accumulate(Telemetry::URLCLASSIFIER_PS_OOM, 1);
3744 : }
3745 0 : return rv;
3746 : }
3747 :
3748 : nsresult
3749 141 : nsUrlClassifierDBServiceWorker::LoadPrefixSet(nsCOMPtr<nsIFile> & aFile)
3750 : {
3751 : bool empty;
3752 141 : nsresult rv = mPrefixSet->IsEmpty(&empty);
3753 141 : NS_ENSURE_SUCCESS(rv, rv);
3754 :
3755 141 : if (!empty) {
3756 85 : LOG(("PrefixSet already loaded, not loading again"));
3757 85 : return NS_OK;
3758 : }
3759 :
3760 : bool exists;
3761 56 : rv = aFile->Exists(&exists);
3762 56 : NS_ENSURE_SUCCESS(rv, rv);
3763 :
3764 : #if defined(PR_LOGGING)
3765 56 : PRIntervalTime clockStart = 0;
3766 56 : if (LOG_ENABLED()) {
3767 0 : clockStart = PR_IntervalNow();
3768 : }
3769 : #endif
3770 :
3771 56 : if (exists) {
3772 0 : Telemetry::AutoTimer<Telemetry::URLCLASSIFIER_PS_FILELOAD_TIME> timer;
3773 0 : LOG(("stored PrefixSet exists, loading from disk"));
3774 0 : rv = mPrefixSet->LoadFromFile(aFile);
3775 : }
3776 56 : if (!exists || NS_FAILED(rv)) {
3777 56 : LOG(("no (usable) stored PrefixSet found, constructing from store"));
3778 56 : rv = ConstructPrefixSet();
3779 56 : NS_ENSURE_SUCCESS(rv, rv);
3780 : }
3781 :
3782 : #ifdef DEBUG
3783 56 : LOG(("SB tree done, size = %d bytes\n",
3784 : mPrefixSet->SizeOfIncludingThis(moz_malloc_size_of)));
3785 : #endif
3786 : #if defined(PR_LOGGING)
3787 56 : if (LOG_ENABLED()) {
3788 0 : PRIntervalTime clockEnd = PR_IntervalNow();
3789 0 : LOG(("Loading took %dms\n",
3790 : PR_IntervalToMilliseconds(clockEnd - clockStart)));
3791 : }
3792 : #endif
3793 :
3794 56 : return NS_OK;
3795 : }
3796 :
3797 : nsresult
3798 141 : nsUrlClassifierDBServiceWorker::MaybeCreateTables(mozIStorageConnection* connection)
3799 : {
3800 141 : LOG(("MaybeCreateTables\n"));
3801 :
3802 : nsresult rv = connection->ExecuteSimpleSQL(
3803 141 : NS_LITERAL_CSTRING("CREATE TABLE IF NOT EXISTS moz_classifier"
3804 : " (id INTEGER PRIMARY KEY,"
3805 : " domain BLOB,"
3806 : " partial_data BLOB,"
3807 : " complete_data BLOB,"
3808 : " chunk_id INTEGER,"
3809 141 : " table_id INTEGER)"));
3810 141 : NS_ENSURE_SUCCESS(rv, rv);
3811 :
3812 : rv = connection->ExecuteSimpleSQL(
3813 141 : NS_LITERAL_CSTRING("CREATE INDEX IF NOT EXISTS"
3814 : " moz_classifier_domain_index"
3815 141 : " ON moz_classifier(domain)"));
3816 141 : NS_ENSURE_SUCCESS(rv, rv);
3817 :
3818 : rv = connection->ExecuteSimpleSQL(
3819 141 : NS_LITERAL_CSTRING("CREATE INDEX IF NOT EXISTS"
3820 : " moz_classifier_chunk_index"
3821 141 : " ON moz_classifier(chunk_id)"));
3822 141 : NS_ENSURE_SUCCESS(rv, rv);
3823 :
3824 : rv = connection->ExecuteSimpleSQL(
3825 141 : NS_LITERAL_CSTRING("CREATE TABLE IF NOT EXISTS moz_subs"
3826 : " (id INTEGER PRIMARY KEY,"
3827 : " domain BLOB,"
3828 : " partial_data BLOB,"
3829 : " complete_data BLOB,"
3830 : " chunk_id INTEGER,"
3831 : " table_id INTEGER,"
3832 141 : " add_chunk_id INTEGER)"));
3833 141 : NS_ENSURE_SUCCESS(rv, rv);
3834 :
3835 : rv = connection->ExecuteSimpleSQL(
3836 141 : NS_LITERAL_CSTRING("CREATE INDEX IF NOT EXISTS"
3837 : " moz_subs_addchunk_index"
3838 141 : " ON moz_subs(add_chunk_id)"));
3839 141 : NS_ENSURE_SUCCESS(rv, rv);
3840 :
3841 : rv = connection->ExecuteSimpleSQL(
3842 141 : NS_LITERAL_CSTRING("CREATE INDEX IF NOT EXISTS"
3843 : " moz_subs_chunk_index"
3844 141 : " ON moz_subs(chunk_id)"));
3845 141 : NS_ENSURE_SUCCESS(rv, rv);
3846 :
3847 : rv = connection->ExecuteSimpleSQL(
3848 141 : NS_LITERAL_CSTRING("CREATE TABLE IF NOT EXISTS moz_tables"
3849 : " (id INTEGER PRIMARY KEY,"
3850 : " name TEXT,"
3851 : " add_chunks TEXT,"
3852 141 : " sub_chunks TEXT);"));
3853 141 : NS_ENSURE_SUCCESS(rv, rv);
3854 :
3855 141 : return rv;
3856 : }
3857 :
3858 : // -------------------------------------------------------------------------
3859 : // nsUrlClassifierLookupCallback
3860 : //
3861 : // This class takes the results of a lookup found on the worker thread
3862 : // and handles any necessary partial hash expansions before calling
3863 : // the client callback.
3864 :
3865 : class nsUrlClassifierLookupCallback : public nsIUrlClassifierLookupCallback
3866 : , public nsIUrlClassifierHashCompleterCallback
3867 141 : {
3868 : public:
3869 : NS_DECL_ISUPPORTS
3870 : NS_DECL_NSIURLCLASSIFIERLOOKUPCALLBACK
3871 : NS_DECL_NSIURLCLASSIFIERHASHCOMPLETERCALLBACK
3872 :
3873 141 : nsUrlClassifierLookupCallback(nsUrlClassifierDBService *dbservice,
3874 : nsIUrlClassifierCallback *c)
3875 : : mDBService(dbservice)
3876 : , mResults(nsnull)
3877 : , mPendingCompletions(0)
3878 141 : , mCallback(c)
3879 141 : {}
3880 :
3881 : private:
3882 : nsresult HandleResults();
3883 :
3884 : nsRefPtr<nsUrlClassifierDBService> mDBService;
3885 : nsAutoPtr<nsTArray<nsUrlClassifierLookupResult> > mResults;
3886 :
3887 : // Completed results to send back to the worker for caching.
3888 : nsAutoPtr<nsTArray<nsUrlClassifierLookupResult> > mCacheResults;
3889 :
3890 : PRUint32 mPendingCompletions;
3891 : nsCOMPtr<nsIUrlClassifierCallback> mCallback;
3892 : };
3893 :
3894 2812 : NS_IMPL_THREADSAFE_ISUPPORTS2(nsUrlClassifierLookupCallback,
3895 : nsIUrlClassifierLookupCallback,
3896 : nsIUrlClassifierHashCompleterCallback)
3897 :
3898 : NS_IMETHODIMP
3899 141 : nsUrlClassifierLookupCallback::LookupComplete(nsTArray<nsUrlClassifierLookupResult>* results)
3900 : {
3901 141 : NS_ASSERTION(mResults == nsnull,
3902 : "Should only get one set of results per nsUrlClassifierLookupCallback!");
3903 :
3904 141 : if (!results) {
3905 0 : HandleResults();
3906 0 : return NS_OK;
3907 : }
3908 :
3909 141 : mResults = results;
3910 141 : mResults->Sort();
3911 :
3912 : // Check the results entries that need to be completed.
3913 277 : for (PRUint32 i = 0; i < results->Length(); i++) {
3914 136 : nsUrlClassifierLookupResult& result = results->ElementAt(i);
3915 :
3916 : // We will complete partial matches and matches that are stale.
3917 136 : if (!result.mConfirmed) {
3918 154 : nsCOMPtr<nsIUrlClassifierHashCompleter> completer;
3919 154 : if (mDBService->GetCompleter(result.mTableName,
3920 154 : getter_AddRefs(completer))) {
3921 114 : nsCAutoString partialHash;
3922 : PRUint8 *buf =
3923 : result.mEntry.mHavePartial ? result.mEntry.mPartialHash.buf
3924 57 : : result.mEntry.mCompleteHash.buf;
3925 57 : partialHash.Assign(reinterpret_cast<char*>(buf), PARTIAL_LENGTH);
3926 :
3927 57 : nsresult rv = completer->Complete(partialHash, this);
3928 57 : if (NS_SUCCEEDED(rv)) {
3929 57 : mPendingCompletions++;
3930 : }
3931 : } else {
3932 : // For tables with no hash completer, a complete hash match is
3933 : // good enough, it doesn't need to be fresh. (we need the
3934 : // mLookupFragment comparison to weed out noise entries, which
3935 : // should never be confirmed).
3936 40 : if (result.mEntry.mHaveComplete
3937 20 : && (result.mLookupFragment == result.mEntry.mCompleteHash)) {
3938 20 : result.mConfirmed = true;
3939 : } else {
3940 0 : NS_WARNING("Partial match in a table without a valid completer, ignoring partial match.");
3941 : }
3942 : }
3943 : }
3944 : }
3945 :
3946 141 : if (mPendingCompletions == 0) {
3947 : // All results were complete, we're ready!
3948 100 : HandleResults();
3949 : }
3950 :
3951 141 : return NS_OK;
3952 : }
3953 :
3954 : NS_IMETHODIMP
3955 57 : nsUrlClassifierLookupCallback::CompletionFinished(nsresult status)
3956 : {
3957 57 : LOG(("nsUrlClassifierLookupCallback::CompletionFinished [%p, %08x]",
3958 : this, status));
3959 57 : if (NS_FAILED(status)) {
3960 0 : NS_WARNING("gethash response failed.");
3961 : }
3962 :
3963 57 : mPendingCompletions--;
3964 57 : if (mPendingCompletions == 0) {
3965 41 : HandleResults();
3966 :
3967 41 : if (mCacheResults) {
3968 : // This hands ownership of the cache results array back to the worker
3969 : // thread.
3970 20 : mDBService->CacheCompletions(mCacheResults.forget());
3971 : }
3972 : }
3973 :
3974 57 : return NS_OK;
3975 : }
3976 :
3977 : NS_IMETHODIMP
3978 51 : nsUrlClassifierLookupCallback::Completion(const nsACString& completeHash,
3979 : const nsACString& tableName,
3980 : PRUint32 chunkId,
3981 : bool verified)
3982 : {
3983 51 : LOG(("nsUrlClassifierLookupCallback::Completion [%p, %s, %d, %d]",
3984 : this, PromiseFlatCString(tableName).get(), chunkId, verified));
3985 : nsUrlClassifierCompleteHash hash;
3986 51 : hash.Assign(completeHash);
3987 :
3988 248 : for (PRUint32 i = 0; i < mResults->Length(); i++) {
3989 197 : nsUrlClassifierLookupResult& result = mResults->ElementAt(i);
3990 :
3991 : // First, see if this result can be used to update an entry.
3992 546 : if (verified &&
3993 196 : !result.mEntry.mHaveComplete &&
3994 113 : hash.StartsWith(result.mEntry.mPartialHash) &&
3995 40 : result.mTableName == tableName &&
3996 : result.mEntry.mChunkId == chunkId) {
3997 : // We have a completion for this entry. Fill it in...
3998 36 : result.mEntry.SetHash(hash);
3999 :
4000 36 : if (!mCacheResults) {
4001 20 : mCacheResults = new nsTArray<nsUrlClassifierLookupResult>();
4002 20 : if (!mCacheResults)
4003 0 : return NS_ERROR_OUT_OF_MEMORY;
4004 : }
4005 :
4006 36 : mCacheResults->AppendElement(result);
4007 : }
4008 :
4009 : // Now, see if it verifies a lookup
4010 197 : if (result.mLookupFragment == hash) {
4011 45 : result.mConfirmed = true;
4012 :
4013 45 : if (result.mTableName != tableName ||
4014 : result.mEntry.mChunkId != chunkId) {
4015 : // The hash we got for this completion matches the hash we
4016 : // looked up, but doesn't match the table/chunk id. This could
4017 : // happen in rare cases where a given URL was moved between
4018 : // lists or added/removed/re-added to the list in the time since
4019 : // we've updated.
4020 : //
4021 : // Update the lookup result, but don't update the entry or try
4022 : // cache the results of this completion, as it might confuse
4023 : // things.
4024 4 : result.mTableName = tableName;
4025 4 : NS_WARNING("Accepting a gethash with an invalid table name or chunk id");
4026 4 : LOG(("Tablename: %s ?= %s, ChunkId %d ?= %d",
4027 : result.mTableName.get(), PromiseFlatCString(tableName).get(),
4028 : result.mEntry.mChunkId, chunkId));
4029 : }
4030 : }
4031 : }
4032 :
4033 51 : return NS_OK;
4034 : }
4035 :
4036 : nsresult
4037 141 : nsUrlClassifierLookupCallback::HandleResults()
4038 : {
4039 141 : if (!mResults) {
4040 : // No results, this URI is clean.
4041 0 : return mCallback->HandleEvent(NS_LITERAL_CSTRING(""));
4042 : }
4043 :
4044 : // Build a stringified list of result tables.
4045 141 : mResults->Sort();
4046 141 : PRUint32 lastTableId = 0;
4047 282 : nsCAutoString tables;
4048 277 : for (PRUint32 i = 0; i < mResults->Length(); i++) {
4049 136 : nsUrlClassifierLookupResult& result = mResults->ElementAt(i);
4050 : // Leave out results that weren't confirmed, as their existence on
4051 : // the list can't be verified. Also leave out randomly-generated
4052 : // noise.
4053 136 : if (!result.mConfirmed || result.mNoise)
4054 12 : continue;
4055 :
4056 124 : if (tables.Length() > 0) {
4057 25 : if (lastTableId == result.mEntry.mTableId)
4058 16 : continue;
4059 9 : tables.Append(",");
4060 : }
4061 :
4062 108 : tables.Append(result.mTableName);
4063 108 : lastTableId = result.mEntry.mTableId;
4064 : }
4065 :
4066 141 : return mCallback->HandleEvent(tables);
4067 : }
4068 :
4069 :
4070 : // -------------------------------------------------------------------------
4071 : // Helper class for nsIURIClassifier implementation, translates table names
4072 : // to nsIURIClassifier enums.
4073 :
4074 : class nsUrlClassifierClassifyCallback : public nsIUrlClassifierCallback
4075 21 : {
4076 : public:
4077 : NS_DECL_ISUPPORTS
4078 : NS_DECL_NSIURLCLASSIFIERCALLBACK
4079 :
4080 21 : nsUrlClassifierClassifyCallback(nsIURIClassifierCallback *c,
4081 : bool checkMalware,
4082 : bool checkPhishing)
4083 : : mCallback(c)
4084 : , mCheckMalware(checkMalware)
4085 21 : , mCheckPhishing(checkPhishing)
4086 21 : {}
4087 :
4088 : private:
4089 : nsCOMPtr<nsIURIClassifierCallback> mCallback;
4090 : bool mCheckMalware;
4091 : bool mCheckPhishing;
4092 : };
4093 :
4094 68 : NS_IMPL_THREADSAFE_ISUPPORTS1(nsUrlClassifierClassifyCallback,
4095 : nsIUrlClassifierCallback)
4096 :
4097 : NS_IMETHODIMP
4098 1 : nsUrlClassifierClassifyCallback::HandleEvent(const nsACString& tables)
4099 : {
4100 : // XXX: we should probably have the wardens tell the service which table
4101 : // names match with which classification. For now the table names give
4102 : // enough information.
4103 1 : nsresult response = NS_OK;
4104 :
4105 1 : nsACString::const_iterator begin, end;
4106 :
4107 1 : tables.BeginReading(begin);
4108 1 : tables.EndReading(end);
4109 4 : if (mCheckMalware &&
4110 3 : FindInReadable(NS_LITERAL_CSTRING("-malware-"), begin, end)) {
4111 0 : response = NS_ERROR_MALWARE_URI;
4112 : } else {
4113 : // Reset begin before checking phishing table
4114 1 : tables.BeginReading(begin);
4115 :
4116 4 : if (mCheckPhishing &&
4117 3 : FindInReadable(NS_LITERAL_CSTRING("-phish-"), begin, end)) {
4118 1 : response = NS_ERROR_PHISHING_URI;
4119 : }
4120 : }
4121 :
4122 1 : mCallback->OnClassifyComplete(response);
4123 :
4124 1 : return NS_OK;
4125 : }
4126 :
4127 :
4128 : // -------------------------------------------------------------------------
4129 : // Proxy class implementation
4130 :
4131 1199 : NS_IMPL_THREADSAFE_ISUPPORTS3(nsUrlClassifierDBService,
4132 : nsIUrlClassifierDBService,
4133 : nsIURIClassifier,
4134 : nsIObserver)
4135 :
4136 : /* static */ nsUrlClassifierDBService*
4137 8 : nsUrlClassifierDBService::GetInstance(nsresult *result)
4138 : {
4139 8 : *result = NS_OK;
4140 8 : if (!sUrlClassifierDBService) {
4141 8 : sUrlClassifierDBService = new nsUrlClassifierDBService();
4142 8 : if (!sUrlClassifierDBService) {
4143 0 : *result = NS_ERROR_OUT_OF_MEMORY;
4144 0 : return nsnull;
4145 : }
4146 :
4147 8 : NS_ADDREF(sUrlClassifierDBService); // addref the global
4148 :
4149 8 : *result = sUrlClassifierDBService->Init();
4150 8 : if (NS_FAILED(*result)) {
4151 0 : NS_RELEASE(sUrlClassifierDBService);
4152 0 : return nsnull;
4153 : }
4154 : } else {
4155 : // Already exists, just add a ref
4156 0 : NS_ADDREF(sUrlClassifierDBService); // addref the return result
4157 : }
4158 8 : return sUrlClassifierDBService;
4159 : }
4160 :
4161 :
4162 8 : nsUrlClassifierDBService::nsUrlClassifierDBService()
4163 : : mCheckMalware(CHECK_MALWARE_DEFAULT)
4164 : , mCheckPhishing(CHECK_PHISHING_DEFAULT)
4165 8 : , mInUpdate(false)
4166 : {
4167 8 : }
4168 :
4169 16 : nsUrlClassifierDBService::~nsUrlClassifierDBService()
4170 : {
4171 8 : sUrlClassifierDBService = nsnull;
4172 8 : }
4173 :
4174 : nsresult
4175 8 : nsUrlClassifierDBService::Init()
4176 : {
4177 : #if defined(PR_LOGGING)
4178 8 : if (!gUrlClassifierDbServiceLog)
4179 8 : gUrlClassifierDbServiceLog = PR_NewLogModule("UrlClassifierDbService");
4180 : #endif
4181 :
4182 : // Force the storage service to be created on the main thread.
4183 : nsresult rv;
4184 : nsCOMPtr<mozIStorageService> storageService =
4185 16 : do_GetService(MOZ_STORAGE_SERVICE_CONTRACTID, &rv);
4186 8 : NS_ENSURE_SUCCESS(rv, rv);
4187 :
4188 : // Force PSM to be loaded on the main thread.
4189 8 : mHash = do_CreateInstance(NS_CRYPTO_HASH_CONTRACTID, &rv);
4190 8 : NS_ENSURE_SUCCESS(rv, rv);
4191 :
4192 8 : mPrefixSet = new nsUrlClassifierPrefixSet();
4193 8 : NS_ENSURE_SUCCESS(rv, rv);
4194 :
4195 : // Should we check document loads for malware URIs?
4196 16 : nsCOMPtr<nsIPrefBranch> prefs = do_GetService(NS_PREFSERVICE_CONTRACTID);
4197 :
4198 8 : PRInt32 gethashNoise = 0;
4199 8 : if (prefs) {
4200 : bool tmpbool;
4201 8 : rv = prefs->GetBoolPref(CHECK_MALWARE_PREF, &tmpbool);
4202 8 : mCheckMalware = NS_SUCCEEDED(rv) ? tmpbool : CHECK_MALWARE_DEFAULT;
4203 :
4204 8 : prefs->AddObserver(CHECK_MALWARE_PREF, this, false);
4205 :
4206 8 : rv = prefs->GetBoolPref(CHECK_PHISHING_PREF, &tmpbool);
4207 8 : mCheckPhishing = NS_SUCCEEDED(rv) ? tmpbool : CHECK_PHISHING_DEFAULT;
4208 :
4209 8 : prefs->AddObserver(CHECK_PHISHING_PREF, this, false);
4210 :
4211 8 : if (NS_FAILED(prefs->GetIntPref(GETHASH_NOISE_PREF, &gethashNoise))) {
4212 0 : gethashNoise = GETHASH_NOISE_DEFAULT;
4213 : }
4214 :
4215 16 : nsXPIDLCString tmpstr;
4216 8 : if (NS_SUCCEEDED(prefs->GetCharPref(GETHASH_TABLES_PREF, getter_Copies(tmpstr)))) {
4217 8 : SplitTables(tmpstr, mGethashWhitelist);
4218 : }
4219 :
4220 8 : prefs->AddObserver(GETHASH_TABLES_PREF, this, false);
4221 :
4222 : PRInt32 tmpint;
4223 8 : rv = prefs->GetIntPref(CONFIRM_AGE_PREF, &tmpint);
4224 8 : PR_ATOMIC_SET(&gFreshnessGuarantee, NS_SUCCEEDED(rv) ? tmpint : CONFIRM_AGE_DEFAULT_SEC);
4225 :
4226 8 : prefs->AddObserver(CONFIRM_AGE_PREF, this, false);
4227 :
4228 8 : rv = prefs->GetIntPref(UPDATE_CACHE_SIZE_PREF, &tmpint);
4229 8 : PR_ATOMIC_SET(&gUpdateCacheSize, NS_SUCCEEDED(rv) ? tmpint : UPDATE_CACHE_SIZE_DEFAULT);
4230 :
4231 8 : rv = prefs->GetIntPref(LOOKUP_CACHE_SIZE_PREF, &tmpint);
4232 8 : PR_ATOMIC_SET(&gLookupCacheSize, NS_SUCCEEDED(rv) ? tmpint : LOOKUP_CACHE_SIZE_DEFAULT);
4233 :
4234 8 : rv = prefs->GetIntPref(UPDATE_WORKING_TIME, &tmpint);
4235 8 : PR_ATOMIC_SET(&gWorkingTimeThreshold,
4236 8 : NS_SUCCEEDED(rv) ? tmpint : UPDATE_WORKING_TIME_DEFAULT);
4237 :
4238 8 : rv = prefs->GetIntPref(UPDATE_DELAY_TIME, &tmpint);
4239 8 : PR_ATOMIC_SET(&gDelayTime,
4240 8 : NS_SUCCEEDED(rv) ? tmpint : UPDATE_DELAY_TIME_DEFAULT);
4241 : }
4242 :
4243 : // Start the background thread.
4244 8 : rv = NS_NewThread(&gDbBackgroundThread);
4245 8 : if (NS_FAILED(rv))
4246 0 : return rv;
4247 :
4248 8 : mWorker = new nsUrlClassifierDBServiceWorker();
4249 8 : if (!mWorker)
4250 0 : return NS_ERROR_OUT_OF_MEMORY;
4251 :
4252 8 : rv = mWorker->Init(gethashNoise, mPrefixSet);
4253 8 : if (NS_FAILED(rv)) {
4254 0 : mWorker = nsnull;
4255 0 : return rv;
4256 : }
4257 :
4258 : // Proxy for calling the worker on the background thread
4259 16 : mWorkerProxy = new UrlClassifierDBServiceWorkerProxy(mWorker);
4260 :
4261 8 : mCompleters.Init();
4262 :
4263 : // Add an observer for shutdown
4264 : nsCOMPtr<nsIObserverService> observerService =
4265 16 : mozilla::services::GetObserverService();
4266 8 : if (!observerService)
4267 0 : return NS_ERROR_FAILURE;
4268 :
4269 8 : observerService->AddObserver(this, "profile-before-change", false);
4270 8 : observerService->AddObserver(this, "xpcom-shutdown-threads", false);
4271 :
4272 8 : return NS_OK;
4273 : }
4274 :
4275 : NS_IMETHODIMP
4276 21 : nsUrlClassifierDBService::Classify(nsIURI *uri,
4277 : nsIURIClassifierCallback* c,
4278 : bool* result)
4279 : {
4280 21 : NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
4281 :
4282 21 : if (!(mCheckMalware || mCheckPhishing)) {
4283 0 : *result = false;
4284 0 : return NS_OK;
4285 : }
4286 :
4287 : nsRefPtr<nsUrlClassifierClassifyCallback> callback =
4288 42 : new nsUrlClassifierClassifyCallback(c, mCheckMalware, mCheckPhishing);
4289 21 : if (!callback) return NS_ERROR_OUT_OF_MEMORY;
4290 :
4291 21 : nsresult rv = LookupURI(uri, callback, false, result);
4292 21 : if (rv == NS_ERROR_MALFORMED_URI) {
4293 0 : *result = false;
4294 : // The URI had no hostname, don't try to classify it.
4295 0 : return NS_OK;
4296 : }
4297 21 : NS_ENSURE_SUCCESS(rv, rv);
4298 :
4299 21 : return NS_OK;
4300 : }
4301 :
4302 : NS_IMETHODIMP
4303 140 : nsUrlClassifierDBService::Lookup(const nsACString& spec,
4304 : nsIUrlClassifierCallback* c)
4305 : {
4306 140 : NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
4307 :
4308 280 : nsCOMPtr<nsIURI> uri;
4309 :
4310 140 : nsresult rv = NS_NewURI(getter_AddRefs(uri), spec);
4311 140 : NS_ENSURE_SUCCESS(rv, rv);
4312 :
4313 140 : uri = NS_GetInnermostURI(uri);
4314 140 : if (!uri) {
4315 0 : return NS_ERROR_FAILURE;
4316 : }
4317 :
4318 : bool didLookup;
4319 140 : return LookupURI(uri, c, true, &didLookup);
4320 : }
4321 :
4322 : nsresult
4323 161 : nsUrlClassifierDBService::LookupURI(nsIURI* uri,
4324 : nsIUrlClassifierCallback* c,
4325 : bool forceLookup,
4326 : bool *didLookup)
4327 : {
4328 161 : NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
4329 :
4330 322 : nsCAutoString key;
4331 : // Canonicalize the url
4332 : nsCOMPtr<nsIUrlClassifierUtils> utilsService =
4333 322 : do_GetService(NS_URLCLASSIFIERUTILS_CONTRACTID);
4334 161 : nsresult rv = utilsService->GetKeyForURI(uri, key);
4335 161 : if (NS_FAILED(rv))
4336 0 : return rv;
4337 :
4338 161 : if (forceLookup) {
4339 140 : *didLookup = true;
4340 : } else {
4341 : // Check if the URI is clean. If so, we don't need to
4342 : // bother queueing up a lookup, we can just return.;
4343 : bool clean;
4344 21 : rv = CheckClean(key, &clean);
4345 21 : NS_ENSURE_SUCCESS(rv, rv);
4346 :
4347 21 : if (!clean) {
4348 : nsCOMPtr<nsIPermissionManager> permissionManager =
4349 2 : do_GetService(NS_PERMISSIONMANAGER_CONTRACTID);
4350 :
4351 1 : if (permissionManager) {
4352 : PRUint32 perm;
4353 1 : permissionManager->TestPermission(uri, "safe-browsing", &perm);
4354 1 : clean |= (perm == nsIPermissionManager::ALLOW_ACTION);
4355 : }
4356 : }
4357 :
4358 21 : *didLookup = !clean;
4359 21 : if (clean) {
4360 20 : return NS_OK;
4361 : }
4362 : }
4363 :
4364 : // Create an nsUrlClassifierLookupCallback object. This object will
4365 : // take care of confirming partial hash matches if necessary before
4366 : // calling the client's callback.
4367 : nsCOMPtr<nsIUrlClassifierLookupCallback> callback =
4368 282 : new nsUrlClassifierLookupCallback(this, c);
4369 141 : if (!callback)
4370 0 : return NS_ERROR_OUT_OF_MEMORY;
4371 :
4372 : nsCOMPtr<nsIUrlClassifierLookupCallback> proxyCallback =
4373 423 : new UrlClassifierLookupCallbackProxy(callback);
4374 :
4375 : // Queue this lookup and call the lookup function to flush the queue if
4376 : // necessary.
4377 141 : rv = mWorker->QueueLookup(key, proxyCallback);
4378 141 : NS_ENSURE_SUCCESS(rv, rv);
4379 :
4380 141 : return mWorkerProxy->Lookup(EmptyCString(), nsnull);
4381 : }
4382 :
4383 : NS_IMETHODIMP
4384 53 : nsUrlClassifierDBService::GetTables(nsIUrlClassifierCallback* c)
4385 : {
4386 53 : NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
4387 :
4388 : // The proxy callback uses the current thread.
4389 : nsCOMPtr<nsIUrlClassifierCallback> proxyCallback =
4390 106 : new UrlClassifierCallbackProxy(c);
4391 :
4392 53 : return mWorkerProxy->GetTables(proxyCallback);
4393 : }
4394 :
4395 : NS_IMETHODIMP
4396 83 : nsUrlClassifierDBService::SetHashCompleter(const nsACString &tableName,
4397 : nsIUrlClassifierHashCompleter *completer)
4398 : {
4399 83 : if (completer) {
4400 29 : if (!mCompleters.Put(tableName, completer)) {
4401 0 : return NS_ERROR_OUT_OF_MEMORY;
4402 : }
4403 : } else {
4404 54 : mCompleters.Remove(tableName);
4405 : }
4406 :
4407 83 : return NS_OK;
4408 : }
4409 :
4410 : NS_IMETHODIMP
4411 87 : nsUrlClassifierDBService::BeginUpdate(nsIUrlClassifierUpdateObserver *observer,
4412 : const nsACString &updateTables,
4413 : const nsACString &clientKey)
4414 : {
4415 87 : NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
4416 :
4417 87 : if (mInUpdate)
4418 0 : return NS_ERROR_NOT_AVAILABLE;
4419 :
4420 87 : mInUpdate = true;
4421 :
4422 : // The proxy observer uses the current thread
4423 : nsCOMPtr<nsIUrlClassifierUpdateObserver> proxyObserver =
4424 174 : new UrlClassifierUpdateObserverProxy(observer);
4425 :
4426 87 : return mWorkerProxy->BeginUpdate(proxyObserver, updateTables, clientKey);
4427 : }
4428 :
4429 : NS_IMETHODIMP
4430 102 : nsUrlClassifierDBService::BeginStream(const nsACString &table,
4431 : const nsACString &serverMAC)
4432 : {
4433 102 : NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
4434 :
4435 102 : return mWorkerProxy->BeginStream(table, serverMAC);
4436 : }
4437 :
4438 : NS_IMETHODIMP
4439 100 : nsUrlClassifierDBService::UpdateStream(const nsACString& aUpdateChunk)
4440 : {
4441 100 : NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
4442 :
4443 100 : return mWorkerProxy->UpdateStream(aUpdateChunk);
4444 : }
4445 :
4446 : NS_IMETHODIMP
4447 100 : nsUrlClassifierDBService::FinishStream()
4448 : {
4449 100 : NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
4450 :
4451 100 : return mWorkerProxy->FinishStream();
4452 : }
4453 :
4454 : NS_IMETHODIMP
4455 85 : nsUrlClassifierDBService::FinishUpdate()
4456 : {
4457 85 : NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
4458 :
4459 85 : mInUpdate = false;
4460 :
4461 85 : return mWorkerProxy->FinishUpdate();
4462 : }
4463 :
4464 :
4465 : NS_IMETHODIMP
4466 2 : nsUrlClassifierDBService::CancelUpdate()
4467 : {
4468 2 : NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
4469 :
4470 2 : mInUpdate = false;
4471 :
4472 2 : return mWorkerProxy->CancelUpdate();
4473 : }
4474 :
4475 : NS_IMETHODIMP
4476 54 : nsUrlClassifierDBService::ResetDatabase()
4477 : {
4478 54 : NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
4479 :
4480 54 : return mWorkerProxy->ResetDatabase();
4481 : }
4482 :
4483 : nsresult
4484 20 : nsUrlClassifierDBService::CacheCompletions(nsTArray<nsUrlClassifierLookupResult> *results)
4485 : {
4486 20 : NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
4487 :
4488 20 : return mWorkerProxy->CacheCompletions(results);
4489 : }
4490 :
4491 : bool
4492 77 : nsUrlClassifierDBService::GetCompleter(const nsACString &tableName,
4493 : nsIUrlClassifierHashCompleter **completer)
4494 : {
4495 77 : if (mCompleters.Get(tableName, completer)) {
4496 57 : return true;
4497 : }
4498 :
4499 20 : if (!mGethashWhitelist.Contains(tableName)) {
4500 20 : return false;
4501 : }
4502 :
4503 0 : return NS_SUCCEEDED(CallGetService(NS_URLCLASSIFIERHASHCOMPLETER_CONTRACTID,
4504 : completer));
4505 : }
4506 :
4507 : NS_IMETHODIMP
4508 20 : nsUrlClassifierDBService::Observe(nsISupports *aSubject, const char *aTopic,
4509 : const PRUnichar *aData)
4510 : {
4511 20 : if (!strcmp(aTopic, NS_PREFBRANCH_PREFCHANGE_TOPIC_ID)) {
4512 : nsresult rv;
4513 8 : nsCOMPtr<nsIPrefBranch> prefs(do_QueryInterface(aSubject, &rv));
4514 4 : NS_ENSURE_SUCCESS(rv, rv);
4515 4 : if (NS_LITERAL_STRING(CHECK_MALWARE_PREF).Equals(aData)) {
4516 : bool tmpbool;
4517 0 : rv = prefs->GetBoolPref(CHECK_MALWARE_PREF, &tmpbool);
4518 0 : mCheckMalware = NS_SUCCEEDED(rv) ? tmpbool : CHECK_MALWARE_DEFAULT;
4519 4 : } else if (NS_LITERAL_STRING(CHECK_PHISHING_PREF).Equals(aData)) {
4520 : bool tmpbool;
4521 0 : rv = prefs->GetBoolPref(CHECK_PHISHING_PREF, &tmpbool);
4522 0 : mCheckPhishing = NS_SUCCEEDED(rv) ? tmpbool : CHECK_PHISHING_DEFAULT;
4523 4 : } else if (NS_LITERAL_STRING(GETHASH_TABLES_PREF).Equals(aData)) {
4524 0 : mGethashWhitelist.Clear();
4525 0 : nsXPIDLCString val;
4526 0 : if (NS_SUCCEEDED(prefs->GetCharPref(GETHASH_TABLES_PREF, getter_Copies(val)))) {
4527 0 : SplitTables(val, mGethashWhitelist);
4528 : }
4529 4 : } else if (NS_LITERAL_STRING(CONFIRM_AGE_PREF).Equals(aData)) {
4530 : PRInt32 tmpint;
4531 4 : rv = prefs->GetIntPref(CONFIRM_AGE_PREF, &tmpint);
4532 4 : PR_ATOMIC_SET(&gFreshnessGuarantee, NS_SUCCEEDED(rv) ? tmpint : CONFIRM_AGE_DEFAULT_SEC);
4533 0 : } else if (NS_LITERAL_STRING(UPDATE_CACHE_SIZE_PREF).Equals(aData)) {
4534 : PRInt32 tmpint;
4535 0 : rv = prefs->GetIntPref(UPDATE_CACHE_SIZE_PREF, &tmpint);
4536 0 : PR_ATOMIC_SET(&gUpdateCacheSize, NS_SUCCEEDED(rv) ? tmpint : UPDATE_CACHE_SIZE_DEFAULT);
4537 0 : } else if (NS_LITERAL_STRING(LOOKUP_CACHE_SIZE_PREF).Equals(aData)) {
4538 : PRInt32 tmpint;
4539 0 : rv = prefs->GetIntPref(LOOKUP_CACHE_SIZE_PREF, &tmpint);
4540 0 : PR_ATOMIC_SET(&gLookupCacheSize, NS_SUCCEEDED(rv) ? tmpint : LOOKUP_CACHE_SIZE_DEFAULT);
4541 0 : } else if (NS_LITERAL_STRING(UPDATE_WORKING_TIME).Equals(aData)) {
4542 : PRInt32 tmpint;
4543 0 : rv = prefs->GetIntPref(UPDATE_WORKING_TIME, &tmpint);
4544 0 : PR_ATOMIC_SET(&gWorkingTimeThreshold,
4545 0 : NS_SUCCEEDED(rv) ? tmpint : UPDATE_WORKING_TIME_DEFAULT);
4546 0 : } else if (NS_LITERAL_STRING(UPDATE_DELAY_TIME).Equals(aData)) {
4547 : PRInt32 tmpint;
4548 0 : rv = prefs->GetIntPref(UPDATE_DELAY_TIME, &tmpint);
4549 0 : PR_ATOMIC_SET(&gDelayTime,
4550 0 : NS_SUCCEEDED(rv) ? tmpint : UPDATE_DELAY_TIME_DEFAULT);
4551 : }
4552 24 : } else if (!strcmp(aTopic, "profile-before-change") ||
4553 8 : !strcmp(aTopic, "xpcom-shutdown-threads")) {
4554 16 : Shutdown();
4555 : } else {
4556 0 : return NS_ERROR_UNEXPECTED;
4557 : }
4558 :
4559 20 : return NS_OK;
4560 : }
4561 :
4562 : // Join the background thread if it exists.
4563 : nsresult
4564 16 : nsUrlClassifierDBService::Shutdown()
4565 : {
4566 16 : LOG(("shutting down db service\n"));
4567 :
4568 16 : if (!gDbBackgroundThread)
4569 8 : return NS_OK;
4570 :
4571 8 : mCompleters.Clear();
4572 :
4573 16 : nsCOMPtr<nsIPrefBranch> prefs = do_GetService(NS_PREFSERVICE_CONTRACTID);
4574 8 : if (prefs) {
4575 8 : prefs->RemoveObserver(CHECK_MALWARE_PREF, this);
4576 8 : prefs->RemoveObserver(CHECK_PHISHING_PREF, this);
4577 8 : prefs->RemoveObserver(GETHASH_TABLES_PREF, this);
4578 8 : prefs->RemoveObserver(CONFIRM_AGE_PREF, this);
4579 : }
4580 :
4581 : nsresult rv;
4582 : // First close the db connection.
4583 8 : if (mWorker) {
4584 8 : rv = mWorkerProxy->CancelUpdate();
4585 8 : NS_ASSERTION(NS_SUCCEEDED(rv), "failed to post cancel update event");
4586 :
4587 8 : rv = mWorkerProxy->CloseDb();
4588 8 : NS_ASSERTION(NS_SUCCEEDED(rv), "failed to post close db event");
4589 : }
4590 :
4591 8 : mWorkerProxy = nsnull;
4592 :
4593 8 : LOG(("joining background thread"));
4594 :
4595 8 : gShuttingDownThread = true;
4596 :
4597 8 : nsIThread *backgroundThread = gDbBackgroundThread;
4598 8 : gDbBackgroundThread = nsnull;
4599 8 : backgroundThread->Shutdown();
4600 8 : NS_RELEASE(backgroundThread);
4601 :
4602 8 : return NS_OK;
4603 : }
4604 :
4605 : nsIThread*
4606 760 : nsUrlClassifierDBService::BackgroundThread()
4607 : {
4608 760 : return gDbBackgroundThread;
4609 : }
4610 :
|