LCOV - code coverage report
Current view: directory - toolkit/components/url-classifier - nsUrlClassifierDBService.cpp (source / functions) Found Hit Coverage
Test: app.info Lines: 1970 1603 81.4 %
Date: 2012-06-02 Functions: 162 148 91.4 %

       1                 : //* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
       2                 : /* ***** BEGIN LICENSE BLOCK *****
       3                 :  * Version: MPL 1.1/GPL 2.0/LGPL 2.1
       4                 :  *
       5                 :  * The contents of this file are subject to the Mozilla Public License Version
       6                 :  * 1.1 (the "License"); you may not use this file except in compliance with
       7                 :  * the License. You may obtain a copy of the License at
       8                 :  * http://www.mozilla.org/MPL/
       9                 :  *
      10                 :  * Software distributed under the License is distributed on an "AS IS" basis,
      11                 :  * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
      12                 :  * for the specific language governing rights and limitations under the
      13                 :  * License.
      14                 :  *
      15                 :  * The Original Code is Url Classifier code
      16                 :  *
      17                 :  * The Initial Developer of the Original Code is
      18                 :  * Google Inc.
      19                 :  * Portions created by the Initial Developer are Copyright (C) 2006
      20                 :  * the Initial Developer. All Rights Reserved.
      21                 :  *
      22                 :  * Contributor(s):
      23                 :  *   Tony Chang <tony@ponderer.org> (original author)
      24                 :  *   Brett Wilson <brettw@gmail.com>
      25                 :  *   Dave Camp <dcamp@mozilla.com>
      26                 :  *   David Dahl <ddahl@mozilla.com>
      27                 :  *   Gian-Carlo Pascutto <gpascutto@mozilla.com>
      28                 :  *
      29                 :  * Alternatively, the contents of this file may be used under the terms of
      30                 :  * either the GNU General Public License Version 2 or later (the "GPL"), or
      31                 :  * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
      32                 :  * in which case the provisions of the GPL or the LGPL are applicable instead
      33                 :  * of those above. If you wish to allow use of your version of this file only
      34                 :  * under the terms of either the GPL or the LGPL, and not to allow others to
      35                 :  * use your version of this file under the terms of the MPL, indicate your
      36                 :  * decision by deleting the provisions above and replace them with the notice
      37                 :  * and other provisions required by the GPL or the LGPL. If you do not delete
      38                 :  * the provisions above, a recipient may use your version of this file under
      39                 :  * the terms of any one of the MPL, the GPL or the LGPL.
      40                 :  *
      41                 :  * ***** END LICENSE BLOCK ***** */
      42                 : 
      43                 : #include "nsAutoPtr.h"
      44                 : #include "nsCOMPtr.h"
      45                 : #include "mozIStorageService.h"
      46                 : #include "mozIStorageConnection.h"
      47                 : #include "mozIStorageStatement.h"
      48                 : #include "mozStorageHelper.h"
      49                 : #include "mozStorageCID.h"
      50                 : #include "nsAppDirectoryServiceDefs.h"
      51                 : #include "nsCRT.h"
      52                 : #include "nsDataHashtable.h"
      53                 : #include "nsICryptoHash.h"
      54                 : #include "nsICryptoHMAC.h"
      55                 : #include "nsIDirectoryService.h"
      56                 : #include "nsIKeyModule.h"
      57                 : #include "nsIObserverService.h"
      58                 : #include "nsIPermissionManager.h"
      59                 : #include "nsIPrefBranch.h"
      60                 : #include "nsIPrefService.h"
      61                 : #include "nsIProperties.h"
      62                 : #include "nsToolkitCompsCID.h"
      63                 : #include "nsIUrlClassifierUtils.h"
      64                 : #include "nsUrlClassifierDBService.h"
      65                 : #include "nsUrlClassifierUtils.h"
      66                 : #include "nsUrlClassifierProxies.h"
      67                 : #include "nsURILoader.h"
      68                 : #include "nsString.h"
      69                 : #include "nsReadableUtils.h"
      70                 : #include "nsTArray.h"
      71                 : #include "nsNetUtil.h"
      72                 : #include "nsNetCID.h"
      73                 : #include "nsThreadUtils.h"
      74                 : #include "nsXPCOMStrings.h"
      75                 : #include "mozilla/Mutex.h"
      76                 : #include "mozilla/Telemetry.h"
      77                 : #include "prlog.h"
      78                 : #include "prprf.h"
      79                 : #include "prnetdb.h"
      80                 : #include "zlib.h"
      81                 : 
      82                 : // Needed to interpert mozIStorageConnection::GetLastError
      83                 : #include <sqlite3.h>
      84                 : 
      85                 : using namespace mozilla;
      86                 : 
      87                 : /**
      88                 :  * The DBServices stores a set of Fragments.  A fragment is one URL
      89                 :  * fragment containing two or more domain components and some number
      90                 :  * of path components.
      91                 :  *
      92                 :  * Fragment examples:
      93                 :  *   example.com/
      94                 :  *   www.example.com/foo/bar
      95                 :  *   www.mail.example.com/mail
      96                 :  *
      97                 :  * Fragments are described in "Simplified Regular Expression Lookup"
      98                 :  * section of the protocol document at
      99                 :  * http://code.google.com/p/google-safe-browsing/wiki/Protocolv2Spec
     100                 :  *
     101                 :  * A fragment is associated with a domain.  The domain for a given
     102                 :  * fragment is the three-host-component domain of the fragment (two
     103                 :  * host components for URLs with only two components) with a trailing
     104                 :  * slash.  So for the fragments listed above, the domains are
     105                 :  * example.com/, www.example.com/ and mail.example.com/.
     106                 :  *
     107                 :  * Fragments and domains are hashed in the database.  The hash is described
     108                 :  * in the protocol document, but it's basically a truncated SHA256 hash.
     109                 :  *
     110                 :  * A (table, chunk id, domain key, fragment) tuple is referred to as
     111                 :  * an Entry.
     112                 :  */
     113                 : 
     114                 : // NSPR_LOG_MODULES=UrlClassifierDbService:5
     115                 : #if defined(PR_LOGGING)
     116                 : static const PRLogModuleInfo *gUrlClassifierDbServiceLog = nsnull;
     117                 : #define LOG(args) PR_LOG(gUrlClassifierDbServiceLog, PR_LOG_DEBUG, args)
     118                 : #define LOG_ENABLED() PR_LOG_TEST(gUrlClassifierDbServiceLog, 4)
     119                 : #else
     120                 : #define LOG(args)
     121                 : #define LOG_ENABLED() (false)
     122                 : #endif
     123                 : 
     124                 : // Schema versioning:  note that we don't bother to migrate between different
     125                 : // versions of the schema, we just start fetching the data freshly with each
     126                 : // migration.
     127                 : 
     128                 : // The database filename is updated when there is an incompatible
     129                 : // schema change and we expect both implementations to continue
     130                 : // accessing the same database (such as between stable versions of the
     131                 : // platform).
     132                 : #define DATABASE_FILENAME "urlclassifier3.sqlite"
     133                 : 
     134                 : // The implementation version is updated during development when we
     135                 : // want to change schema, or to recover from updating bugs.  When an
     136                 : // implementation version change is detected, the database is scrapped
     137                 : // and we start over.
     138                 : #define IMPLEMENTATION_VERSION 7
     139                 : 
     140                 : // Name of the persistent PrefixSet storage
     141                 : #define PREFIXSET_FILENAME  "urlclassifier.pset"
     142                 : 
     143                 : #define MAX_HOST_COMPONENTS 5
     144                 : #define MAX_PATH_COMPONENTS 4
     145                 : 
     146                 : // Updates will fail if fed chunks larger than this
     147                 : #define MAX_CHUNK_SIZE (1024 * 1024)
     148                 : 
     149                 : // Prefs for implementing nsIURIClassifier to block page loads
     150                 : #define CHECK_MALWARE_PREF      "browser.safebrowsing.malware.enabled"
     151                 : #define CHECK_MALWARE_DEFAULT   false
     152                 : 
     153                 : #define CHECK_PHISHING_PREF     "browser.safebrowsing.enabled"
     154                 : #define CHECK_PHISHING_DEFAULT  false
     155                 : 
     156                 : #define GETHASH_NOISE_PREF      "urlclassifier.gethashnoise"
     157                 : #define GETHASH_NOISE_DEFAULT   4
     158                 : 
     159                 : #define GETHASH_TABLES_PREF     "urlclassifier.gethashtables"
     160                 : 
     161                 : #define CONFIRM_AGE_PREF        "urlclassifier.confirm-age"
     162                 : #define CONFIRM_AGE_DEFAULT_SEC (45 * 60)
     163                 : 
     164                 : #define UPDATE_CACHE_SIZE_PREF    "urlclassifier.updatecachemax"
     165                 : #define UPDATE_CACHE_SIZE_DEFAULT -1
     166                 : 
     167                 : #define LOOKUP_CACHE_SIZE_PREF    "urlclassifier.lookupcachemax"
     168                 : #define LOOKUP_CACHE_SIZE_DEFAULT -1
     169                 : 
     170                 : // Amount of time to spend updating before committing and delaying, in
     171                 : // seconds.  This is checked after each update stream, so the actual
     172                 : // time spent can be higher than this, depending on update stream size.
     173                 : #define UPDATE_WORKING_TIME         "urlclassifier.workingtime"
     174                 : #define UPDATE_WORKING_TIME_DEFAULT 5
     175                 : 
     176                 : // The amount of time to delay after hitting UPDATE_WORKING_TIME, in
     177                 : // seconds.
     178                 : #define UPDATE_DELAY_TIME           "urlclassifier.updatetime"
     179                 : #define UPDATE_DELAY_TIME_DEFAULT   60
     180                 : 
     181                 : class nsUrlClassifierDBServiceWorker;
     182                 : 
     183                 : // Singleton instance.
     184                 : static nsUrlClassifierDBService* sUrlClassifierDBService;
     185                 : 
     186                 : nsIThread* nsUrlClassifierDBService::gDbBackgroundThread = nsnull;
     187                 : 
     188                 : // Once we've committed to shutting down, don't do work in the background
     189                 : // thread.
     190                 : static bool gShuttingDownThread = false;
     191                 : 
     192                 : static PRInt32 gFreshnessGuarantee = CONFIRM_AGE_DEFAULT_SEC;
     193                 : 
     194                 : static PRInt32 gUpdateCacheSize = UPDATE_CACHE_SIZE_DEFAULT;
     195                 : static PRInt32 gLookupCacheSize = LOOKUP_CACHE_SIZE_DEFAULT;
     196                 : 
     197                 : static PRInt32 gWorkingTimeThreshold = UPDATE_WORKING_TIME_DEFAULT;
     198                 : static PRInt32 gDelayTime = UPDATE_DELAY_TIME_DEFAULT;
     199                 : 
     200                 : static void
     201              95 : SplitTables(const nsACString& str, nsTArray<nsCString>& tables)
     202                 : {
     203              95 :   tables.Clear();
     204                 : 
     205              95 :   nsACString::const_iterator begin, iter, end;
     206              95 :   str.BeginReading(begin);
     207              95 :   str.EndReading(end);
     208             379 :   while (begin != end) {
     209             189 :     iter = begin;
     210             189 :     FindCharInReadable(',', iter, end);
     211             189 :     tables.AppendElement(Substring(begin, iter));
     212             189 :     begin = iter;
     213             189 :     if (begin != end)
     214              94 :       begin++;
     215                 :   }
     216              95 : }
     217                 : 
     218                 : // -------------------------------------------------------------------------
     219                 : // Hash class implementation
     220                 : 
     221                 : // A convenience wrapper around the potentially-truncated hash for a
     222                 : // domain or fragment.
     223                 : 
     224                 : template <PRUint32 S>
     225                 : struct nsUrlClassifierHash
     226                 : {
     227                 :   static const PRUint32 sHashSize = S;
     228                 :   typedef nsUrlClassifierHash<S> self_type;
     229                 :   PRUint8 buf[S];
     230                 : 
     231             916 :   nsresult FromPlaintext(const nsACString& plainText, nsICryptoHash *hash) {
     232                 :     // From the protocol doc:
     233                 :     // Each entry in the chunk is composed of the 128 most significant bits
     234                 :     // of the SHA 256 hash of a suffix/prefix expression.
     235                 : 
     236             916 :     nsresult rv = hash->Init(nsICryptoHash::SHA256);
     237             916 :     NS_ENSURE_SUCCESS(rv, rv);
     238                 : 
     239             916 :     rv = hash->Update
     240                 :       (reinterpret_cast<const PRUint8*>(plainText.BeginReading()),
     241                 :        plainText.Length());
     242             916 :     NS_ENSURE_SUCCESS(rv, rv);
     243                 : 
     244            1832 :     nsCAutoString hashed;
     245             916 :     rv = hash->Finish(false, hashed);
     246             916 :     NS_ENSURE_SUCCESS(rv, rv);
     247                 : 
     248             916 :     NS_ASSERTION(hashed.Length() >= sHashSize,
     249                 :                  "not enough characters in the hash");
     250                 : 
     251             916 :     memcpy(buf, hashed.BeginReading(), sHashSize);
     252                 : 
     253             916 :     return NS_OK;
     254                 :   }
     255                 : 
     256              51 :   void Assign(const nsACString& str) {
     257              51 :     NS_ASSERTION(str.Length() >= sHashSize,
     258                 :                  "string must be at least sHashSize characters long");
     259              51 :     memcpy(buf, str.BeginReading(), sHashSize);
     260              51 :   }
     261                 : 
     262             136 :   void Clear() {
     263             136 :     memset(buf, 0, sizeof(buf));
     264             136 :   }
     265                 : 
     266             597 :   const bool operator==(const self_type& hash) const {
     267             597 :     return (memcmp(buf, hash.buf, sizeof(buf)) == 0);
     268                 :   }
     269               0 :   const bool operator!=(const self_type& hash) const {
     270               0 :     return !(*this == hash);
     271                 :   }
     272             180 :   const bool operator<(const self_type& hash) const {
     273             180 :     return memcmp(buf, hash.buf, sizeof(self_type)) < 0;
     274                 :   }
     275             362 :   const bool StartsWith(const nsUrlClassifierHash<PARTIAL_LENGTH>& hash) const {
     276                 :     NS_ASSERTION(sHashSize >= PARTIAL_LENGTH, "nsUrlClassifierHash must be at least PARTIAL_LENGTH bytes long");
     277             362 :     return memcmp(buf, hash.buf, PARTIAL_LENGTH) == 0;
     278                 :   }
     279              84 :   PRUint32 ToUint32() const {
     280              84 :     return *(reinterpret_cast<const PRUint32*>(buf));
     281                 :   }
     282                 : };
     283                 : 
     284                 : typedef nsUrlClassifierHash<DOMAIN_LENGTH> nsUrlClassifierDomainHash;
     285                 : typedef nsUrlClassifierHash<PARTIAL_LENGTH> nsUrlClassifierPartialHash;
     286                 : typedef nsUrlClassifierHash<COMPLETE_LENGTH> nsUrlClassifierCompleteHash;
     287                 : 
     288                 : 
     289                 : // -------------------------------------------------------------------------
     290                 : // Entry class implementation
     291                 : 
     292                 : // This class represents one entry in the classifier database.  It consists
     293                 : // of a table id, a chunk id, a domain hash, and a partial or complete hash.
     294                 : class nsUrlClassifierEntry
     295                 : {
     296                 : public:
     297             524 :   nsUrlClassifierEntry()
     298                 :     : mId(-1)
     299                 :     , mHavePartial(false)
     300                 :     , mHaveComplete(false)
     301                 :     , mTableId(0)
     302                 :     , mChunkId(0)
     303             524 :     , mAddChunkId(0)
     304             524 :     {}
     305             560 :   ~nsUrlClassifierEntry() {}
     306                 : 
     307                 :   // Check that this entry could potentially match the complete hash.
     308                 :   bool Match(const nsUrlClassifierCompleteHash &hash);
     309                 : 
     310                 :   // Check that the sub entry should apply to this entry.
     311                 :   bool SubMatch(const nsUrlClassifierEntry& sub);
     312                 : 
     313                 :   // Clear out the entry structure
     314                 :   void Clear();
     315                 : 
     316                 :   // Set the partial hash for this domain.
     317              52 :   void SetHash(const nsUrlClassifierPartialHash &partialHash) {
     318              52 :     mPartialHash = partialHash;
     319              52 :     mHavePartial = true;
     320              52 :   }
     321                 : 
     322                 :   // Set the complete hash for this domain.
     323             173 :   void SetHash(const nsUrlClassifierCompleteHash &completeHash) {
     324             173 :     mCompleteHash = completeHash;
     325             173 :     mHaveComplete = true;
     326             173 :   }
     327                 : 
     328               0 :   bool operator== (const nsUrlClassifierEntry& entry) const {
     329                 :     return ! (mTableId != entry.mTableId ||
     330                 :               mChunkId != entry.mChunkId ||
     331                 :               mHavePartial != entry.mHavePartial ||
     332               0 :               (mHavePartial && mPartialHash != entry.mPartialHash) ||
     333                 :               mHaveComplete != entry.mHaveComplete ||
     334               0 :               (mHaveComplete && mCompleteHash != entry.mCompleteHash));
     335                 :   }
     336                 : 
     337              65 :   bool operator< (const nsUrlClassifierEntry& entry) const {
     338                 :     return (mTableId < entry.mTableId ||
     339                 :             mChunkId < entry.mChunkId ||
     340              47 :             (mHavePartial && !entry.mHavePartial) ||
     341              47 :             (mHavePartial && mPartialHash < entry.mPartialHash) ||
     342              25 :             (mHaveComplete && !entry.mHaveComplete) ||
     343             184 :             (mHaveComplete && mCompleteHash < entry.mCompleteHash));
     344                 :   }
     345                 : 
     346                 :   PRInt64 mId;
     347                 : 
     348                 :   nsUrlClassifierDomainHash mKey;
     349                 : 
     350                 :   bool mHavePartial;
     351                 :   nsUrlClassifierPartialHash mPartialHash;
     352                 : 
     353                 :   bool mHaveComplete;
     354                 :   nsUrlClassifierCompleteHash mCompleteHash;
     355                 : 
     356                 :   PRUint32 mTableId;
     357                 :   PRUint32 mChunkId;
     358                 :   PRUint32 mAddChunkId;
     359                 : };
     360                 : 
     361                 : bool
     362             517 : nsUrlClassifierEntry::Match(const nsUrlClassifierCompleteHash &hash)
     363                 : {
     364             517 :   if (mHaveComplete)
     365             268 :     return mCompleteHash == hash;
     366                 : 
     367             249 :   if (mHavePartial)
     368             249 :     return hash.StartsWith(mPartialHash);
     369                 : 
     370               0 :   return false;
     371                 : }
     372                 : 
     373                 : bool
     374              19 : nsUrlClassifierEntry::SubMatch(const nsUrlClassifierEntry &subEntry)
     375                 : {
     376              19 :   if ((mTableId != subEntry.mTableId) || (mChunkId != subEntry.mAddChunkId))
     377               0 :     return false;
     378                 : 
     379              19 :   if (subEntry.mHaveComplete)
     380              18 :     return mHaveComplete && mCompleteHash == subEntry.mCompleteHash;
     381                 : 
     382               1 :   if (subEntry.mHavePartial)
     383               1 :     return mHavePartial && mPartialHash == subEntry.mPartialHash;
     384                 : 
     385               0 :   return false;
     386                 : }
     387                 : 
     388                 : void
     389               0 : nsUrlClassifierEntry::Clear()
     390                 : {
     391               0 :   mId = -1;
     392               0 :   mHavePartial = false;
     393               0 :   mHaveComplete = false;
     394               0 : }
     395                 : 
     396                 : // -------------------------------------------------------------------------
     397                 : // Lookup result class implementation
     398                 : 
     399                 : // This helper class wraps a nsUrlClassifierEntry found during a lookup.
     400                 : class nsUrlClassifierLookupResult
     401              36 : {
     402                 : public:
     403             136 :   nsUrlClassifierLookupResult() : mConfirmed(false), mNoise(false) {
     404             136 :     mLookupFragment.Clear();
     405             136 :   }
     406             172 :   ~nsUrlClassifierLookupResult() {}
     407                 : 
     408              47 :   bool operator==(const nsUrlClassifierLookupResult &result) const {
     409                 :     // Don't need to compare table name, it's contained by id in the entry.
     410              47 :     return (mLookupFragment == result.mLookupFragment &&
     411                 :             mConfirmed == result.mConfirmed &&
     412              47 :             mEntry == result.mEntry);
     413                 :   }
     414                 : 
     415             108 :   bool operator<(const nsUrlClassifierLookupResult &result) const {
     416                 :     // Don't need to compare table name, it's contained by id in the entry.
     417             108 :     return (mLookupFragment < result.mLookupFragment ||
     418                 :             mConfirmed < result.mConfirmed ||
     419             108 :             mEntry < result.mEntry);
     420                 :   }
     421                 : 
     422                 :   // The hash that matched this entry.
     423                 :   nsUrlClassifierCompleteHash mLookupFragment;
     424                 : 
     425                 :   // The entry that was found during the lookup.
     426                 :   nsUrlClassifierEntry mEntry;
     427                 : 
     428                 :   // TRUE if the lookup matched a complete hash (not just a partial
     429                 :   // one).
     430                 :   bool mConfirmed;
     431                 : 
     432                 :   // TRUE if this lookup is gethash noise.  Does not represent an actual
     433                 :   // result.
     434                 :   bool mNoise;
     435                 : 
     436                 :   // The table name associated with mEntry.mTableId.
     437                 :   nsCString mTableName;
     438                 : };
     439                 : 
     440                 : // -------------------------------------------------------------------------
     441                 : // Store class implementation
     442                 : 
     443                 : // This class mediates access to the classifier and chunk entry tables.
     444                 : class nsUrlClassifierStore
     445                 : {
     446                 : public:
     447              16 :   nsUrlClassifierStore() {}
     448              32 :   virtual ~nsUrlClassifierStore() {}
     449                 : 
     450                 :   // Initialize the statements for the store.
     451                 :   nsresult Init(nsUrlClassifierDBServiceWorker *worker,
     452                 :                 mozIStorageConnection *connection,
     453                 :                 const nsACString& entriesTableName);
     454                 :   // Shut down the store.
     455                 :   void Close();
     456                 : 
     457                 :   // Read an entry from a database statement
     458                 :   virtual bool ReadStatement(mozIStorageStatement* statement,
     459                 :                                nsUrlClassifierEntry& entry);
     460                 : 
     461                 :   // Prepare a statement to write this entry to the database
     462                 :   virtual nsresult BindStatement(const nsUrlClassifierEntry& entry,
     463                 :                                  mozIStorageStatement* statement);
     464                 : 
     465                 :   // Read the entry with a given ID from the database
     466                 :   nsresult ReadEntry(PRInt64 id, nsUrlClassifierEntry& entry, bool *exists);
     467                 : 
     468                 :   // Remove an entry from the database
     469                 :   nsresult DeleteEntry(nsUrlClassifierEntry& entry);
     470                 : 
     471                 :   // Write an entry to the database
     472                 :   nsresult WriteEntry(nsUrlClassifierEntry& entry);
     473                 : 
     474                 :   // Update an entry in the database.  The entry must already exist in the
     475                 :   // database or this method will fail.
     476                 :   nsresult UpdateEntry(nsUrlClassifierEntry& entry);
     477                 : 
     478                 :   // Remove all entries for a given table/chunk pair from the database.
     479                 :   nsresult Expire(PRUint32 tableId,
     480                 :                   PRUint32 chunkNum);
     481                 : 
     482                 :   // Read a certain number of rows adjacent to the requested rowid that
     483                 :   // don't have complete hash data.
     484                 :   nsresult ReadNoiseEntries(PRInt64 rowID,
     485                 :                             PRUint32 numRequested,
     486                 :                             bool before,
     487                 :                             nsTArray<nsUrlClassifierEntry> &entries);
     488                 : 
     489                 :   // Ask the db for a random number.  This is temporary, and should be
     490                 :   // replaced with nsIRandomGenerator when 419739 is fixed.
     491                 :   nsresult RandomNumber(PRInt64 *randomNum);
     492                 :   // Return an array with all Prefixes known
     493                 :   nsresult ReadPrefixes(FallibleTArray<PRUint32>& array, PRUint32 aKey);
     494                 : 
     495                 : 
     496                 : protected:
     497                 :   nsresult ReadEntries(mozIStorageStatement *statement,
     498                 :                        nsTArray<nsUrlClassifierEntry>& entries);
     499                 :   nsUrlClassifierDBServiceWorker *mWorker;
     500                 :   nsCOMPtr<mozIStorageConnection> mConnection;
     501                 : 
     502                 :   nsCOMPtr<mozIStorageStatement> mLookupWithIDStatement;
     503                 : 
     504                 :   nsCOMPtr<mozIStorageStatement> mInsertStatement;
     505                 :   nsCOMPtr<mozIStorageStatement> mUpdateStatement;
     506                 :   nsCOMPtr<mozIStorageStatement> mDeleteStatement;
     507                 :   nsCOMPtr<mozIStorageStatement> mExpireStatement;
     508                 : 
     509                 :   nsCOMPtr<mozIStorageStatement> mPartialEntriesStatement;
     510                 :   nsCOMPtr<mozIStorageStatement> mPartialEntriesAfterStatement;
     511                 :   nsCOMPtr<mozIStorageStatement> mLastPartialEntriesStatement;
     512                 :   nsCOMPtr<mozIStorageStatement> mPartialEntriesBeforeStatement;
     513                 : 
     514                 :   nsCOMPtr<mozIStorageStatement> mRandomStatement;
     515                 :   nsCOMPtr<mozIStorageStatement> mAllPrefixGetStatement;
     516                 :   nsCOMPtr<mozIStorageStatement> mAllPrefixCountStatement;
     517                 : };
     518                 : 
     519                 : nsresult
     520             282 : nsUrlClassifierStore::Init(nsUrlClassifierDBServiceWorker *worker,
     521                 :                            mozIStorageConnection *connection,
     522                 :                            const nsACString& entriesName)
     523                 : {
     524             282 :   mWorker = worker;
     525             282 :   mConnection = connection;
     526                 : 
     527             282 :   nsresult rv = mConnection->CreateStatement
     528             564 :     (NS_LITERAL_CSTRING("SELECT * FROM ") + entriesName +
     529             846 :      NS_LITERAL_CSTRING(" WHERE id=?1"),
     530             846 :      getter_AddRefs(mLookupWithIDStatement));
     531             282 :   NS_ENSURE_SUCCESS(rv, rv);
     532                 : 
     533             282 :   rv = mConnection->CreateStatement
     534             564 :     (NS_LITERAL_CSTRING("DELETE FROM ") + entriesName +
     535             846 :      NS_LITERAL_CSTRING(" WHERE id=?1"),
     536             846 :      getter_AddRefs(mDeleteStatement));
     537             282 :   NS_ENSURE_SUCCESS(rv, rv);
     538                 : 
     539             282 :   rv = mConnection->CreateStatement
     540             564 :     (NS_LITERAL_CSTRING("DELETE FROM ") + entriesName +
     541             846 :      NS_LITERAL_CSTRING(" WHERE table_id=?1 AND chunk_id=?2"),
     542             846 :      getter_AddRefs(mExpireStatement));
     543             282 :   NS_ENSURE_SUCCESS(rv, rv);
     544                 : 
     545             282 :   rv = mConnection->CreateStatement
     546             564 :     (NS_LITERAL_CSTRING("SELECT * FROM ") + entriesName +
     547             846 :      NS_LITERAL_CSTRING(" WHERE complete_data ISNULL"
     548                 :                         " LIMIT ?1"),
     549             846 :      getter_AddRefs(mPartialEntriesStatement));
     550             282 :   NS_ENSURE_SUCCESS(rv, rv);
     551                 : 
     552             282 :   rv = mConnection->CreateStatement
     553             564 :     (NS_LITERAL_CSTRING("SELECT * FROM ") + entriesName +
     554             846 :      NS_LITERAL_CSTRING(" WHERE id > ?1 AND complete_data ISNULL"
     555                 :                         " LIMIT ?2"),
     556             846 :      getter_AddRefs(mPartialEntriesAfterStatement));
     557             282 :   NS_ENSURE_SUCCESS(rv, rv);
     558                 : 
     559             282 :   rv = mConnection->CreateStatement
     560             564 :     (NS_LITERAL_CSTRING("SELECT * FROM ") + entriesName +
     561             846 :      NS_LITERAL_CSTRING(" WHERE complete_data ISNULL"
     562                 :                         " ORDER BY id DESC LIMIT ?1"),
     563             846 :      getter_AddRefs(mLastPartialEntriesStatement));
     564             282 :   NS_ENSURE_SUCCESS(rv, rv);
     565                 : 
     566             282 :   rv = mConnection->CreateStatement
     567             564 :     (NS_LITERAL_CSTRING("SELECT * FROM ") + entriesName +
     568             846 :      NS_LITERAL_CSTRING(" WHERE id < ?1 AND complete_data ISNULL"
     569                 :                         " ORDER BY id DESC LIMIT ?2"),
     570             846 :      getter_AddRefs(mPartialEntriesBeforeStatement));
     571             282 :   NS_ENSURE_SUCCESS(rv, rv);
     572                 : 
     573             282 :   rv = mConnection->CreateStatement
     574             282 :     (NS_LITERAL_CSTRING("SELECT abs(random())"),
     575             564 :      getter_AddRefs(mRandomStatement));
     576             282 :   NS_ENSURE_SUCCESS(rv, rv);
     577                 : 
     578             564 :   rv = mConnection->CreateStatement(NS_LITERAL_CSTRING("SELECT domain, partial_data, complete_data FROM ")
     579             282 :     + entriesName,
     580             846 :     getter_AddRefs(mAllPrefixGetStatement));
     581             282 :   NS_ENSURE_SUCCESS(rv, rv);
     582                 : 
     583             564 :   rv = mConnection->CreateStatement(NS_LITERAL_CSTRING("SELECT COUNT(1) FROM ")
     584             282 :     + entriesName,
     585             846 :     getter_AddRefs(mAllPrefixCountStatement));
     586             282 :   NS_ENSURE_SUCCESS(rv, rv);
     587                 : 
     588             282 :   return NS_OK;
     589                 : }
     590                 : 
     591                 : void
     592             282 : nsUrlClassifierStore::Close()
     593                 : {
     594             282 :   mLookupWithIDStatement = nsnull;
     595                 : 
     596             282 :   mInsertStatement = nsnull;
     597             282 :   mUpdateStatement = nsnull;
     598             282 :   mDeleteStatement = nsnull;
     599             282 :   mExpireStatement = nsnull;
     600                 : 
     601             282 :   mPartialEntriesStatement = nsnull;
     602             282 :   mPartialEntriesAfterStatement = nsnull;
     603             282 :   mPartialEntriesBeforeStatement = nsnull;
     604             282 :   mLastPartialEntriesStatement = nsnull;
     605             282 :   mRandomStatement = nsnull;
     606                 : 
     607             282 :   mAllPrefixGetStatement = nsnull;
     608             282 :   mAllPrefixCountStatement = nsnull;
     609                 : 
     610             282 :   mConnection = nsnull;
     611             282 : }
     612                 : 
     613                 : 
     614                 : bool
     615             204 : nsUrlClassifierStore::ReadStatement(mozIStorageStatement* statement,
     616                 :                                     nsUrlClassifierEntry& entry)
     617                 : {
     618             204 :   entry.mId = statement->AsInt64(0);
     619                 : 
     620                 :   PRUint32 size;
     621             204 :   const PRUint8* blob = statement->AsSharedBlob(1, &size);
     622             204 :   if (!blob || (size != DOMAIN_LENGTH))
     623               0 :     return false;
     624             204 :   memcpy(entry.mKey.buf, blob, DOMAIN_LENGTH);
     625                 : 
     626             204 :   blob = statement->AsSharedBlob(2, &size);
     627             204 :   if (!blob || size == 0) {
     628             147 :     entry.mHavePartial = false;
     629                 :   } else {
     630              57 :     if (size != PARTIAL_LENGTH)
     631               0 :       return false;
     632              57 :     entry.mHavePartial = true;
     633              57 :     memcpy(entry.mPartialHash.buf, blob, PARTIAL_LENGTH);
     634                 :   }
     635                 : 
     636             204 :   blob = statement->AsSharedBlob(3, &size);
     637             204 :   if (!blob || size == 0) {
     638              59 :     entry.mHaveComplete = false;
     639                 :   } else {
     640             145 :     if (size != COMPLETE_LENGTH)
     641               0 :       return false;
     642             145 :     entry.mHaveComplete = true;
     643             145 :     memcpy(entry.mCompleteHash.buf, blob, COMPLETE_LENGTH);
     644                 :   }
     645                 : 
     646                 :   // If we only have a partial entry, and that partial entry matches the
     647                 :   // domain, we don't save the extra copy to the database.
     648             204 :   if (!(entry.mHavePartial || entry.mHaveComplete)) {
     649               5 :     entry.SetHash(entry.mKey);
     650                 :   }
     651                 : 
     652             204 :   entry.mChunkId = statement->AsInt32(4);
     653             204 :   entry.mTableId = statement->AsInt32(5);
     654                 : 
     655             204 :   return true;
     656                 : }
     657                 : 
     658                 : nsresult
     659             199 : nsUrlClassifierStore::BindStatement(const nsUrlClassifierEntry &entry,
     660                 :                                     mozIStorageStatement* statement)
     661                 : {
     662                 :   nsresult rv;
     663                 : 
     664             199 :   if (entry.mId == -1)
     665             163 :     rv = statement->BindNullByIndex(0);
     666                 :   else
     667              36 :     rv = statement->BindInt64ByIndex(0, entry.mId);
     668             199 :   NS_ENSURE_SUCCESS(rv, rv);
     669                 : 
     670             199 :   rv = statement->BindBlobByIndex(1, entry.mKey.buf, DOMAIN_LENGTH);
     671             199 :   NS_ENSURE_SUCCESS(rv, rv);
     672                 : 
     673             199 :   if (entry.mHavePartial) {
     674                 :     // If we only have a partial entry and that entry matches the domain,
     675                 :     // we'll save some space by only storing the domain hash.
     676              82 :     if (!entry.mHaveComplete && entry.mKey == entry.mPartialHash) {
     677               5 :       rv = statement->BindNullByIndex(2);
     678                 :     } else {
     679                 :       rv = statement->BindBlobByIndex(2, entry.mPartialHash.buf,
     680              77 :                                         PARTIAL_LENGTH);
     681                 :     }
     682                 :   } else {
     683             117 :     rv = statement->BindNullByIndex(2);
     684                 :   }
     685             199 :   NS_ENSURE_SUCCESS(rv, rv);
     686                 : 
     687             199 :   if (entry.mHaveComplete) {
     688             153 :     rv = statement->BindBlobByIndex(3, entry.mCompleteHash.buf, COMPLETE_LENGTH);
     689                 :   } else {
     690              46 :     rv = statement->BindNullByIndex(3);
     691                 :   }
     692             199 :   NS_ENSURE_SUCCESS(rv, rv);
     693                 : 
     694             199 :   rv = statement->BindInt32ByIndex(4, entry.mChunkId);
     695             199 :   NS_ENSURE_SUCCESS(rv, rv);
     696                 : 
     697             199 :   rv = statement->BindInt32ByIndex(5, entry.mTableId);
     698             199 :   NS_ENSURE_SUCCESS(rv, rv);
     699                 : 
     700             199 :   return true;
     701                 : }
     702                 : 
     703                 : nsresult
     704             242 : nsUrlClassifierStore::ReadEntries(mozIStorageStatement *statement,
     705                 :                                   nsTArray<nsUrlClassifierEntry>& entries)
     706                 : {
     707                 :   bool exists;
     708             242 :   nsresult rv = statement->ExecuteStep(&exists);
     709             242 :   NS_ENSURE_SUCCESS(rv, rv);
     710                 : 
     711             688 :   while (exists) {
     712             204 :     nsUrlClassifierEntry *entry = entries.AppendElement();
     713             204 :     if (!entry) {
     714               0 :       return NS_ERROR_OUT_OF_MEMORY;
     715                 :     }
     716                 : 
     717             204 :     if (!ReadStatement(statement, *entry))
     718               0 :       return NS_ERROR_FAILURE;
     719                 : 
     720             204 :     statement->ExecuteStep(&exists);
     721                 :   }
     722                 : 
     723             242 :   return NS_OK;
     724                 : }
     725                 : 
     726                 : nsresult
     727               0 : nsUrlClassifierStore::ReadEntry(PRInt64 id,
     728                 :                                 nsUrlClassifierEntry& entry,
     729                 :                                 bool *exists)
     730                 : {
     731               0 :   entry.Clear();
     732                 : 
     733               0 :   mozStorageStatementScoper scoper(mLookupWithIDStatement);
     734                 : 
     735               0 :   nsresult rv = mLookupWithIDStatement->BindInt64ByIndex(0, id);
     736               0 :   NS_ENSURE_SUCCESS(rv, rv);
     737                 : 
     738               0 :   rv = mLookupWithIDStatement->ExecuteStep(exists);
     739               0 :   NS_ENSURE_SUCCESS(rv, rv);
     740                 : 
     741               0 :   if (*exists) {
     742               0 :     if (ReadStatement(mLookupWithIDStatement, entry))
     743               0 :       return NS_ERROR_FAILURE;
     744                 :   }
     745                 : 
     746               0 :   return NS_OK;
     747                 : }
     748                 : 
     749                 : nsresult
     750               0 : nsUrlClassifierStore::ReadNoiseEntries(PRInt64 rowID,
     751                 :                                        PRUint32 numRequested,
     752                 :                                        bool before,
     753                 :                                        nsTArray<nsUrlClassifierEntry> &entries)
     754                 : {
     755               0 :   if (numRequested == 0) {
     756               0 :     return NS_OK;
     757                 :   }
     758                 : 
     759                 :   mozIStorageStatement *statement =
     760               0 :     before ? mPartialEntriesBeforeStatement : mPartialEntriesAfterStatement;
     761               0 :   mozStorageStatementScoper scoper(statement);
     762                 : 
     763               0 :   nsresult rv = statement->BindInt64ByIndex(0, rowID);
     764               0 :   NS_ENSURE_SUCCESS(rv, rv);
     765                 : 
     766               0 :   statement->BindInt32ByIndex(1, numRequested);
     767               0 :   NS_ENSURE_SUCCESS(rv, rv);
     768                 : 
     769               0 :   PRUint32 length = entries.Length();
     770               0 :   rv = ReadEntries(statement, entries);
     771               0 :   NS_ENSURE_SUCCESS(rv, rv);
     772                 : 
     773               0 :   PRUint32 numRead = entries.Length() - length;
     774                 : 
     775               0 :   if (numRead >= numRequested)
     776               0 :     return NS_OK;
     777                 : 
     778                 :   // If we didn't get enough entries, we need the search to wrap around from
     779                 :   // beginning to end (or vice-versa)
     780                 : 
     781                 :   mozIStorageStatement *wraparoundStatement =
     782               0 :     before ? mPartialEntriesStatement : mLastPartialEntriesStatement;
     783               0 :   mozStorageStatementScoper wraparoundScoper(wraparoundStatement);
     784                 : 
     785               0 :   rv = wraparoundStatement->BindInt32ByIndex(0, numRequested - numRead);
     786               0 :   NS_ENSURE_SUCCESS(rv, rv);
     787                 : 
     788               0 :   return ReadEntries(wraparoundStatement, entries);
     789                 : }
     790                 : 
     791                 : nsresult
     792               0 : nsUrlClassifierStore::RandomNumber(PRInt64 *randomNum)
     793                 : {
     794               0 :   mozStorageStatementScoper randScoper(mRandomStatement);
     795                 :   bool exists;
     796               0 :   nsresult rv = mRandomStatement->ExecuteStep(&exists);
     797               0 :   NS_ENSURE_SUCCESS(rv, rv);
     798               0 :   if (!exists)
     799               0 :     return NS_ERROR_NOT_AVAILABLE;
     800                 : 
     801               0 :   *randomNum = mRandomStatement->AsInt64(0);
     802                 : 
     803               0 :   return NS_OK;
     804                 : }
     805                 : 
     806                 : // -------------------------------------------------------------------------
     807                 : // nsUrlClassifierAddStore class implementation
     808                 : 
     809                 : // This class accesses the moz_classifier table.
     810                 : class nsUrlClassifierAddStore: public nsUrlClassifierStore
     811                 : {
     812                 : public:
     813               8 :   nsUrlClassifierAddStore() {};
     814              16 :   virtual ~nsUrlClassifierAddStore() {};
     815                 : 
     816                 :   nsresult Init(nsUrlClassifierDBServiceWorker *worker,
     817                 :                 mozIStorageConnection *connection,
     818                 :                 const nsACString& entriesTableName);
     819                 : 
     820                 :   void Close();
     821                 : 
     822                 :   // Read the entries for a given key/table/chunk from the database
     823                 :   nsresult ReadAddEntries(const nsUrlClassifierDomainHash& key,
     824                 :                           PRUint32 tableId,
     825                 :                           PRUint32 chunkId,
     826                 :                           nsTArray<nsUrlClassifierEntry>& entry);
     827                 : 
     828                 :   // Read the entries for a given host key from the database.
     829                 :   nsresult ReadAddEntries(const nsUrlClassifierDomainHash& key,
     830                 :                           nsTArray<nsUrlClassifierEntry>& entry);
     831                 : 
     832                 : protected:
     833                 :   nsCOMPtr<mozIStorageStatement> mLookupStatement;
     834                 :   nsCOMPtr<mozIStorageStatement> mLookupWithChunkStatement;
     835                 : };
     836                 : 
     837                 : nsresult
     838             141 : nsUrlClassifierAddStore::Init(nsUrlClassifierDBServiceWorker *worker,
     839                 :                               mozIStorageConnection *connection,
     840                 :                               const nsACString &entriesTableName)
     841                 : {
     842                 :   nsresult rv = nsUrlClassifierStore::Init(worker, connection,
     843             141 :                                            entriesTableName);
     844             141 :   NS_ENSURE_SUCCESS(rv, rv);
     845                 : 
     846             141 :   rv = mConnection->CreateStatement
     847             282 :     (NS_LITERAL_CSTRING("INSERT OR REPLACE INTO ") + entriesTableName +
     848             423 :      NS_LITERAL_CSTRING(" VALUES (?1, ?2, ?3, ?4, ?5, ?6)"),
     849             423 :      getter_AddRefs(mInsertStatement));
     850             141 :   NS_ENSURE_SUCCESS(rv, rv);
     851                 : 
     852             141 :   rv = mConnection->CreateStatement
     853             282 :     (NS_LITERAL_CSTRING("UPDATE ") + entriesTableName +
     854             423 :      NS_LITERAL_CSTRING(" SET domain=?2, partial_data=?3, "
     855                 :                         " complete_data=?4, chunk_id=?5, table_id=?6"
     856                 :                         " WHERE id=?1"),
     857             423 :      getter_AddRefs(mUpdateStatement));
     858             141 :   NS_ENSURE_SUCCESS(rv, rv);
     859                 : 
     860             141 :   rv = mConnection->CreateStatement
     861             282 :     (NS_LITERAL_CSTRING("SELECT * FROM ") + entriesTableName +
     862             423 :      NS_LITERAL_CSTRING(" WHERE domain=?1"),
     863             423 :      getter_AddRefs(mLookupStatement));
     864             141 :   NS_ENSURE_SUCCESS(rv, rv);
     865                 : 
     866             141 :   rv = mConnection->CreateStatement
     867             282 :     (NS_LITERAL_CSTRING("SELECT * FROM ") + entriesTableName +
     868             423 :      NS_LITERAL_CSTRING(" WHERE domain=?1 AND table_id=?2 AND chunk_id=?3"),
     869             423 :      getter_AddRefs(mLookupWithChunkStatement));
     870             141 :   NS_ENSURE_SUCCESS(rv, rv);
     871                 : 
     872             141 :   return NS_OK;
     873                 : }
     874                 : 
     875                 : void
     876             141 : nsUrlClassifierAddStore::Close()
     877                 : {
     878             141 :   nsUrlClassifierStore::Close();
     879                 : 
     880             141 :   mLookupStatement = nsnull;
     881             141 :   mLookupWithChunkStatement = nsnull;
     882             141 : }
     883                 : 
     884                 : nsresult
     885              12 : nsUrlClassifierAddStore::ReadAddEntries(const nsUrlClassifierDomainHash& hash,
     886                 :                                         PRUint32 tableId,
     887                 :                                         PRUint32 chunkId,
     888                 :                                         nsTArray<nsUrlClassifierEntry>& entries)
     889                 : {
     890              24 :   mozStorageStatementScoper scoper(mLookupWithChunkStatement);
     891                 : 
     892              12 :   nsresult rv = mLookupWithChunkStatement->BindBlobByIndex
     893              12 :                   (0, hash.buf, DOMAIN_LENGTH);
     894              12 :   NS_ENSURE_SUCCESS(rv, rv);
     895                 : 
     896              12 :   rv = mLookupWithChunkStatement->BindInt32ByIndex(1, tableId);
     897              12 :   NS_ENSURE_SUCCESS(rv, rv);
     898              12 :   rv = mLookupWithChunkStatement->BindInt32ByIndex(2, chunkId);
     899              12 :   NS_ENSURE_SUCCESS(rv, rv);
     900                 : 
     901              12 :   return ReadEntries(mLookupWithChunkStatement, entries);
     902                 : }
     903                 : 
     904                 : nsresult
     905             149 : nsUrlClassifierAddStore::ReadAddEntries(const nsUrlClassifierDomainHash& hash,
     906                 :                                         nsTArray<nsUrlClassifierEntry>& entries)
     907                 : {
     908             298 :   mozStorageStatementScoper scoper(mLookupStatement);
     909                 : 
     910             149 :   nsresult rv = mLookupStatement->BindBlobByIndex
     911             149 :                   (0, hash.buf, DOMAIN_LENGTH);
     912             149 :   NS_ENSURE_SUCCESS(rv, rv);
     913                 : 
     914             149 :   return ReadEntries(mLookupStatement, entries);
     915                 : }
     916                 : 
     917                 : // -------------------------------------------------------------------------
     918                 : // nsUrlClassifierSubStore class implementation
     919                 : 
     920                 : // This class accesses the moz_subs table.
     921                 : class nsUrlClassifierSubStore : public nsUrlClassifierStore
     922                 : {
     923                 : public:
     924               8 :   nsUrlClassifierSubStore() {};
     925              16 :   virtual ~nsUrlClassifierSubStore() {};
     926                 : 
     927                 :   nsresult Init(nsUrlClassifierDBServiceWorker *worker,
     928                 :                 mozIStorageConnection *connection,
     929                 :                 const nsACString& entriesTableName);
     930                 : 
     931                 :   void Close();
     932                 : 
     933                 :   // Read an entry from a database statement
     934                 :   virtual bool ReadStatement(mozIStorageStatement* statement,
     935                 :                                nsUrlClassifierEntry& entry);
     936                 : 
     937                 :   // Prepare a statement to write this entry to the database
     938                 :   virtual nsresult BindStatement(const nsUrlClassifierEntry& entry,
     939                 :                                  mozIStorageStatement* statement);
     940                 : 
     941                 :   // Read sub entries for a given add chunk
     942                 :   nsresult ReadSubEntries(PRUint32 tableId, PRUint32 chunkId,
     943                 :                           nsTArray<nsUrlClassifierEntry> &subEntry);
     944                 : 
     945                 :   // Expire sub entries for a given add chunk
     946                 :   nsresult ExpireAddChunk(PRUint32 tableId, PRUint32 chunkId);
     947                 : 
     948                 : protected:
     949                 :   nsCOMPtr<mozIStorageStatement> mLookupWithAddChunkStatement;
     950                 :   nsCOMPtr<mozIStorageStatement> mExpireAddChunkStatement;
     951                 : };
     952                 : 
     953                 : nsresult
     954             141 : nsUrlClassifierSubStore::Init(nsUrlClassifierDBServiceWorker *worker,
     955                 :                               mozIStorageConnection *connection,
     956                 :                               const nsACString &entriesTableName)
     957                 : {
     958                 :   nsresult rv = nsUrlClassifierStore::Init(worker, connection,
     959             141 :                                            entriesTableName);
     960             141 :   NS_ENSURE_SUCCESS(rv, rv);
     961                 : 
     962             141 :   rv = mConnection->CreateStatement
     963             282 :     (NS_LITERAL_CSTRING("INSERT OR REPLACE INTO ") + entriesTableName +
     964             423 :      NS_LITERAL_CSTRING(" VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)"),
     965             423 :      getter_AddRefs(mInsertStatement));
     966             141 :   NS_ENSURE_SUCCESS(rv, rv);
     967                 : 
     968             141 :   rv = mConnection->CreateStatement
     969             282 :     (NS_LITERAL_CSTRING("UPDATE ") + entriesTableName +
     970             423 :      NS_LITERAL_CSTRING(" SET domain=?2, partial_data=?3, complete_data=?4,"
     971                 :                         " chunk_id=?5, table_id=?6, add_chunk_id=?7"
     972                 :                         " WHERE id=?1"),
     973             423 :      getter_AddRefs(mUpdateStatement));
     974             141 :   NS_ENSURE_SUCCESS(rv, rv);
     975                 : 
     976             141 :   rv = mConnection->CreateStatement
     977             282 :     (NS_LITERAL_CSTRING("SELECT * FROM ") + entriesTableName +
     978             423 :      NS_LITERAL_CSTRING(" WHERE table_id=?1 AND add_chunk_id=?2"),
     979             423 :      getter_AddRefs(mLookupWithAddChunkStatement));
     980             141 :   NS_ENSURE_SUCCESS(rv, rv);
     981                 : 
     982             141 :   rv = mConnection->CreateStatement
     983             282 :     (NS_LITERAL_CSTRING("DELETE FROM ") + entriesTableName +
     984             423 :      NS_LITERAL_CSTRING(" WHERE table_id=?1 AND add_chunk_id=?2"),
     985             423 :      getter_AddRefs(mExpireAddChunkStatement));
     986             141 :   NS_ENSURE_SUCCESS(rv, rv);
     987                 : 
     988             141 :   return NS_OK;
     989                 : }
     990                 : 
     991                 : bool
     992               8 : nsUrlClassifierSubStore::ReadStatement(mozIStorageStatement* statement,
     993                 :                                        nsUrlClassifierEntry& entry)
     994                 : {
     995               8 :   if (!nsUrlClassifierStore::ReadStatement(statement, entry))
     996               0 :     return false;
     997                 : 
     998               8 :   entry.mAddChunkId = statement->AsInt32(6);
     999               8 :   return true;
    1000                 : }
    1001                 : 
    1002                 : nsresult
    1003              15 : nsUrlClassifierSubStore::BindStatement(const nsUrlClassifierEntry& entry,
    1004                 :                                        mozIStorageStatement* statement)
    1005                 : {
    1006              15 :   nsresult rv = nsUrlClassifierStore::BindStatement(entry, statement);
    1007              15 :   NS_ENSURE_SUCCESS(rv, rv);
    1008                 : 
    1009              15 :   return statement->BindInt32ByIndex(6, entry.mAddChunkId);
    1010                 : }
    1011                 : 
    1012                 : nsresult
    1013              81 : nsUrlClassifierSubStore::ReadSubEntries(PRUint32 tableId, PRUint32 addChunkId,
    1014                 :                                         nsTArray<nsUrlClassifierEntry>& entries)
    1015                 : {
    1016             162 :   mozStorageStatementScoper scoper(mLookupWithAddChunkStatement);
    1017                 : 
    1018              81 :   nsresult rv = mLookupWithAddChunkStatement->BindInt32ByIndex(0, tableId);
    1019              81 :   NS_ENSURE_SUCCESS(rv, rv);
    1020              81 :   rv = mLookupWithAddChunkStatement->BindInt32ByIndex(1, addChunkId);
    1021              81 :   NS_ENSURE_SUCCESS(rv, rv);
    1022                 : 
    1023              81 :   return ReadEntries(mLookupWithAddChunkStatement, entries);
    1024                 : }
    1025                 : 
    1026                 : nsresult
    1027              81 : nsUrlClassifierSubStore::ExpireAddChunk(PRUint32 tableId, PRUint32 addChunkId)
    1028                 : {
    1029             162 :   mozStorageStatementScoper scoper(mExpireAddChunkStatement);
    1030                 : 
    1031              81 :   nsresult rv = mExpireAddChunkStatement->BindInt32ByIndex(0, tableId);
    1032              81 :   NS_ENSURE_SUCCESS(rv, rv);
    1033              81 :   rv = mExpireAddChunkStatement->BindInt32ByIndex(1, addChunkId);
    1034              81 :   NS_ENSURE_SUCCESS(rv, rv);
    1035                 : 
    1036              81 :   return mExpireAddChunkStatement->Execute();
    1037                 : }
    1038                 : 
    1039                 : void
    1040             141 : nsUrlClassifierSubStore::Close()
    1041                 : {
    1042             141 :   nsUrlClassifierStore::Close();
    1043             141 :   mLookupWithAddChunkStatement = nsnull;
    1044             141 :   mExpireAddChunkStatement = nsnull;
    1045             141 : }
    1046                 : 
    1047                 : // Similar to GetKey(), but if the domain contains three or more components,
    1048                 : // two keys will be returned:
    1049                 : //  hostname.com/foo/bar -> [hostname.com]
    1050                 : //  mail.hostname.com/foo/bar -> [hostname.com, mail.hostname.com]
    1051                 : //  www.mail.hostname.com/foo/bar -> [hostname.com, mail.hostname.com]
    1052                 : static nsresult GetHostKeys(const nsACString &spec,
    1053                 :                             nsTArray<nsCString> &hostKeys);
    1054                 : 
    1055                 : // take a lookup string (www.hostname.com/path/to/resource.html) and
    1056                 : // expand it into the set of fragments that should be searched for in an
    1057                 : // entry
    1058                 : static nsresult GetLookupFragments(const nsCSubstring& spec,
    1059                 :                                    nsTArray<nsCString>& fragments);
    1060                 : 
    1061                 : // Check for a canonicalized IP address.
    1062                 : static bool IsCanonicalizedIP(const nsACString& host);
    1063                 : 
    1064                 : // Get the database key for a given URI.  This is the top three
    1065                 : // domain components if they exist, otherwise the top two.
    1066                 : //  hostname.com/foo/bar -> hostname.com
    1067                 : //  mail.hostname.com/foo/bar -> mail.hostname.com
    1068                 : //  www.mail.hostname.com/foo/bar -> mail.hostname.com
    1069                 : static nsresult GetKey(const nsACString& spec, nsUrlClassifierDomainHash& hash,
    1070                 :                        nsICryptoHash * aCryptoHash);
    1071                 : 
    1072                 : // We have both a prefix and a domain. Drop the domain, but
    1073                 : // hash the domain, the prefix and a random value together,
    1074                 : // ensuring any collisions happens at a different points for
    1075                 : // different users.
    1076                 : static nsresult KeyedHash(PRUint32 aPref, PRUint32 aDomain,
    1077                 :                           PRUint32 aKey, PRUint32 *aOut);
    1078                 : 
    1079                 : 
    1080                 : // -------------------------------------------------------------------------
    1081                 : // Actual worker implemenatation
    1082                 : class nsUrlClassifierDBServiceWorker : public nsIUrlClassifierDBServiceWorker
    1083                 : {
    1084                 : public:
    1085                 :   nsUrlClassifierDBServiceWorker();
    1086                 : 
    1087                 :   NS_DECL_ISUPPORTS
    1088                 :   NS_DECL_NSIURLCLASSIFIERDBSERVICE
    1089                 :   NS_DECL_NSIURLCLASSIFIERDBSERVICEWORKER
    1090                 : 
    1091                 :   // Initialize, called in the main thread
    1092                 :   nsresult Init(PRInt32 gethashNoise,
    1093                 :                 nsRefPtr<nsUrlClassifierPrefixSet> & prefSet);
    1094                 : 
    1095                 :   // Queue a lookup for the worker to perform, called in the main thread.
    1096                 :   nsresult QueueLookup(const nsACString& lookupKey,
    1097                 :                        nsIUrlClassifierLookupCallback* callback);
    1098                 : 
    1099                 :   // Handle any queued-up lookups.  We call this function during long-running
    1100                 :   // update operations to prevent lookups from blocking for too long.
    1101                 :   nsresult HandlePendingLookups();
    1102                 : 
    1103                 :   // Blocks the PrefixSet from being updated while the main thread is doing
    1104                 :   // its lookups. LockPrefixSet will return whether the PrefixSet is in a
    1105                 :   // usable state. If not, we should fall through to SQLite lookups.
    1106                 :   bool LockPrefixSet();
    1107                 :   void UnlockPrefixSet();
    1108                 : 
    1109                 : private:
    1110                 :   // No subclassing
    1111                 :   ~nsUrlClassifierDBServiceWorker();
    1112                 : 
    1113                 :   // Disallow copy constructor
    1114                 :   nsUrlClassifierDBServiceWorker(nsUrlClassifierDBServiceWorker&);
    1115                 : 
    1116                 :   // Try to open the db, DATABASE_FILENAME.
    1117                 :   nsresult OpenDb();
    1118                 : 
    1119                 :   // Create table in the db if they don't exist.
    1120                 :   nsresult MaybeCreateTables(mozIStorageConnection* connection);
    1121                 : 
    1122                 :   nsresult GetTableName(PRUint32 tableId, nsACString& table);
    1123                 :   nsresult GetTableId(const nsACString& table, PRUint32* tableId);
    1124                 : 
    1125                 :   // Decompress a zlib'ed chunk (used for -exp tables)
    1126                 :   nsresult InflateChunk(nsACString& chunk);
    1127                 : 
    1128                 :   // Expand shavar chunk into its individual entries
    1129                 :   nsresult GetShaEntries(PRUint32 tableId,
    1130                 :                          PRUint32 chunkType,
    1131                 :                          PRUint32 chunkNum,
    1132                 :                          PRUint32 domainSize,
    1133                 :                          PRUint32 hashSize,
    1134                 :                          nsACString& chunk,
    1135                 :                          nsTArray<nsUrlClassifierEntry>& entries);
    1136                 : 
    1137                 :   // Expand a chunk into its individual entries
    1138                 :   nsresult GetChunkEntries(const nsACString& table,
    1139                 :                            PRUint32 tableId,
    1140                 :                            PRUint32 chunkType,
    1141                 :                            PRUint32 chunkNum,
    1142                 :                            PRUint32 hashSize,
    1143                 :                            nsACString& chunk,
    1144                 :                            nsTArray<nsUrlClassifierEntry>& entries);
    1145                 : 
    1146                 :   // Parse one stringified range of chunks of the form "n" or "n-m" from a
    1147                 :   // comma-separated list of chunks.  Upon return, 'begin' will point to the
    1148                 :   // next range of chunks in the list of chunks.
    1149                 :   bool ParseChunkRange(nsACString::const_iterator &begin,
    1150                 :                          const nsACString::const_iterator &end,
    1151                 :                          PRUint32 *first, PRUint32 *last);
    1152                 : 
    1153                 :   // Expand a stringified chunk list into an array of ints.
    1154                 :   nsresult ParseChunkList(const nsACString& chunkStr,
    1155                 :                           nsTArray<PRUint32>& chunks);
    1156                 : 
    1157                 :   // Join an array of ints into a stringified chunk list.
    1158                 :   nsresult JoinChunkList(nsTArray<PRUint32>& chunks, nsCString& chunkStr);
    1159                 : 
    1160                 :   // List the add/subtract chunks that have been applied to a table
    1161                 :   nsresult GetChunkLists(PRUint32 tableId,
    1162                 :                          nsACString& addChunks,
    1163                 :                          nsACString& subChunks);
    1164                 : 
    1165                 :   // Set the list of add/subtract chunks that have been applied to a table
    1166                 :   nsresult SetChunkLists(PRUint32 tableId,
    1167                 :                          const nsACString& addChunks,
    1168                 :                          const nsACString& subChunks);
    1169                 : 
    1170                 :   // Cache the list of add/subtract chunks applied to the table, optionally
    1171                 :   // parsing the add or sub lists.  These lists are cached while updating
    1172                 :   // tables to avoid excessive database reads/writes and parsing.
    1173                 :   nsresult CacheChunkLists(PRUint32 tableId,
    1174                 :                            bool parseAdds,
    1175                 :                            bool parseSubs);
    1176                 : 
    1177                 :   // Clear the cached list of add/subtract chunks.
    1178                 :   void ClearCachedChunkLists();
    1179                 : 
    1180                 :   // Flush the cached add/subtract lists to the database.
    1181                 :   nsresult FlushChunkLists();
    1182                 : 
    1183                 :   // Inserts a chunk id into the list, sorted.  Returns TRUE if the
    1184                 :   // number was successfully added, FALSE if the chunk already exists.
    1185                 :   bool InsertChunkId(nsTArray<PRUint32>& chunks, PRUint32 chunkNum);
    1186                 : 
    1187                 :   // Add a list of entries to the database, merging with
    1188                 :   // existing entries as necessary
    1189                 :   nsresult AddChunk(PRUint32 tableId, PRUint32 chunkNum,
    1190                 :                     nsTArray<nsUrlClassifierEntry>& entries);
    1191                 : 
    1192                 :   // Expire an add chunk
    1193                 :   nsresult ExpireAdd(PRUint32 tableId, PRUint32 chunkNum);
    1194                 : 
    1195                 :   // Subtract a list of entries from the database
    1196                 :   nsresult SubChunk(PRUint32 tableId, PRUint32 chunkNum,
    1197                 :                     nsTArray<nsUrlClassifierEntry>& entries);
    1198                 : 
    1199                 :   // Expire a subtract chunk
    1200                 :   nsresult ExpireSub(PRUint32 tableId, PRUint32 chunkNum);
    1201                 : 
    1202                 :   // Handle line-oriented control information from a stream update
    1203                 :   nsresult ProcessResponseLines(bool* done);
    1204                 :   // Handle chunk data from a stream update
    1205                 :   nsresult ProcessChunk(bool* done);
    1206                 : 
    1207                 :   // Sets up a transaction and begins counting update time.
    1208                 :   nsresult SetupUpdate();
    1209                 : 
    1210                 :   // Applies the current transaction and resets the update/working times.
    1211                 :   nsresult ApplyUpdate();
    1212                 : 
    1213                 :   // Reset the in-progress update stream
    1214                 :   void ResetStream();
    1215                 : 
    1216                 :   // Reset the in-progress update
    1217                 :   void ResetUpdate();
    1218                 : 
    1219                 :   // Look for a given lookup string (www.hostname.com/path/to/resource.html)
    1220                 :   // Returns a list of entries that match.
    1221                 :   nsresult Check(const nsCSubstring& spec,
    1222                 :                  nsTArray<nsUrlClassifierLookupResult>& results);
    1223                 : 
    1224                 :   // Perform a classifier lookup for a given url.
    1225                 :   nsresult DoLookup(const nsACString& spec, nsIUrlClassifierLookupCallback* c);
    1226                 : 
    1227                 :   // Add entries to the results.
    1228                 :   nsresult AddNoise(PRInt64 nearID,
    1229                 :                     PRInt32 count,
    1230                 :                     nsTArray<nsUrlClassifierLookupResult>& results);
    1231                 : 
    1232                 :   // Construct a Prefix Set with known prefixes
    1233                 :   nsresult LoadPrefixSet(nsCOMPtr<nsIFile> & aFile);
    1234                 :   nsresult ConstructPrefixSet();
    1235                 : 
    1236                 :   // Set the SQLite cache size
    1237                 :   nsresult SetCacheSize(mozIStorageConnection * aConnection,
    1238                 :                         PRInt32 aCacheSize);
    1239                 : 
    1240                 :   nsCOMPtr<nsIFile> mDBFile;
    1241                 :   nsCOMPtr<nsIFile> mPSFile;
    1242                 : 
    1243                 :   nsCOMPtr<nsICryptoHash> mCryptoHash;
    1244                 : 
    1245                 :   // Holds a connection to the Db.  We lazily initialize this because it has
    1246                 :   // to be created in the background thread (currently mozStorageConnection
    1247                 :   // isn't thread safe).
    1248                 :   nsCOMPtr<mozIStorageConnection> mConnection;
    1249                 : 
    1250                 :   // The main collection of entries.  This is the store that will be checked
    1251                 :   // when classifying a URL.
    1252                 :   nsUrlClassifierAddStore mMainStore;
    1253                 : 
    1254                 :   // The collection of subs waiting for their accompanying add.
    1255                 :   nsUrlClassifierSubStore mPendingSubStore;
    1256                 : 
    1257                 :   nsCOMPtr<mozIStorageStatement> mGetChunkListsStatement;
    1258                 :   nsCOMPtr<mozIStorageStatement> mSetChunkListsStatement;
    1259                 : 
    1260                 :   nsCOMPtr<mozIStorageStatement> mGetTablesStatement;
    1261                 :   nsCOMPtr<mozIStorageStatement> mGetTableIdStatement;
    1262                 :   nsCOMPtr<mozIStorageStatement> mGetTableNameStatement;
    1263                 :   nsCOMPtr<mozIStorageStatement> mInsertTableIdStatement;
    1264                 :   nsCOMPtr<mozIStorageStatement> mGetPageSizeStatement;
    1265                 : 
    1266                 :   // Stores the last time a given table was updated.
    1267                 :   nsDataHashtable<nsCStringHashKey, PRInt64> mTableFreshness;
    1268                 : 
    1269                 :   // We receive data in small chunks that may be broken in the middle of
    1270                 :   // a line.  So we save the last partial line here.
    1271                 :   nsCString mPendingStreamUpdate;
    1272                 : 
    1273                 :   PRInt32 mUpdateWait;
    1274                 : 
    1275                 :   bool mResetRequested;
    1276                 :   bool mGrewCache;
    1277                 : 
    1278                 :   enum {
    1279                 :     STATE_LINE,
    1280                 :     STATE_CHUNK
    1281                 :   } mState;
    1282                 : 
    1283                 :   enum {
    1284                 :     CHUNK_ADD,
    1285                 :     CHUNK_SUB
    1286                 :   } mChunkType;
    1287                 : 
    1288                 :   PRUint32 mChunkNum;
    1289                 :   PRUint32 mHashSize;
    1290                 :   PRUint32 mChunkLen;
    1291                 : 
    1292                 :   // List of tables included in this update.
    1293                 :   nsTArray<nsCString> mUpdateTables;
    1294                 : 
    1295                 :   nsCString mUpdateTable;
    1296                 :   PRUint32 mUpdateTableId;
    1297                 : 
    1298                 :   nsresult mUpdateStatus;
    1299                 : 
    1300                 :   nsCOMPtr<nsIUrlClassifierUpdateObserver> mUpdateObserver;
    1301                 :   bool mInStream;
    1302                 :   bool mPrimaryStream;
    1303                 : 
    1304                 :   bool mHaveCachedLists;
    1305                 :   PRUint32 mCachedListsTable;
    1306                 :   nsCAutoString mCachedSubsStr;
    1307                 :   nsCAutoString mCachedAddsStr;
    1308                 : 
    1309                 :   bool mHaveCachedAddChunks;
    1310                 :   nsTArray<PRUint32> mCachedAddChunks;
    1311                 : 
    1312                 :   bool mHaveCachedSubChunks;
    1313                 :   nsTArray<PRUint32> mCachedSubChunks;
    1314                 : 
    1315                 :   // The client key with which the data from the server will be MAC'ed.
    1316                 :   nsCString mUpdateClientKey;
    1317                 : 
    1318                 :   // The MAC stated by the server.
    1319                 :   nsCString mServerMAC;
    1320                 : 
    1321                 :   // Start time of the current update interval.  This will be reset
    1322                 :   // every time we apply the update.
    1323                 :   PRIntervalTime mUpdateStartTime;
    1324                 : 
    1325                 :   nsCOMPtr<nsICryptoHMAC> mHMAC;
    1326                 :   // The number of noise entries to add to the set of lookup results.
    1327                 :   PRInt32 mGethashNoise;
    1328                 : 
    1329                 :   // Set of prefixes known to be in the database
    1330                 :   nsRefPtr<nsUrlClassifierPrefixSet> mPrefixSet;
    1331                 :   // Can we use the PrefixSet (low memory conditions)
    1332                 :   bool mPrefixSetEnabled;
    1333                 :   Mutex mPrefixSetEnabledLock;
    1334                 : 
    1335                 :   // Pending lookups are stored in a queue for processing.  The queue
    1336                 :   // is protected by mPendingLookupLock.
    1337                 :   Mutex mPendingLookupLock;
    1338                 : 
    1339             564 :   class PendingLookup {
    1340                 :   public:
    1341                 :     nsCString mKey;
    1342                 :     nsCOMPtr<nsIUrlClassifierLookupCallback> mCallback;
    1343                 :   };
    1344                 : 
    1345                 :   // list of pending lookups
    1346                 :   nsTArray<PendingLookup> mPendingLookups;
    1347                 : };
    1348                 : 
    1349            3631 : NS_IMPL_THREADSAFE_ISUPPORTS2(nsUrlClassifierDBServiceWorker,
    1350                 :                               nsIUrlClassifierDBServiceWorker,
    1351                 :                               nsIUrlClassifierDBService)
    1352                 : 
    1353               8 : nsUrlClassifierDBServiceWorker::nsUrlClassifierDBServiceWorker()
    1354                 :   : mUpdateWait(0)
    1355                 :   , mResetRequested(false)
    1356                 :   , mGrewCache(false)
    1357                 :   , mState(STATE_LINE)
    1358                 :   , mChunkType(CHUNK_ADD)
    1359                 :   , mChunkNum(0)
    1360                 :   , mHashSize(0)
    1361                 :   , mChunkLen(0)
    1362                 :   , mUpdateTableId(0)
    1363                 :   , mUpdateStatus(NS_OK)
    1364                 :   , mInStream(false)
    1365                 :   , mPrimaryStream(false)
    1366                 :   , mHaveCachedLists(false)
    1367                 :   , mCachedListsTable(PR_UINT32_MAX)
    1368                 :   , mHaveCachedAddChunks(false)
    1369                 :   , mHaveCachedSubChunks(false)
    1370                 :   , mUpdateStartTime(0)
    1371                 :   , mGethashNoise(0)
    1372                 :   , mPrefixSet(0)
    1373                 :   , mPrefixSetEnabled(true)
    1374                 :   , mPrefixSetEnabledLock("mPrefixSetEnabledLock")
    1375               8 :   , mPendingLookupLock("nsUrlClassifierDBServerWorker.mPendingLookupLock")
    1376                 : {
    1377               8 : }
    1378                 : 
    1379              16 : nsUrlClassifierDBServiceWorker::~nsUrlClassifierDBServiceWorker()
    1380                 : {
    1381               8 :   NS_ASSERTION(!mConnection,
    1382                 :                "Db connection not closed, leaking memory!  Call CloseDb "
    1383                 :                "to close the connection.");
    1384               8 : }
    1385                 : 
    1386                 : nsresult
    1387               8 : nsUrlClassifierDBServiceWorker::Init(PRInt32 gethashNoise,
    1388                 :                                      nsRefPtr<nsUrlClassifierPrefixSet> & prefSet)
    1389                 : {
    1390               8 :   mGethashNoise = gethashNoise;
    1391               8 :   mPrefixSet = prefSet;
    1392                 : 
    1393                 :   // Compute database filename
    1394                 : 
    1395                 :   // Because we dump raw integers into the database, this database isn't
    1396                 :   // portable between machine types, so store it in the local profile dir.
    1397                 :   nsresult rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_LOCAL_50_DIR,
    1398               8 :                                        getter_AddRefs(mDBFile));
    1399                 : 
    1400               8 :   if (NS_FAILED(rv)) {
    1401                 :     rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR,
    1402               0 :                                 getter_AddRefs(mDBFile));
    1403                 :   }
    1404                 : 
    1405               8 :   if (NS_FAILED(rv)) return NS_ERROR_NOT_AVAILABLE;
    1406                 : 
    1407               8 :   rv = mDBFile->Clone(getter_AddRefs(mPSFile));
    1408               8 :   NS_ENSURE_SUCCESS(rv, rv);
    1409                 : 
    1410               8 :   rv = mDBFile->Append(NS_LITERAL_STRING(DATABASE_FILENAME));
    1411               8 :   NS_ENSURE_SUCCESS(rv, rv);
    1412                 : 
    1413               8 :   rv = mPSFile->Append(NS_LITERAL_STRING(PREFIXSET_FILENAME));
    1414               8 :   NS_ENSURE_SUCCESS(rv, rv);
    1415                 : 
    1416               8 :   ResetUpdate();
    1417                 : 
    1418               8 :   mTableFreshness.Init();
    1419                 : 
    1420               8 :   return NS_OK;
    1421                 : }
    1422                 : 
    1423                 : nsresult
    1424             141 : nsUrlClassifierDBServiceWorker::QueueLookup(const nsACString& spec,
    1425                 :                                             nsIUrlClassifierLookupCallback* callback)
    1426                 : {
    1427             282 :   MutexAutoLock lock(mPendingLookupLock);
    1428                 : 
    1429             141 :   PendingLookup* lookup = mPendingLookups.AppendElement();
    1430             141 :   if (!lookup) return NS_ERROR_OUT_OF_MEMORY;
    1431                 : 
    1432             141 :   lookup->mKey = spec;
    1433             141 :   lookup->mCallback = callback;
    1434                 : 
    1435             141 :   return NS_OK;
    1436                 : }
    1437                 : 
    1438                 : nsresult
    1439              21 : nsUrlClassifierDBService::CheckClean(const nsACString &spec,
    1440                 :                                      bool *clean)
    1441                 : {
    1442              42 :   Telemetry::AutoTimer<Telemetry::URLCLASSIFIER_PS_LOOKUP_TIME> timer;
    1443                 : 
    1444                 :   // Is the PrefixSet usable?
    1445              21 :   bool usePrefixSet = mWorker->LockPrefixSet();
    1446                 : 
    1447                 :   // No, bail out and pretend the URL is not clean. We will do
    1448                 :   // a database lookup and get the correct result.
    1449              21 :   if (!usePrefixSet) {
    1450               0 :     mWorker->UnlockPrefixSet();
    1451               0 :     *clean = false;
    1452               0 :     return NS_OK;
    1453                 :   }
    1454                 : 
    1455                 :   // Get the set of fragments to look up.
    1456              42 :   nsTArray<nsCString> fragments;
    1457              21 :   nsresult rv = GetLookupFragments(spec, fragments);
    1458              21 :   if (NS_FAILED(rv)) {
    1459               0 :     goto error_checkclean;
    1460                 :   }
    1461                 : 
    1462                 :   PRUint32 prefixkey;
    1463              21 :   rv = mPrefixSet->GetKey(&prefixkey);
    1464              21 :   if (NS_FAILED(rv)) {
    1465               0 :     goto error_checkclean;
    1466                 :   }
    1467                 : 
    1468              21 :   *clean = true;
    1469                 : 
    1470              63 :   for (PRUint32 i = 0; i < fragments.Length(); i++) {
    1471                 :     nsUrlClassifierDomainHash fragmentKeyHash;
    1472              42 :     fragmentKeyHash.FromPlaintext(fragments[i], mHash);
    1473                 : 
    1474                 :     // Find the corresponding host key
    1475                 :     nsUrlClassifierDomainHash hostkey;
    1476              42 :     rv = GetKey(fragments[i], hostkey, mHash);
    1477              42 :     if (NS_FAILED(rv)) {
    1478                 :       /* This happens for hosts on the local network,
    1479                 :          can't check these against the DB */
    1480               0 :       continue;
    1481                 :     }
    1482                 : 
    1483              42 :     PRUint32 hostprefix = hostkey.ToUint32();
    1484              42 :     PRUint32 fragkey = fragmentKeyHash.ToUint32();
    1485                 :     PRUint32 codedkey;
    1486              42 :     rv = KeyedHash(fragkey, hostprefix, prefixkey, &codedkey);
    1487              42 :     if (NS_FAILED(rv)) {
    1488               0 :       goto error_checkclean;
    1489                 :     }
    1490                 : 
    1491              42 :     bool found = false;
    1492              42 :     bool ready = false;  /* opportunistic probe */
    1493              42 :     rv = mPrefixSet->Probe(codedkey, prefixkey, &ready, &found);
    1494              42 :     if (NS_FAILED(rv)) {
    1495               0 :       goto error_checkclean;
    1496                 :     }
    1497              42 :     LOG(("CheckClean Probed %X ready: %d found: %d ",
    1498                 :          codedkey, ready, found));
    1499              42 :     if (found || !ready) {
    1500               1 :       *clean = false;
    1501                 :     }
    1502                 :   }
    1503                 : 
    1504              21 :   mWorker->UnlockPrefixSet();
    1505              21 :   return NS_OK;
    1506                 : 
    1507                 :  error_checkclean:
    1508               0 :   mWorker->UnlockPrefixSet();
    1509               0 :   return rv;
    1510                 : }
    1511                 : 
    1512             141 : static nsresult GetHostKeys(const nsACString &spec,
    1513                 :                             nsTArray<nsCString> &hostKeys)
    1514                 : {
    1515             141 :   nsACString::const_iterator begin, end, iter;
    1516             141 :   spec.BeginReading(begin);
    1517             141 :   spec.EndReading(end);
    1518                 : 
    1519             141 :   iter = begin;
    1520             141 :   if (!FindCharInReadable('/', iter, end)) {
    1521               0 :     return NS_OK;
    1522                 :   }
    1523                 : 
    1524             282 :   const nsCSubstring& host = Substring(begin, iter);
    1525                 : 
    1526             141 :   if (IsCanonicalizedIP(host)) {
    1527               2 :     nsCString *key = hostKeys.AppendElement();
    1528               2 :     if (!key)
    1529               0 :       return NS_ERROR_OUT_OF_MEMORY;
    1530                 : 
    1531               2 :     key->Assign(host);
    1532               2 :     key->Append("/");
    1533               2 :     return NS_OK;
    1534                 :   }
    1535                 : 
    1536             278 :   nsTArray<nsCString> hostComponents;
    1537             139 :   ParseString(PromiseFlatCString(host), '.', hostComponents);
    1538                 : 
    1539             139 :   if (hostComponents.Length() < 2) {
    1540                 :     // no host or toplevel host, this won't match anything in the db
    1541               0 :     return NS_OK;
    1542                 :   }
    1543                 : 
    1544                 :   // First check with two domain components
    1545             139 :   PRInt32 last = PRInt32(hostComponents.Length()) - 1;
    1546             139 :   nsCString *lookupHost = hostKeys.AppendElement();
    1547             139 :   if (!lookupHost)
    1548               0 :     return NS_ERROR_OUT_OF_MEMORY;
    1549                 : 
    1550             139 :   lookupHost->Assign(hostComponents[last - 1]);
    1551             139 :   lookupHost->Append(".");
    1552             139 :   lookupHost->Append(hostComponents[last]);
    1553             139 :   lookupHost->Append("/");
    1554                 : 
    1555                 :   // Now check with three domain components
    1556             139 :   if (hostComponents.Length() > 2) {
    1557               8 :     nsCString *lookupHost2 = hostKeys.AppendElement();
    1558               8 :     if (!lookupHost2)
    1559               0 :       return NS_ERROR_OUT_OF_MEMORY;
    1560               8 :     lookupHost2->Assign(hostComponents[last - 2]);
    1561               8 :     lookupHost2->Append(".");
    1562               8 :     lookupHost2->Append(*lookupHost);
    1563                 :   }
    1564                 : 
    1565             139 :  return NS_OK;
    1566                 : }
    1567                 : 
    1568                 : nsresult
    1569             162 : GetLookupFragments(const nsACString& spec,
    1570                 :                    nsTArray<nsCString>& fragments)
    1571                 : {
    1572             162 :   fragments.Clear();
    1573                 : 
    1574             162 :   nsACString::const_iterator begin, end, iter;
    1575             162 :   spec.BeginReading(begin);
    1576             162 :   spec.EndReading(end);
    1577                 : 
    1578             162 :   iter = begin;
    1579             162 :   if (!FindCharInReadable('/', iter, end)) {
    1580               0 :     return NS_OK;
    1581                 :   }
    1582                 : 
    1583             324 :   const nsCSubstring& host = Substring(begin, iter++);
    1584             324 :   nsCAutoString path;
    1585             162 :   path.Assign(Substring(iter, end));
    1586                 : 
    1587                 :   /**
    1588                 :    * From the protocol doc:
    1589                 :    * For the hostname, the client will try at most 5 different strings.  They
    1590                 :    * are:
    1591                 :    * a) The exact hostname of the url
    1592                 :    * b) The 4 hostnames formed by starting with the last 5 components and
    1593                 :    *    successivly removing the leading component.  The top-level component
    1594                 :    *    can be skipped. This is not done if the hostname is a numerical IP.
    1595                 :    */
    1596             324 :   nsTArray<nsCString> hosts;
    1597             162 :   hosts.AppendElement(host);
    1598                 : 
    1599             162 :   if (!IsCanonicalizedIP(host)) {
    1600             160 :     host.BeginReading(begin);
    1601             160 :     host.EndReading(end);
    1602             160 :     int numHostComponents = 0;
    1603             510 :     while (RFindInReadable(NS_LITERAL_CSTRING("."), begin, end) &&
    1604                 :            numHostComponents < MAX_HOST_COMPONENTS) {
    1605                 :       // don't bother checking toplevel domains
    1606             190 :       if (++numHostComponents >= 2) {
    1607              30 :         host.EndReading(iter);
    1608              30 :         hosts.AppendElement(Substring(end, iter));
    1609                 :       }
    1610             190 :       end = begin;
    1611             190 :       host.BeginReading(begin);
    1612                 :     }
    1613                 :   }
    1614                 : 
    1615                 :   /**
    1616                 :    * From the protocol doc:
    1617                 :    * For the path, the client will also try at most 6 different strings.
    1618                 :    * They are:
    1619                 :    * a) the exact path of the url, including query parameters
    1620                 :    * b) the exact path of the url, without query parameters
    1621                 :    * c) the 4 paths formed by starting at the root (/) and
    1622                 :    *    successively appending path components, including a trailing
    1623                 :    *    slash.  This behavior should only extend up to the next-to-last
    1624                 :    *    path component, that is, a trailing slash should never be
    1625                 :    *    appended that was not present in the original url.
    1626                 :    */
    1627             324 :   nsTArray<nsCString> paths;
    1628             324 :   nsCAutoString pathToAdd;
    1629                 : 
    1630             162 :   path.BeginReading(begin);
    1631             162 :   path.EndReading(end);
    1632             162 :   iter = begin;
    1633             162 :   if (FindCharInReadable('?', iter, end)) {
    1634               3 :     pathToAdd = Substring(begin, iter);
    1635               3 :     paths.AppendElement(pathToAdd);
    1636               3 :     end = iter;
    1637                 :   }
    1638                 : 
    1639             162 :   int numPathComponents = 1;
    1640             162 :   iter = begin;
    1641             332 :   while (FindCharInReadable('/', iter, end) &&
    1642                 :          numPathComponents < MAX_PATH_COMPONENTS) {
    1643               8 :     iter++;
    1644               8 :     pathToAdd.Assign(Substring(begin, iter));
    1645               8 :     paths.AppendElement(pathToAdd);
    1646               8 :     numPathComponents++;
    1647                 :   }
    1648                 : 
    1649                 :   // If we haven't already done so, add the full path
    1650             162 :   if (!pathToAdd.Equals(path)) {
    1651             137 :     paths.AppendElement(path);
    1652                 :   }
    1653                 :   // Check an empty path (for whole-domain blacklist entries)
    1654             162 :   paths.AppendElement(EmptyCString());
    1655                 : 
    1656             354 :   for (PRUint32 hostIndex = 0; hostIndex < hosts.Length(); hostIndex++) {
    1657             549 :     for (PRUint32 pathIndex = 0; pathIndex < paths.Length(); pathIndex++) {
    1658             714 :       nsCString key;
    1659             357 :       key.Assign(hosts[hostIndex]);
    1660             357 :       key.Append('/');
    1661             357 :       key.Append(paths[pathIndex]);
    1662             357 :       LOG(("Chking %s", key.get()));
    1663                 : 
    1664             357 :       fragments.AppendElement(key);
    1665                 :     }
    1666                 :   }
    1667                 : 
    1668             162 :   return NS_OK;
    1669                 : }
    1670                 : 
    1671                 : nsresult
    1672             141 : nsUrlClassifierDBServiceWorker::Check(const nsACString& spec,
    1673                 :                                       nsTArray<nsUrlClassifierLookupResult>& results)
    1674                 : {
    1675             141 :   PRInt64 now = (PR_Now() / PR_USEC_PER_SEC);
    1676                 : 
    1677                 :   // Get list of host keys to look up
    1678             282 :   nsAutoTArray<nsCString, 2> lookupHosts;
    1679             141 :   nsresult rv = GetHostKeys(spec, lookupHosts);
    1680                 : 
    1681             282 :   nsTArray<nsUrlClassifierEntry> mCachedEntries;
    1682                 : 
    1683                 :   // Gather host's prefixes
    1684             290 :   for (PRUint32 i = 0; i < lookupHosts.Length(); i++) {
    1685                 :     // Find the corresponding host key
    1686                 :     nsUrlClassifierDomainHash hostKey;
    1687             149 :     nsresult rv = GetKey(lookupHosts[i], hostKey, mCryptoHash);
    1688             149 :     NS_ENSURE_SUCCESS(rv, rv);
    1689                 :     // Read the entries for this fragments host from SQLite
    1690             149 :     mMainStore.ReadAddEntries(hostKey, mCachedEntries);
    1691                 :   }
    1692                 : 
    1693                 :   // Now get the set of fragments to look up.
    1694             282 :   nsTArray<nsCString> fragments;
    1695             141 :   rv = GetLookupFragments(spec, fragments);
    1696             141 :   NS_ENSURE_SUCCESS(rv, rv);
    1697                 : 
    1698                 :   // Now check each lookup fragment against the entries in the DB.
    1699             456 :   for (PRUint32 i = 0; i < fragments.Length(); i++) {
    1700                 :     nsUrlClassifierCompleteHash lookupHash;
    1701             315 :     lookupHash.FromPlaintext(fragments[i], mCryptoHash);
    1702                 : 
    1703             832 :     for (PRUint32 j = 0; j < mCachedEntries.Length(); j++) {
    1704             517 :       nsUrlClassifierEntry &entry = mCachedEntries[j];
    1705             517 :       if (entry.Match(lookupHash)) {
    1706                 :         // If the entry doesn't contain a complete hash, we need to
    1707                 :         // save it here so that it can be compared against the
    1708                 :         // complete hash.  However, we don't set entry.mHaveComplete
    1709                 :         // because it isn't a verified part of the entry yet.
    1710             136 :         nsUrlClassifierLookupResult *result = results.AppendElement();
    1711             136 :         if (!result)
    1712               0 :           return NS_ERROR_OUT_OF_MEMORY;
    1713                 : 
    1714             136 :         result->mLookupFragment = lookupHash;
    1715             136 :         result->mEntry = entry;
    1716                 : 
    1717                 :         // Fill in the table name.
    1718             136 :         GetTableName(entry.mTableId, result->mTableName);
    1719                 : 
    1720                 :         bool fresh;
    1721                 :         PRInt64 tableUpdateTime;
    1722             136 :         if (mTableFreshness.Get(result->mTableName, &tableUpdateTime)) {
    1723             112 :           LOG(("tableUpdateTime: %lld, now: %lld, freshnessGuarantee: %d\n",
    1724                 :                tableUpdateTime, now, gFreshnessGuarantee));
    1725             112 :           fresh = ((now - tableUpdateTime) <= gFreshnessGuarantee);
    1726                 :         } else {
    1727              24 :           LOG(("No expiration time for this table.\n"));
    1728              24 :           fresh = false;
    1729                 :         }
    1730                 : 
    1731                 :         // This is a confirmed result if we match a complete fragment in
    1732                 :         // an up-to-date table.
    1733             136 :         result->mConfirmed = entry.mHaveComplete && fresh;
    1734                 : 
    1735             136 :         LOG(("Found a result.  complete=%d, fresh=%d",
    1736                 :              entry.mHaveComplete, fresh));
    1737                 :       }
    1738                 :     }
    1739                 :   }
    1740                 : 
    1741             141 :   return NS_OK;
    1742                 : }
    1743                 : 
    1744                 : /**
    1745                 :  * Lookup up a key in the database is a two step process:
    1746                 :  *
    1747                 :  * a) First we look for any Entries in the database that might apply to this
    1748                 :  *    url.  For each URL there are one or two possible domain names to check:
    1749                 :  *    the two-part domain name (example.com) and the three-part name
    1750                 :  *    (www.example.com).  We check the database for both of these.
    1751                 :  * b) If we find any entries, we check the list of fragments for that entry
    1752                 :  *    against the possible subfragments of the URL as described in the
    1753                 :  *    "Simplified Regular Expression Lookup" section of the protocol doc.
    1754                 :  */
    1755                 : nsresult
    1756             141 : nsUrlClassifierDBServiceWorker::DoLookup(const nsACString& spec,
    1757                 :                                          nsIUrlClassifierLookupCallback* c)
    1758                 : {
    1759             141 :   if (gShuttingDownThread) {
    1760               0 :     c->LookupComplete(nsnull);
    1761               0 :     return NS_ERROR_NOT_INITIALIZED;
    1762                 :   }
    1763                 : 
    1764             141 :   nsresult rv = OpenDb();
    1765             141 :   if (NS_FAILED(rv)) {
    1766               0 :     c->LookupComplete(nsnull);
    1767               0 :     return NS_ERROR_FAILURE;
    1768                 :   }
    1769                 : 
    1770                 : #if defined(PR_LOGGING)
    1771             141 :   PRIntervalTime clockStart = 0;
    1772             141 :   if (LOG_ENABLED()) {
    1773               0 :     clockStart = PR_IntervalNow();
    1774                 :   }
    1775                 : #endif
    1776                 : 
    1777             282 :   nsAutoPtr<nsTArray<nsUrlClassifierLookupResult> > results;
    1778             141 :   results = new nsTArray<nsUrlClassifierLookupResult>();
    1779             141 :   if (!results) {
    1780               0 :     c->LookupComplete(nsnull);
    1781               0 :     return NS_ERROR_OUT_OF_MEMORY;
    1782                 :   }
    1783                 : 
    1784                 :   // we ignore failures from Check because we'd rather return the
    1785                 :   // results that were found than fail.
    1786             141 :   Check(spec, *results);
    1787                 : 
    1788                 : #if defined(PR_LOGGING)
    1789             141 :   if (LOG_ENABLED()) {
    1790               0 :     PRIntervalTime clockEnd = PR_IntervalNow();
    1791               0 :     LOG(("query took %dms\n",
    1792                 :          PR_IntervalToMilliseconds(clockEnd - clockStart)));
    1793                 :   }
    1794                 : #endif
    1795                 : 
    1796             200 :   for (PRUint32 i = 0; i < results->Length(); i++) {
    1797             111 :     if (!results->ElementAt(i).mConfirmed) {
    1798                 :       // We're going to be doing a gethash request, add some extra entries.
    1799              52 :       AddNoise(results->ElementAt(i).mEntry.mId, mGethashNoise, *results);
    1800              52 :       break;
    1801                 :     }
    1802                 :   }
    1803                 : 
    1804                 :   // At this point ownership of 'results' is handed to the callback.
    1805             141 :   c->LookupComplete(results.forget());
    1806                 : 
    1807             141 :   return NS_OK;
    1808                 : }
    1809                 : 
    1810                 : nsresult
    1811             595 : nsUrlClassifierDBServiceWorker::HandlePendingLookups()
    1812                 : {
    1813            1190 :   MutexAutoLock lock(mPendingLookupLock);
    1814            1331 :   while (mPendingLookups.Length() > 0) {
    1815             282 :     PendingLookup lookup = mPendingLookups[0];
    1816             141 :     mPendingLookups.RemoveElementAt(0);
    1817                 :     {
    1818             282 :       MutexAutoUnlock unlock(mPendingLookupLock);
    1819             141 :       DoLookup(lookup.mKey, lookup.mCallback);
    1820                 :     }
    1821                 :   }
    1822                 : 
    1823             595 :   return NS_OK;
    1824                 : }
    1825                 : 
    1826                 : nsresult
    1827              52 : nsUrlClassifierDBServiceWorker::AddNoise(PRInt64 nearID,
    1828                 :                                          PRInt32 count,
    1829                 :                                          nsTArray<nsUrlClassifierLookupResult>& results)
    1830                 : {
    1831              52 :   if (count < 1) {
    1832              52 :     return NS_OK;
    1833                 :   }
    1834                 : 
    1835                 :   PRInt64 randomNum;
    1836               0 :   nsresult rv = mMainStore.RandomNumber(&randomNum);
    1837               0 :   NS_ENSURE_SUCCESS(rv, rv);
    1838                 : 
    1839               0 :   PRInt32 numBefore = randomNum % count;
    1840                 : 
    1841               0 :   nsTArray<nsUrlClassifierEntry> noiseEntries;
    1842               0 :   rv = mMainStore.ReadNoiseEntries(nearID, numBefore, true, noiseEntries);
    1843               0 :   NS_ENSURE_SUCCESS(rv, rv);
    1844                 : 
    1845               0 :   rv = mMainStore.ReadNoiseEntries(nearID, count - numBefore, false, noiseEntries);
    1846               0 :   NS_ENSURE_SUCCESS(rv, rv);
    1847                 : 
    1848               0 :   for (PRUint32 i = 0; i < noiseEntries.Length(); i++) {
    1849               0 :     nsUrlClassifierLookupResult *result = results.AppendElement();
    1850               0 :     if (!result)
    1851               0 :       return NS_ERROR_OUT_OF_MEMORY;
    1852                 : 
    1853               0 :     result->mEntry = noiseEntries[i];
    1854               0 :     result->mConfirmed = false;
    1855               0 :     result->mNoise = true;
    1856                 : 
    1857                 :     // Fill in the table name.
    1858               0 :     GetTableName(noiseEntries[i].mTableId, result->mTableName);
    1859                 :   }
    1860                 : 
    1861               0 :   return NS_OK;
    1862                 : }
    1863                 : 
    1864                 : 
    1865                 : // Lookup a key in the db.
    1866                 : NS_IMETHODIMP
    1867             141 : nsUrlClassifierDBServiceWorker::Lookup(const nsACString& spec,
    1868                 :                                        nsIUrlClassifierCallback* c)
    1869                 : {
    1870             141 :   return HandlePendingLookups();
    1871                 : }
    1872                 : 
    1873                 : NS_IMETHODIMP
    1874              53 : nsUrlClassifierDBServiceWorker::GetTables(nsIUrlClassifierCallback* c)
    1875                 : {
    1876              53 :   if (gShuttingDownThread)
    1877               0 :     return NS_ERROR_NOT_INITIALIZED;
    1878                 : 
    1879              53 :   nsresult rv = OpenDb();
    1880              53 :   if (NS_FAILED(rv)) {
    1881               0 :     NS_ERROR("Unable to open database");
    1882               0 :     return NS_ERROR_FAILURE;
    1883                 :   }
    1884                 : 
    1885             106 :   mozStorageStatementScoper scoper(mGetTablesStatement);
    1886                 : 
    1887             106 :   nsCAutoString response;
    1888                 :   bool hasMore;
    1889             163 :   while (NS_SUCCEEDED(rv = mGetTablesStatement->ExecuteStep(&hasMore)) &&
    1890                 :          hasMore) {
    1891             114 :     nsCAutoString val;
    1892              57 :     mGetTablesStatement->GetUTF8String(0, val);
    1893                 : 
    1894              57 :     if (val.IsEmpty()) {
    1895               0 :       continue;
    1896                 :     }
    1897                 : 
    1898              57 :     response.Append(val);
    1899              57 :     response.Append(';');
    1900                 : 
    1901              57 :     mGetTablesStatement->GetUTF8String(1, val);
    1902                 : 
    1903              57 :     bool haveAdds = false;
    1904              57 :     if (!val.IsEmpty()) {
    1905              56 :       response.Append("a:");
    1906              56 :       response.Append(val);
    1907              56 :       haveAdds = true;
    1908                 :     }
    1909                 : 
    1910              57 :     mGetTablesStatement->GetUTF8String(2, val);
    1911              57 :     if (!val.IsEmpty()) {
    1912              12 :       if (haveAdds)
    1913              12 :         response.Append(":");
    1914                 : 
    1915              12 :       response.Append("s:");
    1916              12 :       response.Append(val);
    1917                 :     }
    1918                 : 
    1919              57 :     response.Append('\n');
    1920                 :   }
    1921                 : 
    1922              53 :   if (NS_FAILED(rv)) {
    1923               0 :     response.Truncate();
    1924                 :   }
    1925                 : 
    1926              53 :   c->HandleEvent(response);
    1927                 : 
    1928              53 :   return rv;
    1929                 : }
    1930                 : 
    1931                 : nsresult
    1932             102 : nsUrlClassifierDBServiceWorker::GetTableId(const nsACString& table,
    1933                 :                                            PRUint32* tableId)
    1934                 : {
    1935             204 :   mozStorageStatementScoper findScoper(mGetTableIdStatement);
    1936                 : 
    1937             102 :   nsresult rv = mGetTableIdStatement->BindUTF8StringByIndex(0, table);
    1938             102 :   NS_ENSURE_SUCCESS(rv, rv);
    1939                 : 
    1940                 :   bool exists;
    1941             102 :   rv = mGetTableIdStatement->ExecuteStep(&exists);
    1942             102 :   NS_ENSURE_SUCCESS(rv, rv);
    1943             102 :   if (exists) {
    1944              43 :     *tableId = mGetTableIdStatement->AsInt32(0);
    1945              43 :     return NS_OK;
    1946                 :   }
    1947                 : 
    1948             118 :   mozStorageStatementScoper insertScoper(mInsertTableIdStatement);
    1949              59 :   rv = mInsertTableIdStatement->BindUTF8StringByIndex(0, table);
    1950              59 :   NS_ENSURE_SUCCESS(rv, rv);
    1951                 : 
    1952              59 :   rv = mInsertTableIdStatement->Execute();
    1953              59 :   NS_ENSURE_SUCCESS(rv, rv);
    1954                 : 
    1955                 :   PRInt64 rowId;
    1956              59 :   rv = mConnection->GetLastInsertRowID(&rowId);
    1957              59 :   NS_ENSURE_SUCCESS(rv, rv);
    1958                 : 
    1959              59 :   if (rowId > PR_UINT32_MAX)
    1960               0 :     return NS_ERROR_FAILURE;
    1961                 : 
    1962              59 :   *tableId = rowId;
    1963                 : 
    1964              59 :   return NS_OK;
    1965                 : }
    1966                 : 
    1967                 : nsresult
    1968             136 : nsUrlClassifierDBServiceWorker::GetTableName(PRUint32 tableId,
    1969                 :                                              nsACString& tableName)
    1970                 : {
    1971             272 :   mozStorageStatementScoper findScoper(mGetTableNameStatement);
    1972             136 :   nsresult rv = mGetTableNameStatement->BindInt32ByIndex(0, tableId);
    1973             136 :   NS_ENSURE_SUCCESS(rv, rv);
    1974                 :   bool exists;
    1975             136 :   rv = mGetTableNameStatement->ExecuteStep(&exists);
    1976             136 :   NS_ENSURE_SUCCESS(rv, rv);
    1977             136 :   if (!exists) return NS_ERROR_FAILURE;
    1978                 : 
    1979             136 :   return mGetTableNameStatement->GetUTF8String(0, tableName);
    1980                 : }
    1981                 : 
    1982                 : nsresult
    1983               0 : nsUrlClassifierDBServiceWorker::InflateChunk(nsACString& chunk)
    1984                 : {
    1985               0 :   nsCAutoString inflated;
    1986                 :   char buf[4096];
    1987                 : 
    1988               0 :   const nsPromiseFlatCString& flat = PromiseFlatCString(chunk);
    1989                 : 
    1990                 :   z_stream stream;
    1991               0 :   memset(&stream, 0, sizeof(stream));
    1992               0 :   stream.next_in = (Bytef*)flat.get();
    1993               0 :   stream.avail_in = flat.Length();
    1994                 : 
    1995               0 :   if (inflateInit(&stream) != Z_OK) {
    1996               0 :     return NS_ERROR_FAILURE;
    1997                 :   }
    1998                 : 
    1999                 :   int code;
    2000               0 :   do {
    2001               0 :     stream.next_out = (Bytef*)buf;
    2002               0 :     stream.avail_out = sizeof(buf);
    2003                 : 
    2004               0 :     code = inflate(&stream, Z_NO_FLUSH);
    2005               0 :     PRUint32 numRead = sizeof(buf) - stream.avail_out;
    2006                 : 
    2007               0 :     if (code == Z_OK || code == Z_STREAM_END) {
    2008               0 :       inflated.Append(buf, numRead);
    2009                 :     }
    2010                 :   } while (code == Z_OK);
    2011                 : 
    2012               0 :   inflateEnd(&stream);
    2013                 : 
    2014               0 :   if (code != Z_STREAM_END) {
    2015               0 :     return NS_ERROR_FAILURE;
    2016                 :   }
    2017                 : 
    2018               0 :   chunk = inflated;
    2019                 : 
    2020               0 :   return NS_OK;
    2021                 : }
    2022                 : 
    2023                 : nsresult
    2024               8 : nsUrlClassifierStore::DeleteEntry(nsUrlClassifierEntry& entry)
    2025                 : {
    2026               8 :   if (entry.mId == -1) {
    2027               0 :     return NS_OK;
    2028                 :   }
    2029                 : 
    2030              16 :   mozStorageStatementScoper scoper(mDeleteStatement);
    2031               8 :   mDeleteStatement->BindInt64ByIndex(0, entry.mId);
    2032               8 :   nsresult rv = mDeleteStatement->Execute();
    2033               8 :   NS_ENSURE_SUCCESS(rv, rv);
    2034                 : 
    2035               8 :   entry.mId = -1;
    2036                 : 
    2037               8 :   return NS_OK;
    2038                 : }
    2039                 : 
    2040                 : nsresult
    2041             163 : nsUrlClassifierStore::WriteEntry(nsUrlClassifierEntry& entry)
    2042                 : {
    2043             163 :   if (entry.mId != -1) {
    2044                 :     // existing entry, just ignore it
    2045               0 :     return NS_OK;
    2046                 :   }
    2047                 : 
    2048             326 :   mozStorageStatementScoper scoper(mInsertStatement);
    2049                 : 
    2050             163 :   nsresult rv = BindStatement(entry, mInsertStatement);
    2051             163 :   NS_ENSURE_SUCCESS(rv, rv);
    2052                 : 
    2053             163 :   rv = mInsertStatement->Execute();
    2054             163 :   NS_ENSURE_SUCCESS(rv, rv);
    2055                 : 
    2056                 :   PRInt64 rowId;
    2057             163 :   rv = mConnection->GetLastInsertRowID(&rowId);
    2058             163 :   NS_ENSURE_SUCCESS(rv, rv);
    2059                 : 
    2060             163 :   if (rowId > PR_UINT32_MAX) {
    2061               0 :     return NS_ERROR_FAILURE;
    2062                 :   }
    2063                 : 
    2064             163 :   entry.mId = rowId;
    2065                 : 
    2066             163 :   return NS_OK;
    2067                 : }
    2068                 : 
    2069                 : nsresult
    2070              36 : nsUrlClassifierStore::UpdateEntry(nsUrlClassifierEntry& entry)
    2071                 : {
    2072              72 :   mozStorageStatementScoper scoper(mUpdateStatement);
    2073                 : 
    2074              36 :   NS_ENSURE_ARG(entry.mId != -1);
    2075                 : 
    2076              36 :   nsresult rv = BindStatement(entry, mUpdateStatement);
    2077              36 :   NS_ENSURE_SUCCESS(rv, rv);
    2078                 : 
    2079              36 :   rv = mUpdateStatement->Execute();
    2080              36 :   NS_ENSURE_SUCCESS(rv, rv);
    2081                 : 
    2082              36 :   return NS_OK;
    2083                 : }
    2084                 : 
    2085                 : static bool
    2086             678 : IsCanonicalizedIP(const nsACString& host)
    2087                 : {
    2088                 :   // The canonicalization process will have left IP addresses in dotted
    2089                 :   // decimal with no surprises.
    2090                 :   PRUint32 i1, i2, i3, i4;
    2091                 :   char c;
    2092             678 :   if (PR_sscanf(PromiseFlatCString(host).get(), "%u.%u.%u.%u%c",
    2093             678 :                 &i1, &i2, &i3, &i4, &c) == 4) {
    2094               8 :     return (i1 <= 0xFF && i2 <= 0xFF && i3 <= 0xFF && i4 <= 0xFF);
    2095                 :   }
    2096                 : 
    2097             670 :   return false;
    2098                 : }
    2099                 : 
    2100                 : static nsresult
    2101             375 : GetKey(const nsACString& spec,
    2102                 :        nsUrlClassifierDomainHash& hash,
    2103                 :        nsICryptoHash * aCryptoHash)
    2104                 : {
    2105             375 :   nsACString::const_iterator begin, end, iter;
    2106             375 :   spec.BeginReading(begin);
    2107             375 :   spec.EndReading(end);
    2108                 : 
    2109             375 :   iter = begin;
    2110             375 :   if (!FindCharInReadable('/', iter, end)) {
    2111               0 :     return NS_OK;
    2112                 :   }
    2113                 : 
    2114             750 :   const nsCSubstring& host = Substring(begin, iter);
    2115                 : 
    2116             375 :   if (IsCanonicalizedIP(host)) {
    2117               8 :     nsCAutoString key;
    2118               4 :     key.Assign(host);
    2119               4 :     key.Append("/");
    2120               4 :     return hash.FromPlaintext(key, aCryptoHash);
    2121                 :   }
    2122                 : 
    2123             742 :   nsTArray<nsCString> hostComponents;
    2124             371 :   ParseString(PromiseFlatCString(host), '.', hostComponents);
    2125                 : 
    2126             371 :   if (hostComponents.Length() < 2)
    2127               0 :     return NS_ERROR_FAILURE;
    2128                 : 
    2129             371 :   PRInt32 last = PRInt32(hostComponents.Length()) - 1;
    2130             742 :   nsCAutoString lookupHost;
    2131                 : 
    2132             371 :   if (hostComponents.Length() > 2) {
    2133              43 :     lookupHost.Append(hostComponents[last - 2]);
    2134              43 :     lookupHost.Append(".");
    2135                 :   }
    2136                 : 
    2137             371 :   lookupHost.Append(hostComponents[last - 1]);
    2138             371 :   lookupHost.Append(".");
    2139             371 :   lookupHost.Append(hostComponents[last]);
    2140             371 :   lookupHost.Append("/");
    2141                 : 
    2142             371 :   return hash.FromPlaintext(lookupHost, aCryptoHash);
    2143                 : }
    2144                 : 
    2145                 : nsresult
    2146               0 : nsUrlClassifierDBServiceWorker::GetShaEntries(PRUint32 tableId,
    2147                 :                                               PRUint32 chunkType,
    2148                 :                                               PRUint32 chunkNum,
    2149                 :                                               PRUint32 domainSize,
    2150                 :                                               PRUint32 fragmentSize,
    2151                 :                                               nsACString& chunk,
    2152                 :                                               nsTArray<nsUrlClassifierEntry>& entries)
    2153                 : {
    2154               0 :   PRUint32 start = 0;
    2155               0 :   while (start + domainSize + 1 <= chunk.Length()) {
    2156                 :     nsUrlClassifierDomainHash domain;
    2157               0 :     domain.Assign(Substring(chunk, start, DOMAIN_LENGTH));
    2158               0 :     start += domainSize;
    2159                 : 
    2160                 :     // then there is a one-byte count of fragments
    2161               0 :     PRUint8 numEntries = static_cast<PRUint8>(chunk[start]);
    2162               0 :     start++;
    2163                 : 
    2164               0 :     if (numEntries == 0) {
    2165                 :       // if there are no fragments, the domain itself is treated as a
    2166                 :       // fragment.  This will only work if domainHashSize == hashSize
    2167               0 :       if (domainSize != fragmentSize) {
    2168               0 :         NS_WARNING("Received 0-fragment entry where domainSize != fragmentSize");
    2169               0 :         return NS_ERROR_FAILURE;
    2170                 :       }
    2171                 : 
    2172               0 :       nsUrlClassifierEntry* entry = entries.AppendElement();
    2173               0 :       if (!entry) return NS_ERROR_OUT_OF_MEMORY;
    2174                 : 
    2175               0 :       entry->mKey = domain;
    2176               0 :       entry->mTableId = tableId;
    2177               0 :       entry->mChunkId = chunkNum;
    2178               0 :       entry->SetHash(domain);
    2179                 : 
    2180               0 :       if (chunkType == CHUNK_SUB) {
    2181               0 :         if (start + 4 > chunk.Length()) {
    2182                 :           // there isn't as much data as there should be.
    2183               0 :           NS_WARNING("Received a zero-entry sub chunk without an associated add.");
    2184               0 :           return NS_ERROR_FAILURE;
    2185                 :         }
    2186               0 :         const nsCSubstring& str = Substring(chunk, start, 4);
    2187                 :         PRUint32 p;
    2188               0 :         memcpy(&p, str.BeginReading(), 4);
    2189               0 :         entry->mAddChunkId = PR_ntohl(p);
    2190               0 :         if (entry->mAddChunkId == 0) {
    2191               0 :           NS_WARNING("Received invalid chunk number.");
    2192               0 :           return NS_ERROR_FAILURE;
    2193                 :         }
    2194               0 :         start += 4;
    2195                 :       }
    2196                 :     } else {
    2197               0 :       PRUint32 entrySize = fragmentSize;
    2198               0 :       if (chunkType == CHUNK_SUB) {
    2199               0 :         entrySize += 4;
    2200                 :       }
    2201               0 :       if (start + (numEntries * entrySize) > chunk.Length()) {
    2202                 :         // there isn't as much data as they said there would be.
    2203               0 :         NS_WARNING("Received a chunk without enough data");
    2204               0 :         return NS_ERROR_FAILURE;
    2205                 :       }
    2206                 : 
    2207               0 :       for (PRUint8 i = 0; i < numEntries; i++) {
    2208               0 :         nsUrlClassifierEntry* entry = entries.AppendElement();
    2209               0 :         if (!entry) return NS_ERROR_OUT_OF_MEMORY;
    2210                 : 
    2211               0 :         entry->mKey = domain;
    2212               0 :         entry->mTableId = tableId;
    2213               0 :         entry->mChunkId = chunkNum;
    2214                 : 
    2215               0 :         if (chunkType == CHUNK_SUB) {
    2216               0 :           const nsCSubstring& str = Substring(chunk, start, 4);
    2217                 :           PRUint32 p;
    2218               0 :           memcpy(&p, str.BeginReading(), 4);
    2219               0 :           entry->mAddChunkId = PR_ntohl(p);
    2220               0 :           if (entry->mAddChunkId == 0) {
    2221               0 :             NS_WARNING("Received invalid chunk number.");
    2222               0 :             return NS_ERROR_FAILURE;
    2223                 :           }
    2224               0 :           start += 4;
    2225                 :         }
    2226                 : 
    2227               0 :         if (fragmentSize == PARTIAL_LENGTH) {
    2228                 :           nsUrlClassifierPartialHash hash;
    2229               0 :           hash.Assign(Substring(chunk, start, PARTIAL_LENGTH));
    2230               0 :           entry->SetHash(hash);
    2231               0 :         } else if (fragmentSize == COMPLETE_LENGTH) {
    2232                 :           nsUrlClassifierCompleteHash hash;
    2233               0 :           hash.Assign(Substring(chunk, start, COMPLETE_LENGTH));
    2234               0 :           entry->SetHash(hash);
    2235                 :         } else {
    2236               0 :           NS_ASSERTION(false, "Invalid fragment size!");
    2237               0 :           return NS_ERROR_FAILURE;
    2238                 :         }
    2239                 : 
    2240               0 :         start += fragmentSize;
    2241                 :       }
    2242                 :     }
    2243                 :   }
    2244                 : 
    2245               0 :   return NS_OK;
    2246                 : }
    2247                 : 
    2248                 : nsresult
    2249             103 : nsUrlClassifierDBServiceWorker::GetChunkEntries(const nsACString& table,
    2250                 :                                                 PRUint32 tableId,
    2251                 :                                                 PRUint32 chunkType,
    2252                 :                                                 PRUint32 chunkNum,
    2253                 :                                                 PRUint32 hashSize,
    2254                 :                                                 nsACString& chunk,
    2255                 :                                                 nsTArray<nsUrlClassifierEntry>& entries)
    2256                 : {
    2257                 :   nsresult rv;
    2258             103 :   if (StringEndsWith(table, NS_LITERAL_CSTRING("-exp"))) {
    2259                 :     // regexp tables need to be ungzipped
    2260               0 :     rv = InflateChunk(chunk);
    2261               0 :     NS_ENSURE_SUCCESS(rv, rv);
    2262                 :   }
    2263                 : 
    2264             103 :   if (StringEndsWith(table, NS_LITERAL_CSTRING("-shavar"))) {
    2265                 :     rv = GetShaEntries(tableId, chunkType, chunkNum, DOMAIN_LENGTH, hashSize,
    2266               0 :                        chunk, entries);
    2267               0 :     NS_ENSURE_SUCCESS(rv, rv);
    2268                 :   } else {
    2269             206 :     nsTArray<nsCString> lines;
    2270             103 :     ParseString(PromiseFlatCString(chunk), '\n', lines);
    2271                 : 
    2272                 :     // non-hashed tables need to be hashed
    2273             287 :     for (PRInt32 i = 0; i < PRInt32(lines.Length()); i++) {
    2274             184 :       nsUrlClassifierEntry *entry = entries.AppendElement();
    2275             184 :       if (!entry)
    2276               0 :         return NS_ERROR_OUT_OF_MEMORY;
    2277                 : 
    2278             368 :       nsCAutoString entryStr;
    2279             184 :       if (chunkType == CHUNK_SUB) {
    2280              27 :         nsCString::const_iterator begin, iter, end;
    2281              27 :         lines[i].BeginReading(begin);
    2282              27 :         lines[i].EndReading(end);
    2283              27 :         iter = begin;
    2284              54 :         if (!FindCharInReadable(':', iter, end) ||
    2285              27 :             PR_sscanf(lines[i].get(), "%d:", &entry->mAddChunkId) != 1) {
    2286               0 :           NS_WARNING("Received sub chunk without associated add chunk.");
    2287               0 :           return NS_ERROR_FAILURE;
    2288                 :         }
    2289              27 :         iter++;
    2290              27 :         entryStr = Substring(iter, end);
    2291                 :       } else {
    2292             157 :         entryStr = lines[i];
    2293                 :       }
    2294                 : 
    2295             184 :       rv = GetKey(entryStr, entry->mKey, mCryptoHash);
    2296             184 :       NS_ENSURE_SUCCESS(rv, rv);
    2297                 : 
    2298             184 :       entry->mTableId = tableId;
    2299             184 :       entry->mChunkId = chunkNum;
    2300             184 :       if (hashSize == PARTIAL_LENGTH) {
    2301                 :         nsUrlClassifierPartialHash hash;
    2302              47 :         hash.FromPlaintext(entryStr, mCryptoHash);
    2303              47 :         entry->SetHash(hash);
    2304             137 :       } else if (hashSize == COMPLETE_LENGTH) {
    2305                 :         nsUrlClassifierCompleteHash hash;
    2306             137 :         hash.FromPlaintext(entryStr, mCryptoHash);
    2307             137 :         entry->SetHash(hash);
    2308                 :       } else {
    2309               0 :         NS_ASSERTION(false, "Invalid fragment size!");
    2310               0 :         return NS_ERROR_FAILURE;
    2311                 :       }
    2312                 :     }
    2313                 :   }
    2314                 : 
    2315             103 :   return NS_OK;
    2316                 : }
    2317                 : 
    2318                 : bool
    2319              39 : nsUrlClassifierDBServiceWorker::ParseChunkRange(nsACString::const_iterator &begin,
    2320                 :                                                 const nsACString::const_iterator &end,
    2321                 :                                                 PRUint32 *first,
    2322                 :                                                 PRUint32 *last)
    2323                 : {
    2324              39 :   nsACString::const_iterator iter = begin;
    2325              39 :   FindCharInReadable(',', iter, end);
    2326                 : 
    2327              78 :   nsCAutoString element(Substring(begin, iter));
    2328              39 :   begin = iter;
    2329              39 :   if (begin != end)
    2330               6 :     begin++;
    2331                 : 
    2332              39 :   PRUint32 numRead = PR_sscanf(element.get(), "%u-%u", first, last);
    2333              39 :   if (numRead == 2) {
    2334              11 :     if (*first > *last) {
    2335               0 :       PRUint32 tmp = *first;
    2336               0 :       *first = *last;
    2337               0 :       *last = tmp;
    2338                 :     }
    2339              11 :     return true;
    2340                 :   }
    2341                 : 
    2342              28 :   if (numRead == 1) {
    2343              28 :     *last = *first;
    2344              28 :     return true;
    2345                 :   }
    2346                 : 
    2347               0 :   return false;
    2348                 : }
    2349                 : 
    2350                 : nsresult
    2351             104 : nsUrlClassifierDBServiceWorker::ParseChunkList(const nsACString& chunkStr,
    2352                 :                                                nsTArray<PRUint32>& chunks)
    2353                 : {
    2354             104 :   LOG(("Parsing %s", PromiseFlatCString(chunkStr).get()));
    2355                 : 
    2356             104 :   nsACString::const_iterator begin, end;
    2357             104 :   chunkStr.BeginReading(begin);
    2358             104 :   chunkStr.EndReading(end);
    2359             236 :   while (begin != end) {
    2360                 :     PRUint32 first, last;
    2361              28 :     if (ParseChunkRange(begin, end, &first, &last)) {
    2362              68 :       for (PRUint32 num = first; num <= last; num++) {
    2363              40 :         chunks.AppendElement(num);
    2364                 :       }
    2365                 :     }
    2366                 :   }
    2367                 : 
    2368             104 :   LOG(("Got %d elements.", chunks.Length()));
    2369                 : 
    2370             104 :   return NS_OK;
    2371                 : }
    2372                 : 
    2373                 : nsresult
    2374             104 : nsUrlClassifierDBServiceWorker::JoinChunkList(nsTArray<PRUint32>& chunks,
    2375                 :                                               nsCString& chunkStr)
    2376                 : {
    2377             104 :   chunkStr.Truncate();
    2378             104 :   chunks.Sort();
    2379                 : 
    2380             104 :   PRUint32 i = 0;
    2381             305 :   while (i < chunks.Length()) {
    2382              97 :     if (i != 0) {
    2383               4 :       chunkStr.Append(',');
    2384                 :     }
    2385              97 :     chunkStr.AppendInt(chunks[i]);
    2386                 : 
    2387              97 :     PRUint32 first = i;
    2388              97 :     PRUint32 last = first;
    2389              97 :     i++;
    2390             221 :     while (i < chunks.Length() && (chunks[i] == chunks[i - 1] + 1 || chunks[i] == chunks[i - 1])) {
    2391              27 :       last = i++;
    2392                 :     }
    2393                 : 
    2394              97 :     if (last != first) {
    2395              21 :       chunkStr.Append('-');
    2396              21 :       chunkStr.AppendInt(chunks[last]);
    2397                 :     }
    2398                 :   }
    2399                 : 
    2400             104 :   return NS_OK;
    2401                 : }
    2402                 : 
    2403                 : 
    2404                 : nsresult
    2405              86 : nsUrlClassifierDBServiceWorker::GetChunkLists(PRUint32 tableId,
    2406                 :                                               nsACString& addChunks,
    2407                 :                                               nsACString& subChunks)
    2408                 : {
    2409              86 :   addChunks.Truncate();
    2410              86 :   subChunks.Truncate();
    2411                 : 
    2412             172 :   mozStorageStatementScoper scoper(mGetChunkListsStatement);
    2413                 : 
    2414              86 :   nsresult rv = mGetChunkListsStatement->BindInt32ByIndex(0, tableId);
    2415              86 :   NS_ENSURE_SUCCESS(rv, rv);
    2416                 : 
    2417              86 :   bool hasMore = false;
    2418              86 :   rv = mGetChunkListsStatement->ExecuteStep(&hasMore);
    2419              86 :   NS_ENSURE_SUCCESS(rv, rv);
    2420                 : 
    2421              86 :   if (!hasMore) {
    2422               0 :     LOG(("Getting chunks for %d, found nothing", tableId));
    2423               0 :     return NS_OK;
    2424                 :   }
    2425                 : 
    2426              86 :   rv = mGetChunkListsStatement->GetUTF8String(0, addChunks);
    2427              86 :   NS_ENSURE_SUCCESS(rv, rv);
    2428                 : 
    2429              86 :   rv = mGetChunkListsStatement->GetUTF8String(1, subChunks);
    2430              86 :   NS_ENSURE_SUCCESS(rv, rv);
    2431                 : 
    2432              86 :   LOG(("Getting chunks for %d, got %s/%s",
    2433                 :        tableId,
    2434                 :        PromiseFlatCString(addChunks).get(),
    2435                 :        PromiseFlatCString(subChunks).get()));
    2436                 : 
    2437              86 :   return NS_OK;
    2438                 : }
    2439                 : 
    2440                 : nsresult
    2441              86 : nsUrlClassifierDBServiceWorker::SetChunkLists(PRUint32 tableId,
    2442                 :                                               const nsACString& addChunks,
    2443                 :                                               const nsACString& subChunks)
    2444                 : {
    2445             172 :   mozStorageStatementScoper scoper(mSetChunkListsStatement);
    2446                 : 
    2447              86 :   mSetChunkListsStatement->BindUTF8StringByIndex(0, addChunks);
    2448              86 :   mSetChunkListsStatement->BindUTF8StringByIndex(1, subChunks);
    2449              86 :   mSetChunkListsStatement->BindInt32ByIndex(2, tableId);
    2450              86 :   nsresult rv = mSetChunkListsStatement->Execute();
    2451              86 :   NS_ENSURE_SUCCESS(rv, rv);
    2452                 : 
    2453              86 :   return NS_OK;
    2454                 : }
    2455                 : 
    2456                 : nsresult
    2457             120 : nsUrlClassifierDBServiceWorker::CacheChunkLists(PRUint32 tableId,
    2458                 :                                                 bool parseAdds,
    2459                 :                                                 bool parseSubs)
    2460                 : {
    2461                 :   nsresult rv;
    2462                 : 
    2463             120 :   if (mHaveCachedLists && mCachedListsTable != tableId) {
    2464               3 :     rv = FlushChunkLists();
    2465               3 :     NS_ENSURE_SUCCESS(rv, rv);
    2466                 :   }
    2467                 : 
    2468             120 :   if (!mHaveCachedLists) {
    2469              86 :     rv = GetChunkLists(tableId, mCachedAddsStr, mCachedSubsStr);
    2470              86 :     NS_ENSURE_SUCCESS(rv, rv);
    2471                 : 
    2472              86 :     mHaveCachedLists = true;
    2473              86 :     mCachedListsTable = tableId;
    2474                 :   }
    2475                 : 
    2476             120 :   if (parseAdds && !mHaveCachedAddChunks) {
    2477              84 :     ParseChunkList(mCachedAddsStr, mCachedAddChunks);
    2478              84 :     mHaveCachedAddChunks = true;
    2479                 :   }
    2480                 : 
    2481             120 :   if (parseSubs && !mHaveCachedSubChunks) {
    2482              20 :     ParseChunkList(mCachedSubsStr, mCachedSubChunks);
    2483              20 :     mHaveCachedSubChunks = true;
    2484                 :   }
    2485                 : 
    2486             120 :   return NS_OK;
    2487                 : }
    2488                 : 
    2489                 : nsresult
    2490              87 : nsUrlClassifierDBServiceWorker::FlushChunkLists()
    2491                 : {
    2492              87 :   if (!mHaveCachedLists) {
    2493               1 :     return NS_OK;
    2494                 :   }
    2495                 : 
    2496              86 :   if (mHaveCachedAddChunks) {
    2497              84 :     JoinChunkList(mCachedAddChunks, mCachedAddsStr);
    2498                 :   }
    2499                 : 
    2500              86 :   if (mHaveCachedSubChunks) {
    2501              20 :     JoinChunkList(mCachedSubChunks, mCachedSubsStr);
    2502                 :   }
    2503                 : 
    2504                 :   nsresult rv = SetChunkLists(mCachedListsTable,
    2505              86 :                               mCachedAddsStr, mCachedSubsStr);
    2506                 : 
    2507                 :   // clear out the cache before checking/returning the error here.
    2508              86 :   ClearCachedChunkLists();
    2509                 : 
    2510              86 :   return rv;
    2511                 : }
    2512                 : 
    2513                 : void
    2514             143 : nsUrlClassifierDBServiceWorker::ClearCachedChunkLists()
    2515                 : {
    2516             143 :   mCachedAddsStr.Truncate();
    2517             143 :   mCachedSubsStr.Truncate();
    2518             143 :   mCachedListsTable = PR_UINT32_MAX;
    2519             143 :   mHaveCachedLists = false;
    2520                 : 
    2521             143 :   mCachedAddChunks.Clear();
    2522             143 :   mHaveCachedAddChunks = false;
    2523                 : 
    2524             143 :   mCachedSubChunks.Clear();
    2525             143 :   mHaveCachedSubChunks = false;
    2526             143 : }
    2527                 : 
    2528                 : bool
    2529             103 : nsUrlClassifierDBServiceWorker::InsertChunkId(nsTArray<PRUint32> &chunks,
    2530                 :                                               PRUint32 chunkNum)
    2531                 : {
    2532             103 :   PRUint32 low = 0, high = chunks.Length();
    2533             235 :   while (high > low) {
    2534              31 :     PRUint32 mid = (high + low) >> 1;
    2535              31 :     if (chunks[mid] == chunkNum)
    2536               2 :       return false;
    2537              29 :     if (chunks[mid] < chunkNum)
    2538              29 :       low = mid + 1;
    2539                 :     else
    2540               0 :       high = mid;
    2541                 :   }
    2542                 : 
    2543             101 :   PRUint32 *item = chunks.InsertElementAt(low, chunkNum);
    2544             101 :   return (item != nsnull);
    2545                 : }
    2546                 : 
    2547                 : nsresult
    2548              83 : nsUrlClassifierDBServiceWorker::AddChunk(PRUint32 tableId,
    2549                 :                                          PRUint32 chunkNum,
    2550                 :                                          nsTArray<nsUrlClassifierEntry>& entries)
    2551                 : {
    2552                 : #if defined(PR_LOGGING)
    2553              83 :   PRIntervalTime clockStart = 0;
    2554              83 :   if (LOG_ENABLED()) {
    2555               0 :     clockStart = PR_IntervalNow();
    2556                 :   }
    2557                 : #endif
    2558                 : 
    2559              83 :   nsresult rv = CacheChunkLists(tableId, true, false);
    2560              83 :   NS_ENSURE_SUCCESS(rv, rv);
    2561                 : 
    2562              83 :   if (!InsertChunkId(mCachedAddChunks, chunkNum)) {
    2563               2 :     LOG(("Ignoring duplicate add chunk %d in table %d", chunkNum, tableId));
    2564               2 :     return NS_OK;
    2565                 :   }
    2566                 : 
    2567              81 :   LOG(("Adding %d entries to chunk %d in table %d", entries.Length(), chunkNum, tableId));
    2568                 : 
    2569             162 :   nsTArray<PRUint32> entryIDs;
    2570                 : 
    2571             162 :   nsAutoTArray<nsUrlClassifierEntry, 5> subEntries;
    2572              81 :   rv = mPendingSubStore.ReadSubEntries(tableId, chunkNum, subEntries);
    2573              81 :   NS_ENSURE_SUCCESS(rv, rv);
    2574                 : 
    2575             236 :   for (PRUint32 i = 0; i < entries.Length(); i++) {
    2576             155 :     nsUrlClassifierEntry& thisEntry = entries[i];
    2577                 : 
    2578             155 :     HandlePendingLookups();
    2579                 : 
    2580             155 :     bool writeEntry = true;
    2581             156 :     for (PRUint32 j = 0; j < subEntries.Length(); j++) {
    2582               8 :       if (thisEntry.SubMatch(subEntries[j])) {
    2583               7 :         subEntries.RemoveElementAt(j);
    2584                 : 
    2585               7 :         writeEntry = false;
    2586               7 :         break;
    2587                 :       }
    2588                 :     }
    2589                 : 
    2590             155 :     HandlePendingLookups();
    2591                 : 
    2592             155 :     if (writeEntry) {
    2593             148 :       rv = mMainStore.WriteEntry(thisEntry);
    2594             148 :       NS_ENSURE_SUCCESS(rv, rv);
    2595                 :     }
    2596                 :   }
    2597                 : 
    2598              81 :   rv = mPendingSubStore.ExpireAddChunk(tableId, chunkNum);
    2599              81 :   NS_ENSURE_SUCCESS(rv, rv);
    2600                 : 
    2601                 : #if defined(PR_LOGGING)
    2602              81 :   if (LOG_ENABLED()) {
    2603               0 :     PRIntervalTime clockEnd = PR_IntervalNow();
    2604               0 :     LOG(("adding chunk %d took %dms\n", chunkNum,
    2605                 :          PR_IntervalToMilliseconds(clockEnd - clockStart)));
    2606                 :   }
    2607                 : #endif
    2608                 : 
    2609              81 :   return rv;
    2610                 : }
    2611                 : 
    2612                 : nsresult
    2613              17 : nsUrlClassifierStore::Expire(PRUint32 tableId, PRUint32 chunkNum)
    2614                 : {
    2615              17 :   LOG(("Expiring chunk %d\n", chunkNum));
    2616                 : 
    2617              34 :   mozStorageStatementScoper expireScoper(mExpireStatement);
    2618                 : 
    2619              17 :   nsresult rv = mExpireStatement->BindInt32ByIndex(0, tableId);
    2620              17 :   NS_ENSURE_SUCCESS(rv, rv);
    2621              17 :   rv = mExpireStatement->BindInt32ByIndex(1, chunkNum);
    2622              17 :   NS_ENSURE_SUCCESS(rv, rv);
    2623                 : 
    2624              17 :   mWorker->HandlePendingLookups();
    2625                 : 
    2626              17 :   rv = mExpireStatement->Execute();
    2627              17 :   NS_ENSURE_SUCCESS(rv, rv);
    2628                 : 
    2629              17 :   return NS_OK;
    2630                 : }
    2631                 : 
    2632                 : nsresult
    2633              11 : nsUrlClassifierDBServiceWorker::ExpireAdd(PRUint32 tableId,
    2634                 :                                           PRUint32 chunkNum)
    2635                 : {
    2636              11 :   nsresult rv = CacheChunkLists(tableId, true, false);
    2637              11 :   NS_ENSURE_SUCCESS(rv, rv);
    2638              11 :   mCachedAddChunks.RemoveElement(chunkNum);
    2639                 : 
    2640              11 :   return mMainStore.Expire(tableId, chunkNum);
    2641                 : }
    2642                 : 
    2643                 : nsresult
    2644              20 : nsUrlClassifierDBServiceWorker::SubChunk(PRUint32 tableId,
    2645                 :                                          PRUint32 chunkNum,
    2646                 :                                          nsTArray<nsUrlClassifierEntry>& entries)
    2647                 : {
    2648              20 :   nsresult rv = CacheChunkLists(tableId, true, true);
    2649                 : 
    2650              20 :   if (!InsertChunkId(mCachedSubChunks, chunkNum)) {
    2651               0 :     LOG(("Ignoring duplicate sub chunk %d in table %d", chunkNum, tableId));
    2652               0 :     return NS_OK;
    2653                 :   }
    2654                 : 
    2655              20 :   LOG(("Subbing %d entries in chunk %d in table %d", entries.Length(), chunkNum, tableId));
    2656                 : 
    2657              47 :   for (PRUint32 i = 0; i < entries.Length(); i++) {
    2658              54 :     nsAutoTArray<nsUrlClassifierEntry, 5> existingEntries;
    2659              27 :     nsUrlClassifierEntry& thisEntry = entries[i];
    2660                 : 
    2661              27 :     HandlePendingLookups();
    2662                 : 
    2663                 :     // Check if we have the add chunk associated with the sub.
    2664              27 :     bool haveAdds = (mCachedAddChunks.BinaryIndexOf(thisEntry.mAddChunkId) !=
    2665              27 :                        mCachedAddChunks.NoIndex);
    2666                 : 
    2667              27 :     if (haveAdds) {
    2668                 :       rv = mMainStore.ReadAddEntries(thisEntry.mKey, thisEntry.mTableId,
    2669              12 :                                      thisEntry.mAddChunkId, existingEntries);
    2670              12 :       NS_ENSURE_SUCCESS(rv, rv);
    2671                 :     }
    2672                 : 
    2673              30 :     for (PRUint32 j = 0; j < existingEntries.Length(); j++) {
    2674              11 :       if (existingEntries[j].SubMatch(thisEntry)) {
    2675               8 :         rv = mMainStore.DeleteEntry(existingEntries[j]);
    2676               8 :         NS_ENSURE_SUCCESS(rv, rv);
    2677               8 :         existingEntries.RemoveElementAt(j);
    2678               8 :         break;
    2679                 :       }
    2680                 :     }
    2681                 : 
    2682              27 :     if (!haveAdds) {
    2683                 :       // Save this entry in the pending subtraction store.
    2684              15 :       rv = mPendingSubStore.WriteEntry(thisEntry);
    2685              15 :       NS_ENSURE_SUCCESS(rv, rv);
    2686                 :     }
    2687                 :   }
    2688                 : 
    2689              20 :   return NS_OK;
    2690                 : }
    2691                 : 
    2692                 : nsresult
    2693               6 : nsUrlClassifierDBServiceWorker::ExpireSub(PRUint32 tableId, PRUint32 chunkNum)
    2694                 : {
    2695               6 :   nsresult rv = CacheChunkLists(tableId, false, true);
    2696               6 :   NS_ENSURE_SUCCESS(rv, rv);
    2697               6 :   mCachedSubChunks.RemoveElement(chunkNum);
    2698                 : 
    2699               6 :   return mPendingSubStore.Expire(tableId, chunkNum);
    2700                 : }
    2701                 : 
    2702                 : nsresult
    2703             103 : nsUrlClassifierDBServiceWorker::ProcessChunk(bool* done)
    2704                 : {
    2705                 :   // wait until the chunk has been read
    2706             103 :   if (mPendingStreamUpdate.Length() < static_cast<PRUint32>(mChunkLen)) {
    2707               0 :     *done = true;
    2708               0 :     return NS_OK;
    2709                 :   }
    2710                 : 
    2711             206 :   nsCAutoString chunk;
    2712             103 :   chunk.Assign(Substring(mPendingStreamUpdate, 0, mChunkLen));
    2713             103 :   mPendingStreamUpdate = Substring(mPendingStreamUpdate, mChunkLen);
    2714                 : 
    2715             103 :   LOG(("Handling a chunk sized %d", chunk.Length()));
    2716                 : 
    2717             206 :   nsTArray<nsUrlClassifierEntry> entries;
    2718                 :   nsresult rv = GetChunkEntries(mUpdateTable, mUpdateTableId, mChunkType,
    2719             103 :                                 mChunkNum, mHashSize, chunk, entries);
    2720             103 :   NS_ENSURE_SUCCESS(rv, rv);
    2721                 : 
    2722             103 :   if (mChunkType == CHUNK_ADD) {
    2723              83 :     rv = AddChunk(mUpdateTableId, mChunkNum, entries);
    2724                 :   } else {
    2725              20 :     rv = SubChunk(mUpdateTableId, mChunkNum, entries);
    2726                 :   }
    2727                 : 
    2728             103 :   mState = STATE_LINE;
    2729             103 :   *done = false;
    2730                 : 
    2731             103 :   return rv;
    2732                 : }
    2733                 : 
    2734                 : nsresult
    2735             203 : nsUrlClassifierDBServiceWorker::ProcessResponseLines(bool* done)
    2736                 : {
    2737             203 :   PRUint32 cur = 0;
    2738                 :   PRInt32 next;
    2739                 : 
    2740                 :   nsresult rv;
    2741                 :   // We will run to completion unless we find a chunk line
    2742             203 :   *done = true;
    2743                 : 
    2744             203 :   nsACString& updateString = mPendingStreamUpdate;
    2745                 : 
    2746             726 :   while(cur < updateString.Length() &&
    2747                 :         (next = updateString.FindChar('\n', cur)) != kNotFound) {
    2748             848 :     const nsCSubstring& line = Substring(updateString, cur, next - cur);
    2749             424 :     cur = next + 1;
    2750                 : 
    2751             424 :     LOG(("Processing %s\n", PromiseFlatCString(line).get()));
    2752                 : 
    2753             424 :     if (mHMAC && mServerMAC.IsEmpty()) {
    2754                 :       // If we did not receive a server MAC during BeginStream(), we
    2755                 :       // require the first line of the update to be either a MAC or
    2756                 :       // a request to rekey.
    2757               2 :       if (StringBeginsWith(line, NS_LITERAL_CSTRING("m:"))) {
    2758               2 :         mServerMAC = Substring(line, 2);
    2759               2 :         nsUrlClassifierUtils::UnUrlsafeBase64(mServerMAC);
    2760                 : 
    2761                 :         // The remainder of the pending update needs to be digested.
    2762               4 :         const nsCSubstring &toDigest = Substring(updateString, cur);
    2763               4 :         rv = mHMAC->Update(reinterpret_cast<const PRUint8*>(toDigest.BeginReading()),
    2764               4 :                            toDigest.Length());
    2765               2 :         NS_ENSURE_SUCCESS(rv, rv);
    2766               0 :       } else if (line.EqualsLiteral("e:pleaserekey")) {
    2767               0 :         mUpdateObserver->RekeyRequested();
    2768                 :       } else {
    2769               0 :         LOG(("No MAC specified!"));
    2770               0 :         return NS_ERROR_FAILURE;
    2771                 :       }
    2772             422 :     } else if (StringBeginsWith(line, NS_LITERAL_CSTRING("n:"))) {
    2773             101 :       if (PR_sscanf(PromiseFlatCString(line).get(), "n:%d",
    2774             101 :                     &mUpdateWait) != 1) {
    2775               0 :         LOG(("Error parsing n: field: %s", PromiseFlatCString(line).get()));
    2776               0 :         mUpdateWait = 0;
    2777                 :       }
    2778             321 :     } else if (line.EqualsLiteral("r:pleasereset")) {
    2779               1 :       mResetRequested = true;
    2780             320 :     } else if (line.EqualsLiteral("e:pleaserekey")) {
    2781               0 :       mUpdateObserver->RekeyRequested();
    2782             320 :     } else if (StringBeginsWith(line, NS_LITERAL_CSTRING("i:"))) {
    2783              87 :       mUpdateTable.Assign(Substring(line, 2));
    2784              87 :       GetTableId(mUpdateTable, &mUpdateTableId);
    2785              87 :       LOG(("update table: '%s' (%d)", mUpdateTable.get(), mUpdateTableId));
    2786             233 :     } else if (StringBeginsWith(line, NS_LITERAL_CSTRING("u:"))) {
    2787              17 :       if (!mPrimaryStream) {
    2788               0 :         LOG(("Forwarded update tried to add its own forwarded update."));
    2789               0 :         return NS_ERROR_FAILURE;
    2790                 :       }
    2791                 : 
    2792              34 :       const nsCSubstring& data = Substring(line, 2);
    2793              17 :       if (mHMAC) {
    2794                 :         // We're expecting MACs alongside any url forwards.
    2795               3 :         nsCSubstring::const_iterator begin, end, sepBegin, sepEnd;
    2796               3 :         data.BeginReading(begin);
    2797               3 :         sepBegin = begin;
    2798                 : 
    2799               3 :         data.EndReading(end);
    2800               3 :         sepEnd = end;
    2801                 : 
    2802               3 :         if (!RFindInReadable(NS_LITERAL_CSTRING(","), sepBegin, sepEnd)) {
    2803               0 :           NS_WARNING("No MAC specified for a redirect in a request that expects a MAC");
    2804               0 :           return NS_ERROR_FAILURE;
    2805                 :         }
    2806                 : 
    2807               6 :         nsCString serverMAC(Substring(sepEnd, end));
    2808               3 :         nsUrlClassifierUtils::UnUrlsafeBase64(serverMAC);
    2809               6 :         mUpdateObserver->UpdateUrlRequested(Substring(begin, sepBegin),
    2810                 :                                             mUpdateTable,
    2811               3 :                                             serverMAC);
    2812                 :       } else {
    2813                 :         // We didn't ask for a MAC, none should have been specified.
    2814              14 :         mUpdateObserver->UpdateUrlRequested(data, mUpdateTable,
    2815              14 :                                             NS_LITERAL_CSTRING(""));
    2816                 :       }
    2817             696 :     } else if (StringBeginsWith(line, NS_LITERAL_CSTRING("a:")) ||
    2818             480 :                StringBeginsWith(line, NS_LITERAL_CSTRING("s:"))) {
    2819             104 :       mState = STATE_CHUNK;
    2820                 :       char command;
    2821             104 :       if (PR_sscanf(PromiseFlatCString(line).get(),
    2822             104 :                     "%c:%d:%d:%d", &command, &mChunkNum, &mHashSize, &mChunkLen) != 4) {
    2823               0 :         return NS_ERROR_FAILURE;
    2824                 :       }
    2825                 : 
    2826             104 :       if (mChunkLen > MAX_CHUNK_SIZE) {
    2827               0 :         return NS_ERROR_FAILURE;
    2828                 :       }
    2829                 : 
    2830             104 :       if (!(mHashSize == PARTIAL_LENGTH || mHashSize == COMPLETE_LENGTH)) {
    2831               1 :         NS_WARNING("Invalid hash size specified in update.");
    2832               1 :         return NS_ERROR_FAILURE;
    2833                 :       }
    2834                 : 
    2835             103 :       mChunkType = (command == 'a') ? CHUNK_ADD : CHUNK_SUB;
    2836                 : 
    2837                 :       // Done parsing lines, move to chunk state now
    2838             103 :       *done = false;
    2839                 :       break;
    2840             112 :     } else if (StringBeginsWith(line, NS_LITERAL_CSTRING("ad:"))) {
    2841              12 :       const nsCSubstring &list = Substring(line, 3);
    2842               6 :       nsACString::const_iterator begin, end;
    2843               6 :       list.BeginReading(begin);
    2844               6 :       list.EndReading(end);
    2845               6 :       while (begin != end) {
    2846                 :         PRUint32 first, last;
    2847               7 :         if (ParseChunkRange(begin, end, &first, &last)) {
    2848              18 :           for (PRUint32 num = first; num <= last; num++) {
    2849              11 :             rv = ExpireAdd(mUpdateTableId, num);
    2850              11 :             NS_ENSURE_SUCCESS(rv, rv);
    2851                 :           }
    2852                 :         } else {
    2853               0 :           return NS_ERROR_FAILURE;
    2854                 :         }
    2855                 :       }
    2856             106 :     } else if (StringBeginsWith(line, NS_LITERAL_CSTRING("sd:"))) {
    2857               6 :       const nsCSubstring &list = Substring(line, 3);
    2858               3 :       nsACString::const_iterator begin, end;
    2859               3 :       list.BeginReading(begin);
    2860               3 :       list.EndReading(end);
    2861               3 :       while (begin != end) {
    2862                 :         PRUint32 first, last;
    2863               4 :         if (ParseChunkRange(begin, end, &first, &last)) {
    2864              10 :           for (PRUint32 num = first; num <= last; num++) {
    2865               6 :             rv = ExpireSub(mUpdateTableId, num);
    2866               6 :             NS_ENSURE_SUCCESS(rv, rv);
    2867                 :           }
    2868                 :         } else {
    2869               0 :           return NS_ERROR_FAILURE;
    2870                 :         }
    2871                 :       }
    2872                 :     } else {
    2873             103 :       LOG(("ignoring unknown line: '%s'", PromiseFlatCString(line).get()));
    2874                 :     }
    2875                 :   }
    2876                 : 
    2877             202 :   mPendingStreamUpdate = Substring(updateString, cur);
    2878                 : 
    2879             202 :   return NS_OK;
    2880                 : }
    2881                 : 
    2882                 : void
    2883             102 : nsUrlClassifierDBServiceWorker::ResetStream()
    2884                 : {
    2885             102 :   mState = STATE_LINE;
    2886             102 :   mChunkNum = 0;
    2887             102 :   mHashSize = 0;
    2888             102 :   mChunkLen = 0;
    2889             102 :   mInStream = false;
    2890             102 :   mPrimaryStream = false;
    2891             102 :   mUpdateTable.Truncate();
    2892             102 :   mPendingStreamUpdate.Truncate();
    2893             102 :   mServerMAC.Truncate();
    2894             102 :   mHMAC = nsnull;
    2895             102 : }
    2896                 : 
    2897                 : void
    2898              95 : nsUrlClassifierDBServiceWorker::ResetUpdate()
    2899                 : {
    2900              95 :   mUpdateWait = 0;
    2901              95 :   mUpdateStatus = NS_OK;
    2902              95 :   mUpdateObserver = nsnull;
    2903              95 :   mUpdateClientKey.Truncate();
    2904              95 :   mResetRequested = false;
    2905              95 :   mUpdateTables.Clear();
    2906              95 : }
    2907                 : 
    2908                 : NS_IMETHODIMP
    2909               0 : nsUrlClassifierDBServiceWorker::SetHashCompleter(const nsACString &tableName,
    2910                 :                                                  nsIUrlClassifierHashCompleter *completer)
    2911                 : {
    2912               0 :   return NS_ERROR_NOT_IMPLEMENTED;
    2913                 : }
    2914                 : 
    2915                 : NS_IMETHODIMP
    2916              87 : nsUrlClassifierDBServiceWorker::BeginUpdate(nsIUrlClassifierUpdateObserver *observer,
    2917                 :                                             const nsACString &tables,
    2918                 :                                             const nsACString &clientKey)
    2919                 : {
    2920              87 :   if (gShuttingDownThread)
    2921               0 :     return NS_ERROR_NOT_INITIALIZED;
    2922                 : 
    2923              87 :   NS_ENSURE_STATE(!mUpdateObserver);
    2924                 : 
    2925              87 :   nsresult rv = OpenDb();
    2926              87 :   if (NS_FAILED(rv)) {
    2927               0 :     NS_ERROR("Unable to open database");
    2928               0 :     return NS_ERROR_FAILURE;
    2929                 :   }
    2930                 : 
    2931                 :   bool transaction;
    2932              87 :   rv = mConnection->GetTransactionInProgress(&transaction);
    2933              87 :   if (NS_FAILED(rv)) {
    2934               0 :     mUpdateStatus = rv;
    2935               0 :     return rv;
    2936                 :   }
    2937                 : 
    2938              87 :   if (transaction) {
    2939               0 :     NS_WARNING("Transaction already in progress in nsUrlClassifierDBServiceWorker::BeginUpdate.  Cancelling update.");
    2940               0 :     mUpdateStatus = NS_ERROR_FAILURE;
    2941               0 :     return rv;
    2942                 :   }
    2943                 : 
    2944              87 :   rv = SetupUpdate();
    2945              87 :   if (NS_FAILED(rv)) {
    2946               0 :     mUpdateStatus = rv;
    2947               0 :     return rv;
    2948                 :   }
    2949                 : 
    2950              87 :   mUpdateObserver = observer;
    2951                 : 
    2952              87 :   if (!clientKey.IsEmpty()) {
    2953               2 :     rv = nsUrlClassifierUtils::DecodeClientKey(clientKey, mUpdateClientKey);
    2954               2 :     NS_ENSURE_SUCCESS(rv, rv);
    2955                 :   }
    2956                 : 
    2957                 :   // The first stream in an update is the only stream that may request
    2958                 :   // forwarded updates.
    2959              87 :   mPrimaryStream = true;
    2960                 : 
    2961              87 :   SplitTables(tables, mUpdateTables);
    2962                 : 
    2963              87 :   return NS_OK;
    2964                 : }
    2965                 : 
    2966                 : NS_IMETHODIMP
    2967             102 : nsUrlClassifierDBServiceWorker::BeginStream(const nsACString &table,
    2968                 :                                             const nsACString &serverMAC)
    2969                 : {
    2970             102 :   if (gShuttingDownThread)
    2971               0 :     return NS_ERROR_NOT_INITIALIZED;
    2972                 : 
    2973             102 :   NS_ENSURE_STATE(mUpdateObserver);
    2974             102 :   NS_ENSURE_STATE(!mInStream);
    2975                 : 
    2976                 :   // We may have committed the update in FinishStream, if so set it up
    2977                 :   // again here.
    2978             102 :   nsresult rv = SetupUpdate();
    2979             102 :   if (NS_FAILED(rv)) {
    2980               0 :     mUpdateStatus = rv;
    2981               0 :     return rv;
    2982                 :   }
    2983                 : 
    2984             102 :   mInStream = true;
    2985                 : 
    2986                 :   // If we're expecting a MAC, create the nsICryptoHMAC component now.
    2987             102 :   if (!mUpdateClientKey.IsEmpty()) {
    2988                 :     nsCOMPtr<nsIKeyObjectFactory> keyObjectFactory(do_GetService(
    2989              10 :         "@mozilla.org/security/keyobjectfactory;1", &rv));
    2990               5 :     if (NS_FAILED(rv)) {
    2991               0 :       NS_WARNING("Failed to get nsIKeyObjectFactory service");
    2992               0 :       mUpdateStatus = rv;
    2993               0 :       return mUpdateStatus;
    2994                 :     }
    2995                 : 
    2996              10 :     nsCOMPtr<nsIKeyObject> keyObject;
    2997               5 :     rv = keyObjectFactory->KeyFromString(nsIKeyObject::HMAC, mUpdateClientKey, 
    2998               5 :         getter_AddRefs(keyObject));
    2999               5 :     if (NS_FAILED(rv)) {
    3000               0 :       NS_WARNING("Failed to create key object, maybe not FIPS compliant?");
    3001               0 :       mUpdateStatus = rv;
    3002               0 :       return mUpdateStatus;
    3003                 :     }
    3004                 : 
    3005               5 :     mHMAC = do_CreateInstance(NS_CRYPTO_HMAC_CONTRACTID, &rv);
    3006               5 :     if (NS_FAILED(rv)) {
    3007               0 :       NS_WARNING("Failed to create nsICryptoHMAC instance");
    3008               0 :       mUpdateStatus = rv;
    3009               0 :       return mUpdateStatus;
    3010                 :     }
    3011                 : 
    3012               5 :     rv = mHMAC->Init(nsICryptoHMAC::SHA1, keyObject);
    3013               5 :     if (NS_FAILED(rv)) {
    3014               0 :       NS_WARNING("Failed to initialize nsICryptoHMAC instance");
    3015               0 :       mUpdateStatus = rv;
    3016               0 :       return mUpdateStatus;
    3017                 :     }
    3018                 :   }
    3019                 : 
    3020             102 :   mServerMAC = serverMAC;
    3021                 : 
    3022             102 :   if (!table.IsEmpty()) {
    3023              15 :     mUpdateTable = table;
    3024              15 :     GetTableId(mUpdateTable, &mUpdateTableId);
    3025              15 :     LOG(("update table: '%s' (%d)", mUpdateTable.get(), mUpdateTableId));
    3026                 :   }
    3027                 : 
    3028             102 :   return NS_OK;
    3029                 : }
    3030                 : 
    3031                 : /**
    3032                 :  * Updating the database:
    3033                 :  *
    3034                 :  * The Update() method takes a series of chunks separated with control data,
    3035                 :  * as described in
    3036                 :  * http://code.google.com/p/google-safe-browsing/wiki/Protocolv2Spec
    3037                 :  *
    3038                 :  * It will iterate through the control data until it reaches a chunk.  By
    3039                 :  * the time it reaches a chunk, it should have received
    3040                 :  * a) the table to which this chunk applies
    3041                 :  * b) the type of chunk (add, delete, expire add, expire delete).
    3042                 :  * c) the chunk ID
    3043                 :  * d) the length of the chunk.
    3044                 :  *
    3045                 :  * For add and subtract chunks, it needs to read the chunk data (expires
    3046                 :  * don't have any data).  Chunk data is a list of URI fragments whose
    3047                 :  * encoding depends on the type of table (which is indicated by the end
    3048                 :  * of the table name):
    3049                 :  * a) tables ending with -exp are a zlib-compressed list of URI fragments
    3050                 :  *    separated by newlines.
    3051                 :  * b) tables ending with -sha128 have the form
    3052                 :  *    [domain][N][frag0]...[fragN]
    3053                 :  *       16    1   16        16
    3054                 :  *    If N is 0, the domain is reused as a fragment.
    3055                 :  * c) any other tables are assumed to be a plaintext list of URI fragments
    3056                 :  *    separated by newlines.
    3057                 :  *
    3058                 :  * Update() can be fed partial data;  It will accumulate data until there is
    3059                 :  * enough to act on.  Finish() should be called when there will be no more
    3060                 :  * data.
    3061                 :  */
    3062                 : NS_IMETHODIMP
    3063             100 : nsUrlClassifierDBServiceWorker::UpdateStream(const nsACString& chunk)
    3064                 : {
    3065             100 :   if (gShuttingDownThread)
    3066               0 :     return NS_ERROR_NOT_INITIALIZED;
    3067                 : 
    3068             100 :   NS_ENSURE_STATE(mInStream);
    3069                 : 
    3070             100 :   HandlePendingLookups();
    3071                 : 
    3072             100 :   LOG(("Update from Stream."));
    3073             100 :   nsresult rv = OpenDb();
    3074             100 :   if (NS_FAILED(rv)) {
    3075               0 :     NS_ERROR("Unable to open database");
    3076               0 :     return NS_ERROR_FAILURE;
    3077                 :   }
    3078                 : 
    3079                 :   // if something has gone wrong during this update, just throw it away
    3080             100 :   if (NS_FAILED(mUpdateStatus)) {
    3081               0 :     return mUpdateStatus;
    3082                 :   }
    3083                 : 
    3084             100 :   if (mHMAC && !mServerMAC.IsEmpty()) {
    3085               6 :     rv = mHMAC->Update(reinterpret_cast<const PRUint8*>(chunk.BeginReading()),
    3086               6 :                        chunk.Length());
    3087               3 :     if (NS_FAILED(rv)) {
    3088               0 :       mUpdateStatus = rv;
    3089               0 :       return mUpdateStatus;
    3090                 :     }
    3091                 :   }
    3092                 : 
    3093             100 :   LOG(("Got %s\n", PromiseFlatCString(chunk).get()));
    3094                 : 
    3095             100 :   mPendingStreamUpdate.Append(chunk);
    3096                 : 
    3097             100 :   bool done = false;
    3098             505 :   while (!done) {
    3099             306 :     if (mState == STATE_CHUNK) {
    3100             103 :       rv = ProcessChunk(&done);
    3101                 :     } else {
    3102             203 :       rv = ProcessResponseLines(&done);
    3103                 :     }
    3104             306 :     if (NS_FAILED(rv)) {
    3105               1 :       mUpdateStatus = rv;
    3106               1 :       return rv;
    3107                 :     }
    3108                 :   }
    3109                 : 
    3110              99 :   return NS_OK;
    3111                 : }
    3112                 : 
    3113                 : NS_IMETHODIMP
    3114             100 : nsUrlClassifierDBServiceWorker::FinishStream()
    3115                 : {
    3116             100 :   if (gShuttingDownThread)
    3117               0 :     return NS_ERROR_NOT_INITIALIZED;
    3118                 : 
    3119             100 :   NS_ENSURE_STATE(mInStream);
    3120             100 :   NS_ENSURE_STATE(mUpdateObserver);
    3121                 : 
    3122             100 :   PRInt32 nextStreamDelay = 0;
    3123                 : 
    3124             100 :   if (NS_SUCCEEDED(mUpdateStatus) && mHMAC) {
    3125              10 :     nsCAutoString clientMAC;
    3126               5 :     mHMAC->Finish(true, clientMAC);
    3127                 : 
    3128               5 :     if (clientMAC != mServerMAC) {
    3129               0 :       NS_WARNING("Invalid update MAC!");
    3130               0 :       LOG(("Invalid update MAC: expected %s, got %s",
    3131                 :            mServerMAC.get(), clientMAC.get()));
    3132               0 :       mUpdateStatus = NS_ERROR_FAILURE;
    3133                 :     }
    3134               5 :     PRIntervalTime updateTime = PR_IntervalNow() - mUpdateStartTime;
    3135               5 :     if (PR_IntervalToSeconds(updateTime) >=
    3136                 :         static_cast<PRUint32>(gWorkingTimeThreshold)) {
    3137                 :       // We've spent long enough working that we should commit what we
    3138                 :       // have and hold off for a bit.
    3139               0 :       nsresult rv = ApplyUpdate();
    3140               0 :       if (NS_FAILED(rv)) {
    3141               0 :         if (rv == NS_ERROR_FILE_CORRUPTED) {
    3142               0 :           ResetDatabase();
    3143                 :         }
    3144               0 :         return rv;
    3145                 :       }
    3146               0 :       nextStreamDelay = gDelayTime * 1000;
    3147                 :     }
    3148                 :   }
    3149                 : 
    3150             100 :   mUpdateObserver->StreamFinished(mUpdateStatus,
    3151             100 :                                   static_cast<PRUint32>(nextStreamDelay));
    3152                 : 
    3153             100 :   ResetStream();
    3154                 : 
    3155             100 :   return NS_OK;
    3156                 : }
    3157                 : 
    3158                 : nsresult
    3159             228 : nsUrlClassifierDBServiceWorker::SetCacheSize(
    3160                 :   mozIStorageConnection * aConnection, PRInt32 aCacheSize)
    3161                 : {
    3162             456 :   mozStorageStatementScoper scoper(mGetPageSizeStatement);
    3163                 :   bool hasResult;
    3164             228 :   nsresult rv = mGetPageSizeStatement->ExecuteStep(&hasResult);
    3165             228 :   NS_ENSURE_SUCCESS(rv, rv);
    3166                 : 
    3167             228 :   NS_ASSERTION(hasResult, "Should always be able to get page size from sqlite");
    3168             228 :   PRUint32 pageSize = mGetPageSizeStatement->AsInt32(0);
    3169             228 :   PRUint32 cachePages = aCacheSize / pageSize;
    3170                 :   nsCAutoString cacheSizePragma(MOZ_STORAGE_UNIQUIFY_QUERY_STR
    3171             456 :                                 "PRAGMA cache_size=");
    3172             228 :   cacheSizePragma.AppendInt(cachePages);
    3173             228 :   rv = aConnection->ExecuteSimpleSQL(cacheSizePragma);
    3174             228 :   NS_ENSURE_SUCCESS(rv, rv);
    3175                 : 
    3176             228 :   return NS_OK;
    3177                 : }
    3178                 : 
    3179                 : nsresult
    3180             189 : nsUrlClassifierDBServiceWorker::SetupUpdate()
    3181                 : {
    3182             189 :   LOG(("nsUrlClassifierDBServiceWorker::SetupUpdate"));
    3183                 :   bool inProgress;
    3184             189 :   nsresult rv = mConnection->GetTransactionInProgress(&inProgress);
    3185             189 :   if (inProgress) {
    3186             102 :     return NS_OK;
    3187                 :   }
    3188                 : 
    3189              87 :   mUpdateStartTime = PR_IntervalNow();
    3190                 : 
    3191              87 :   rv = mConnection->BeginTransaction();
    3192              87 :   NS_ENSURE_SUCCESS(rv, rv);
    3193                 : 
    3194              87 :   if (gUpdateCacheSize > 0) {
    3195              87 :     rv = SetCacheSize(mConnection, gUpdateCacheSize);
    3196              87 :     NS_ENSURE_SUCCESS(rv, rv);
    3197              87 :     if (gUpdateCacheSize != gLookupCacheSize) {
    3198              87 :       mGrewCache = true;
    3199                 :     }
    3200                 :   }
    3201                 : 
    3202              87 :   return NS_OK;
    3203                 : }
    3204                 : 
    3205                 : nsresult
    3206              85 : nsUrlClassifierDBServiceWorker::ApplyUpdate()
    3207                 : {
    3208              85 :   LOG(("nsUrlClassifierDBServiceWorker::ApplyUpdate"));
    3209                 : 
    3210              85 :   if (mConnection) {
    3211              85 :     if (NS_FAILED(mUpdateStatus)) {
    3212               1 :       mConnection->RollbackTransaction();
    3213                 :     } else {
    3214              84 :       mUpdateStatus = FlushChunkLists();
    3215              84 :       if (NS_SUCCEEDED(mUpdateStatus)) {
    3216              84 :         mUpdateStatus = mConnection->CommitTransaction();
    3217                 :       }
    3218                 :     }
    3219                 :   }
    3220                 : 
    3221              85 :   if (NS_SUCCEEDED(mUpdateStatus)) {
    3222                 :     // Reconstruct the prefix tree from the DB
    3223              84 :     nsresult rv = ConstructPrefixSet();
    3224              84 :     NS_ENSURE_SUCCESS(rv, rv);
    3225                 :   }
    3226                 : 
    3227              85 :   if (mGrewCache) {
    3228                 :     // During the update we increased the page cache to bigger than we
    3229                 :     // want to keep around.  At the moment, the only reliable way to make
    3230                 :     // sure that the page cache is freed is to reopen the connection.
    3231              85 :     LOG(("GrewCache true, reopening DB"));
    3232              85 :     mGrewCache = false;
    3233              85 :     CloseDb();
    3234              85 :     OpenDb();
    3235                 :   }
    3236                 : 
    3237              85 :   mUpdateStartTime = 0;
    3238                 : 
    3239              85 :   return NS_OK;
    3240                 : }
    3241                 : 
    3242                 : NS_IMETHODIMP
    3243              85 : nsUrlClassifierDBServiceWorker::FinishUpdate()
    3244                 : {
    3245              85 :   LOG(("nsUrlClassifierDBServiceWorker::FinishUpdate()"));
    3246              85 :   if (gShuttingDownThread)
    3247               0 :     return NS_ERROR_NOT_INITIALIZED;
    3248                 : 
    3249              85 :   NS_ENSURE_STATE(!mInStream);
    3250              85 :   NS_ENSURE_STATE(mUpdateObserver);
    3251                 : 
    3252                 :   // We need to get the error code before ApplyUpdate, because it might
    3253                 :   // close/open the connection.
    3254              85 :   PRInt32 errcode = SQLITE_OK;
    3255              85 :   if (mConnection)
    3256              85 :     mConnection->GetLastError(&errcode);
    3257                 : 
    3258              85 :   nsresult rv = ApplyUpdate();
    3259              85 :   if (NS_FAILED(rv)) {
    3260               0 :     if (rv == NS_ERROR_FILE_CORRUPTED) {
    3261               0 :       ResetDatabase();
    3262                 :     }
    3263               0 :     return rv;
    3264                 :   }
    3265                 : 
    3266              85 :   if (NS_SUCCEEDED(mUpdateStatus)) {
    3267              84 :     mUpdateObserver->UpdateSuccess(mUpdateWait);
    3268                 :   } else {
    3269               1 :     mUpdateObserver->UpdateError(mUpdateStatus);
    3270                 :   }
    3271                 : 
    3272                 :   // It's important that we only reset the database on an update
    3273                 :   // command if the update was successful, otherwise unauthenticated
    3274                 :   // updates could cause a database reset.
    3275              85 :   bool resetDB = (NS_SUCCEEDED(mUpdateStatus) && mResetRequested) ||
    3276              85 :                     errcode == SQLITE_CORRUPT;
    3277                 : 
    3278              85 :   if (!resetDB) {
    3279              84 :     if (NS_SUCCEEDED(mUpdateStatus)) {
    3280              83 :       PRInt64 now = (PR_Now() / PR_USEC_PER_SEC);
    3281             249 :       for (PRUint32 i = 0; i < mUpdateTables.Length(); i++) {
    3282             166 :         LOG(("Successfully updated %s", mUpdateTables[i].get()));
    3283             166 :         mTableFreshness.Put(mUpdateTables[i], now);
    3284                 :       }
    3285                 :     } else {
    3286               3 :       for (PRUint32 i = 0; i < mUpdateTables.Length(); i++) {
    3287               2 :         LOG(("Failed updating %s", mUpdateTables[i].get()));
    3288               2 :         mTableFreshness.Remove(mUpdateTables[i]);
    3289                 :       }
    3290                 :     }
    3291                 :   }
    3292                 : 
    3293              85 :   ResetUpdate();
    3294                 : 
    3295              85 :   if (resetDB) {
    3296               1 :     ResetDatabase();
    3297                 :   }
    3298                 : 
    3299              85 :   return NS_OK;
    3300                 : }
    3301                 : 
    3302                 : NS_IMETHODIMP
    3303              55 : nsUrlClassifierDBServiceWorker::ResetDatabase()
    3304                 : {
    3305              55 :   LOG(("nsUrlClassifierDBServiceWorker::ResetDatabase [%p]", this));
    3306              55 :   ClearCachedChunkLists();
    3307                 : 
    3308              55 :   mTableFreshness.Clear();
    3309                 : 
    3310              55 :   nsresult rv = CloseDb();
    3311              55 :   NS_ENSURE_SUCCESS(rv, rv);
    3312                 : 
    3313              55 :   rv = mPrefixSet->SetPrefixes(nsnull, 0);
    3314              55 :   NS_ENSURE_SUCCESS(rv, rv);
    3315                 : 
    3316              55 :   mDBFile->Remove(false);
    3317              55 :   mPSFile->Remove(false);
    3318                 : 
    3319              55 :   return NS_OK;
    3320                 : }
    3321                 : 
    3322                 : NS_IMETHODIMP
    3323              10 : nsUrlClassifierDBServiceWorker::CancelUpdate()
    3324                 : {
    3325              10 :   LOG(("CancelUpdate"));
    3326                 : 
    3327              10 :   if (mUpdateObserver) {
    3328               2 :     mUpdateStatus = NS_BINDING_ABORTED;
    3329                 : 
    3330               2 :     ClearCachedChunkLists();
    3331               2 :     mConnection->RollbackTransaction();
    3332               2 :     mUpdateObserver->UpdateError(mUpdateStatus);
    3333                 : 
    3334               5 :     for (PRUint32 i = 0; i < mUpdateTables.Length(); i++) {
    3335               3 :       LOG(("Failed updating %s", mUpdateTables[i].get()));
    3336               3 :       mTableFreshness.Remove(mUpdateTables[i]);
    3337                 :     }
    3338                 : 
    3339               2 :     ResetStream();
    3340               2 :     ResetUpdate();
    3341                 :   }
    3342                 : 
    3343              10 :   return NS_OK;
    3344                 : }
    3345                 : 
    3346                 : // Allows the main thread to delete the connection which may be in
    3347                 : // a background thread.
    3348                 : // XXX This could be turned into a single shutdown event so the logic
    3349                 : // is simpler in nsUrlClassifierDBService::Shutdown.
    3350                 : NS_IMETHODIMP
    3351             148 : nsUrlClassifierDBServiceWorker::CloseDb()
    3352                 : {
    3353             148 :   if (mConnection) {
    3354             141 :     mMainStore.Close();
    3355             141 :     mPendingSubStore.Close();
    3356                 : 
    3357             141 :     mGetChunkListsStatement = nsnull;
    3358             141 :     mSetChunkListsStatement = nsnull;
    3359                 : 
    3360             141 :     mGetTablesStatement = nsnull;
    3361             141 :     mGetTableIdStatement = nsnull;
    3362             141 :     mGetTableNameStatement = nsnull;
    3363             141 :     mInsertTableIdStatement = nsnull;
    3364             141 :     mGetPageSizeStatement = nsnull;
    3365                 : 
    3366             141 :     mConnection = nsnull;
    3367             141 :     LOG(("urlclassifier db closed\n"));
    3368                 :   }
    3369                 : 
    3370             148 :   mCryptoHash = nsnull;
    3371                 : 
    3372             148 :   return NS_OK;
    3373                 : }
    3374                 : 
    3375                 : NS_IMETHODIMP
    3376              20 : nsUrlClassifierDBServiceWorker::CacheCompletions(nsTArray<nsUrlClassifierLookupResult> *results)
    3377                 : {
    3378              20 :   LOG(("nsUrlClassifierDBServiceWorker::CacheCompletions [%p]", this));
    3379                 : 
    3380              40 :   nsAutoPtr<nsTArray<nsUrlClassifierLookupResult> > resultsPtr(results);
    3381                 : 
    3382                 :   // Start a new transaction.  If a transaction is open for an update
    3383                 :   // this will be a noop, and this cache will be included in the
    3384                 :   // update's transaction.
    3385              40 :   mozStorageTransaction trans(mConnection, true);
    3386                 : 
    3387              56 :   for (PRUint32 i = 0; i < results->Length(); i++) {
    3388              36 :     nsUrlClassifierLookupResult& result = results->ElementAt(i);
    3389                 :     // Failing to update here shouldn't be fatal (and might be common,
    3390                 :     // if we're updating entries that were removed since they were
    3391                 :     // returned after a lookup).
    3392              36 :     mMainStore.UpdateEntry(result.mEntry);
    3393                 :   }
    3394                 : 
    3395              20 :   return NS_OK;
    3396                 : }
    3397                 : 
    3398                 : nsresult
    3399             466 : nsUrlClassifierDBServiceWorker::OpenDb()
    3400                 : {
    3401                 :   // Connection already open, don't do anything.
    3402             466 :   if (mConnection) {
    3403             325 :     return NS_OK;
    3404                 :   }
    3405                 : 
    3406             141 :   LOG(("Opening db\n"));
    3407                 : 
    3408                 :   nsresult rv;
    3409                 :   // open the connection
    3410                 :   nsCOMPtr<mozIStorageService> storageService =
    3411             282 :     do_GetService(MOZ_STORAGE_SERVICE_CONTRACTID, &rv);
    3412             141 :   NS_ENSURE_SUCCESS(rv, rv);
    3413                 : 
    3414                 :   bool exists;
    3415             141 :   rv = mDBFile->Exists(&exists);
    3416             141 :   NS_ENSURE_SUCCESS(rv, rv);
    3417             141 :   bool newDB = !exists;
    3418                 : 
    3419             282 :   nsCOMPtr<mozIStorageConnection> connection;
    3420             141 :   rv = storageService->OpenDatabase(mDBFile, getter_AddRefs(connection));
    3421             141 :   if (rv == NS_ERROR_FILE_CORRUPTED) {
    3422                 :     // delete the db and try opening again
    3423               0 :     rv = mDBFile->Remove(false);
    3424               0 :     NS_ENSURE_SUCCESS(rv, rv);
    3425                 : 
    3426               0 :     newDB = true;
    3427                 : 
    3428               0 :     rv = storageService->OpenDatabase(mDBFile, getter_AddRefs(connection));
    3429                 :   }
    3430             141 :   NS_ENSURE_SUCCESS(rv, rv);
    3431                 : 
    3432             141 :   if (!newDB) {
    3433                 :     PRInt32 databaseVersion;
    3434              85 :     rv = connection->GetSchemaVersion(&databaseVersion);
    3435              85 :     NS_ENSURE_SUCCESS(rv, rv);
    3436                 : 
    3437              85 :     if (databaseVersion != IMPLEMENTATION_VERSION) {
    3438               0 :       LOG(("Incompatible database, removing."));
    3439                 : 
    3440               0 :       rv = connection->Close();
    3441               0 :       NS_ENSURE_SUCCESS(rv, rv);
    3442                 : 
    3443               0 :       rv = mDBFile->Remove(false);
    3444               0 :       NS_ENSURE_SUCCESS(rv, rv);
    3445                 : 
    3446               0 :       newDB = true;
    3447                 : 
    3448               0 :       rv = storageService->OpenDatabase(mDBFile, getter_AddRefs(connection));
    3449               0 :       NS_ENSURE_SUCCESS(rv, rv);
    3450                 :     }
    3451                 :   }
    3452                 : 
    3453             141 :   connection->SetGrowthIncrement(5 * 1024 * 1024, EmptyCString());
    3454             141 :   rv = connection->ExecuteSimpleSQL(NS_LITERAL_CSTRING("PRAGMA synchronous=OFF"));
    3455             141 :   NS_ENSURE_SUCCESS(rv, rv);
    3456                 : 
    3457             141 :   rv = connection->CreateStatement
    3458             141 :     (NS_LITERAL_CSTRING(MOZ_STORAGE_UNIQUIFY_QUERY_STR "PRAGMA page_size"),
    3459             282 :      getter_AddRefs(mGetPageSizeStatement));
    3460             141 :   NS_ENSURE_SUCCESS(rv, rv);
    3461                 : 
    3462             141 :   rv = SetCacheSize(connection, gLookupCacheSize);
    3463             141 :   NS_ENSURE_SUCCESS(rv, rv);
    3464                 : 
    3465             141 :   if (newDB) {
    3466              56 :     rv = connection->SetSchemaVersion(IMPLEMENTATION_VERSION);
    3467              56 :     NS_ENSURE_SUCCESS(rv, rv);
    3468                 :   }
    3469                 : 
    3470                 :   // Create the table
    3471             141 :   rv = MaybeCreateTables(connection);
    3472             141 :   NS_ENSURE_SUCCESS(rv, rv);
    3473                 : 
    3474                 :   rv = mMainStore.Init(this, connection,
    3475             141 :                        NS_LITERAL_CSTRING("moz_classifier"));
    3476             141 :   NS_ENSURE_SUCCESS(rv, rv);
    3477                 : 
    3478                 :   rv = mPendingSubStore.Init(this, connection,
    3479             141 :                              NS_LITERAL_CSTRING("moz_subs"));
    3480             141 :   NS_ENSURE_SUCCESS(rv, rv);
    3481                 : 
    3482             141 :   rv = connection->CreateStatement
    3483             141 :          (NS_LITERAL_CSTRING("SELECT add_chunks, sub_chunks FROM moz_tables"
    3484                 :                              " WHERE id=?1"),
    3485             282 :           getter_AddRefs(mGetChunkListsStatement));
    3486             141 :   NS_ENSURE_SUCCESS(rv, rv);
    3487                 : 
    3488             141 :   rv = connection->CreateStatement
    3489             141 :          (NS_LITERAL_CSTRING("UPDATE moz_tables"
    3490                 :                              " SET add_chunks=?1, sub_chunks=?2"
    3491                 :                              " WHERE id=?3"),
    3492             282 :           getter_AddRefs(mSetChunkListsStatement));
    3493             141 :   NS_ENSURE_SUCCESS(rv, rv);
    3494                 : 
    3495             141 :   rv = connection->CreateStatement
    3496             141 :          (NS_LITERAL_CSTRING("SELECT name, add_chunks, sub_chunks"
    3497                 :                              " FROM moz_tables"),
    3498             282 :           getter_AddRefs(mGetTablesStatement));
    3499             141 :   NS_ENSURE_SUCCESS(rv, rv);
    3500                 : 
    3501             141 :   rv = connection->CreateStatement
    3502             141 :     (NS_LITERAL_CSTRING("SELECT id FROM moz_tables"
    3503                 :                         " WHERE name = ?1"),
    3504             282 :      getter_AddRefs(mGetTableIdStatement));
    3505             141 :   NS_ENSURE_SUCCESS(rv, rv);
    3506                 : 
    3507             141 :   rv = connection->CreateStatement
    3508             141 :     (NS_LITERAL_CSTRING("SELECT name FROM moz_tables"
    3509                 :                         " WHERE id = ?1"),
    3510             282 :      getter_AddRefs(mGetTableNameStatement));
    3511             141 :   NS_ENSURE_SUCCESS(rv, rv);
    3512                 : 
    3513             141 :   rv = connection->CreateStatement
    3514             141 :     (NS_LITERAL_CSTRING("INSERT INTO moz_tables(id, name, add_chunks, sub_chunks)"
    3515                 :                         " VALUES (null, ?1, null, null)"),
    3516             282 :      getter_AddRefs(mInsertTableIdStatement));
    3517             141 :   NS_ENSURE_SUCCESS(rv, rv);
    3518                 : 
    3519             141 :   mConnection = connection;
    3520                 : 
    3521             141 :   mCryptoHash = do_CreateInstance(NS_CRYPTO_HASH_CONTRACTID, &rv);
    3522             141 :   NS_ENSURE_SUCCESS(rv, rv);
    3523                 : 
    3524             141 :   LOG(("loading Prefix Set\n"));
    3525             141 :   rv = LoadPrefixSet(mPSFile);
    3526             141 :   if (NS_FAILED(rv)) {
    3527               0 :     if (rv == NS_ERROR_FILE_CORRUPTED) {
    3528               0 :       ResetDatabase();
    3529                 :     }
    3530               0 :     return rv;
    3531                 :   }
    3532                 : 
    3533             141 :   return NS_OK;
    3534                 : }
    3535                 : 
    3536                 : // We have both a prefix and a domain. Drop the domain, but
    3537                 : // hash the domain, the prefix and a random value together,
    3538                 : // ensuring any collisions happens at a different points for
    3539                 : // different users.
    3540                 : // We need to calculate +- 500k hashes each update.
    3541                 : // The extensive initialization and finalization of normal
    3542                 : // cryptographic hashes, as well as fairly low speed, causes them
    3543                 : // to be prohibitively slow here, hence we can't use them.
    3544                 : // We use MurmurHash3 instead because it's reasonably well
    3545                 : // researched, trusted inside some other big projects, extremely
    3546                 : // fast and with a specific a 32-bit output version, and fairly
    3547                 : // compact. Upon testing with the actual prefix data, it does
    3548                 : // not appear to increase the number of collisions by any
    3549                 : // meaningful amount.
    3550             224 : static nsresult KeyedHash(PRUint32 aPref, PRUint32 aDomain,
    3551                 :                           PRUint32 aKey, PRUint32 *aOut)
    3552                 : {
    3553                 :   // This is a reimplementation of MurmurHash3 32-bit
    3554                 :   // based on the public domain C++ sources.
    3555                 :   // http://code.google.com/p/smhasher/source/browse/trunk/MurmurHash3.cpp
    3556                 :   // for nblocks = 2
    3557             224 :   PRUint32 c1 = 0xCC9E2D51;
    3558             224 :   PRUint32 c2 = 0x1B873593;
    3559             224 :   PRUint32 c3 = 0xE6546B64;
    3560             224 :   PRUint32 c4 = 0x85EBCA6B;
    3561             224 :   PRUint32 c5 = 0xC2B2AE35;
    3562             224 :   PRUint32 h1 = aPref; // seed
    3563                 :   PRUint32 k1;
    3564                 :   PRUint32 karr[2];
    3565                 : 
    3566             224 :   karr[0] = aDomain;
    3567             224 :   karr[1] = aKey;
    3568                 : 
    3569             672 :   for (PRUint32 i = 0; i < 2; i++) {
    3570             448 :     k1 = karr[i];
    3571             448 :     k1 *= c1;
    3572             448 :     k1 = (k1 << 15) | (k1 >> (32-15));
    3573             448 :     k1 *= c2;
    3574                 : 
    3575             448 :     h1 ^= k1;
    3576             448 :     h1 = (h1 << 13) | (h1 >> (32-13));
    3577             448 :     h1 *= 5;
    3578             448 :     h1 += c3;
    3579                 :   }
    3580                 : 
    3581             224 :   h1 ^= 2; // len
    3582                 :   // fmix
    3583             224 :   h1 ^= h1 >> 16;
    3584             224 :   h1 *= c4;
    3585             224 :   h1 ^= h1 >> 13;
    3586             224 :   h1 *= c5;
    3587             224 :   h1 ^= h1 >> 16;
    3588                 : 
    3589             224 :   *aOut = h1;
    3590                 : 
    3591             224 :   return NS_OK;
    3592                 : }
    3593                 : 
    3594             140 : nsresult nsUrlClassifierStore::ReadPrefixes(FallibleTArray<PRUint32>& array,
    3595                 :                                             PRUint32 aKey)
    3596                 : {
    3597             280 :   mozStorageStatementScoper scoper(mAllPrefixGetStatement);
    3598             280 :   mozStorageStatementScoper scoperToo(mAllPrefixCountStatement);
    3599                 :   bool hasMoreData;
    3600             140 :   PRUint32 pcnt = 0;
    3601             140 :   PRUint32 fcnt = 0;
    3602                 : 
    3603                 : #if defined(PR_LOGGING)
    3604             140 :   PRIntervalTime clockStart = 0;
    3605             140 :   if (LOG_ENABLED()) {
    3606               0 :     clockStart = PR_IntervalNow();
    3607                 :   }
    3608                 : #endif
    3609                 : 
    3610                 :   // Make sure we allocate no more than we really need, so first
    3611                 :   // check how much entries there are
    3612             140 :   if (NS_SUCCEEDED(mAllPrefixCountStatement->ExecuteStep(&hasMoreData)) && hasMoreData) {
    3613             140 :     PRUint32 count = mAllPrefixCountStatement->AsInt32(0);
    3614             140 :     if (!array.SetCapacity(count)) {
    3615               0 :       return NS_ERROR_OUT_OF_MEMORY;
    3616                 :     }
    3617                 :   } else {
    3618               0 :     return NS_ERROR_FILE_CORRUPTED;
    3619                 :   }
    3620                 : 
    3621             462 :   while (NS_SUCCEEDED(mAllPrefixGetStatement->ExecuteStep(&hasMoreData)) && hasMoreData) {
    3622                 :     PRUint32 prefixval;
    3623                 :     PRUint32 domainval;
    3624                 :     PRUint32 size;
    3625                 : 
    3626             182 :     const PRUint8 *blobdomain = mAllPrefixGetStatement->AsSharedBlob(0, &size);
    3627             182 :     if (!blobdomain || (size != DOMAIN_LENGTH))
    3628               0 :       return false;
    3629                 : 
    3630             182 :     domainval = *(reinterpret_cast<const PRUint32*>(blobdomain));
    3631                 : 
    3632             182 :     const PRUint8 *blobprefix = mAllPrefixGetStatement->AsSharedBlob(1, &size);
    3633             182 :     if (!blobprefix || (size != PARTIAL_LENGTH)) {
    3634             139 :       const PRUint8 *blobfull = mAllPrefixGetStatement->AsSharedBlob(2, &size);
    3635             139 :       if (!blobfull || (size != COMPLETE_LENGTH)) {
    3636               5 :         prefixval = domainval;
    3637               5 :         fcnt++;
    3638                 :       } else {
    3639             134 :         prefixval = *(reinterpret_cast<const PRUint32*>(blobfull));
    3640             139 :       }
    3641                 :     } else {
    3642              43 :       prefixval = *(reinterpret_cast<const PRUint32*>(blobprefix));
    3643                 :     }
    3644                 : 
    3645                 :     PRUint32 keyedVal;
    3646             182 :     nsresult rv = KeyedHash(prefixval, domainval, aKey, &keyedVal);
    3647             182 :     NS_ENSURE_SUCCESS(rv, rv);
    3648                 : 
    3649             182 :     PRUint32 *res = array.AppendElement(keyedVal);
    3650             182 :     MOZ_ASSERT(res != nsnull);
    3651             182 :     pcnt++;
    3652                 :     // Normal DB size is about 500k entries. If we are getting 10x
    3653                 :     // as much, the database must be corrupted.
    3654             182 :     if (pcnt > 5000000) {
    3655               0 :       return NS_ERROR_FILE_CORRUPTED;
    3656                 :     }
    3657                 :   }
    3658                 : 
    3659             140 :   LOG(("SB prefixes: %d fulldomain: %d\n", pcnt, fcnt));
    3660                 : 
    3661                 : #if defined(PR_LOGGING)
    3662             140 :   if (LOG_ENABLED()) {
    3663               0 :     PRIntervalTime clockEnd = PR_IntervalNow();
    3664               0 :     LOG(("Gathering took %dms\n",
    3665                 :          PR_IntervalToMilliseconds(clockEnd - clockStart)));
    3666                 :   }
    3667                 : #endif
    3668                 : 
    3669             140 :   return NS_OK;
    3670                 : }
    3671                 : 
    3672              21 : bool nsUrlClassifierDBServiceWorker::LockPrefixSet()
    3673                 : {
    3674              21 :   mPrefixSetEnabledLock.Lock();
    3675              21 :   return mPrefixSetEnabled;
    3676                 : }
    3677                 : 
    3678              21 : void nsUrlClassifierDBServiceWorker::UnlockPrefixSet()
    3679                 : {
    3680              21 :   mPrefixSetEnabledLock.Unlock();
    3681              21 : }
    3682                 : 
    3683                 : nsresult
    3684             140 : nsUrlClassifierDBServiceWorker::ConstructPrefixSet()
    3685                 : {
    3686             280 :   Telemetry::AutoTimer<Telemetry::URLCLASSIFIER_PS_CONSTRUCT_TIME> timer;
    3687                 : 
    3688                 :   PRUint32 key;
    3689             140 :   nsresult rv = mPrefixSet->GetKey(&key);
    3690             140 :   NS_ENSURE_SUCCESS(rv, rv);
    3691                 : 
    3692             280 :   FallibleTArray<PRUint32> array;
    3693             140 :   rv = mMainStore.ReadPrefixes(array, key);
    3694             140 :   if (NS_FAILED(rv)) {
    3695               0 :     goto error_bailout;
    3696                 :   }
    3697                 : 
    3698                 : #ifdef HASHFUNCTION_COLLISION_TEST
    3699                 :   array.Sort();
    3700                 :   PRUint32 collisions = 0;
    3701                 :   for (int i = 1; i < array.Length(); i++) {
    3702                 :     if (array[i - 1] == array[i]) {
    3703                 :       collisions++;
    3704                 :     }
    3705                 :   }
    3706                 :   LOG(("%d collisions in the set", collisions));
    3707                 : #endif
    3708                 : 
    3709             140 :   if (array.IsEmpty()) {
    3710                 :     // DB is empty, put a sentinel to show that we loaded it
    3711              69 :     if (!array.AppendElement(0)) {
    3712               0 :       goto error_bailout;
    3713                 :     }
    3714                 :   }
    3715                 :   // SetPrefixes requires sorted arrays
    3716             140 :   array.Sort();
    3717                 : 
    3718                 :   // construct new prefixset
    3719             140 :   rv = mPrefixSet->SetPrefixes(array.Elements(), array.Length());
    3720             140 :   if (NS_FAILED(rv)) {
    3721               0 :     goto error_bailout;
    3722                 :   }
    3723                 : 
    3724                 :   // store the new tree to disk
    3725             140 :   rv = mPrefixSet->StoreToFile(mPSFile);
    3726             140 :   NS_WARN_IF_FALSE(NS_SUCCEEDED(rv), "failed to store the prefixset");
    3727                 : 
    3728                 :   // re-enable prefixset usage if disabled earlier
    3729             140 :   mPrefixSetEnabled = true;
    3730                 : 
    3731             140 :   return NS_OK;
    3732                 : 
    3733                 :  error_bailout:
    3734                 :   // disable prefixset usage
    3735               0 :   MutexAutoLock lock(mPrefixSetEnabledLock);
    3736               0 :   mPrefixSetEnabled = false;
    3737                 :   // load an empty prefixset
    3738               0 :   nsAutoTArray<PRUint32, 1> sentinel;
    3739               0 :   sentinel.Clear();
    3740               0 :   sentinel.AppendElement(0);
    3741               0 :   mPrefixSet->SetPrefixes(sentinel.Elements(), sentinel.Length());
    3742               0 :   if (rv == NS_ERROR_OUT_OF_MEMORY) {
    3743               0 :     Telemetry::Accumulate(Telemetry::URLCLASSIFIER_PS_OOM, 1);
    3744                 :   }
    3745               0 :   return rv;
    3746                 : }
    3747                 : 
    3748                 : nsresult
    3749             141 : nsUrlClassifierDBServiceWorker::LoadPrefixSet(nsCOMPtr<nsIFile> & aFile)
    3750                 : {
    3751                 :   bool empty;
    3752             141 :   nsresult rv = mPrefixSet->IsEmpty(&empty);
    3753             141 :   NS_ENSURE_SUCCESS(rv, rv);
    3754                 : 
    3755             141 :   if (!empty) {
    3756              85 :     LOG(("PrefixSet already loaded, not loading again"));
    3757              85 :     return NS_OK;
    3758                 :   }
    3759                 : 
    3760                 :   bool exists;
    3761              56 :   rv = aFile->Exists(&exists);
    3762              56 :   NS_ENSURE_SUCCESS(rv, rv);
    3763                 : 
    3764                 : #if defined(PR_LOGGING)
    3765              56 :   PRIntervalTime clockStart = 0;
    3766              56 :   if (LOG_ENABLED()) {
    3767               0 :     clockStart = PR_IntervalNow();
    3768                 :   }
    3769                 : #endif
    3770                 : 
    3771              56 :   if (exists) {
    3772               0 :     Telemetry::AutoTimer<Telemetry::URLCLASSIFIER_PS_FILELOAD_TIME> timer;
    3773               0 :     LOG(("stored PrefixSet exists, loading from disk"));
    3774               0 :     rv = mPrefixSet->LoadFromFile(aFile);
    3775                 :   }
    3776              56 :   if (!exists || NS_FAILED(rv)) {
    3777              56 :     LOG(("no (usable) stored PrefixSet found, constructing from store"));
    3778              56 :     rv = ConstructPrefixSet();
    3779              56 :     NS_ENSURE_SUCCESS(rv, rv);
    3780                 :   }
    3781                 : 
    3782                 : #ifdef DEBUG
    3783              56 :   LOG(("SB tree done, size = %d bytes\n",
    3784                 :        mPrefixSet->SizeOfIncludingThis(moz_malloc_size_of)));
    3785                 : #endif
    3786                 : #if defined(PR_LOGGING)
    3787              56 :   if (LOG_ENABLED()) {
    3788               0 :     PRIntervalTime clockEnd = PR_IntervalNow();
    3789               0 :     LOG(("Loading took %dms\n",
    3790                 :          PR_IntervalToMilliseconds(clockEnd - clockStart)));
    3791                 :   }
    3792                 : #endif
    3793                 : 
    3794              56 :   return NS_OK;
    3795                 : }
    3796                 : 
    3797                 : nsresult
    3798             141 : nsUrlClassifierDBServiceWorker::MaybeCreateTables(mozIStorageConnection* connection)
    3799                 : {
    3800             141 :   LOG(("MaybeCreateTables\n"));
    3801                 : 
    3802                 :   nsresult rv = connection->ExecuteSimpleSQL(
    3803             141 :     NS_LITERAL_CSTRING("CREATE TABLE IF NOT EXISTS moz_classifier"
    3804                 :                        " (id INTEGER PRIMARY KEY,"
    3805                 :                        "  domain BLOB,"
    3806                 :                        "  partial_data BLOB,"
    3807                 :                        "  complete_data BLOB,"
    3808                 :                        "  chunk_id INTEGER,"
    3809             141 :                        "  table_id INTEGER)"));
    3810             141 :   NS_ENSURE_SUCCESS(rv, rv);
    3811                 : 
    3812                 :   rv = connection->ExecuteSimpleSQL(
    3813             141 :     NS_LITERAL_CSTRING("CREATE INDEX IF NOT EXISTS"
    3814                 :                        " moz_classifier_domain_index"
    3815             141 :                        " ON moz_classifier(domain)"));
    3816             141 :   NS_ENSURE_SUCCESS(rv, rv);
    3817                 : 
    3818                 :   rv = connection->ExecuteSimpleSQL(
    3819             141 :     NS_LITERAL_CSTRING("CREATE INDEX IF NOT EXISTS"
    3820                 :                        " moz_classifier_chunk_index"
    3821             141 :                        " ON moz_classifier(chunk_id)"));
    3822             141 :   NS_ENSURE_SUCCESS(rv, rv);
    3823                 : 
    3824                 :   rv = connection->ExecuteSimpleSQL(
    3825             141 :     NS_LITERAL_CSTRING("CREATE TABLE IF NOT EXISTS moz_subs"
    3826                 :                        " (id INTEGER PRIMARY KEY,"
    3827                 :                        "  domain BLOB,"
    3828                 :                        "  partial_data BLOB,"
    3829                 :                        "  complete_data BLOB,"
    3830                 :                        "  chunk_id INTEGER,"
    3831                 :                        "  table_id INTEGER,"
    3832             141 :                        "  add_chunk_id INTEGER)"));
    3833             141 :   NS_ENSURE_SUCCESS(rv, rv);
    3834                 : 
    3835                 :   rv = connection->ExecuteSimpleSQL(
    3836             141 :     NS_LITERAL_CSTRING("CREATE INDEX IF NOT EXISTS"
    3837                 :                        " moz_subs_addchunk_index"
    3838             141 :                        " ON moz_subs(add_chunk_id)"));
    3839             141 :   NS_ENSURE_SUCCESS(rv, rv);
    3840                 : 
    3841                 :   rv = connection->ExecuteSimpleSQL(
    3842             141 :     NS_LITERAL_CSTRING("CREATE INDEX IF NOT EXISTS"
    3843                 :                        " moz_subs_chunk_index"
    3844             141 :                        " ON moz_subs(chunk_id)"));
    3845             141 :   NS_ENSURE_SUCCESS(rv, rv);
    3846                 : 
    3847                 :   rv = connection->ExecuteSimpleSQL(
    3848             141 :     NS_LITERAL_CSTRING("CREATE TABLE IF NOT EXISTS moz_tables"
    3849                 :                        " (id INTEGER PRIMARY KEY,"
    3850                 :                        "  name TEXT,"
    3851                 :                        "  add_chunks TEXT,"
    3852             141 :                        "  sub_chunks TEXT);"));
    3853             141 :   NS_ENSURE_SUCCESS(rv, rv);
    3854                 : 
    3855             141 :   return rv;
    3856                 : }
    3857                 : 
    3858                 : // -------------------------------------------------------------------------
    3859                 : // nsUrlClassifierLookupCallback
    3860                 : //
    3861                 : // This class takes the results of a lookup found on the worker thread
    3862                 : // and handles any necessary partial hash expansions before calling
    3863                 : // the client callback.
    3864                 : 
    3865                 : class nsUrlClassifierLookupCallback : public nsIUrlClassifierLookupCallback
    3866                 :                                     , public nsIUrlClassifierHashCompleterCallback
    3867             141 : {
    3868                 : public:
    3869                 :   NS_DECL_ISUPPORTS
    3870                 :   NS_DECL_NSIURLCLASSIFIERLOOKUPCALLBACK
    3871                 :   NS_DECL_NSIURLCLASSIFIERHASHCOMPLETERCALLBACK
    3872                 : 
    3873             141 :   nsUrlClassifierLookupCallback(nsUrlClassifierDBService *dbservice,
    3874                 :                                 nsIUrlClassifierCallback *c)
    3875                 :     : mDBService(dbservice)
    3876                 :     , mResults(nsnull)
    3877                 :     , mPendingCompletions(0)
    3878             141 :     , mCallback(c)
    3879             141 :     {}
    3880                 : 
    3881                 : private:
    3882                 :   nsresult HandleResults();
    3883                 : 
    3884                 :   nsRefPtr<nsUrlClassifierDBService> mDBService;
    3885                 :   nsAutoPtr<nsTArray<nsUrlClassifierLookupResult> > mResults;
    3886                 : 
    3887                 :   // Completed results to send back to the worker for caching.
    3888                 :   nsAutoPtr<nsTArray<nsUrlClassifierLookupResult> > mCacheResults;
    3889                 : 
    3890                 :   PRUint32 mPendingCompletions;
    3891                 :   nsCOMPtr<nsIUrlClassifierCallback> mCallback;
    3892                 : };
    3893                 : 
    3894            2812 : NS_IMPL_THREADSAFE_ISUPPORTS2(nsUrlClassifierLookupCallback,
    3895                 :                               nsIUrlClassifierLookupCallback,
    3896                 :                               nsIUrlClassifierHashCompleterCallback)
    3897                 : 
    3898                 : NS_IMETHODIMP
    3899             141 : nsUrlClassifierLookupCallback::LookupComplete(nsTArray<nsUrlClassifierLookupResult>* results)
    3900                 : {
    3901             141 :   NS_ASSERTION(mResults == nsnull,
    3902                 :                "Should only get one set of results per nsUrlClassifierLookupCallback!");
    3903                 : 
    3904             141 :   if (!results) {
    3905               0 :     HandleResults();
    3906               0 :     return NS_OK;
    3907                 :   }
    3908                 : 
    3909             141 :   mResults = results;
    3910             141 :   mResults->Sort();
    3911                 : 
    3912                 :   // Check the results entries that need to be completed.
    3913             277 :   for (PRUint32 i = 0; i < results->Length(); i++) {
    3914             136 :     nsUrlClassifierLookupResult& result = results->ElementAt(i);
    3915                 : 
    3916                 :     // We will complete partial matches and matches that are stale.
    3917             136 :     if (!result.mConfirmed) {
    3918             154 :       nsCOMPtr<nsIUrlClassifierHashCompleter> completer;
    3919             154 :       if (mDBService->GetCompleter(result.mTableName,
    3920             154 :                                    getter_AddRefs(completer))) {
    3921             114 :         nsCAutoString partialHash;
    3922                 :         PRUint8 *buf =
    3923                 :           result.mEntry.mHavePartial ? result.mEntry.mPartialHash.buf
    3924              57 :                                      : result.mEntry.mCompleteHash.buf;
    3925              57 :         partialHash.Assign(reinterpret_cast<char*>(buf), PARTIAL_LENGTH);
    3926                 : 
    3927              57 :         nsresult rv = completer->Complete(partialHash, this);
    3928              57 :         if (NS_SUCCEEDED(rv)) {
    3929              57 :           mPendingCompletions++;
    3930                 :         }
    3931                 :       } else {
    3932                 :         // For tables with no hash completer, a complete hash match is
    3933                 :         // good enough, it doesn't need to be fresh. (we need the
    3934                 :         // mLookupFragment comparison to weed out noise entries, which
    3935                 :         // should never be confirmed).
    3936              40 :         if (result.mEntry.mHaveComplete
    3937              20 :             && (result.mLookupFragment == result.mEntry.mCompleteHash)) {
    3938              20 :           result.mConfirmed = true;
    3939                 :         } else {
    3940               0 :           NS_WARNING("Partial match in a table without a valid completer, ignoring partial match.");
    3941                 :         }
    3942                 :       }
    3943                 :     }
    3944                 :   }
    3945                 : 
    3946             141 :   if (mPendingCompletions == 0) {
    3947                 :     // All results were complete, we're ready!
    3948             100 :     HandleResults();
    3949                 :   }
    3950                 : 
    3951             141 :   return NS_OK;
    3952                 : }
    3953                 : 
    3954                 : NS_IMETHODIMP
    3955              57 : nsUrlClassifierLookupCallback::CompletionFinished(nsresult status)
    3956                 : {
    3957              57 :   LOG(("nsUrlClassifierLookupCallback::CompletionFinished [%p, %08x]",
    3958                 :        this, status));
    3959              57 :   if (NS_FAILED(status)) {
    3960               0 :     NS_WARNING("gethash response failed.");
    3961                 :   }
    3962                 : 
    3963              57 :   mPendingCompletions--;
    3964              57 :   if (mPendingCompletions == 0) {
    3965              41 :     HandleResults();
    3966                 : 
    3967              41 :     if (mCacheResults) {
    3968                 :       // This hands ownership of the cache results array back to the worker
    3969                 :       // thread.
    3970              20 :       mDBService->CacheCompletions(mCacheResults.forget());
    3971                 :     }
    3972                 :   }
    3973                 : 
    3974              57 :   return NS_OK;
    3975                 : }
    3976                 : 
    3977                 : NS_IMETHODIMP
    3978              51 : nsUrlClassifierLookupCallback::Completion(const nsACString& completeHash,
    3979                 :                                           const nsACString& tableName,
    3980                 :                                           PRUint32 chunkId,
    3981                 :                                           bool verified)
    3982                 : {
    3983              51 :   LOG(("nsUrlClassifierLookupCallback::Completion [%p, %s, %d, %d]",
    3984                 :        this, PromiseFlatCString(tableName).get(), chunkId, verified));
    3985                 :   nsUrlClassifierCompleteHash hash;
    3986              51 :   hash.Assign(completeHash);
    3987                 : 
    3988             248 :   for (PRUint32 i = 0; i < mResults->Length(); i++) {
    3989             197 :     nsUrlClassifierLookupResult& result = mResults->ElementAt(i);
    3990                 : 
    3991                 :     // First, see if this result can be used to update an entry.
    3992             546 :     if (verified &&
    3993             196 :         !result.mEntry.mHaveComplete &&
    3994             113 :         hash.StartsWith(result.mEntry.mPartialHash) &&
    3995              40 :         result.mTableName == tableName &&
    3996                 :         result.mEntry.mChunkId == chunkId) {
    3997                 :       // We have a completion for this entry.  Fill it in...
    3998              36 :       result.mEntry.SetHash(hash);
    3999                 : 
    4000              36 :       if (!mCacheResults) {
    4001              20 :         mCacheResults = new nsTArray<nsUrlClassifierLookupResult>();
    4002              20 :         if (!mCacheResults)
    4003               0 :           return NS_ERROR_OUT_OF_MEMORY;
    4004                 :       }
    4005                 : 
    4006              36 :       mCacheResults->AppendElement(result);
    4007                 :     }
    4008                 : 
    4009                 :     // Now, see if it verifies a lookup
    4010             197 :     if (result.mLookupFragment == hash) {
    4011              45 :       result.mConfirmed = true;
    4012                 : 
    4013              45 :       if (result.mTableName != tableName ||
    4014                 :           result.mEntry.mChunkId != chunkId) {
    4015                 :         // The hash we got for this completion matches the hash we
    4016                 :         // looked up, but doesn't match the table/chunk id.  This could
    4017                 :         // happen in rare cases where a given URL was moved between
    4018                 :         // lists or added/removed/re-added to the list in the time since
    4019                 :         // we've updated.
    4020                 :         //
    4021                 :         // Update the lookup result, but don't update the entry or try
    4022                 :         // cache the results of this completion, as it might confuse
    4023                 :         // things.
    4024               4 :         result.mTableName = tableName;
    4025               4 :         NS_WARNING("Accepting a gethash with an invalid table name or chunk id");
    4026               4 :         LOG(("Tablename: %s ?= %s, ChunkId %d ?= %d",
    4027                 :              result.mTableName.get(), PromiseFlatCString(tableName).get(),
    4028                 :              result.mEntry.mChunkId, chunkId));
    4029                 :       }
    4030                 :     }
    4031                 :   }
    4032                 : 
    4033              51 :   return NS_OK;
    4034                 : }
    4035                 : 
    4036                 : nsresult
    4037             141 : nsUrlClassifierLookupCallback::HandleResults()
    4038                 : {
    4039             141 :   if (!mResults) {
    4040                 :     // No results, this URI is clean.
    4041               0 :     return mCallback->HandleEvent(NS_LITERAL_CSTRING(""));
    4042                 :   }
    4043                 : 
    4044                 :   // Build a stringified list of result tables.
    4045             141 :   mResults->Sort();
    4046             141 :   PRUint32 lastTableId = 0;
    4047             282 :   nsCAutoString tables;
    4048             277 :   for (PRUint32 i = 0; i < mResults->Length(); i++) {
    4049             136 :     nsUrlClassifierLookupResult& result = mResults->ElementAt(i);
    4050                 :     // Leave out results that weren't confirmed, as their existence on
    4051                 :     // the list can't be verified.  Also leave out randomly-generated
    4052                 :     // noise.
    4053             136 :     if (!result.mConfirmed || result.mNoise)
    4054              12 :       continue;
    4055                 : 
    4056             124 :     if (tables.Length() > 0) {
    4057              25 :       if (lastTableId == result.mEntry.mTableId)
    4058              16 :         continue;
    4059               9 :       tables.Append(",");
    4060                 :     }
    4061                 : 
    4062             108 :     tables.Append(result.mTableName);
    4063             108 :     lastTableId = result.mEntry.mTableId;
    4064                 :   }
    4065                 : 
    4066             141 :   return mCallback->HandleEvent(tables);
    4067                 : }
    4068                 : 
    4069                 : 
    4070                 : // -------------------------------------------------------------------------
    4071                 : // Helper class for nsIURIClassifier implementation, translates table names
    4072                 : // to nsIURIClassifier enums.
    4073                 : 
    4074                 : class nsUrlClassifierClassifyCallback : public nsIUrlClassifierCallback
    4075              21 : {
    4076                 : public:
    4077                 :   NS_DECL_ISUPPORTS
    4078                 :   NS_DECL_NSIURLCLASSIFIERCALLBACK
    4079                 : 
    4080              21 :   nsUrlClassifierClassifyCallback(nsIURIClassifierCallback *c,
    4081                 :                                   bool checkMalware,
    4082                 :                                   bool checkPhishing)
    4083                 :     : mCallback(c)
    4084                 :     , mCheckMalware(checkMalware)
    4085              21 :     , mCheckPhishing(checkPhishing)
    4086              21 :     {}
    4087                 : 
    4088                 : private:
    4089                 :   nsCOMPtr<nsIURIClassifierCallback> mCallback;
    4090                 :   bool mCheckMalware;
    4091                 :   bool mCheckPhishing;
    4092                 : };
    4093                 : 
    4094              68 : NS_IMPL_THREADSAFE_ISUPPORTS1(nsUrlClassifierClassifyCallback,
    4095                 :                               nsIUrlClassifierCallback)
    4096                 : 
    4097                 : NS_IMETHODIMP
    4098               1 : nsUrlClassifierClassifyCallback::HandleEvent(const nsACString& tables)
    4099                 : {
    4100                 :   // XXX: we should probably have the wardens tell the service which table
    4101                 :   // names match with which classification.  For now the table names give
    4102                 :   // enough information.
    4103               1 :   nsresult response = NS_OK;
    4104                 : 
    4105               1 :   nsACString::const_iterator begin, end;
    4106                 : 
    4107               1 :   tables.BeginReading(begin);
    4108               1 :   tables.EndReading(end);
    4109               4 :   if (mCheckMalware &&
    4110               3 :       FindInReadable(NS_LITERAL_CSTRING("-malware-"), begin, end)) {
    4111               0 :     response = NS_ERROR_MALWARE_URI;
    4112                 :   } else {
    4113                 :     // Reset begin before checking phishing table
    4114               1 :     tables.BeginReading(begin);
    4115                 : 
    4116               4 :     if (mCheckPhishing &&
    4117               3 :         FindInReadable(NS_LITERAL_CSTRING("-phish-"), begin, end)) {
    4118               1 :       response = NS_ERROR_PHISHING_URI;
    4119                 :     }
    4120                 :   }
    4121                 : 
    4122               1 :   mCallback->OnClassifyComplete(response);
    4123                 : 
    4124               1 :   return NS_OK;
    4125                 : }
    4126                 : 
    4127                 : 
    4128                 : // -------------------------------------------------------------------------
    4129                 : // Proxy class implementation
    4130                 : 
    4131            1199 : NS_IMPL_THREADSAFE_ISUPPORTS3(nsUrlClassifierDBService,
    4132                 :                               nsIUrlClassifierDBService,
    4133                 :                               nsIURIClassifier,
    4134                 :                               nsIObserver)
    4135                 : 
    4136                 : /* static */ nsUrlClassifierDBService*
    4137               8 : nsUrlClassifierDBService::GetInstance(nsresult *result)
    4138                 : {
    4139               8 :   *result = NS_OK;
    4140               8 :   if (!sUrlClassifierDBService) {
    4141               8 :     sUrlClassifierDBService = new nsUrlClassifierDBService();
    4142               8 :     if (!sUrlClassifierDBService) {
    4143               0 :       *result = NS_ERROR_OUT_OF_MEMORY;
    4144               0 :       return nsnull;
    4145                 :     }
    4146                 : 
    4147               8 :     NS_ADDREF(sUrlClassifierDBService);   // addref the global
    4148                 : 
    4149               8 :     *result = sUrlClassifierDBService->Init();
    4150               8 :     if (NS_FAILED(*result)) {
    4151               0 :       NS_RELEASE(sUrlClassifierDBService);
    4152               0 :       return nsnull;
    4153                 :     }
    4154                 :   } else {
    4155                 :     // Already exists, just add a ref
    4156               0 :     NS_ADDREF(sUrlClassifierDBService);   // addref the return result
    4157                 :   }
    4158               8 :   return sUrlClassifierDBService;
    4159                 : }
    4160                 : 
    4161                 : 
    4162               8 : nsUrlClassifierDBService::nsUrlClassifierDBService()
    4163                 :  : mCheckMalware(CHECK_MALWARE_DEFAULT)
    4164                 :  , mCheckPhishing(CHECK_PHISHING_DEFAULT)
    4165               8 :  , mInUpdate(false)
    4166                 : {
    4167               8 : }
    4168                 : 
    4169              16 : nsUrlClassifierDBService::~nsUrlClassifierDBService()
    4170                 : {
    4171               8 :   sUrlClassifierDBService = nsnull;
    4172               8 : }
    4173                 : 
    4174                 : nsresult
    4175               8 : nsUrlClassifierDBService::Init()
    4176                 : {
    4177                 : #if defined(PR_LOGGING)
    4178               8 :   if (!gUrlClassifierDbServiceLog)
    4179               8 :     gUrlClassifierDbServiceLog = PR_NewLogModule("UrlClassifierDbService");
    4180                 : #endif
    4181                 : 
    4182                 :   // Force the storage service to be created on the main thread.
    4183                 :   nsresult rv;
    4184                 :   nsCOMPtr<mozIStorageService> storageService =
    4185              16 :     do_GetService(MOZ_STORAGE_SERVICE_CONTRACTID, &rv);
    4186               8 :   NS_ENSURE_SUCCESS(rv, rv);
    4187                 : 
    4188                 :   // Force PSM to be loaded on the main thread.
    4189               8 :   mHash = do_CreateInstance(NS_CRYPTO_HASH_CONTRACTID, &rv);
    4190               8 :   NS_ENSURE_SUCCESS(rv, rv);
    4191                 : 
    4192               8 :   mPrefixSet = new nsUrlClassifierPrefixSet();
    4193               8 :   NS_ENSURE_SUCCESS(rv, rv);
    4194                 : 
    4195                 :   // Should we check document loads for malware URIs?
    4196              16 :   nsCOMPtr<nsIPrefBranch> prefs = do_GetService(NS_PREFSERVICE_CONTRACTID);
    4197                 : 
    4198               8 :   PRInt32 gethashNoise = 0;
    4199               8 :   if (prefs) {
    4200                 :     bool tmpbool;
    4201               8 :     rv = prefs->GetBoolPref(CHECK_MALWARE_PREF, &tmpbool);
    4202               8 :     mCheckMalware = NS_SUCCEEDED(rv) ? tmpbool : CHECK_MALWARE_DEFAULT;
    4203                 : 
    4204               8 :     prefs->AddObserver(CHECK_MALWARE_PREF, this, false);
    4205                 : 
    4206               8 :     rv = prefs->GetBoolPref(CHECK_PHISHING_PREF, &tmpbool);
    4207               8 :     mCheckPhishing = NS_SUCCEEDED(rv) ? tmpbool : CHECK_PHISHING_DEFAULT;
    4208                 : 
    4209               8 :     prefs->AddObserver(CHECK_PHISHING_PREF, this, false);
    4210                 : 
    4211               8 :     if (NS_FAILED(prefs->GetIntPref(GETHASH_NOISE_PREF, &gethashNoise))) {
    4212               0 :       gethashNoise = GETHASH_NOISE_DEFAULT;
    4213                 :     }
    4214                 : 
    4215              16 :     nsXPIDLCString tmpstr;
    4216               8 :     if (NS_SUCCEEDED(prefs->GetCharPref(GETHASH_TABLES_PREF, getter_Copies(tmpstr)))) {
    4217               8 :       SplitTables(tmpstr, mGethashWhitelist);
    4218                 :     }
    4219                 : 
    4220               8 :     prefs->AddObserver(GETHASH_TABLES_PREF, this, false);
    4221                 : 
    4222                 :     PRInt32 tmpint;
    4223               8 :     rv = prefs->GetIntPref(CONFIRM_AGE_PREF, &tmpint);
    4224               8 :     PR_ATOMIC_SET(&gFreshnessGuarantee, NS_SUCCEEDED(rv) ? tmpint : CONFIRM_AGE_DEFAULT_SEC);
    4225                 : 
    4226               8 :     prefs->AddObserver(CONFIRM_AGE_PREF, this, false);
    4227                 : 
    4228               8 :     rv = prefs->GetIntPref(UPDATE_CACHE_SIZE_PREF, &tmpint);
    4229               8 :     PR_ATOMIC_SET(&gUpdateCacheSize, NS_SUCCEEDED(rv) ? tmpint : UPDATE_CACHE_SIZE_DEFAULT);
    4230                 : 
    4231               8 :     rv = prefs->GetIntPref(LOOKUP_CACHE_SIZE_PREF, &tmpint);
    4232               8 :     PR_ATOMIC_SET(&gLookupCacheSize, NS_SUCCEEDED(rv) ? tmpint : LOOKUP_CACHE_SIZE_DEFAULT);
    4233                 : 
    4234               8 :     rv = prefs->GetIntPref(UPDATE_WORKING_TIME, &tmpint);
    4235               8 :     PR_ATOMIC_SET(&gWorkingTimeThreshold,
    4236               8 :                   NS_SUCCEEDED(rv) ? tmpint : UPDATE_WORKING_TIME_DEFAULT);
    4237                 : 
    4238               8 :     rv = prefs->GetIntPref(UPDATE_DELAY_TIME, &tmpint);
    4239               8 :     PR_ATOMIC_SET(&gDelayTime,
    4240               8 :                   NS_SUCCEEDED(rv) ? tmpint : UPDATE_DELAY_TIME_DEFAULT);
    4241                 :   }
    4242                 : 
    4243                 :   // Start the background thread.
    4244               8 :   rv = NS_NewThread(&gDbBackgroundThread);
    4245               8 :   if (NS_FAILED(rv))
    4246               0 :     return rv;
    4247                 : 
    4248               8 :   mWorker = new nsUrlClassifierDBServiceWorker();
    4249               8 :   if (!mWorker)
    4250               0 :     return NS_ERROR_OUT_OF_MEMORY;
    4251                 : 
    4252               8 :   rv = mWorker->Init(gethashNoise, mPrefixSet);
    4253               8 :   if (NS_FAILED(rv)) {
    4254               0 :     mWorker = nsnull;
    4255               0 :     return rv;
    4256                 :   }
    4257                 : 
    4258                 :   // Proxy for calling the worker on the background thread
    4259              16 :   mWorkerProxy = new UrlClassifierDBServiceWorkerProxy(mWorker);
    4260                 : 
    4261               8 :   mCompleters.Init();
    4262                 : 
    4263                 :   // Add an observer for shutdown
    4264                 :   nsCOMPtr<nsIObserverService> observerService =
    4265              16 :       mozilla::services::GetObserverService();
    4266               8 :   if (!observerService)
    4267               0 :     return NS_ERROR_FAILURE;
    4268                 : 
    4269               8 :   observerService->AddObserver(this, "profile-before-change", false);
    4270               8 :   observerService->AddObserver(this, "xpcom-shutdown-threads", false);
    4271                 : 
    4272               8 :   return NS_OK;
    4273                 : }
    4274                 : 
    4275                 : NS_IMETHODIMP
    4276              21 : nsUrlClassifierDBService::Classify(nsIURI *uri,
    4277                 :                                    nsIURIClassifierCallback* c,
    4278                 :                                    bool* result)
    4279                 : {
    4280              21 :   NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
    4281                 : 
    4282              21 :   if (!(mCheckMalware || mCheckPhishing)) {
    4283               0 :     *result = false;
    4284               0 :     return NS_OK;
    4285                 :   }
    4286                 : 
    4287                 :   nsRefPtr<nsUrlClassifierClassifyCallback> callback =
    4288              42 :     new nsUrlClassifierClassifyCallback(c, mCheckMalware, mCheckPhishing);
    4289              21 :   if (!callback) return NS_ERROR_OUT_OF_MEMORY;
    4290                 : 
    4291              21 :   nsresult rv = LookupURI(uri, callback, false, result);
    4292              21 :   if (rv == NS_ERROR_MALFORMED_URI) {
    4293               0 :     *result = false;
    4294                 :     // The URI had no hostname, don't try to classify it.
    4295               0 :     return NS_OK;
    4296                 :   }
    4297              21 :   NS_ENSURE_SUCCESS(rv, rv);
    4298                 : 
    4299              21 :   return NS_OK;
    4300                 : }
    4301                 : 
    4302                 : NS_IMETHODIMP
    4303             140 : nsUrlClassifierDBService::Lookup(const nsACString& spec,
    4304                 :                                  nsIUrlClassifierCallback* c)
    4305                 : {
    4306             140 :   NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
    4307                 : 
    4308             280 :   nsCOMPtr<nsIURI> uri;
    4309                 : 
    4310             140 :   nsresult rv = NS_NewURI(getter_AddRefs(uri), spec);
    4311             140 :   NS_ENSURE_SUCCESS(rv, rv);
    4312                 : 
    4313             140 :   uri = NS_GetInnermostURI(uri);
    4314             140 :   if (!uri) {
    4315               0 :     return NS_ERROR_FAILURE;
    4316                 :   }
    4317                 : 
    4318                 :   bool didLookup;
    4319             140 :   return LookupURI(uri, c, true, &didLookup);
    4320                 : }
    4321                 : 
    4322                 : nsresult
    4323             161 : nsUrlClassifierDBService::LookupURI(nsIURI* uri,
    4324                 :                                     nsIUrlClassifierCallback* c,
    4325                 :                                     bool forceLookup,
    4326                 :                                     bool *didLookup)
    4327                 : {
    4328             161 :   NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
    4329                 : 
    4330             322 :   nsCAutoString key;
    4331                 :   // Canonicalize the url
    4332                 :   nsCOMPtr<nsIUrlClassifierUtils> utilsService =
    4333             322 :     do_GetService(NS_URLCLASSIFIERUTILS_CONTRACTID);
    4334             161 :   nsresult rv = utilsService->GetKeyForURI(uri, key);
    4335             161 :   if (NS_FAILED(rv))
    4336               0 :     return rv;
    4337                 : 
    4338             161 :   if (forceLookup) {
    4339             140 :     *didLookup = true;
    4340                 :   } else {
    4341                 :     // Check if the URI is clean.  If so, we don't need to
    4342                 :     // bother queueing up a lookup, we can just return.;
    4343                 :     bool clean;
    4344              21 :     rv = CheckClean(key, &clean);
    4345              21 :     NS_ENSURE_SUCCESS(rv, rv);
    4346                 : 
    4347              21 :     if (!clean) {
    4348                 :       nsCOMPtr<nsIPermissionManager> permissionManager =
    4349               2 :         do_GetService(NS_PERMISSIONMANAGER_CONTRACTID);
    4350                 : 
    4351               1 :       if (permissionManager) {
    4352                 :         PRUint32 perm;
    4353               1 :         permissionManager->TestPermission(uri, "safe-browsing", &perm);
    4354               1 :         clean |= (perm == nsIPermissionManager::ALLOW_ACTION);
    4355                 :       }
    4356                 :     }
    4357                 : 
    4358              21 :     *didLookup = !clean;
    4359              21 :     if (clean) {
    4360              20 :       return NS_OK;
    4361                 :     }
    4362                 :   }
    4363                 : 
    4364                 :   // Create an nsUrlClassifierLookupCallback object.  This object will
    4365                 :   // take care of confirming partial hash matches if necessary before
    4366                 :   // calling the client's callback.
    4367                 :   nsCOMPtr<nsIUrlClassifierLookupCallback> callback =
    4368             282 :     new nsUrlClassifierLookupCallback(this, c);
    4369             141 :   if (!callback)
    4370               0 :     return NS_ERROR_OUT_OF_MEMORY;
    4371                 : 
    4372                 :   nsCOMPtr<nsIUrlClassifierLookupCallback> proxyCallback =
    4373             423 :     new UrlClassifierLookupCallbackProxy(callback);
    4374                 : 
    4375                 :   // Queue this lookup and call the lookup function to flush the queue if
    4376                 :   // necessary.
    4377             141 :   rv = mWorker->QueueLookup(key, proxyCallback);
    4378             141 :   NS_ENSURE_SUCCESS(rv, rv);
    4379                 : 
    4380             141 :   return mWorkerProxy->Lookup(EmptyCString(), nsnull);
    4381                 : }
    4382                 : 
    4383                 : NS_IMETHODIMP
    4384              53 : nsUrlClassifierDBService::GetTables(nsIUrlClassifierCallback* c)
    4385                 : {
    4386              53 :   NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
    4387                 : 
    4388                 :   // The proxy callback uses the current thread.
    4389                 :   nsCOMPtr<nsIUrlClassifierCallback> proxyCallback =
    4390             106 :     new UrlClassifierCallbackProxy(c);
    4391                 : 
    4392              53 :   return mWorkerProxy->GetTables(proxyCallback);
    4393                 : }
    4394                 : 
    4395                 : NS_IMETHODIMP
    4396              83 : nsUrlClassifierDBService::SetHashCompleter(const nsACString &tableName,
    4397                 :                                            nsIUrlClassifierHashCompleter *completer)
    4398                 : {
    4399              83 :   if (completer) {
    4400              29 :     if (!mCompleters.Put(tableName, completer)) {
    4401               0 :       return NS_ERROR_OUT_OF_MEMORY;
    4402                 :     }
    4403                 :   } else {
    4404              54 :     mCompleters.Remove(tableName);
    4405                 :   }
    4406                 : 
    4407              83 :   return NS_OK;
    4408                 : }
    4409                 : 
    4410                 : NS_IMETHODIMP
    4411              87 : nsUrlClassifierDBService::BeginUpdate(nsIUrlClassifierUpdateObserver *observer,
    4412                 :                                       const nsACString &updateTables,
    4413                 :                                       const nsACString &clientKey)
    4414                 : {
    4415              87 :   NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
    4416                 : 
    4417              87 :   if (mInUpdate)
    4418               0 :     return NS_ERROR_NOT_AVAILABLE;
    4419                 : 
    4420              87 :   mInUpdate = true;
    4421                 : 
    4422                 :   // The proxy observer uses the current thread
    4423                 :   nsCOMPtr<nsIUrlClassifierUpdateObserver> proxyObserver =
    4424             174 :     new UrlClassifierUpdateObserverProxy(observer);
    4425                 : 
    4426              87 :   return mWorkerProxy->BeginUpdate(proxyObserver, updateTables, clientKey);
    4427                 : }
    4428                 : 
    4429                 : NS_IMETHODIMP
    4430             102 : nsUrlClassifierDBService::BeginStream(const nsACString &table,
    4431                 :                                       const nsACString &serverMAC)
    4432                 : {
    4433             102 :   NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
    4434                 : 
    4435             102 :   return mWorkerProxy->BeginStream(table, serverMAC);
    4436                 : }
    4437                 : 
    4438                 : NS_IMETHODIMP
    4439             100 : nsUrlClassifierDBService::UpdateStream(const nsACString& aUpdateChunk)
    4440                 : {
    4441             100 :   NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
    4442                 : 
    4443             100 :   return mWorkerProxy->UpdateStream(aUpdateChunk);
    4444                 : }
    4445                 : 
    4446                 : NS_IMETHODIMP
    4447             100 : nsUrlClassifierDBService::FinishStream()
    4448                 : {
    4449             100 :   NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
    4450                 : 
    4451             100 :   return mWorkerProxy->FinishStream();
    4452                 : }
    4453                 : 
    4454                 : NS_IMETHODIMP
    4455              85 : nsUrlClassifierDBService::FinishUpdate()
    4456                 : {
    4457              85 :   NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
    4458                 : 
    4459              85 :   mInUpdate = false;
    4460                 : 
    4461              85 :   return mWorkerProxy->FinishUpdate();
    4462                 : }
    4463                 : 
    4464                 : 
    4465                 : NS_IMETHODIMP
    4466               2 : nsUrlClassifierDBService::CancelUpdate()
    4467                 : {
    4468               2 :   NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
    4469                 : 
    4470               2 :   mInUpdate = false;
    4471                 : 
    4472               2 :   return mWorkerProxy->CancelUpdate();
    4473                 : }
    4474                 : 
    4475                 : NS_IMETHODIMP
    4476              54 : nsUrlClassifierDBService::ResetDatabase()
    4477                 : {
    4478              54 :   NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
    4479                 : 
    4480              54 :   return mWorkerProxy->ResetDatabase();
    4481                 : }
    4482                 : 
    4483                 : nsresult
    4484              20 : nsUrlClassifierDBService::CacheCompletions(nsTArray<nsUrlClassifierLookupResult> *results)
    4485                 : {
    4486              20 :   NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
    4487                 : 
    4488              20 :   return mWorkerProxy->CacheCompletions(results);
    4489                 : }
    4490                 : 
    4491                 : bool
    4492              77 : nsUrlClassifierDBService::GetCompleter(const nsACString &tableName,
    4493                 :                                        nsIUrlClassifierHashCompleter **completer)
    4494                 : {
    4495              77 :   if (mCompleters.Get(tableName, completer)) {
    4496              57 :     return true;
    4497                 :   }
    4498                 : 
    4499              20 :   if (!mGethashWhitelist.Contains(tableName)) {
    4500              20 :     return false;
    4501                 :   }
    4502                 : 
    4503               0 :   return NS_SUCCEEDED(CallGetService(NS_URLCLASSIFIERHASHCOMPLETER_CONTRACTID,
    4504                 :                                      completer));
    4505                 : }
    4506                 : 
    4507                 : NS_IMETHODIMP
    4508              20 : nsUrlClassifierDBService::Observe(nsISupports *aSubject, const char *aTopic,
    4509                 :                                   const PRUnichar *aData)
    4510                 : {
    4511              20 :   if (!strcmp(aTopic, NS_PREFBRANCH_PREFCHANGE_TOPIC_ID)) {
    4512                 :     nsresult rv;
    4513               8 :     nsCOMPtr<nsIPrefBranch> prefs(do_QueryInterface(aSubject, &rv));
    4514               4 :     NS_ENSURE_SUCCESS(rv, rv);
    4515               4 :     if (NS_LITERAL_STRING(CHECK_MALWARE_PREF).Equals(aData)) {
    4516                 :       bool tmpbool;
    4517               0 :       rv = prefs->GetBoolPref(CHECK_MALWARE_PREF, &tmpbool);
    4518               0 :       mCheckMalware = NS_SUCCEEDED(rv) ? tmpbool : CHECK_MALWARE_DEFAULT;
    4519               4 :     } else if (NS_LITERAL_STRING(CHECK_PHISHING_PREF).Equals(aData)) {
    4520                 :       bool tmpbool;
    4521               0 :       rv = prefs->GetBoolPref(CHECK_PHISHING_PREF, &tmpbool);
    4522               0 :       mCheckPhishing = NS_SUCCEEDED(rv) ? tmpbool : CHECK_PHISHING_DEFAULT;
    4523               4 :     } else if (NS_LITERAL_STRING(GETHASH_TABLES_PREF).Equals(aData)) {
    4524               0 :       mGethashWhitelist.Clear();
    4525               0 :       nsXPIDLCString val;
    4526               0 :       if (NS_SUCCEEDED(prefs->GetCharPref(GETHASH_TABLES_PREF, getter_Copies(val)))) {
    4527               0 :         SplitTables(val, mGethashWhitelist);
    4528                 :       }
    4529               4 :     } else if (NS_LITERAL_STRING(CONFIRM_AGE_PREF).Equals(aData)) {
    4530                 :       PRInt32 tmpint;
    4531               4 :       rv = prefs->GetIntPref(CONFIRM_AGE_PREF, &tmpint);
    4532               4 :       PR_ATOMIC_SET(&gFreshnessGuarantee, NS_SUCCEEDED(rv) ? tmpint : CONFIRM_AGE_DEFAULT_SEC);
    4533               0 :     } else if (NS_LITERAL_STRING(UPDATE_CACHE_SIZE_PREF).Equals(aData)) {
    4534                 :       PRInt32 tmpint;
    4535               0 :       rv = prefs->GetIntPref(UPDATE_CACHE_SIZE_PREF, &tmpint);
    4536               0 :       PR_ATOMIC_SET(&gUpdateCacheSize, NS_SUCCEEDED(rv) ? tmpint : UPDATE_CACHE_SIZE_DEFAULT);
    4537               0 :     } else if (NS_LITERAL_STRING(LOOKUP_CACHE_SIZE_PREF).Equals(aData)) {
    4538                 :       PRInt32 tmpint;
    4539               0 :       rv = prefs->GetIntPref(LOOKUP_CACHE_SIZE_PREF, &tmpint);
    4540               0 :       PR_ATOMIC_SET(&gLookupCacheSize, NS_SUCCEEDED(rv) ? tmpint : LOOKUP_CACHE_SIZE_DEFAULT);
    4541               0 :     } else if (NS_LITERAL_STRING(UPDATE_WORKING_TIME).Equals(aData)) {
    4542                 :       PRInt32 tmpint;
    4543               0 :       rv = prefs->GetIntPref(UPDATE_WORKING_TIME, &tmpint);
    4544               0 :       PR_ATOMIC_SET(&gWorkingTimeThreshold,
    4545               0 :                     NS_SUCCEEDED(rv) ? tmpint : UPDATE_WORKING_TIME_DEFAULT);
    4546               0 :     } else if (NS_LITERAL_STRING(UPDATE_DELAY_TIME).Equals(aData)) {
    4547                 :       PRInt32 tmpint;
    4548               0 :       rv = prefs->GetIntPref(UPDATE_DELAY_TIME, &tmpint);
    4549               0 :       PR_ATOMIC_SET(&gDelayTime,
    4550               0 :                     NS_SUCCEEDED(rv) ? tmpint : UPDATE_DELAY_TIME_DEFAULT);
    4551                 :     }
    4552              24 :   } else if (!strcmp(aTopic, "profile-before-change") ||
    4553               8 :              !strcmp(aTopic, "xpcom-shutdown-threads")) {
    4554              16 :     Shutdown();
    4555                 :   } else {
    4556               0 :     return NS_ERROR_UNEXPECTED;
    4557                 :   }
    4558                 : 
    4559              20 :   return NS_OK;
    4560                 : }
    4561                 : 
    4562                 : // Join the background thread if it exists.
    4563                 : nsresult
    4564              16 : nsUrlClassifierDBService::Shutdown()
    4565                 : {
    4566              16 :   LOG(("shutting down db service\n"));
    4567                 : 
    4568              16 :   if (!gDbBackgroundThread)
    4569               8 :     return NS_OK;
    4570                 : 
    4571               8 :   mCompleters.Clear();
    4572                 : 
    4573              16 :   nsCOMPtr<nsIPrefBranch> prefs = do_GetService(NS_PREFSERVICE_CONTRACTID);
    4574               8 :   if (prefs) {
    4575               8 :     prefs->RemoveObserver(CHECK_MALWARE_PREF, this);
    4576               8 :     prefs->RemoveObserver(CHECK_PHISHING_PREF, this);
    4577               8 :     prefs->RemoveObserver(GETHASH_TABLES_PREF, this);
    4578               8 :     prefs->RemoveObserver(CONFIRM_AGE_PREF, this);
    4579                 :   }
    4580                 : 
    4581                 :   nsresult rv;
    4582                 :   // First close the db connection.
    4583               8 :   if (mWorker) {
    4584               8 :     rv = mWorkerProxy->CancelUpdate();
    4585               8 :     NS_ASSERTION(NS_SUCCEEDED(rv), "failed to post cancel update event");
    4586                 : 
    4587               8 :     rv = mWorkerProxy->CloseDb();
    4588               8 :     NS_ASSERTION(NS_SUCCEEDED(rv), "failed to post close db event");
    4589                 :   }
    4590                 : 
    4591               8 :   mWorkerProxy = nsnull;
    4592                 : 
    4593               8 :   LOG(("joining background thread"));
    4594                 : 
    4595               8 :   gShuttingDownThread = true;
    4596                 : 
    4597               8 :   nsIThread *backgroundThread = gDbBackgroundThread;
    4598               8 :   gDbBackgroundThread = nsnull;
    4599               8 :   backgroundThread->Shutdown();
    4600               8 :   NS_RELEASE(backgroundThread);
    4601                 : 
    4602               8 :   return NS_OK;
    4603                 : }
    4604                 : 
    4605                 : nsIThread*
    4606             760 : nsUrlClassifierDBService::BackgroundThread()
    4607                 : {
    4608             760 :   return gDbBackgroundThread;
    4609                 : }
    4610                 : 

Generated by: LCOV version 1.7