1 : /* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
2 : /* vim:set ts=4 sw=4 sts=4 cin et: */
3 : /* ***** BEGIN LICENSE BLOCK *****
4 : * Version: MPL 1.1/GPL 2.0/LGPL 2.1
5 : *
6 : * The contents of this file are subject to the Mozilla Public License Version
7 : * 1.1 (the "License"); you may not use this file except in compliance with
8 : * the License. You may obtain a copy of the License at
9 : * http://www.mozilla.org/MPL/
10 : *
11 : * Software distributed under the License is distributed on an "AS IS" basis,
12 : * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
13 : * for the specific language governing rights and limitations under the
14 : * License.
15 : *
16 : * The Original Code is nsDiskCacheMap.h, released
17 : * March 23, 2001.
18 : *
19 : * The Initial Developer of the Original Code is
20 : * Netscape Communications Corporation.
21 : * Portions created by the Initial Developer are Copyright (C) 2001
22 : * the Initial Developer. All Rights Reserved.
23 : *
24 : * Contributor(s):
25 : * Patrick C. Beard <beard@netscape.com>
26 : * Gordon Sheridan <gordon@netscape.com>
27 : *
28 : * Alternatively, the contents of this file may be used under the terms of
29 : * either the GNU General Public License Version 2 or later (the "GPL"), or
30 : * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
31 : * in which case the provisions of the GPL or the LGPL are applicable instead
32 : * of those above. If you wish to allow use of your version of this file only
33 : * under the terms of either the GPL or the LGPL, and not to allow others to
34 : * use your version of this file under the terms of the MPL, indicate your
35 : * decision by deleting the provisions above and replace them with the notice
36 : * and other provisions required by the GPL or the LGPL. If you do not delete
37 : * the provisions above, a recipient may use your version of this file under
38 : * the terms of any one of the MPL, the GPL or the LGPL.
39 : *
40 : * ***** END LICENSE BLOCK ***** */
41 :
42 : #ifndef _nsDiskCacheMap_h_
43 : #define _nsDiskCacheMap_h_
44 :
45 : #include <limits.h>
46 :
47 : #include "prtypes.h"
48 : #include "prnetdb.h"
49 : #include "nsDebug.h"
50 : #include "nsError.h"
51 : #include "nsILocalFile.h"
52 :
53 : #include "nsDiskCache.h"
54 : #include "nsDiskCacheBlockFile.h"
55 :
56 :
57 : class nsDiskCacheBinding;
58 : struct nsDiskCacheEntry;
59 :
60 : /******************************************************************************
61 : * nsDiskCacheRecord
62 : *
63 : * Cache Location Format
64 : *
65 : * 1000 0000 0000 0000 0000 0000 0000 0000 : initialized bit
66 : *
67 : * 0011 0000 0000 0000 0000 0000 0000 0000 : File Selector (0 = separate file)
68 : * 0000 0011 0000 0000 0000 0000 0000 0000 : number of extra contiguous blocks 1-4
69 : * 0100 1100 0000 0000 0000 0000 0000 0000 : reserved bits
70 : * 0000 0000 1111 1111 1111 1111 1111 1111 : block# 0-16777216 (2^24)
71 : *
72 : * 0000 0000 1111 1111 1111 1111 0000 0000 : eFileSizeMask (size of file in k: see note)
73 : * 0000 0000 0000 0000 0000 0000 1111 1111 : eFileGenerationMask
74 : *
75 : * File Selector:
76 : * 0 = separate file on disk
77 : * 1 = 256 byte block file
78 : * 2 = 1k block file
79 : * 3 = 4k block file
80 : *
81 : * eFileSizeMask note: Files larger than 65535 KiB have this limit stored in
82 : * the location. The file itself must be examined to
83 : * determine its actual size if necessary.
84 : *
85 : *****************************************************************************/
86 :
87 : /*
88 : We have 3 block files with roughly the same max size (32MB)
89 : 1 - block size 256B, number of blocks 131072
90 : 2 - block size 1kB, number of blocks 32768
91 : 3 - block size 4kB, number of blocks 8192
92 : */
93 : #define kNumBlockFiles 3
94 : #define SIZE_SHIFT(idx) (2 * ((idx) - 1))
95 : #define BLOCK_SIZE_FOR_INDEX(idx) ((idx) ? (256 << SIZE_SHIFT(idx)) : 0)
96 : #define BITMAP_SIZE_FOR_INDEX(idx) ((idx) ? (131072 >> SIZE_SHIFT(idx)) : 0)
97 :
98 : // Min and max values for the number of records in the DiskCachemap
99 : #define kMinRecordCount 512
100 :
101 : #define kSeparateFile 0
102 : #define kBuckets (1 << 5) // must be a power of 2!
103 :
104 : // Maximum size in K which can be stored in the location (see eFileSizeMask).
105 : // Both data and metadata can be larger, but only up to kMaxDataSizeK can be
106 : // counted into total cache size. I.e. if there are entries where either data or
107 : // metadata is larger than kMaxDataSizeK, the total cache size will be
108 : // inaccurate (smaller) than the actual cache size. The alternative is to stat
109 : // the files to find the real size, which was decided against for performance
110 : // reasons. See bug #651100 comment #21.
111 : #define kMaxDataSizeK 0xFFFF
112 :
113 : // preallocate up to 1MB of separate cache file
114 : #define kPreallocateLimit 1 * 1024 * 1024
115 :
116 : class nsDiskCacheRecord {
117 :
118 : private:
119 : PRUint32 mHashNumber;
120 : PRUint32 mEvictionRank;
121 : PRUint32 mDataLocation;
122 : PRUint32 mMetaLocation;
123 :
124 : enum {
125 : eLocationInitializedMask = 0x80000000,
126 :
127 : eLocationSelectorMask = 0x30000000,
128 : eLocationSelectorOffset = 28,
129 :
130 : eExtraBlocksMask = 0x03000000,
131 : eExtraBlocksOffset = 24,
132 :
133 : eReservedMask = 0x4C000000,
134 :
135 : eBlockNumberMask = 0x00FFFFFF,
136 :
137 : eFileSizeMask = 0x00FFFF00,
138 : eFileSizeOffset = 8,
139 : eFileGenerationMask = 0x000000FF,
140 : eFileReservedMask = 0x4F000000
141 :
142 : };
143 :
144 : public:
145 3360 : nsDiskCacheRecord()
146 3360 : : mHashNumber(0), mEvictionRank(0), mDataLocation(0), mMetaLocation(0)
147 : {
148 3360 : }
149 :
150 2969 : bool ValidRecord()
151 : {
152 2969 : if ((mDataLocation & eReservedMask) || (mMetaLocation & eReservedMask))
153 0 : return false;
154 2969 : return true;
155 : }
156 :
157 : // HashNumber accessors
158 99549 : PRUint32 HashNumber() const { return mHashNumber; }
159 1272 : void SetHashNumber( PRUint32 hashNumber) { mHashNumber = hashNumber; }
160 :
161 : // EvictionRank accessors
162 13446 : PRUint32 EvictionRank() const { return mEvictionRank; }
163 1211 : void SetEvictionRank( PRUint32 rank) { mEvictionRank = rank ? rank : 1; }
164 :
165 : // DataLocation accessors
166 1236 : bool DataLocationInitialized() const { return 0 != (mDataLocation & eLocationInitializedMask); }
167 178 : void ClearDataLocation() { mDataLocation = 0; }
168 :
169 1039 : PRUint32 DataFile() const
170 : {
171 1039 : return (PRUint32)(mDataLocation & eLocationSelectorMask) >> eLocationSelectorOffset;
172 : }
173 :
174 476 : void SetDataBlocks( PRUint32 index, PRUint32 startBlock, PRUint32 blockCount)
175 : {
176 : // clear everything
177 476 : mDataLocation = 0;
178 :
179 : // set file index
180 476 : NS_ASSERTION( index < (kNumBlockFiles + 1), "invalid location index");
181 476 : NS_ASSERTION( index > 0,"invalid location index");
182 476 : mDataLocation |= (index << eLocationSelectorOffset) & eLocationSelectorMask;
183 :
184 : // set startBlock
185 476 : NS_ASSERTION(startBlock == (startBlock & eBlockNumberMask), "invalid block number");
186 476 : mDataLocation |= startBlock & eBlockNumberMask;
187 :
188 : // set blockCount
189 476 : NS_ASSERTION( (blockCount>=1) && (blockCount<=4),"invalid block count");
190 476 : --blockCount;
191 476 : mDataLocation |= (blockCount << eExtraBlocksOffset) & eExtraBlocksMask;
192 :
193 476 : mDataLocation |= eLocationInitializedMask;
194 476 : }
195 :
196 304 : PRUint32 DataBlockCount() const
197 : {
198 304 : return (PRUint32)((mDataLocation & eExtraBlocksMask) >> eExtraBlocksOffset) + 1;
199 : }
200 :
201 304 : PRUint32 DataStartBlock() const
202 : {
203 304 : return (mDataLocation & eBlockNumberMask);
204 : }
205 :
206 : PRUint32 DataBlockSize() const
207 : {
208 : return BLOCK_SIZE_FOR_INDEX(DataFile());
209 : }
210 :
211 232 : PRUint32 DataFileSize() const { return (mDataLocation & eFileSizeMask) >> eFileSizeOffset; }
212 155 : void SetDataFileSize(PRUint32 size)
213 : {
214 155 : NS_ASSERTION((mDataLocation & eFileReservedMask) == 0, "bad location");
215 155 : mDataLocation &= ~eFileSizeMask; // clear eFileSizeMask
216 155 : mDataLocation |= (size << eFileSizeOffset) & eFileSizeMask;
217 155 : }
218 :
219 75 : PRUint8 DataFileGeneration() const
220 : {
221 75 : return (mDataLocation & eFileGenerationMask);
222 : }
223 :
224 37 : void SetDataFileGeneration( PRUint8 generation)
225 : {
226 : // clear everything, (separate file index = 0)
227 37 : mDataLocation = 0;
228 37 : mDataLocation |= generation & eFileGenerationMask;
229 37 : mDataLocation |= eLocationInitializedMask;
230 37 : }
231 :
232 : // MetaLocation accessors
233 939 : bool MetaLocationInitialized() const { return 0 != (mMetaLocation & eLocationInitializedMask); }
234 333 : void ClearMetaLocation() { mMetaLocation = 0; }
235 660 : PRUint32 MetaLocation() const { return mMetaLocation; }
236 :
237 1051 : PRUint32 MetaFile() const
238 : {
239 1051 : return (PRUint32)(mMetaLocation & eLocationSelectorMask) >> eLocationSelectorOffset;
240 : }
241 :
242 660 : void SetMetaBlocks( PRUint32 index, PRUint32 startBlock, PRUint32 blockCount)
243 : {
244 : // clear everything
245 660 : mMetaLocation = 0;
246 :
247 : // set file index
248 660 : NS_ASSERTION( index < (kNumBlockFiles + 1), "invalid location index");
249 660 : NS_ASSERTION( index > 0, "invalid location index");
250 660 : mMetaLocation |= (index << eLocationSelectorOffset) & eLocationSelectorMask;
251 :
252 : // set startBlock
253 660 : NS_ASSERTION(startBlock == (startBlock & eBlockNumberMask), "invalid block number");
254 660 : mMetaLocation |= startBlock & eBlockNumberMask;
255 :
256 : // set blockCount
257 660 : NS_ASSERTION( (blockCount>=1) && (blockCount<=4),"invalid block count");
258 660 : --blockCount;
259 660 : mMetaLocation |= (blockCount << eExtraBlocksOffset) & eExtraBlocksMask;
260 :
261 660 : mMetaLocation |= eLocationInitializedMask;
262 660 : }
263 :
264 558 : PRUint32 MetaBlockCount() const
265 : {
266 558 : return (PRUint32)((mMetaLocation & eExtraBlocksMask) >> eExtraBlocksOffset) + 1;
267 : }
268 :
269 558 : PRUint32 MetaStartBlock() const
270 : {
271 558 : return (mMetaLocation & eBlockNumberMask);
272 : }
273 :
274 : PRUint32 MetaBlockSize() const
275 : {
276 : return BLOCK_SIZE_FOR_INDEX(MetaFile());
277 : }
278 :
279 54 : PRUint32 MetaFileSize() const { return (mMetaLocation & eFileSizeMask) >> eFileSizeOffset; }
280 0 : void SetMetaFileSize(PRUint32 size)
281 : {
282 0 : mMetaLocation &= ~eFileSizeMask; // clear eFileSizeMask
283 0 : mMetaLocation |= (size << eFileSizeOffset) & eFileSizeMask;
284 0 : }
285 :
286 0 : PRUint8 MetaFileGeneration() const
287 : {
288 0 : return (mMetaLocation & eFileGenerationMask);
289 : }
290 :
291 0 : void SetMetaFileGeneration( PRUint8 generation)
292 : {
293 : // clear everything, (separate file index = 0)
294 0 : mMetaLocation = 0;
295 0 : mMetaLocation |= generation & eFileGenerationMask;
296 0 : mMetaLocation |= eLocationInitializedMask;
297 0 : }
298 :
299 960 : PRUint8 Generation() const
300 : {
301 1311 : if ((mDataLocation & eLocationInitializedMask) &&
302 351 : (DataFile() == 0))
303 75 : return DataFileGeneration();
304 :
305 1161 : if ((mMetaLocation & eLocationInitializedMask) &&
306 276 : (MetaFile() == 0))
307 0 : return MetaFileGeneration();
308 :
309 885 : return 0; // no generation
310 : }
311 :
312 : #if defined(IS_LITTLE_ENDIAN)
313 376 : void Swap()
314 : {
315 376 : mHashNumber = htonl(mHashNumber);
316 376 : mEvictionRank = htonl(mEvictionRank);
317 376 : mDataLocation = htonl(mDataLocation);
318 376 : mMetaLocation = htonl(mMetaLocation);
319 376 : }
320 : #endif
321 :
322 : #if defined(IS_LITTLE_ENDIAN)
323 0 : void Unswap()
324 : {
325 0 : mHashNumber = ntohl(mHashNumber);
326 0 : mEvictionRank = ntohl(mEvictionRank);
327 0 : mDataLocation = ntohl(mDataLocation);
328 0 : mMetaLocation = ntohl(mMetaLocation);
329 0 : }
330 : #endif
331 :
332 : };
333 :
334 :
335 : /******************************************************************************
336 : * nsDiskCacheRecordVisitor
337 : *****************************************************************************/
338 :
339 : enum { kDeleteRecordAndContinue = -1,
340 : kStopVisitingRecords = 0,
341 : kVisitNextRecord = 1
342 : };
343 :
344 17 : class nsDiskCacheRecordVisitor {
345 : public:
346 :
347 : virtual PRInt32 VisitRecord( nsDiskCacheRecord * mapRecord) = 0;
348 : };
349 :
350 :
351 : /******************************************************************************
352 : * nsDiskCacheHeader
353 : *****************************************************************************/
354 :
355 : struct nsDiskCacheHeader {
356 : PRUint32 mVersion; // cache version.
357 : PRUint32 mDataSize; // size of cache in units of 1024bytes.
358 : PRInt32 mEntryCount; // number of entries stored in cache.
359 : PRUint32 mIsDirty; // dirty flag.
360 : PRInt32 mRecordCount; // Number of records
361 : PRUint32 mEvictionRank[kBuckets]; // Highest EvictionRank of the bucket
362 : PRUint32 mBucketUsage[kBuckets]; // Number of used entries in the bucket
363 :
364 175 : nsDiskCacheHeader()
365 : : mVersion(nsDiskCache::kCurrentVersion)
366 : , mDataSize(0)
367 : , mEntryCount(0)
368 : , mIsDirty(true)
369 175 : , mRecordCount(0)
370 175 : {}
371 :
372 411 : void Swap()
373 : {
374 : #if defined(IS_LITTLE_ENDIAN)
375 411 : mVersion = htonl(mVersion);
376 411 : mDataSize = htonl(mDataSize);
377 411 : mEntryCount = htonl(mEntryCount);
378 411 : mIsDirty = htonl(mIsDirty);
379 411 : mRecordCount = htonl(mRecordCount);
380 :
381 13563 : for (PRUint32 i = 0; i < kBuckets ; i++) {
382 13152 : mEvictionRank[i] = htonl(mEvictionRank[i]);
383 13152 : mBucketUsage[i] = htonl(mBucketUsage[i]);
384 : }
385 : #endif
386 411 : }
387 :
388 411 : void Unswap()
389 : {
390 : #if defined(IS_LITTLE_ENDIAN)
391 411 : mVersion = ntohl(mVersion);
392 411 : mDataSize = ntohl(mDataSize);
393 411 : mEntryCount = ntohl(mEntryCount);
394 411 : mIsDirty = ntohl(mIsDirty);
395 411 : mRecordCount = ntohl(mRecordCount);
396 :
397 13563 : for (PRUint32 i = 0; i < kBuckets ; i++) {
398 13152 : mEvictionRank[i] = ntohl(mEvictionRank[i]);
399 13152 : mBucketUsage[i] = ntohl(mBucketUsage[i]);
400 : }
401 : #endif
402 411 : }
403 : };
404 :
405 :
406 : /******************************************************************************
407 : * nsDiskCacheMap
408 : *****************************************************************************/
409 :
410 : class nsDiskCacheMap {
411 : public:
412 :
413 175 : nsDiskCacheMap() :
414 : mCacheDirectory(nsnull),
415 : mMapFD(nsnull),
416 : mRecordArray(nsnull),
417 : mBufferSize(0),
418 : mBuffer(nsnull),
419 175 : mMaxRecordCount(16384) // this default value won't matter
420 175 : { }
421 :
422 1225 : ~nsDiskCacheMap() {
423 175 : (void) Close(true);
424 1050 : }
425 :
426 : /**
427 : * File Operations
428 : *
429 : * Open
430 : *
431 : * Creates a new cache map file if one doesn't exist.
432 : * Returns error if it detects change in format or cache wasn't closed.
433 : */
434 : nsresult Open( nsILocalFile * cacheDirectory);
435 : nsresult Close(bool flush);
436 : nsresult Trim();
437 :
438 : nsresult FlushHeader();
439 : nsresult FlushRecords( bool unswap);
440 :
441 : void NotifyCapacityChange(PRUint32 capacity);
442 :
443 : /**
444 : * Record operations
445 : */
446 : nsresult AddRecord( nsDiskCacheRecord * mapRecord, nsDiskCacheRecord * oldRecord);
447 : nsresult UpdateRecord( nsDiskCacheRecord * mapRecord);
448 : nsresult FindRecord( PRUint32 hashNumber, nsDiskCacheRecord * mapRecord);
449 : nsresult DeleteRecord( nsDiskCacheRecord * mapRecord);
450 : nsresult VisitRecords( nsDiskCacheRecordVisitor * visitor);
451 : nsresult EvictRecords( nsDiskCacheRecordVisitor * visitor);
452 :
453 : /**
454 : * Disk Entry operations
455 : */
456 : nsresult DeleteStorage( nsDiskCacheRecord * record);
457 :
458 : nsresult GetFileForDiskCacheRecord( nsDiskCacheRecord * record,
459 : bool meta,
460 : bool createPath,
461 : nsIFile ** result);
462 :
463 : nsresult GetLocalFileForDiskCacheRecord( nsDiskCacheRecord * record,
464 : bool meta,
465 : bool createPath,
466 : nsILocalFile ** result);
467 :
468 : // On success, this returns the buffer owned by nsDiskCacheMap,
469 : // so it must not be deleted by the caller.
470 : nsDiskCacheEntry * ReadDiskCacheEntry( nsDiskCacheRecord * record);
471 :
472 : nsresult WriteDiskCacheEntry( nsDiskCacheBinding * binding);
473 :
474 : nsresult ReadDataCacheBlocks(nsDiskCacheBinding * binding, char * buffer, PRUint32 size);
475 : nsresult WriteDataCacheBlocks(nsDiskCacheBinding * binding, char * buffer, PRUint32 size);
476 : nsresult DeleteStorage( nsDiskCacheRecord * record, bool metaData);
477 :
478 : /**
479 : * Statistical Operations
480 : */
481 1291 : void IncrementTotalSize( PRUint32 delta)
482 : {
483 1291 : mHeader.mDataSize += delta;
484 1291 : mHeader.mIsDirty = true;
485 1291 : }
486 :
487 666 : void DecrementTotalSize( PRUint32 delta)
488 : {
489 666 : NS_ASSERTION(mHeader.mDataSize >= delta, "disk cache size negative?");
490 666 : mHeader.mDataSize = mHeader.mDataSize > delta ? mHeader.mDataSize - delta : 0;
491 666 : mHeader.mIsDirty = true;
492 666 : }
493 :
494 1136 : inline void IncrementTotalSize( PRUint32 blocks, PRUint32 blockSize)
495 : {
496 : // Round up to nearest K
497 1136 : IncrementTotalSize(((blocks*blockSize) + 0x03FF) >> 10);
498 1136 : }
499 :
500 423 : inline void DecrementTotalSize( PRUint32 blocks, PRUint32 blockSize)
501 : {
502 : // Round up to nearest K
503 423 : DecrementTotalSize(((blocks*blockSize) + 0x03FF) >> 10);
504 423 : }
505 :
506 1547 : PRUint32 TotalSize() { return mHeader.mDataSize; }
507 :
508 0 : PRInt32 EntryCount() { return mHeader.mEntryCount; }
509 :
510 :
511 : private:
512 :
513 : /**
514 : * Private methods
515 : */
516 : nsresult OpenBlockFiles();
517 : nsresult CloseBlockFiles(bool flush);
518 : bool CacheFilesExist();
519 :
520 : nsresult CreateCacheSubDirectories();
521 :
522 : PRUint32 CalculateFileIndex(PRUint32 size);
523 :
524 : nsresult GetBlockFileForIndex( PRUint32 index, nsILocalFile ** result);
525 2546 : PRUint32 GetBlockSizeForIndex( PRUint32 index) const {
526 2546 : return BLOCK_SIZE_FOR_INDEX(index);
527 : }
528 708 : PRUint32 GetBitMapSizeForIndex( PRUint32 index) const {
529 708 : return BITMAP_SIZE_FOR_INDEX(index);
530 : }
531 :
532 : // returns the bucket number
533 4654 : PRUint32 GetBucketIndex( PRUint32 hashNumber) const {
534 4654 : return (hashNumber & (kBuckets - 1));
535 : }
536 :
537 : // Gets the size of the bucket (in number of records)
538 9877 : PRUint32 GetRecordsPerBucket() const {
539 9877 : return mHeader.mRecordCount / kBuckets;
540 : }
541 :
542 : // Gets the first record in the bucket
543 8775 : nsDiskCacheRecord *GetFirstRecordInBucket(PRUint32 bucket) const {
544 8775 : return mRecordArray + bucket * GetRecordsPerBucket();
545 : }
546 :
547 : PRUint32 GetBucketRank(PRUint32 bucketIndex, PRUint32 targetRank);
548 :
549 : PRInt32 VisitEachRecord(PRUint32 bucketIndex,
550 : nsDiskCacheRecordVisitor * visitor,
551 : PRUint32 evictionRank);
552 :
553 : nsresult GrowRecords();
554 : nsresult ShrinkRecords();
555 :
556 : nsresult EnsureBuffer(PRUint32 bufSize);
557 :
558 : // The returned structure will point to the buffer owned by nsDiskCacheMap,
559 : // so it must not be deleted by the caller.
560 : nsDiskCacheEntry * CreateDiskCacheEntry(nsDiskCacheBinding * binding,
561 : PRUint32 * size);
562 :
563 : /**
564 : * data members
565 : */
566 : private:
567 : nsCOMPtr<nsILocalFile> mCacheDirectory;
568 : PRFileDesc * mMapFD;
569 : nsDiskCacheRecord * mRecordArray;
570 : nsDiskCacheBlockFile mBlockFile[kNumBlockFiles];
571 : PRUint32 mBufferSize;
572 : char * mBuffer;
573 : nsDiskCacheHeader mHeader;
574 : PRInt32 mMaxRecordCount;
575 : };
576 :
577 : #endif // _nsDiskCacheMap_h_
|