From b75dbc4bbe73133b18c3b2d64f457d8d0bd50d35 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 5 Dec 2007 13:41:56 +0000 Subject: [PATCH] 2007-12-05 Tatsuhiro Tsujikawa Enable direct I/O support in checksum checking. * src/IteratableChunkChecksumValidator.{h, cc} * test/IteratableChunkChecksumValidatorTest.cc * src/CheckIntegrityEntry.cc * src/PieceHashCheckIntegrityEntry.cc * src/IteratableChecksumValidator.{h, cc} * src/BtCheckIntegrityEntry.cc: Added doc. --- ChangeLog | 10 ++++ src/BtCheckIntegrityEntry.cc | 2 + src/CheckIntegrityEntry.cc | 17 +++++- src/IteratableChecksumValidator.cc | 23 ++++++-- src/IteratableChecksumValidator.h | 2 + src/IteratableChunkChecksumValidator.cc | 58 ++++++++++++++++++-- src/IteratableChunkChecksumValidator.h | 7 +++ src/PieceHashCheckIntegrityEntry.cc | 1 + test/IteratableChunkChecksumValidatorTest.cc | 3 +- 9 files changed, 109 insertions(+), 14 deletions(-) diff --git a/ChangeLog b/ChangeLog index 88829448..934e9874 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,13 @@ +2007-12-05 Tatsuhiro Tsujikawa + + Enable direct I/O support in checksum checking. + * src/IteratableChunkChecksumValidator.{h, cc} + * test/IteratableChunkChecksumValidatorTest.cc + * src/CheckIntegrityEntry.cc + * src/PieceHashCheckIntegrityEntry.cc + * src/IteratableChecksumValidator.{h, cc} + * src/BtCheckIntegrityEntry.cc: Added doc. + 2007-12-04 Tatsuhiro Tsujikawa Added --allow-piece-length-change option. diff --git a/src/BtCheckIntegrityEntry.cc b/src/BtCheckIntegrityEntry.cc index 1481e86e..e12b7313 100644 --- a/src/BtCheckIntegrityEntry.cc +++ b/src/BtCheckIntegrityEntry.cc @@ -66,5 +66,7 @@ Commands BtCheckIntegrityEntry::onDownloadFinished(DownloadEngine* e) // are valid, then aira2 goes to seeding mode. Sometimes it is better // to exit rather than doing seeding. So, it would be good to toggle this // behavior. + // TODO If FileAllocationEntry is not going to be used, call + // DiskAdaptor::disableDirectIO() manually here. return onDownloadIncomplete(e); } diff --git a/src/CheckIntegrityEntry.cc b/src/CheckIntegrityEntry.cc index 43f5a6a1..f05ba3d1 100644 --- a/src/CheckIntegrityEntry.cc +++ b/src/CheckIntegrityEntry.cc @@ -36,14 +36,27 @@ #include "Command.h" #include "RequestGroup.h" #include "IteratableValidator.h" +#include "prefs.h" +#include "PieceStorage.h" +#include "DownloadContext.h" +#include "DiskAdaptor.h" +#include "Option.h" CheckIntegrityEntry::CheckIntegrityEntry(RequestGroup* requestGroup, Command* nextCommand): RequestGroupEntry(requestGroup, nextCommand), _validator(0) -{} +{ + if(_requestGroup->getOption()->getAsBool(PREF_ENABLE_DIRECT_IO) && + _requestGroup->getDownloadContext()->getFileEntries().size() == 1) { + _requestGroup->getPieceStorage()->getDiskAdaptor()->enableDirectIO(); + } +} -CheckIntegrityEntry::~CheckIntegrityEntry() {} +CheckIntegrityEntry::~CheckIntegrityEntry() +{ + // TODO DiskAdaptor::disableIO() will be called from ~FileAllocationEntry +} void CheckIntegrityEntry::validateChunk() { diff --git a/src/IteratableChecksumValidator.cc b/src/IteratableChecksumValidator.cc index fc68474b..b3db4b8b 100644 --- a/src/IteratableChecksumValidator.cc +++ b/src/IteratableChecksumValidator.cc @@ -43,7 +43,8 @@ #include "DiskAdaptor.h" #include "BitfieldMan.h" -#define BUFSIZE 16*1024 +#define BUFSIZE (256*1024) +#define ALIGNMENT 512 IteratableChecksumValidator::IteratableChecksumValidator(const SingleFileDownloadContextHandle& dctx, const PieceStorageHandle& pieceStorage): @@ -51,16 +52,21 @@ IteratableChecksumValidator::IteratableChecksumValidator(const SingleFileDownloa _pieceStorage(pieceStorage), _currentOffset(0), _ctx(0), - _logger(LogFactory::getInstance()) {} + _logger(LogFactory::getInstance()), + _buffer(0) {} -IteratableChecksumValidator::~IteratableChecksumValidator() {} +IteratableChecksumValidator::~IteratableChecksumValidator() +{ + delete [] _buffer; +} void IteratableChecksumValidator::validateChunk() { if(!finished()) { - unsigned char data[BUFSIZE]; - int32_t length = _pieceStorage->getDiskAdaptor()->readData(data, sizeof(data), _currentOffset); - _ctx->digestUpdate(data, length); + int32_t length = _pieceStorage->getDiskAdaptor()->readData(_buffer, + BUFSIZE, + _currentOffset); + _ctx->digestUpdate(_buffer, length); _currentOffset += length; if(finished()) { string actualChecksum = Util::toHex((const unsigned char*)_ctx->digestFinal().c_str(), _ctx->digestLength()); @@ -86,6 +92,11 @@ int64_t IteratableChecksumValidator::getTotalLength() const void IteratableChecksumValidator::init() { +#ifdef HAVE_POSIX_MEMALIGN + _buffer = (unsigned char*)Util::allocateAlignedMemory(ALIGNMENT, BUFSIZE); +#else + _buffer = new unsigned char[BUFSIZE]; +#endif // HAVE_POSIX_MEMALIGN _currentOffset = 0; _ctx = new MessageDigestContext(); _ctx->trySetAlgo(_dctx->getChecksumHashAlgo()); diff --git a/src/IteratableChecksumValidator.h b/src/IteratableChecksumValidator.h index de7d3cc1..0ef098a8 100644 --- a/src/IteratableChecksumValidator.h +++ b/src/IteratableChecksumValidator.h @@ -58,6 +58,8 @@ private: const Logger* _logger; + unsigned char* _buffer; + public: IteratableChecksumValidator(const SingleFileDownloadContextHandle& dctx, const PieceStorageHandle& pieceStorage); diff --git a/src/IteratableChunkChecksumValidator.cc b/src/IteratableChunkChecksumValidator.cc index 725331a0..bd58e89c 100644 --- a/src/IteratableChunkChecksumValidator.cc +++ b/src/IteratableChunkChecksumValidator.cc @@ -43,6 +43,11 @@ #include "BitfieldMan.h" #include "LogFactory.h" #include "Logger.h" +#include "messageDigest.h" +#include + +#define BUFSIZE (256*1024) +#define ALIGNMENT 512 IteratableChunkChecksumValidator:: IteratableChunkChecksumValidator(const DownloadContextHandle& dctx, @@ -51,9 +56,14 @@ IteratableChunkChecksumValidator(const DownloadContextHandle& dctx, _pieceStorage(pieceStorage), _bitfield(new BitfieldMan(_dctx->getPieceLength(), _dctx->getTotalLength())), _currentIndex(0), - _logger(LogFactory::getInstance()) {} + _logger(LogFactory::getInstance()), + _ctx(0), + _buffer(0) {} -IteratableChunkChecksumValidator::~IteratableChunkChecksumValidator() {} +IteratableChunkChecksumValidator::~IteratableChunkChecksumValidator() +{ + delete [] _buffer; +} void IteratableChunkChecksumValidator::validateChunk() @@ -96,17 +106,55 @@ string IteratableChunkChecksumValidator::calculateActualChecksum() } else { length = _dctx->getPieceLength(); } - return MessageDigestHelper::digest(_dctx->getPieceHashAlgo(), - _pieceStorage->getDiskAdaptor(), - offset, length); + return digest(offset, length); } void IteratableChunkChecksumValidator::init() { +#ifdef HAVE_POSIX_MEMALIGN + _buffer = (unsigned char*)Util::allocateAlignedMemory(ALIGNMENT, BUFSIZE); +#else + _buffer = new unsigned char[BUFSIZE]; +#endif // HAVE_POSIX_MEMALIGN + _ctx = new MessageDigestContext(); + _ctx->trySetAlgo(_dctx->getPieceHashAlgo()); + _ctx->digestInit(); _bitfield->clearAllBit(); _currentIndex = 0; } +string IteratableChunkChecksumValidator::digest(int64_t offset, int32_t length) +{ + _ctx->digestReset(); + int64_t curoffset = offset/ALIGNMENT*ALIGNMENT; + int64_t max = offset+length; + int32_t woffset; + if(curoffset < offset) { + woffset = offset-curoffset; + } else { + woffset = 0; + } + while(curoffset < max) { + int32_t r = _pieceStorage->getDiskAdaptor()->readData(_buffer, BUFSIZE, + curoffset); + if(r == 0) { + throw new DlAbortEx(EX_FILE_READ, _dctx->getActualBasePath().c_str(), + strerror(errno)); + } + int32_t wlength; + if(max < curoffset+r) { + wlength = max-curoffset-woffset; + } else { + wlength = r-woffset; + } + _ctx->digestUpdate(_buffer+woffset, wlength); + curoffset += r; + woffset = 0; + } + return Util::toHex((const unsigned char*)_ctx->digestFinal().c_str(), _ctx->digestLength()); +} + + bool IteratableChunkChecksumValidator::finished() const { return _currentIndex >= (uint32_t)_dctx->getNumPieces(); diff --git a/src/IteratableChunkChecksumValidator.h b/src/IteratableChunkChecksumValidator.h index b8de18f7..0ec824c5 100644 --- a/src/IteratableChunkChecksumValidator.h +++ b/src/IteratableChunkChecksumValidator.h @@ -43,6 +43,8 @@ class PieceStorage; typedef SharedHandle PieceStorageHandle; class BitfieldMan; class Logger; +class MessageDigestContext; +typedef SharedHandle MessageDigestContextHandle; class IteratableChunkChecksumValidator:public IteratableValidator { @@ -52,8 +54,13 @@ private: BitfieldMan* _bitfield; uint32_t _currentIndex; const Logger* _logger; + MessageDigestContextHandle _ctx; + unsigned char* _buffer; string calculateActualChecksum(); + + string digest(int64_t offset, int32_t length); + public: IteratableChunkChecksumValidator(const DownloadContextHandle& dctx, const PieceStorageHandle& pieceStorage); diff --git a/src/PieceHashCheckIntegrityEntry.cc b/src/PieceHashCheckIntegrityEntry.cc index 248147de..1fdb07c6 100644 --- a/src/PieceHashCheckIntegrityEntry.cc +++ b/src/PieceHashCheckIntegrityEntry.cc @@ -58,6 +58,7 @@ void PieceHashCheckIntegrityEntry::initValidator() IteratableChunkChecksumValidatorHandle validator = new IteratableChunkChecksumValidator(_requestGroup->getDownloadContext(), _requestGroup->getPieceStorage()); + validator->init(); _validator = validator; #endif // ENABLE_MESSAGE_DIGEST } diff --git a/test/IteratableChunkChecksumValidatorTest.cc b/test/IteratableChunkChecksumValidatorTest.cc index 340494d2..c410c665 100644 --- a/test/IteratableChunkChecksumValidatorTest.cc +++ b/test/IteratableChunkChecksumValidatorTest.cc @@ -40,7 +40,8 @@ void IteratableChunkChecksumValidatorTest::testValidate() { ps->getDiskAdaptor()->openFile(); IteratableChunkChecksumValidator validator(dctx, ps); - + validator.init(); + validator.validateChunk(); CPPUNIT_ASSERT(!validator.finished()); validator.validateChunk();