diff --git a/ChangeLog b/ChangeLog index ef41ce86..b4d4c0c4 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,205 @@ +2007-01-23 Tatsuhiro Tsujikawa + + To add chunk checksum validation: + * src/MetalinkEntry.h + (MetalinkChunkChecksum.h): New include. + (chunkChecksum): New variable. + * src/Request.h + (method): New variable. + (setMethod): New function. + (getMethod): New function. + (METHOD_GET): New static constant. + (METHOD_HEAD): New static constant. + * src/Xml2MetalinkProcessor.h + (getPieceHash): New function. + * src/PieceStorage.h + (markAllPiecesDone): New function. + (checkIntegrity): New function. + * src/FileAllocator.h + (NullFileAllocationMonitor.h): New include. + (FileAllocator): Initialize fileAllocationMonitor with new + NullFileAllocationMonitor(). + * src/MultiDiskAdaptor.h + (messageDigest.h): Remove include. + (ctx): Removed. + (hashUpdate): Added ctx. + (MultiDiskAdaptor): Removed ctx. + (sha1Sum): Renamed as messageDigest. + (messageDigest): New function. + * src/UrlRequestInfo.h + (HeadResult): New class. + (digestAlgo): New variable. + (chunkChecksumLength): New variable. + (chunkChecksums): New variable. + (getHeadResult): New function. + (UrlRequestInfo): Added digestAlgo, chunkChecksumLength. + (setDigestAlgo): New function. + (setChunkChecksumLength): New function. + (setChunkChecksums): New function. + * src/DefaultPieceStorage.cc + (DiskAdaptorWriter.h): New include. + (ChunkChecksumValidator.h): New include. + (markAllPiecesDone): New function. + (checkIntegrity): New function. + * src/DefaultBtContext.h + (getPieceHashes): New function. + * src/TorrentRequestInfo.cc + (execute): Try to validate chunk checksum if file already exists and + .aria2 file doesn't there and user allows aria2 to overwrite it. + * src/messageDigest.h + (~MessageDigestContext): Added digestFree(). + * src/MetalinkRequestInfo.cc + (execute): Set digestAlgo, chunkChecksum, chunkChecksums to reqInfo. + * src/DiskAdaptor.h + (messageDigest.h): New include. + (sha1Sum): Renamed as messageDigest. + (messageDigest): New function. + * src/DownloadCommand.h + (PeerStat.h): New include. + (maxDownloadSpeedLimit): New variable. + (startupIdleTime): New variable. + (lowestDownloadSpeedLimit): New variable. + (peerStat): New variable. + (setMaxDownloadSpeedLimit): New function. + (setStartupIdleTime): New function. + (setLowestDownloadSPeedLimit): New function. + * src/BtContext.h + (getPieceHashes): New function. + * src/main.cc + (main): Set PREF_REALTIME_CHUNK_CHECKSUM and PREF_CHECK_INTEGRITY + option to true for testing purpose. + * src/BtPieceMessage.cc + (checkPieceHash): Use messageDigest + * src/DownloadEngine.cc + (SetDescriptor): Removed. + (AccumulateActiveCommand): Removed. + (waitData): Rewritten. + (updateFdSet): Rewritten. + * src/MultiDiskAdaptor.cc + (hashUpdate): Added ctx. + (sha1Sum): Renamed as messageDigest. + (messageDigest): New function. + * src/BitfieldMan.h + (isBitRangeSet): New function. + (unsetBitRange): New function. + * src/ByteArrayDiskWriter.h + (sha1Sum): Renamed as messageDigest. + (messageDigest): New function. + * src/ConsoleDownloadEngine.cc + (calculateStatistics): If nspeed < 0 then set nspeed to 0. + * src/DiskWriter.h + (messageDigest.h): New include. + (sha1Sum): Renamed as messageDigest. + (messageDigest): New function. + * src/ChunkChecksumValidator.h: New class. + * src/DiskAdaptorWriter.h: New class. + * src/prefs.h + (PREF_REALTIME_CHUNK_CHECKSUM): New definition. + (PREF_CHECK_INTEGRITY): New definition. + * src/HttpResponseCommand.cc + (handleDefaultEncoding): Added method "HEAD" handling. + Removed the call to e->segmentMan->shouldCancelDownloadForSafety(). + (handleOtherEncoding): + Added the call to e->segmentMan->shouldCancelDownloadForSafety(). + (createHttpDownloadCommand): Set maxDownloadSpeedLimit, + startupIdleTime, lowestDownloadSpeedLimit to command. + * src/SegmentMan.h + (getSegmentEntryByIndex): New function. + (getSegmentEntryByCuid): New function. + (getSegmentEntryIteratorByCuid): New function. + (diskWriter): DiskWriter -> DiskWriterHandle + (pieceHashes): New variable. + (chunkHashLength): New variable. + (digestAlgo): New variable. + (FindPeerStat): Removed. + (getPeerStat): Rewritten. + (markAllPiecesDone): New function. + (checkIntegrity): New function. + (tryChunkChecksumValidation): New function. + (isChunkChecksumValidationReady): New function. + * src/BitfieldMan.cc + (BitfieldMan): Initialized bitfieldLength, blocks to 0. + (BitfieldMan): Initialized blockLength, totalLength, bitfieldLength, + blocks to 0. + (isBitRangeSet): New function. + (unsetBitRange): New function. + * src/FtpNegotiateCommand.cc + (executeInternal): Set maxDownloadSpeedLimit, + startupIdleTime, lowestDownloadSpeedLimit to command. + (recvSize): Added method "HEAD" handling. + Removed the call to e->segmentMan->shouldCancelDownloadForSafety(). + * src/AbstractSingleDiskAdaptor.cc + (sha1Sum): Renamed as messageDigest. + (messageDigest): New function. + * src/AbstractSingleDiskAdaptor.h + (sha1Sum): Renamed as messageDigest. + (messageDigest): New function. + * src/Util.h + (indexRange): New function. + * src/MetalinkEntry.cc + (MetalinkEntry): Initialized chunkChecksum to 0. + * src/ShaVisitor.cc + (~ShaVisitor): Removed the call to ctx.digestFree(). + * src/SegmentMan.cc + (ChunkChecksumValidator.h): New include. + (SegmentMan): Initialized chunkHashLength to 0. Initialized digestAlgo + to DIGEST_ALGO_SHA1. + (~SegmentMan): Removed diskWriter. + (FindSegmentEntryByIndex): Removed. + (FindSegmentEntryByCuid): Removed. + (checkoutSegment): Rewritten. + (findSlowerSegmentEntry): Rewritten. + (getSegment): Rewritten. + (updateSegment): Rewritten. + (completeSegment): Rewritten. + (markAllPiecesDone): New function. + (checkIntegrity): New function. + (isChunkChecksumValidationReady): New function. + (tryChunkChecksumValidation): New function. + * src/Xml2MetalinkProcessor.cc + (getEntry): Get size and set it to entry. + Get chunk checksum and set it to entry. + (getPieceHash): New function. + * src/Util.cc + (sha1Sum): Removed ctx.digestFree() + (fileChecksum): Removed ctx.digestFree() + (indexRange): New function. + * src/Request.cc + (METHOD_GET): New variable. + (METHOD_HEAD): New variable. + (Request): Added method. + * src/UrlRequestInfo.cc + (FatalException.h): New include. + (message.h): New include. + (operator<<): Added operator<< for class HeadResult. + (getHeadResult): New function. + (execute): Get filename and size in separate download engine. + * src/ChunkChecksumValidator.cc: New class. + * src/DownloadCommand.cc: + (DownloadCommand): Added peerStat. + (executeInternal): Use maxDownloadSpeedLimit member instead of getting + the value from Option. + The buffer size is now 16KB. + Use peerStat member instead of getting it from SegmentMan. + Use startupIdleTime member instead of gettingit from Option. + Added chunk checksum validation. + * src/AbstractDiskWriter.cc + (AbstractDiskWriter): Removed ctx. + (~AbstractDiskWriter): Removed ctx.digestFree() + (writeDataInternal): Returns the return value of write. + (readDataInternal): Returns the return value of read. + (sha1Sum): Renamed as messageDigest + (messageDigest): New function. + * src/AbstractDiwkWriter.h + (messageDigest.h): Removed include. + (ctx): Removed. + (sha1Sum): Renamed as messageDigest + (messageDigest): New function. + * src/DefaultPieceStorage.h + (markAllPiecesDone): New function. + (checkIntegrity): New function. + * src/NullFileAllocationMonitor.h: New class. + 2007-01-16 Tatsuhiro Tsujikawa To decrease CPU usage in bittorrent download, calculation results in diff --git a/TODO b/TODO index b07db9b9..23d7a8e4 100644 --- a/TODO +++ b/TODO @@ -24,5 +24,7 @@ * remove blockIndex * Add an ability of seeding -* Add piece hash checking +* Stopping while piece hash checking and file allocation * Stop download after selective download completes +* Remove -pg option in Makefile.am +* Continue file allocation with existing file \ No newline at end of file diff --git a/src/AbstractDiskWriter.cc b/src/AbstractDiskWriter.cc index e0d958bf..383a3a88 100644 --- a/src/AbstractDiskWriter.cc +++ b/src/AbstractDiskWriter.cc @@ -48,20 +48,11 @@ AbstractDiskWriter::AbstractDiskWriter(): fd(0), fileAllocator(0), logger(LogFactory::getInstance()) -#ifdef ENABLE_MESSAGE_DIGEST - ,ctx(DIGEST_ALGO_SHA1) -#endif // ENABLE_MESSAGE_DIGEST -{ -#ifdef ENABLE_MESSAGE_DIGEST - ctx.digestInit(); -#endif // ENABLE_MESSAGE_DIGEST -} +{} -AbstractDiskWriter::~AbstractDiskWriter() { +AbstractDiskWriter::~AbstractDiskWriter() +{ closeFile(); -#ifdef ENABLE_MESSAGE_DIGEST - ctx.digestFree(); -#endif // ENABLE_MESSAGE_DIGEST } void AbstractDiskWriter::openFile(const string& filename, uint64_t totalLength) { @@ -104,42 +95,42 @@ void AbstractDiskWriter::createFile(const string& filename, int32_t addFlags) { } } -void AbstractDiskWriter::writeDataInternal(const char* data, uint32_t len) { - if(write(fd, data, len) < 0) { - throw new DlAbortEx(EX_FILE_WRITE, filename.c_str(), strerror(errno)); - } +int32_t AbstractDiskWriter::writeDataInternal(const char* data, uint32_t len) { + return write(fd, data, len); } int AbstractDiskWriter::readDataInternal(char* data, uint32_t len) { - int32_t ret; - if((ret = read(fd, data, len)) < 0) { - throw new DlAbortEx(EX_FILE_READ, filename.c_str(), strerror(errno)); - } - return ret; + return read(fd, data, len); } -string AbstractDiskWriter::sha1Sum(int64_t offset, uint64_t length) { +string AbstractDiskWriter::messageDigest(int64_t offset, uint64_t length, + const MessageDigestContext::DigestAlgo& algo) +{ #ifdef ENABLE_MESSAGE_DIGEST - ctx.digestReset(); + MessageDigestContext ctx(algo); + ctx.digestInit(); - uint32_t BUFSIZE = 16*1024; + int32_t BUFSIZE = 16*1024; char buf[BUFSIZE]; for(uint64_t i = 0; i < length/BUFSIZE; i++) { - if((int32_t)BUFSIZE != readData(buf, BUFSIZE, offset)) { + int32_t rs = readData(buf, BUFSIZE, offset); + if(BUFSIZE != readData(buf, BUFSIZE, offset)) { throw new DlAbortEx(EX_FILE_SHA1SUM, filename.c_str(), strerror(errno)); } ctx.digestUpdate(buf, BUFSIZE); offset += BUFSIZE; } - uint32_t r = length%BUFSIZE; + int32_t r = length%BUFSIZE; if(r > 0) { - if((int32_t)r != readData(buf, r, offset)) { + int32_t rs = readData(buf, r, offset); + if(r != readData(buf, r, offset)) { throw new DlAbortEx(EX_FILE_SHA1SUM, filename.c_str(), strerror(errno)); } ctx.digestUpdate(buf, r); } unsigned char hashValue[20]; ctx.digestFinal(hashValue); + return Util::toHex(hashValue, 20); #else return ""; @@ -154,11 +145,17 @@ void AbstractDiskWriter::seek(int64_t offset) { void AbstractDiskWriter::writeData(const char* data, uint32_t len, int64_t offset) { seek(offset); - writeDataInternal(data, len); + if(writeDataInternal(data, len) < 0) { + throw new DlAbortEx(EX_FILE_WRITE, filename.c_str(), strerror(errno)); + } } int AbstractDiskWriter::readData(char* data, uint32_t len, int64_t offset) { + int32_t ret; seek(offset); - return readDataInternal(data, len); + if((ret = readDataInternal(data, len)) < 0) { + throw new DlAbortEx(EX_FILE_READ, filename.c_str(), strerror(errno)); + } + return ret; } diff --git a/src/AbstractDiskWriter.h b/src/AbstractDiskWriter.h index 2019bc76..d340cdef 100644 --- a/src/AbstractDiskWriter.h +++ b/src/AbstractDiskWriter.h @@ -36,9 +36,6 @@ #define _D_ABSTRACT_DISK_WRITER_H_ #include "DiskWriter.h" -#ifdef ENABLE_MESSAGE_DIGEST -#include "messageDigest.h" -#endif // ENABLE_MESSAGE_DIGEST #include "FileAllocator.h" #include "Logger.h" @@ -48,13 +45,11 @@ protected: int32_t fd; FileAllocatorHandle fileAllocator; const Logger* logger; -#ifdef ENABLE_MESSAGE_DIGEST - MessageDigestContext ctx; -#endif // ENABLE_MESSAGE_DIGEST void createFile(const string& filename, int32_t addFlags = 0); - void writeDataInternal(const char* data, uint32_t len); +private: + int writeDataInternal(const char* data, uint32_t len); int readDataInternal(char* data, uint32_t len); void seek(int64_t offset); @@ -69,7 +64,8 @@ public: virtual void openExistingFile(const string& filename); - virtual string sha1Sum(int64_t offset, uint64_t length); + virtual string messageDigest(int64_t offset, uint64_t length, + const MessageDigestContext::DigestAlgo& algo); virtual void writeData(const char* data, uint32_t len, int64_t offset); diff --git a/src/AbstractSingleDiskAdaptor.cc b/src/AbstractSingleDiskAdaptor.cc index 41b6c4b4..beff651d 100644 --- a/src/AbstractSingleDiskAdaptor.cc +++ b/src/AbstractSingleDiskAdaptor.cc @@ -59,8 +59,8 @@ int AbstractSingleDiskAdaptor::readData(unsigned char* data, uint32_t len, int64 return diskWriter->readData(data, len, offset); } -string AbstractSingleDiskAdaptor::sha1Sum(int64_t offset, uint64_t length) { - return diskWriter->sha1Sum(offset, length); +string AbstractSingleDiskAdaptor::messageDigest(int64_t offset, uint64_t length, const MessageDigestContext::DigestAlgo& algo) { + return diskWriter->messageDigest(offset, length, algo); } bool AbstractSingleDiskAdaptor::fileExists() diff --git a/src/AbstractSingleDiskAdaptor.h b/src/AbstractSingleDiskAdaptor.h index db67d2ef..e3581d9d 100644 --- a/src/AbstractSingleDiskAdaptor.h +++ b/src/AbstractSingleDiskAdaptor.h @@ -60,7 +60,8 @@ public: virtual int readData(unsigned char* data, uint32_t len, int64_t offset); - virtual string sha1Sum(int64_t offset, uint64_t length); + virtual string messageDigest(int64_t offset, uint64_t length, + const MessageDigestContext::DigestAlgo& algo); virtual bool fileExists(); diff --git a/src/BitfieldMan.cc b/src/BitfieldMan.cc index 253dff26..3e695075 100644 --- a/src/BitfieldMan.cc +++ b/src/BitfieldMan.cc @@ -42,6 +42,8 @@ BitfieldMan::BitfieldMan(uint32_t blockLength, uint64_t totalLength) bitfield(0), useBitfield(0), filterBitfield(0), + bitfieldLength(0), + blocks(0), filterEnabled(false), randomizer(0), cachedNumMissingBlock(0), @@ -62,9 +64,13 @@ BitfieldMan::BitfieldMan(uint32_t blockLength, uint64_t totalLength) } BitfieldMan::BitfieldMan(const BitfieldMan& bitfieldMan) - :bitfield(0), + :blockLength(0), + totalLength(0), + bitfield(0), useBitfield(0), filterBitfield(0), + bitfieldLength(0), + blocks(0), filterEnabled(false), randomizer(0), cachedNumMissingBlock(0), @@ -616,3 +622,21 @@ void BitfieldMan::updateCache() cachedCompletedLength = getCompletedLengthNow(); cachedFilteredComletedLength = getFilteredCompletedLengthNow(); } + +bool BitfieldMan::isBitRangeSet(int32_t startIndex, int32_t endIndex) const +{ + for(int32_t i = startIndex; i <= endIndex; ++i) { + if(!isBitSet(i)) { + return false; + } + } + return true; +} + +void BitfieldMan::unsetBitRange(int32_t startIndex, int32_t endIndex) +{ + for(int32_t i = startIndex; i <= endIndex; ++i) { + unsetBit(i); + } + updateCache(); +} diff --git a/src/BitfieldMan.h b/src/BitfieldMan.h index 1703e00a..2a3153c6 100644 --- a/src/BitfieldMan.h +++ b/src/BitfieldMan.h @@ -248,6 +248,10 @@ public: } void updateCache(); + + bool isBitRangeSet(int32_t startIndex, int32_t endIndex) const; + + void unsetBitRange(int32_t startIndex, int32_t endIndex); }; #endif // _D_BITFIELD_MAN_H_ diff --git a/src/BtContext.h b/src/BtContext.h index 7ab6197e..c94088cc 100644 --- a/src/BtContext.h +++ b/src/BtContext.h @@ -62,6 +62,8 @@ public: virtual string getPieceHash(int index) const = 0; + virtual const Strings& getPieceHashes() const = 0; + virtual long long int getTotalLength() const = 0; virtual FILE_MODE getFileMode() const = 0; diff --git a/src/BtPieceMessage.cc b/src/BtPieceMessage.cc index cd861e91..9d52ef75 100644 --- a/src/BtPieceMessage.cc +++ b/src/BtPieceMessage.cc @@ -191,7 +191,7 @@ string BtPieceMessage::toString() const { bool BtPieceMessage::checkPieceHash(const PieceHandle& piece) { int64_t offset = ((int64_t)piece->getIndex())*btContext->getPieceLength(); - return pieceStorage->getDiskAdaptor()->sha1Sum(offset, piece->getLength()) == + return pieceStorage->getDiskAdaptor()->messageDigest(offset, piece->getLength(), DIGEST_ALGO_SHA1) == btContext->getPieceHash(piece->getIndex()); } diff --git a/src/ByteArrayDiskWriter.h b/src/ByteArrayDiskWriter.h index 4abdd256..c9819692 100644 --- a/src/ByteArrayDiskWriter.h +++ b/src/ByteArrayDiskWriter.h @@ -61,7 +61,8 @@ public: virtual void writeData(const char* data, uint32_t len, int64_t position = 0); virtual int readData(char* data, uint32_t len, int64_t position); // not implemented yet - virtual string sha1Sum(int64_t offset, uint64_t length) { return ""; } + virtual string messageDigest(int64_t offset, uint64_t length, + const MessageDigestContext::DigestAlgo& algo) { return ""; } const char* getByteArray() const { return buf; diff --git a/src/ChunkChecksumValidator.cc b/src/ChunkChecksumValidator.cc new file mode 100644 index 00000000..a571d9fd --- /dev/null +++ b/src/ChunkChecksumValidator.cc @@ -0,0 +1,117 @@ +/* */ +#include "ChunkChecksumValidator.h" +#include "Util.h" +#include "Exception.h" +#include "TimeA2.h" + +void ChunkChecksumValidator::validateSameLengthChecksum(BitfieldMan* bitfieldMan, + int32_t index, + const string& expectedChecksum, + uint32_t dataLength, + uint32_t checksumLength) +{ + int64_t offset = index*checksumLength; + string actualChecksum = diskWriter->messageDigest(offset, dataLength, algo); + if(actualChecksum != expectedChecksum) { + logger->error("Chunk checksum validation failed. checksumIndex=%d, offset=%lld, length=%u, expected=%s, actual=%s", + index, offset, dataLength, expectedChecksum.c_str(), actualChecksum.c_str()); + bitfieldMan->unsetBit(index); + } +} + +void ChunkChecksumValidator::validateDifferentLengthChecksum(BitfieldMan* bitfieldMan, + int32_t index, + const string& expectedChecksum, + uint32_t dataLength, + uint32_t checksumLength) +{ + int64_t offset = index*checksumLength; + int32_t startIndex; + int32_t endIndex; + Util::indexRange(startIndex, endIndex, offset, + checksumLength, bitfieldMan->getBlockLength()); + if(bitfieldMan->isBitRangeSet(startIndex, endIndex)) { + string actualChecksum = diskWriter->messageDigest(offset, dataLength, algo); + if(expectedChecksum != actualChecksum) { + // wrong checksum + logger->error("Chunk checksum validation failed. checksumIndex=%d, offset=%lld, length=%u, expected=%s, actual=%s", + index, offset, dataLength, + expectedChecksum.c_str(), actualChecksum.c_str()); + bitfieldMan->unsetBitRange(startIndex, endIndex); + } + } +} + +void ChunkChecksumValidator::validate(BitfieldMan* bitfieldMan, + const Strings& checksums, + uint32_t checksumLength) +{ + // We assume file is already opened using DiskWriter::open or openExistingFile. + if(checksumLength*checksums.size() < bitfieldMan->getTotalLength()) { + // insufficient checksums. + logger->error("Insufficient checksums. checksumLength=%u, numChecksum=%u", + checksumLength, checksums.size()); + return; + } + uint32_t x = bitfieldMan->getTotalLength()/checksumLength; + uint32_t r = bitfieldMan->getTotalLength()%checksumLength; + void (ChunkChecksumValidator::*f)(BitfieldMan*, int32_t, const string&, uint32_t, uint32_t); + + if(checksumLength == bitfieldMan->getBlockLength()) { + f = &ChunkChecksumValidator::validateSameLengthChecksum; + } else { + f = &ChunkChecksumValidator::validateDifferentLengthChecksum; + } + + fileAllocationMonitor->setMinValue(0); + fileAllocationMonitor->setMaxValue(bitfieldMan->getTotalLength()); + fileAllocationMonitor->setCurrentValue(0); + fileAllocationMonitor->showProgress(); + Time cp; + for(uint32_t i = 0; i < x; ++i) { + (this->*f)(bitfieldMan, i, checksums.at(i), checksumLength, checksumLength); + if(cp.elapsedInMillis(500)) { + fileAllocationMonitor->setCurrentValue(i*checksumLength); + fileAllocationMonitor->showProgress(); + cp.reset(); + } + } + if(r) { + (this->*f)(bitfieldMan, x, checksums.at(x), r, checksumLength); + } + fileAllocationMonitor->setCurrentValue(bitfieldMan->getTotalLength()); + fileAllocationMonitor->showProgress(); +} diff --git a/src/ChunkChecksumValidator.h b/src/ChunkChecksumValidator.h new file mode 100644 index 00000000..d6c47056 --- /dev/null +++ b/src/ChunkChecksumValidator.h @@ -0,0 +1,94 @@ +/* */ +#ifndef _D_CHUNK_CHECKSUM_VALIDATOR_H_ +#define _D_CHUNK_CHECKSUM_VALIDATOR_H_ + +#include "common.h" +#include "DiskWriter.h" +#include "BitfieldMan.h" +#include "messageDigest.h" +#include "LogFactory.h" +#include "FileAllocationMonitor.h" +#include "NullFileAllocationMonitor.h" + +class ChunkChecksumValidator { +private: + DiskWriterHandle diskWriter; + + MessageDigestContext::DigestAlgo algo; + + FileAllocationMonitorHandle fileAllocationMonitor; + + const Logger* logger; + + void validateSameLengthChecksum(BitfieldMan* bitfieldMan, + int32_t index, + const string& expectedChecksum, + uint32_t thisLength, + uint32_t checksumLength); + + void validateDifferentLengthChecksum(BitfieldMan* bitfieldMan, + int32_t index, + const string& expectedChecksum, + uint32_t thisLength, + uint32_t checksumLength); +public: + ChunkChecksumValidator(): + diskWriter(0), + algo(DIGEST_ALGO_SHA1), + fileAllocationMonitor(new NullFileAllocationMonitor()), + logger(LogFactory::getInstance()) + {} + + ~ChunkChecksumValidator() {} + + void validate(BitfieldMan* bitfieldMan, + const Strings& checksums, + uint32_t checksumLength); + + void setDiskWriter(const DiskWriterHandle& diskWriter) { + this->diskWriter = diskWriter; + } + + void setDigestAlgo(const MessageDigestContext::DigestAlgo& algo) { + this->algo = algo; + } + + void setFileAllocationMonitor(const FileAllocationMonitorHandle& monitor) { + this->fileAllocationMonitor = monitor; + } +}; + +#endif // _D_CHUNK_CHECKSUM_VALIDATOR_H_ diff --git a/src/ConsoleDownloadEngine.cc b/src/ConsoleDownloadEngine.cc index 0dc4e466..94669e64 100644 --- a/src/ConsoleDownloadEngine.cc +++ b/src/ConsoleDownloadEngine.cc @@ -76,6 +76,9 @@ void ConsoleDownloadEngine::calculateStatistics() { int elapsed = cp.difference(); if(elapsed >= 1) { int nspeed = (int)((dlSize-psize)/elapsed); + if(nspeed < 0) { + nspeed = 0; + } speed = (nspeed+speed)/2; cp.reset(); psize = dlSize; diff --git a/src/DefaultBtContext.h b/src/DefaultBtContext.h index 4133d7ab..acec81f9 100644 --- a/src/DefaultBtContext.h +++ b/src/DefaultBtContext.h @@ -78,6 +78,11 @@ private: virtual string getPieceHash(int index) const; + virtual const Strings& getPieceHashes() const + { + return pieceHashes; + } + virtual long long int getTotalLength() const; virtual FILE_MODE getFileMode() const; diff --git a/src/DefaultPieceStorage.cc b/src/DefaultPieceStorage.cc index 3d47759d..bb73552d 100644 --- a/src/DefaultPieceStorage.cc +++ b/src/DefaultPieceStorage.cc @@ -42,6 +42,8 @@ #include "DlAbortEx.h" #include "BitfieldManFactory.h" #include "FileAllocationMonitor.h" +#include "DiskAdaptorWriter.h" +#include "ChunkChecksumValidator.h" DefaultPieceStorage::DefaultPieceStorage(BtContextHandle btContext, const Option* option): btContext(btContext), @@ -428,3 +430,20 @@ void DefaultPieceStorage::removeAdvertisedPiece(int elapsed) { haves.erase(itr, haves.end()); } } + +void DefaultPieceStorage::markAllPiecesDone() +{ + bitfieldMan->setAllBit(); +} + +void DefaultPieceStorage::checkIntegrity() +{ + logger->notice("Validating file %s", + diskAdaptor->getFilePath().c_str()); + ChunkChecksumValidator v; + v.setDigestAlgo(DIGEST_ALGO_SHA1); + v.setDiskWriter(new DiskAdaptorWriter(diskAdaptor)); + v.setFileAllocationMonitor(FileAllocationMonitorFactory::getFactory()->createNewMonitor()); + v.validate(bitfieldMan, btContext->getPieceHashes(), + btContext->getPieceLength()); +} diff --git a/src/DefaultPieceStorage.h b/src/DefaultPieceStorage.h index da68fc80..edaa2bf5 100644 --- a/src/DefaultPieceStorage.h +++ b/src/DefaultPieceStorage.h @@ -152,10 +152,15 @@ public: virtual void removeAdvertisedPiece(int elapsed); + virtual void markAllPiecesDone(); + + virtual void checkIntegrity(); + /** * This method is made private for test purpose only. */ void addUsedPiece(const PieceHandle& piece); + }; #endif // _D_DEFAULT_PIECE_STORAGE_H_ diff --git a/src/DiskAdaptor.h b/src/DiskAdaptor.h index 8ba6e66b..b3852e1a 100644 --- a/src/DiskAdaptor.h +++ b/src/DiskAdaptor.h @@ -38,6 +38,7 @@ #include "common.h" #include "FileEntry.h" #include "Logger.h" +#include "messageDigest.h" class DiskAdaptor { protected: @@ -60,7 +61,8 @@ public: virtual int readData(unsigned char* data, uint32_t len, int64_t offset) = 0; - virtual string sha1Sum(int64_t offset, uint64_t length) = 0; + virtual string messageDigest(int64_t offset, uint64_t length, + const MessageDigestContext::DigestAlgo& algo) = 0; virtual void onDownloadComplete() = 0; diff --git a/src/DiskAdaptorWriter.h b/src/DiskAdaptorWriter.h new file mode 100644 index 00000000..6aefe3cc --- /dev/null +++ b/src/DiskAdaptorWriter.h @@ -0,0 +1,87 @@ +/* */ +#ifndef _D_DISK_ADAPTOR_WRITER_H_ +#define _D_DISK_ADAPTOR_WRITER_H_ + +#include "DiskWriter.h" +#include "DiskAdaptor.h" + +class DiskAdaptorWriter : public DiskWriter { +private: + DiskAdaptorHandle diskAdaptor; +public: + DiskAdaptorWriter(const DiskAdaptorHandle& diskAdaptor): + diskAdaptor(diskAdaptor) {} + + virtual ~DiskAdaptorWriter() {} + + virtual void initAndOpenFile(const string& filename, uint64_t totalLength = 0) + { + diskAdaptor->initAndOpenFile(); + } + + virtual void openFile(const string& filename, uint64_t totalLength = 0) + { + diskAdaptor->openFile(); + } + + virtual void closeFile() + { + diskAdaptor->closeFile(); + } + + virtual void openExistingFile(const string& filename) + { + diskAdaptor->openExistingFile(); + } + + virtual void writeData(const char* data, uint32_t len, int64_t position = 0) + { + diskAdaptor->writeData((const unsigned char*)data, len, position); + } + + virtual int readData(char* data, uint32_t len, int64_t position) + { + return diskAdaptor->readData((unsigned char*)data, len, position); + } + + virtual string messageDigest(int64_t offset, uint64_t length, + const MessageDigestContext::DigestAlgo& algo) + { + return diskAdaptor->messageDigest(offset, length, algo); + } +}; + +#endif // _D_DISK_ADAPTOR_WRITER_H_ diff --git a/src/DiskWriter.h b/src/DiskWriter.h index 73d5a9a3..be176d36 100644 --- a/src/DiskWriter.h +++ b/src/DiskWriter.h @@ -37,6 +37,9 @@ #include #include "common.h" +#ifdef ENABLE_MESSAGE_DIGEST +#include "messageDigest.h" +#endif // ENABLE_MESSAGE_DIGEST using namespace std; @@ -89,7 +92,8 @@ public: return readData((char*)data, len, position); } - virtual string sha1Sum(int64_t offset, uint64_t length) = 0; + virtual string messageDigest(int64_t offset, uint64_t length, + const MessageDigestContext::DigestAlgo& algo) = 0; }; typedef SharedHandle DiskWriterHandle; diff --git a/src/DownloadCommand.cc b/src/DownloadCommand.cc index 31c914ca..250bd3b9 100644 --- a/src/DownloadCommand.cc +++ b/src/DownloadCommand.cc @@ -46,25 +46,23 @@ DownloadCommand::DownloadCommand(int cuid, const RequestHandle req, DownloadEngine* e, const SocketHandle& s): - AbstractCommand(cuid, req, e, s), lastSize(0) { - PeerStatHandle peerStat = this->e->segmentMan->getPeerStat(cuid); + AbstractCommand(cuid, req, e, s), lastSize(0), peerStat(0) { + peerStat = this->e->segmentMan->getPeerStat(cuid); if(!peerStat.get()) { - peerStat = PeerStatHandle(new PeerStat(cuid)); + peerStat = new PeerStat(cuid); this->e->segmentMan->registerPeerStat(peerStat); } peerStat->downloadStart(); } DownloadCommand::~DownloadCommand() { - PeerStatHandle peerStat = e->segmentMan->getPeerStat(cuid); assert(peerStat.get()); peerStat->downloadStop(); } bool DownloadCommand::executeInternal(Segment& segment) { - int maxSpeedLimit = e->option->getAsInt(PREF_MAX_DOWNLOAD_LIMIT); - if(maxSpeedLimit > 0 && - maxSpeedLimit < e->segmentMan->calculateDownloadSpeed()) { + if(maxDownloadSpeedLimit > 0 && + maxDownloadSpeedLimit < e->segmentMan->calculateDownloadSpeed()) { usleep(1); e->commands.push_back(this); return false; @@ -74,13 +72,11 @@ bool DownloadCommand::executeInternal(Segment& segment) { te = getTransferEncoding(transferEncoding); assert(te != NULL); } - int bufSize = 4096; + int bufSize = 16*1024;//4096; char buf[bufSize]; socket->readData(buf, bufSize); - PeerStatHandle peerStat = e->segmentMan->getPeerStat(cuid); - assert(peerStat.get()); if(te != NULL) { - int infbufSize = 4096; + int infbufSize = 16*1024;//4096; char infbuf[infbufSize]; te->inflate(infbuf, infbufSize, buf, bufSize); e->segmentMan->diskWriter->writeData(infbuf, infbufSize, @@ -94,14 +90,13 @@ bool DownloadCommand::executeInternal(Segment& segment) { peerStat->updateDownloadLength(bufSize); } // calculate downloading speed - if(peerStat->getDownloadStartTime().elapsed(e->option->getAsInt(PREF_STARTUP_IDLE_TIME))) { - int lowestLimit = e->option->getAsInt(PREF_LOWEST_SPEED_LIMIT); - int nowSpeed = peerStat->calculateDownloadSpeed(); - if(lowestLimit > 0 && nowSpeed <= lowestLimit) { + if(peerStat->getDownloadStartTime().elapsed(startupIdleTime)) { + uint32_t nowSpeed = peerStat->calculateDownloadSpeed(); + if(lowestDownloadSpeedLimit > 0 && nowSpeed <= lowestDownloadSpeedLimit) { throw new DlAbortEx("CUID#%d - Too slow Downloading speed: %d <= %d(B/s)", cuid, nowSpeed, - lowestLimit); + lowestDownloadSpeedLimit); } } if(e->segmentMan->totalSize != 0 && bufSize == 0) { @@ -113,6 +108,9 @@ bool DownloadCommand::executeInternal(Segment& segment) { if(te != NULL) te->end(); logger->info(MSG_DOWNLOAD_COMPLETED, cuid); e->segmentMan->completeSegment(cuid, segment); + if(e->option->get(PREF_REALTIME_CHUNK_CHECKSUM) == V_TRUE) { + e->segmentMan->tryChunkChecksumValidation(segment); + } // this unit is going to download another segment. return prepareForNextSegment(segment); } else { diff --git a/src/DownloadCommand.h b/src/DownloadCommand.h index dc1f68c4..8ef5b5fe 100644 --- a/src/DownloadCommand.h +++ b/src/DownloadCommand.h @@ -38,12 +38,17 @@ #include "AbstractCommand.h" #include "TransferEncoding.h" #include "TimeA2.h" +#include "PeerStat.h" using namespace std; class DownloadCommand : public AbstractCommand { private: long long int lastSize; + uint32_t maxDownloadSpeedLimit; + uint32_t startupIdleTime; + uint32_t lowestDownloadSpeedLimit; + PeerStatHandle peerStat; protected: bool executeInternal(Segment& segment); @@ -57,6 +62,17 @@ public: string transferEncoding; + void setMaxDownloadSpeedLimit(uint32_t maxDownloadSpeedLimit) { + this->maxDownloadSpeedLimit = maxDownloadSpeedLimit; + } + + void setStartupIdleTime(uint32_t startupIdleTime) { + this->startupIdleTime = startupIdleTime; + } + + void setLowestDownloadSpeedLimit(uint32_t lowestDownloadSpeedLimit) { + this->lowestDownloadSpeedLimit = lowestDownloadSpeedLimit; + } }; #endif // _D_DOWNLOAD_COMMAND_H_ diff --git a/src/DownloadEngine.cc b/src/DownloadEngine.cc index 4c3f59d3..132d06dc 100644 --- a/src/DownloadEngine.cc +++ b/src/DownloadEngine.cc @@ -108,89 +108,6 @@ void DownloadEngine::shortSleep() const { select(0, &rfds, NULL, NULL, &tv); } -class SetDescriptor { -private: - int* max_ptr; - fd_set* rfds_ptr; - fd_set* wfds_ptr; -public: - SetDescriptor(int* max_ptr, fd_set* rfds_ptr, fd_set* wfds_ptr): - max_ptr(max_ptr), - rfds_ptr(rfds_ptr), - wfds_ptr(wfds_ptr) {} - - void operator()(const SocketEntry& entry) { - int fd = entry.socket->getSockfd(); - switch(entry.type) { - case SocketEntry::TYPE_RD: - FD_SET(fd, rfds_ptr); - break; - case SocketEntry::TYPE_WR: - FD_SET(fd, wfds_ptr); - break; - } - if(*max_ptr < fd) { - *max_ptr = fd; - } - } -#ifdef ENABLE_ASYNC_DNS - void operator()(const NameResolverEntry& entry) { - int tempFd = entry.nameResolver->getFds(rfds_ptr, wfds_ptr); - if(*max_ptr < tempFd) { - *max_ptr = tempFd; - } - } -#endif // ENABLE_ASYNC_DNS -}; - -class AccumulateActiveCommand { -private: - Commands* activeCommands_ptr; - fd_set* rfds_ptr; - fd_set* wfds_ptr; -public: - AccumulateActiveCommand(Commands* activeCommands_ptr, - fd_set* rfds_ptr, - fd_set* wfds_ptr): - activeCommands_ptr(activeCommands_ptr), - rfds_ptr(rfds_ptr), - wfds_ptr(wfds_ptr) {} - - void operator()(const SocketEntry& entry) { - if(FD_ISSET(entry.socket->getSockfd(), rfds_ptr) || - FD_ISSET(entry.socket->getSockfd(), wfds_ptr)) { - activeCommands_ptr->push_back(entry.command); - } - /* - switch(entry.type) { - case SocketEntry::TYPE_RD: - if(FD_ISSET(entry.socket->getSockfd(), rfds_ptr)) { - activeCommands_ptr->push_back(entry.command); - } - break; - case SocketEntry::TYPE_WR: - if(FD_ISSET(entry.socket->getSockfd(), wfds_ptr)) { - activeCommands_ptr->push_back(entry.command); - } - break; - } - */ - } -#ifdef ENABLE_ASYNC_DNS - void operator()(const NameResolverEntry& entry) { - entry.nameResolver->process(rfds_ptr, wfds_ptr); - switch(entry.nameResolver->getStatus()) { - case NameResolver::STATUS_SUCCESS: - case NameResolver::STATUS_ERROR: - activeCommands_ptr->push_back(entry.command); - break; - default: - break; - } - } -#endif // ENABLE_ASYNC_DNS -}; - void DownloadEngine::waitData(Commands& activeCommands) { fd_set rfds; fd_set wfds; @@ -204,16 +121,33 @@ void DownloadEngine::waitData(Commands& activeCommands) { tv.tv_usec = 0; retval = select(fdmax+1, &rfds, &wfds, NULL, &tv); if(retval > 0) { - for_each(socketEntries.begin(), socketEntries.end(), - AccumulateActiveCommand(&activeCommands, &rfds, &wfds)); + for(SocketEntries::iterator itr = socketEntries.begin(); + itr != socketEntries.end(); ++itr) { + SocketEntry& entry = *itr; + if(FD_ISSET(entry.socket->getSockfd(), &rfds) || + FD_ISSET(entry.socket->getSockfd(), &wfds)) { + if(find(activeCommands.begin(), activeCommands.end(), entry.command) == activeCommands.end()) { + activeCommands.push_back(entry.command); + } + } + } #ifdef ENABLE_ASYNC_DNS - for_each(nameResolverEntries.begin(), nameResolverEntries.end(), - AccumulateActiveCommand(&activeCommands, &rfds, &wfds)); + for(NameResolverEntries::iterator itr = nameResolverEntries.begin(); + itr != nameResolverEntries.end(); ++itr) { + NameResolverEntry& entry = *itr; + entry.nameResolver->process(&rfds, &wfds); + switch(entry.nameResolver->getStatus()) { + case NameResolver::STATUS_SUCCESS: + case NameResolver::STATUS_ERROR: + if(find(activeCommands.begin(), activeCommands.end(), entry.command) == activeCommands.end()) { + activeCommands.push_back(entry.command); + } + break; + default: + break; + } + } #endif // ENABLE_ASYNC_DNS - sort(activeCommands.begin(), activeCommands.end()); - activeCommands.erase(unique(activeCommands.begin(), - activeCommands.end()), - activeCommands.end()); } } @@ -222,11 +156,31 @@ void DownloadEngine::updateFdSet() { FD_ZERO(&rfdset); FD_ZERO(&wfdset); #ifdef ENABLE_ASYNC_DNS - for_each(nameResolverEntries.begin(), nameResolverEntries.end(), - SetDescriptor(&fdmax, &rfdset, &wfdset)); + for(NameResolverEntries::iterator itr = nameResolverEntries.begin(); + itr != nameResolverEntries.end(); ++itr) { + NameResolverEntry& entry = *itr; + int fd = entry.nameResolver->getFds(&rfdset, &wfdset); + if(fdmax < fd) { + fdmax = fd; + } + } #endif // ENABLE_ASYNC_DNS - for_each(socketEntries.begin(), socketEntries.end(), - SetDescriptor(&fdmax, &rfdset, &wfdset)); + for(SocketEntries::iterator itr = socketEntries.begin(); + itr != socketEntries.end(); ++itr) { + SocketEntry& entry = *itr; + int fd = entry.socket->getSockfd(); + switch(entry.type) { + case SocketEntry::TYPE_RD: + FD_SET(fd, &rfdset); + break; + case SocketEntry::TYPE_WR: + FD_SET(fd, &wfdset); + break; + } + if(fdmax < fd) { + fdmax = fd; + } + } } bool DownloadEngine::addSocket(const SocketEntry& entry) { diff --git a/src/FileAllocator.h b/src/FileAllocator.h index 267160d9..ee36c16a 100644 --- a/src/FileAllocator.h +++ b/src/FileAllocator.h @@ -37,12 +37,13 @@ #include "common.h" #include "FileAllocationMonitor.h" +#include "NullFileAllocationMonitor.h" class FileAllocator { private: FileAllocationMonitorHandle fileAllocationMonitor; public: - FileAllocator():fileAllocationMonitor(0) {} + FileAllocator():fileAllocationMonitor(new NullFileAllocationMonitor()) {} ~FileAllocator() {} diff --git a/src/FtpNegotiationCommand.cc b/src/FtpNegotiationCommand.cc index 12e0bbb3..f7dc8c50 100644 --- a/src/FtpNegotiationCommand.cc +++ b/src/FtpNegotiationCommand.cc @@ -62,6 +62,9 @@ bool FtpNegotiationCommand::executeInternal(Segment& segment) { } else if(sequence == SEQ_NEGOTIATION_COMPLETED) { FtpDownloadCommand* command = new FtpDownloadCommand(cuid, req, e, dataSocket, socket); + command->setMaxDownloadSpeedLimit(e->option->getAsInt(PREF_MAX_DOWNLOAD_LIMIT)); + command->setStartupIdleTime(e->option->getAsInt(PREF_STARTUP_IDLE_TIME)); + command->setLowestDownloadSpeedLimit(e->option->getAsInt(PREF_LOWEST_SPEED_LIMIT)); e->commands.push_back(command); return true; } else { @@ -183,17 +186,21 @@ bool FtpNegotiationCommand::recvSize() { throw new DlAbortEx(EX_TOO_LARGE_FILE, size); } if(!e->segmentMan->downloadStarted) { + if(req->getMethod() == Request::METHOD_HEAD) { + e->segmentMan->downloadStarted = true; + e->segmentMan->totalSize = size; + e->segmentMan->initBitfield(e->option->getAsInt(PREF_SEGMENT_SIZE), + e->segmentMan->totalSize); + e->segmentMan->markAllPiecesDone(); + e->segmentMan->isSplittable = false; // TODO because we don't want segment file to be saved. + return true; + } e->segmentMan->downloadStarted = true; e->segmentMan->totalSize = size; e->segmentMan->initBitfield(e->option->getAsInt(PREF_SEGMENT_SIZE), e->segmentMan->totalSize); e->segmentMan->filename = Util::urldecode(req->getFile()); - if(e->segmentMan->shouldCancelDownloadForSafety()) { - throw new FatalException(EX_FILE_ALREADY_EXISTS, - e->segmentMan->getFilePath().c_str(), - e->segmentMan->getSegmentFilePath().c_str()); - } bool segFileExists = e->segmentMan->segmentFileExists(); if(segFileExists) { e->segmentMan->load(); diff --git a/src/HttpResponseCommand.cc b/src/HttpResponseCommand.cc index b0970e8e..d0f75404 100644 --- a/src/HttpResponseCommand.cc +++ b/src/HttpResponseCommand.cc @@ -153,6 +153,17 @@ bool HttpResponseCommand::handleDefaultEncoding(const HttpHeader& headers) { } e->segmentMan->isSplittable = !(size == 0); e->segmentMan->filename = determinFilename(headers); + + // quick hack for method 'head' + if(req->getMethod() == Request::METHOD_HEAD) { + e->segmentMan->downloadStarted = true; + e->segmentMan->totalSize = size; + e->segmentMan->initBitfield(e->option->getAsInt(PREF_SEGMENT_SIZE), + e->segmentMan->totalSize); + e->segmentMan->markAllPiecesDone(); + e->segmentMan->isSplittable = false; // TODO because we don't want segment file to be saved. + return true; + } bool segFileExists = e->segmentMan->segmentFileExists(); e->segmentMan->downloadStarted = true; if(segFileExists) { @@ -161,11 +172,6 @@ bool HttpResponseCommand::handleDefaultEncoding(const HttpHeader& headers) { // send request again to the server with Range header return prepareForRetry(0); } else { - if(e->segmentMan->shouldCancelDownloadForSafety()) { - throw new FatalException(EX_FILE_ALREADY_EXISTS, - e->segmentMan->getFilePath().c_str(), - e->segmentMan->getSegmentFilePath().c_str()); - } e->segmentMan->totalSize = size; e->segmentMan->initBitfield(e->option->getAsInt(PREF_SEGMENT_SIZE), e->segmentMan->totalSize); @@ -176,6 +182,11 @@ bool HttpResponseCommand::handleDefaultEncoding(const HttpHeader& headers) { } bool HttpResponseCommand::handleOtherEncoding(const string& transferEncoding, const HttpHeader& headers) { + if(e->segmentMan->shouldCancelDownloadForSafety()) { + throw new FatalException(EX_FILE_ALREADY_EXISTS, + e->segmentMan->getFilePath().c_str(), + e->segmentMan->getSegmentFilePath().c_str()); + } // we ignore content-length when transfer-encoding is set e->segmentMan->downloadStarted = true; e->segmentMan->isSplittable = false; @@ -192,6 +203,9 @@ bool HttpResponseCommand::handleOtherEncoding(const string& transferEncoding, co void HttpResponseCommand::createHttpDownloadCommand(const string& transferEncoding) { HttpDownloadCommand* command = new HttpDownloadCommand(cuid, req, e, socket); + command->setMaxDownloadSpeedLimit(e->option->getAsInt(PREF_MAX_DOWNLOAD_LIMIT)); + command->setStartupIdleTime(e->option->getAsInt(PREF_STARTUP_IDLE_TIME)); + command->setLowestDownloadSpeedLimit(e->option->getAsInt(PREF_LOWEST_SPEED_LIMIT)); TransferEncoding* enc = NULL; if(transferEncoding.size() && (enc = command->getTransferEncoding(transferEncoding)) == NULL) { delete(command); diff --git a/src/Makefile.am b/src/Makefile.am index f38be47b..df0cdd22 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -63,7 +63,8 @@ SRCS = Socket.h\ SimpleRandomizer.cc SimpleRandomizer.h\ FileAllocator.cc FileAllocator.h\ FileAllocationMonitor.cc FileAllocationMonitor.h\ - ConsoleFileAllocationMonitor.cc ConsoleFileAllocationMonitor.h + ConsoleFileAllocationMonitor.cc ConsoleFileAllocationMonitor.h\ + ChunkChecksumValidator.cc ChunkChecksumValidator.h # debug_new.cpp if ENABLE_ASYNC_DNS diff --git a/src/Makefile.in b/src/Makefile.in index ddb5a07f..3a7f446d 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -212,6 +212,7 @@ am__libaria2c_a_SOURCES_DIST = Socket.h SocketCore.cc SocketCore.h \ SimpleRandomizer.h FileAllocator.cc FileAllocator.h \ FileAllocationMonitor.cc FileAllocationMonitor.h \ ConsoleFileAllocationMonitor.cc ConsoleFileAllocationMonitor.h \ + ChunkChecksumValidator.cc ChunkChecksumValidator.h \ NameResolver.cc NameResolver.h MetaEntry.h Data.cc Data.h \ Dictionary.cc Dictionary.h List.cc List.h MetaFileUtil.cc \ MetaFileUtil.h MetaEntryVisitor.h ShaVisitor.cc ShaVisitor.h \ @@ -375,7 +376,8 @@ am__objects_4 = SocketCore.$(OBJEXT) Command.$(OBJEXT) \ SpeedCalc.$(OBJEXT) BitfieldMan.$(OBJEXT) \ BitfieldManFactory.$(OBJEXT) SimpleRandomizer.$(OBJEXT) \ FileAllocator.$(OBJEXT) FileAllocationMonitor.$(OBJEXT) \ - ConsoleFileAllocationMonitor.$(OBJEXT) $(am__objects_1) \ + ConsoleFileAllocationMonitor.$(OBJEXT) \ + ChunkChecksumValidator.$(OBJEXT) $(am__objects_1) \ $(am__objects_2) $(am__objects_3) am_libaria2c_a_OBJECTS = $(am__objects_4) libaria2c_a_OBJECTS = $(am_libaria2c_a_OBJECTS) @@ -582,6 +584,7 @@ SRCS = Socket.h SocketCore.cc SocketCore.h Command.cc Command.h \ SimpleRandomizer.h FileAllocator.cc FileAllocator.h \ FileAllocationMonitor.cc FileAllocationMonitor.h \ ConsoleFileAllocationMonitor.cc ConsoleFileAllocationMonitor.h \ + ChunkChecksumValidator.cc ChunkChecksumValidator.h \ $(am__append_1) $(am__append_2) $(am__append_3) noinst_LIBRARIES = libaria2c.a libaria2c_a_SOURCES = $(SRCS) @@ -697,6 +700,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/BtSuggestPieceMessage.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/BtUnchokeMessage.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ByteArrayDiskWriter.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ChunkChecksumValidator.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ChunkedEncoding.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Command.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/CompactPeerListProcessor.Po@am__quote@ diff --git a/src/MetalinkEntry.cc b/src/MetalinkEntry.cc index ffaa762e..7e48aeb7 100644 --- a/src/MetalinkEntry.cc +++ b/src/MetalinkEntry.cc @@ -36,7 +36,7 @@ #include "Util.h" #include -MetalinkEntry::MetalinkEntry() {} +MetalinkEntry::MetalinkEntry():chunkChecksum(0) {} MetalinkEntry::~MetalinkEntry() {} diff --git a/src/MetalinkEntry.h b/src/MetalinkEntry.h index 8bc3b473..4852a5b7 100644 --- a/src/MetalinkEntry.h +++ b/src/MetalinkEntry.h @@ -38,6 +38,7 @@ #include "common.h" #include "MetalinkResource.h" #include "Checksum.h" +#include "MetalinkChunkChecksum.h" #include class MetalinkEntry { @@ -46,10 +47,11 @@ public: string version; string language; string os; - long long int size; + uint64_t size; Checksum checksum; public: MetalinkResources resources; + MetalinkChunkChecksumHandle chunkChecksum; public: MetalinkEntry(); ~MetalinkEntry(); diff --git a/src/MetalinkRequestInfo.cc b/src/MetalinkRequestInfo.cc index a03d9ff8..f1e5578d 100644 --- a/src/MetalinkRequestInfo.cc +++ b/src/MetalinkRequestInfo.cc @@ -121,8 +121,11 @@ RequestInfos MetalinkRequestInfo::execute() { // BitTorrent downloading urls.push_back((*itr)->url); } - RequestInfoHandle reqInfo(new UrlRequestInfo(urls, maxConnection, op)); + UrlRequestInfoHandle reqInfo = new UrlRequestInfo(urls, maxConnection, op); reqInfo->setChecksum(checksum); + reqInfo->setDigestAlgo(entry->chunkChecksum->digestAlgo); + reqInfo->setChunkChecksumLength(entry->chunkChecksum->pieceLength); + reqInfo->setChunkChecksums(entry->chunkChecksum->pieceHashes); nextReqInfos.push_front(reqInfo); } } catch(RecoverableException* ex) { diff --git a/src/MultiDiskAdaptor.cc b/src/MultiDiskAdaptor.cc index c03416f7..b6f0d5d7 100644 --- a/src/MultiDiskAdaptor.cc +++ b/src/MultiDiskAdaptor.cc @@ -169,7 +169,8 @@ int MultiDiskAdaptor::readData(unsigned char* data, uint32_t len, int64_t offset return totalReadLength; } -void MultiDiskAdaptor::hashUpdate(const DiskWriterEntryHandle entry, +void MultiDiskAdaptor::hashUpdate(MessageDigestContext& ctx, + const DiskWriterEntryHandle& entry, int64_t offset, uint64_t length) { uint32_t BUFSIZE = 16*1024; @@ -190,17 +191,19 @@ void MultiDiskAdaptor::hashUpdate(const DiskWriterEntryHandle entry, } } -string MultiDiskAdaptor::sha1Sum(int64_t offset, uint64_t length) { +string MultiDiskAdaptor::messageDigest(int64_t offset, uint64_t length, + const MessageDigestContext::DigestAlgo& algo) { int64_t fileOffset = offset; bool reading = false; uint32_t rem = length; - ctx.digestReset(); + MessageDigestContext ctx(algo); + ctx.digestInit(); for(DiskWriterEntries::iterator itr = diskWriterEntries.begin(); itr != diskWriterEntries.end() && rem != 0; itr++) { if(isInRange(*itr, offset) || reading) { uint32_t readLength = calculateLength((*itr), fileOffset, rem); - hashUpdate(*itr, fileOffset, readLength); + hashUpdate(ctx, *itr, fileOffset, readLength); rem -= readLength; reading = true; fileOffset = 0; diff --git a/src/MultiDiskAdaptor.h b/src/MultiDiskAdaptor.h index 1e58d4b8..8ac95ecb 100644 --- a/src/MultiDiskAdaptor.h +++ b/src/MultiDiskAdaptor.h @@ -38,7 +38,6 @@ #include "DiskAdaptor.h" #include "Option.h" #include "DiskWriter.h" -#include "messageDigest.h" #include "File.h" class DiskWriterEntry { @@ -103,7 +102,6 @@ class MultiDiskAdaptor : public DiskAdaptor { private: string topDir; uint32_t pieceLength; - MessageDigestContext ctx; DiskWriterEntries diskWriterEntries; const Option* option; @@ -117,17 +115,15 @@ private: int64_t fileOffset, uint32_t rem) const; - void hashUpdate(const DiskWriterEntryHandle entry, + void hashUpdate(MessageDigestContext& ctx, + const DiskWriterEntryHandle& entry, int64_t offset, uint64_t length); string getTopDirPath() const; public: MultiDiskAdaptor():pieceLength(0), - ctx(DIGEST_ALGO_SHA1), option(0) - { - ctx.digestInit(); - } + {} virtual ~MultiDiskAdaptor() {} @@ -146,7 +142,8 @@ public: virtual int readData(unsigned char* data, uint32_t len, int64_t offset); - virtual string sha1Sum(int64_t offset, uint64_t length); + virtual string messageDigest(int64_t offset, uint64_t length, + const MessageDigestContext::DigestAlgo& algo); virtual bool fileExists(); diff --git a/src/MultiDiskWriter.cc b/src/MultiDiskWriter.cc index ff3b03ae..89ae72bf 100644 --- a/src/MultiDiskWriter.cc +++ b/src/MultiDiskWriter.cc @@ -47,7 +47,6 @@ MultiDiskWriter::MultiDiskWriter(int pieceLength): MultiDiskWriter::~MultiDiskWriter() { clearEntries(); - ctx.digestFree(); } void MultiDiskWriter::clearEntries() { diff --git a/src/NullFileAllocationMonitor.h b/src/NullFileAllocationMonitor.h new file mode 100644 index 00000000..fa8678d0 --- /dev/null +++ b/src/NullFileAllocationMonitor.h @@ -0,0 +1,58 @@ +/* */ +#ifndef _D_NULL_FILE_ALLOCATION_MONITOR_H_ +#define _D_NULL_FILE_ALLOCATION_MONITOR_H_ + +#include "FileAllocationMonitor.h" + +class NullFileAllocationMonitor : public FileAllocationMonitor { +public: + NullFileAllocationMonitor() {} + + virtual ~NullFileAllocationMonitor() {} + + virtual void setFilename(const string& filename) {} + + virtual void setMinValue(const uint64_t& min) {} + + virtual void setMaxValue(const uint64_t& max) {} + + virtual void setCurrentValue(const uint64_t& current) {} + + virtual void showProgress() {} + +}; + +#endif // _D_NULL_FILE_ALLOCATION_MONITOR_H_ diff --git a/src/PieceStorage.h b/src/PieceStorage.h index c53f3aa8..f3c2d07e 100644 --- a/src/PieceStorage.h +++ b/src/PieceStorage.h @@ -146,6 +146,15 @@ public: */ virtual void removeAdvertisedPiece(int elapsed) = 0; + /** + * Sets all bits in bitfield to 1. + */ + virtual void markAllPiecesDone() = 0; + + /** + * Validates file integrity by comparing checksums. + */ + virtual void checkIntegrity() = 0; }; typedef SharedHandle PieceStorageHandle; diff --git a/src/Request.cc b/src/Request.cc index 1f7e4160..62248826 100644 --- a/src/Request.cc +++ b/src/Request.cc @@ -36,7 +36,11 @@ #include "Util.h" #include "FeatureConfig.h" -Request::Request():port(0), tryCount(0), keepAlive(true), isTorrent(false) { +const string Request::METHOD_GET = "get"; + +const string Request::METHOD_HEAD = "head"; + +Request::Request():port(0), tryCount(0), keepAlive(true), method(METHOD_GET), isTorrent(false) { cookieBox = new CookieBox(); } diff --git a/src/Request.h b/src/Request.h index 7b998d4a..c62c5c7d 100644 --- a/src/Request.h +++ b/src/Request.h @@ -74,6 +74,7 @@ private: int tryCount; int trackerEvent; bool keepAlive; + string method; bool parseUrl(const string& url); public: Segment segment; @@ -111,6 +112,17 @@ public: void setTrackerEvent(int event) { trackerEvent = event; } int getTrackerEvent() const { return trackerEvent; } + void setMethod(const string& method) { + this->method = method; + } + + const string& getMethod() const { + return method; + } + + static const string METHOD_GET; + static const string METHOD_HEAD; + enum TRACKER_EVENT { AUTO, STARTED, diff --git a/src/SegmentMan.cc b/src/SegmentMan.cc index 5d5c2356..a5967312 100644 --- a/src/SegmentMan.cc +++ b/src/SegmentMan.cc @@ -40,6 +40,7 @@ #include "prefs.h" #include "LogFactory.h" #include "BitfieldManFactory.h" +#include "ChunkChecksumValidator.h" #include #include #include @@ -51,13 +52,15 @@ SegmentMan::SegmentMan():bitfield(0), downloadStarted(false), dir("."), errors(0), - diskWriter(0) { + diskWriter(0), + chunkHashLength(0), + digestAlgo(DIGEST_ALGO_SHA1) +{ logger = LogFactory::getInstance(); } SegmentMan::~SegmentMan() { delete bitfield; - delete diskWriter; } bool SegmentMan::segmentFileExists() const { @@ -233,47 +236,20 @@ void SegmentMan::initBitfield(int segmentLength, long long int totalLength) { this->bitfield = BitfieldManFactory::getNewFactory()->createBitfieldMan(segmentLength, totalLength); } -class FindSegmentEntryByIndex { -private: - int index; -public: - FindSegmentEntryByIndex(int index):index(index) {} - - bool operator()(const SegmentEntryHandle& entry) { - return entry->segment.index == index; - } -}; - -class FindSegmentEntryByCuid { -private: - int cuid; -public: - FindSegmentEntryByCuid(int cuid):cuid(cuid) {} - - bool operator()(const SegmentEntryHandle& entry) { - return entry->cuid == cuid; - } -}; - Segment SegmentMan::checkoutSegment(int cuid, int index) { logger->debug("Attach segment#%d to CUID#%d.", index, cuid); bitfield->setUseBit(index); - - SegmentEntries::iterator itr = find_if(usedSegmentEntries.begin(), - usedSegmentEntries.end(), - FindSegmentEntryByIndex(index)); + SegmentEntryHandle segmentEntry = getSegmentEntryByIndex(index); Segment segment; - if(itr == usedSegmentEntries.end()) { + if(segmentEntry.isNull()) { segment = Segment(index, bitfield->getBlockLength(index), bitfield->getBlockLength()); - SegmentEntryHandle entry = - SegmentEntryHandle(new SegmentEntry(cuid, segment)); + SegmentEntryHandle entry = new SegmentEntry(cuid, segment); usedSegmentEntries.push_back(entry); } else { - (*itr)->cuid = cuid; - segment = (*itr)->segment; + segmentEntry->cuid = cuid; + segment = segmentEntry->segment; } - logger->debug("index=%d, length=%d, segmentLength=%d, writtenLength=%d", segment.index, segment.length, segment.segmentLength, segment.writtenLength); @@ -286,13 +262,11 @@ bool SegmentMan::onNullBitfield(Segment& segment, int cuid) { usedSegmentEntries.push_back(SegmentEntryHandle(new SegmentEntry(cuid, segment))); return true; } else { - SegmentEntries::iterator itr = find_if(usedSegmentEntries.begin(), - usedSegmentEntries.end(), - FindSegmentEntryByCuid(cuid)); - if(itr == usedSegmentEntries.end()) { + SegmentEntryHandle segmentEntry = getSegmentEntryByCuid(cuid); + if(segmentEntry.isNull()) { return false; } else { - segment = (*itr)->segment; + segment = segmentEntry->segment; return true; } } @@ -302,7 +276,7 @@ SegmentEntryHandle SegmentMan::findSlowerSegmentEntry(const PeerStatHandle& peer int speed = (int)(peerStat->getAvgDownloadSpeed()*0.8); SegmentEntryHandle slowSegmentEntry(0); for(SegmentEntries::const_iterator itr = usedSegmentEntries.begin(); - itr != usedSegmentEntries.end(); itr++) { + itr != usedSegmentEntries.end(); ++itr) { const SegmentEntryHandle& segmentEntry = *itr; if(segmentEntry->cuid == 0) { continue; @@ -326,11 +300,10 @@ bool SegmentMan::getSegment(Segment& segment, int cuid) { if(!bitfield) { return onNullBitfield(segment, cuid); } - SegmentEntries::iterator itr = find_if(usedSegmentEntries.begin(), - usedSegmentEntries.end(), - FindSegmentEntryByCuid(cuid)); - if(itr != usedSegmentEntries.end()) { - segment = (*itr)->segment; + + SegmentEntryHandle segmentEntry = getSegmentEntryByCuid(cuid); + if(!segmentEntry.isNull()) { + segment = segmentEntry->segment; return true; } int index = bitfield->getSparseMissingUnusedIndex(); @@ -378,13 +351,11 @@ bool SegmentMan::updateSegment(int cuid, const Segment& segment) { if(segment.isNull()) { return false; } - SegmentEntries::iterator itr = find_if(usedSegmentEntries.begin(), - usedSegmentEntries.end(), - FindSegmentEntryByCuid(cuid)); - if(itr == usedSegmentEntries.end()) { + SegmentEntryHandle segmentEntry = getSegmentEntryByCuid(cuid); + if(segmentEntry.isNull()) { return false; } else { - (*itr)->segment = segment; + segmentEntry->segment = segment; return true; } } @@ -425,9 +396,7 @@ bool SegmentMan::completeSegment(int cuid, const Segment& segment) { initBitfield(option->getAsInt(PREF_SEGMENT_SIZE), segment.writtenLength); bitfield->setAllBit(); } - SegmentEntries::iterator itr = find_if(usedSegmentEntries.begin(), - usedSegmentEntries.end(), - FindSegmentEntryByCuid(cuid)); + SegmentEntries::iterator itr = getSegmentEntryIteratorByCuid(cuid); if(itr == usedSegmentEntries.end()) { return false; } else { @@ -463,7 +432,7 @@ void SegmentMan::registerPeerStat(const PeerStatHandle& peerStat) { } } -int SegmentMan::calculateDownloadSpeed() const { +uint32_t SegmentMan::calculateDownloadSpeed() const { int speed = 0; for(PeerStats::const_iterator itr = peerStats.begin(); itr != peerStats.end(); itr++) { @@ -483,3 +452,78 @@ bool SegmentMan::shouldCancelDownloadForSafety() { return fileExists() && !segmentFileExists() && option->get(PREF_FORCE_TRUNCATE) != V_TRUE; } + +void SegmentMan::markAllPiecesDone() +{ + if(bitfield) { + bitfield->setAllBit(); + } +} + +void SegmentMan::checkIntegrity() +{ + logger->notice("Validating file %s", + getFilePath().c_str()); + ChunkChecksumValidator v; + v.setDigestAlgo(digestAlgo); + v.setDiskWriter(diskWriter); + v.setFileAllocationMonitor(FileAllocationMonitorFactory::getFactory()->createNewMonitor()); + v.validate(bitfield, pieceHashes, chunkHashLength); +} + +bool SegmentMan::isChunkChecksumValidationReady() const { + return bitfield && + pieceHashes.size()*chunkHashLength == bitfield->getBlockLength()*(bitfield->getMaxIndex()+1); +} + +void SegmentMan::tryChunkChecksumValidation(const Segment& segment) +{ + if(!isChunkChecksumValidationReady()) { + return; + } + int32_t hashStartIndex; + int32_t hashEndIndex; + Util::indexRange(hashStartIndex, hashEndIndex, + segment.getPosition(), + segment.writtenLength, + chunkHashLength); + if(hashStartIndex*chunkHashLength < segment.getPosition() && !bitfield->isBitSet(segment.index-1)) { + ++hashStartIndex; + } + if(hashEndIndex*(chunkHashLength+1) > segment.getPosition()+segment.segmentLength && !bitfield->isBitSet(segment.index+1)) { + --hashEndIndex; + } + logger->debug("hashStartIndex=%d, hashEndIndex=%d", + hashStartIndex, hashEndIndex); + if(hashStartIndex > hashEndIndex) { + logger->debug("No chunk to verify."); + return; + } + int64_t hashOffset = hashStartIndex*chunkHashLength; + int32_t startIndex; + int32_t endIndex; + Util::indexRange(startIndex, endIndex, + hashOffset, + (hashEndIndex-hashStartIndex+1)*chunkHashLength, + segment.segmentLength); + logger->debug("startIndex=%d, endIndex=%d", startIndex, endIndex); + if(bitfield->isBitRangeSet(startIndex, endIndex)) { + for(int32_t index = hashStartIndex; index <= hashEndIndex; ++index) { + int64_t offset = index*chunkHashLength; + uint32_t dataLength = + offset+chunkHashLength <= totalSize ? chunkHashLength : totalSize-offset; + string actualChecksum = diskWriter->messageDigest(offset, dataLength, digestAlgo); + string expectedChecksum = pieceHashes.at(index); + if(expectedChecksum == actualChecksum) { + logger->info("Chunk checksum validation succeeded."); + } else { + logger->error("Chunk checksum validation failed. checksumIndex=%d, offset=%lld, length=%u, expected=%s, actual=%s", + index, offset, dataLength, + expectedChecksum.c_str(), actualChecksum.c_str()); + logger->info("Unset bit from %d to %d(inclusive)", startIndex, endIndex); + bitfield->unsetBitRange(startIndex, endIndex); + break; + } + } + } +} diff --git a/src/SegmentMan.h b/src/SegmentMan.h index ba4cca1e..37f53f0e 100644 --- a/src/SegmentMan.h +++ b/src/SegmentMan.h @@ -77,6 +77,39 @@ private: bool onNullBitfield(Segment& segment, int cuid); Segment checkoutSegment(int cuid, int index); SegmentEntryHandle findSlowerSegmentEntry(const PeerStatHandle& peerStat) const; + SegmentEntryHandle getSegmentEntryByIndex(int index) { + for(SegmentEntries::const_iterator itr = usedSegmentEntries.begin(); + itr != usedSegmentEntries.end(); ++itr) { + const SegmentEntryHandle& segmentEntry = *itr; + if(segmentEntry->segment.index == index) { + return segmentEntry; + } + } + return 0; + } + + SegmentEntryHandle getSegmentEntryByCuid(int cuid) { + for(SegmentEntries::const_iterator itr = usedSegmentEntries.begin(); + itr != usedSegmentEntries.end(); ++itr) { + const SegmentEntryHandle& segmentEntry = *itr; + if(segmentEntry->cuid == cuid) { + return segmentEntry; + } + } + return 0; + } + + SegmentEntries::iterator getSegmentEntryIteratorByCuid(int cuid) { + for(SegmentEntries::iterator itr = usedSegmentEntries.begin(); + itr != usedSegmentEntries.end(); ++itr) { + const SegmentEntryHandle& segmentEntry = *itr; + if(segmentEntry->cuid == cuid) { + return itr; + } + } + return usedSegmentEntries.end(); + } + public: /** * The total number of bytes to download. @@ -121,9 +154,13 @@ public: int errors; const Option* option; - DiskWriter* diskWriter; + DiskWriterHandle diskWriter; Requests reserved; + Strings pieceHashes; + uint32_t chunkHashLength; + MessageDigestContext::DigestAlgo digestAlgo; + SegmentMan(); ~SegmentMan(); @@ -221,45 +258,37 @@ public: */ void registerPeerStat(const PeerStatHandle& peerStat); - class FindPeerStat { - private: - int cuid; - public: - FindPeerStat(int cuid):cuid(cuid) {} - - bool operator()(const PeerStatHandle& peerStat) { - if(peerStat->getCuid() == cuid) { - return true; - } else { - return false; - } - } - }; - /** * Returns peerStat whose cuid is given cuid. If it is not found, returns - * PeerStatHandle(0). + * 0. */ PeerStatHandle getPeerStat(int cuid) const { - PeerStats::const_iterator itr = find_if(peerStats.begin(), peerStats.end(), - FindPeerStat(cuid)); - if(itr == peerStats.end()) { - // TODO - return PeerStatHandle(0); - } else { - return *itr; + for(PeerStats::const_iterator itr = peerStats.begin(); itr != peerStats.end(); ++itr) { + const PeerStatHandle& peerStat = *itr; + if(peerStat->getCuid() == cuid) { + return peerStat; + } } + return 0; } /** * Returns current download speed in bytes per sec. */ - int calculateDownloadSpeed() const; + uint32_t calculateDownloadSpeed() const; bool fileExists(); bool shouldCancelDownloadForSafety(); + void markAllPiecesDone(); + + void checkIntegrity(); + + void tryChunkChecksumValidation(const Segment& segment); + + bool isChunkChecksumValidationReady() const; + }; #endif // _D_SEGMENT_MAN_H_ diff --git a/src/ShaVisitor.cc b/src/ShaVisitor.cc index 6cb711c8..b760d72c 100644 --- a/src/ShaVisitor.cc +++ b/src/ShaVisitor.cc @@ -40,9 +40,8 @@ ShaVisitor::ShaVisitor(): ctx.digestInit(); } -ShaVisitor::~ShaVisitor() { - ctx.digestFree(); -} +ShaVisitor::~ShaVisitor() {} + void ShaVisitor::visit(const Data* d) { if(d->isNumber()) { diff --git a/src/TorrentRequestInfo.cc b/src/TorrentRequestInfo.cc index 7fcc7bf6..dbf29c3e 100644 --- a/src/TorrentRequestInfo.cc +++ b/src/TorrentRequestInfo.cc @@ -67,12 +67,22 @@ RequestInfos TorrentRequestInfo::execute() { // load .aria2 file if it exists. BT_PROGRESS_INFO_FILE(btContext)->load(); PIECE_STORAGE(btContext)->getDiskAdaptor()->openExistingFile(); + if(op->get(PREF_CHECK_INTEGRITY) == V_TRUE) { + PIECE_STORAGE(btContext)->checkIntegrity(); + } } else { - if(PIECE_STORAGE(btContext)->getDiskAdaptor()->fileExists() && - op->get(PREF_FORCE_TRUNCATE) != V_TRUE) { - throw new FatalException(EX_FILE_ALREADY_EXISTS, - PIECE_STORAGE(btContext)->getDiskAdaptor()->getFilePath().c_str(), - BT_PROGRESS_INFO_FILE(btContext)->getFilename().c_str()); + if(PIECE_STORAGE(btContext)->getDiskAdaptor()->fileExists()) { + if(op->get(PREF_FORCE_TRUNCATE) != V_TRUE) { + throw new FatalException(EX_FILE_ALREADY_EXISTS, + PIECE_STORAGE(btContext)->getDiskAdaptor()->getFilePath().c_str(), + BT_PROGRESS_INFO_FILE(btContext)->getFilename().c_str()); + } else { + PIECE_STORAGE(btContext)->getDiskAdaptor()->openExistingFile(); + if(op->get(PREF_CHECK_INTEGRITY) == V_TRUE) { + PIECE_STORAGE(btContext)->markAllPiecesDone(); + PIECE_STORAGE(btContext)->checkIntegrity(); + } + } } else { PIECE_STORAGE(btContext)->getDiskAdaptor()->initAndOpenFile(); } diff --git a/src/UrlRequestInfo.cc b/src/UrlRequestInfo.cc index 3d849037..620ceaca 100644 --- a/src/UrlRequestInfo.cc +++ b/src/UrlRequestInfo.cc @@ -38,6 +38,13 @@ #include "prefs.h" #include "DownloadEngineFactory.h" #include "RecoverableException.h" +#include "FatalException.h" +#include "message.h" + +std::ostream& operator<<(std::ostream& o, const HeadResult& hr) { + o << "filename = " << hr.filename << ", " << "totalLength = " << hr.totalLength; + return o; +} extern volatile sig_atomic_t haltRequested; @@ -81,18 +88,22 @@ private: Requests* requestsPtr; string referer; int split; + string method; public: CreateRequest(Requests* requestsPtr, const string& referer, - int split) + int split, + const string& method = Request::METHOD_GET) :requestsPtr(requestsPtr), referer(referer), - split(split) {} + split(split), + method(method) {} void operator()(const string& url) { for(int s = 1; s <= split; s++) { RequestHandle req; req->setReferer(referer); + req->setMethod(method); if(req->setUrl(url)) { requestsPtr->push_back(req); } else { @@ -109,6 +120,32 @@ void UrlRequestInfo::printUrls(const Strings& urls) const { } } +HeadResult UrlRequestInfo::getHeadResult() { + Requests requests; + for_each(urls.begin(), urls.end(), + CreateRequest(&requests, + op->get(PREF_REFERER), + 1, + Request::METHOD_HEAD)); + Requests reserved(requests.begin()+1, requests.end()); + requests.erase(requests.begin()+1, requests.end()); + + SharedHandle e(DownloadEngineFactory::newConsoleEngine(op, requests, reserved)); + + HeadResult hr; + try { + e->run(); + hr.filename = e->segmentMan->filename; + hr.totalLength = e->segmentMan->totalSize; + } catch(RecoverableException *ex) { + logger->error("Exception caught", ex); + delete ex; + fail = true; + } + return hr; +} + + RequestInfos UrlRequestInfo::execute() { Requests requests; Requests reserved; @@ -117,11 +154,49 @@ RequestInfos UrlRequestInfo::execute() { CreateRequest(&requests, op->get(PREF_REFERER), op->getAsInt(PREF_SPLIT))); + + HeadResult hr = getHeadResult(); + + if(fail) { + return RequestInfos(); + } + + logger->info("Head result: filename=%s, total length=%s", + hr.filename.c_str(), Util::ullitos(hr.totalLength, true).c_str()); adjustRequestSize(requests, reserved, maxConnections); SharedHandle e(DownloadEngineFactory::newConsoleEngine(op, requests, reserved)); - + e->segmentMan->filename = hr.filename; + e->segmentMan->totalSize = hr.totalLength; + e->segmentMan->downloadStarted = true; + e->segmentMan->digestAlgo = digestAlgo; + e->segmentMan->chunkHashLength = chunkChecksumLength; + e->segmentMan->pieceHashes = chunkChecksums; + + if(e->segmentMan->segmentFileExists()) { + e->segmentMan->load(); + e->segmentMan->diskWriter->openExistingFile(e->segmentMan->getFilePath()); + if(e->option->get(PREF_CHECK_INTEGRITY) == V_TRUE) { + e->segmentMan->checkIntegrity(); + } + } else { + if(e->segmentMan->shouldCancelDownloadForSafety()) { + throw new FatalException(EX_FILE_ALREADY_EXISTS, + e->segmentMan->getFilePath().c_str(), + e->segmentMan->getSegmentFilePath().c_str()); + } + e->segmentMan->initBitfield(e->option->getAsInt(PREF_SEGMENT_SIZE), + e->segmentMan->totalSize); + if(e->segmentMan->fileExists() && e->option->get(PREF_CHECK_INTEGRITY) == V_TRUE) { + e->segmentMan->diskWriter->openExistingFile(e->segmentMan->getFilePath()); + e->segmentMan->markAllPiecesDone(); + e->segmentMan->checkIntegrity(); + } else { + e->segmentMan->diskWriter->initAndOpenFile(e->segmentMan->getFilePath(), + e->segmentMan->totalSize); + } + } Util::setGlobalSignalHandler(SIGINT, handler, 0); Util::setGlobalSignalHandler(SIGTERM, handler, 0); diff --git a/src/UrlRequestInfo.h b/src/UrlRequestInfo.h index 3c52ef4a..e4ce39ea 100644 --- a/src/UrlRequestInfo.h +++ b/src/UrlRequestInfo.h @@ -37,24 +37,53 @@ #include "RequestInfo.h" +class HeadResult { +public: + HeadResult():totalLength(0) {} + string filename; + uint64_t totalLength; +}; + +std::ostream& operator<<(std::ostream& o, const HeadResult& hr); + class UrlRequestInfo : public RequestInfo { private: Strings urls; int maxConnections; + MessageDigestContext::DigestAlgo digestAlgo; + uint32_t chunkChecksumLength; + Strings chunkChecksums; + RequestInfo* createNextRequestInfo() const; void adjustRequestSize(Requests& requests, Requests& reserved, int maxConnections) const; void printUrls(const Strings& urls) const; + HeadResult getHeadResult(); public: UrlRequestInfo(const Strings& urls, int maxConnections, Option* op): RequestInfo(op), urls(urls), - maxConnections(maxConnections) {} + maxConnections(maxConnections), + digestAlgo(DIGEST_ALGO_SHA1), + chunkChecksumLength(0) {} virtual ~UrlRequestInfo() {} virtual RequestInfos execute(); + + void setDigestAlgo(const MessageDigestContext::DigestAlgo& algo) { + this->digestAlgo = algo; + } + + void setChunkChecksumLength(uint32_t chunkChecksumLength) { + this->chunkChecksumLength = chunkChecksumLength; + } + + void setChunkChecksums(const Strings& chunkChecksums) { + this->chunkChecksums = chunkChecksums; + } }; +typedef SharedHandle UrlRequestInfoHandle; #endif // _D_URL_REQUEST_INFO_H_ diff --git a/src/Util.cc b/src/Util.cc index 3ba201df..07106530 100644 --- a/src/Util.cc +++ b/src/Util.cc @@ -460,7 +460,6 @@ void Util::sha1Sum(unsigned char* digest, const void* data, int dataLength) { ctx.digestInit(); ctx.digestUpdate(data, dataLength); ctx.digestFinal(digest); - ctx.digestFree(); } string Util::simpleMessageDigest(const string& data) { @@ -501,7 +500,6 @@ void Util::fileChecksum(const string& filename, unsigned char* digest, } } ctx.digestFinal(digest); - ctx.digestFree(); } #endif // ENABLE_MESSAGE_DIGEST @@ -638,3 +636,11 @@ void Util::setGlobalSignalHandler(int signal, void (*handler)(int), int flags) { sigemptyset(&sigact.sa_mask); sigaction(signal, &sigact, NULL); } + +void Util::indexRange(int32_t& startIndex, int32_t& endIndex, + int64_t offset, uint32_t srcLength, uint32_t destLength) +{ + startIndex = offset/destLength; + endIndex = (offset+srcLength-1)/destLength; +} + diff --git a/src/Util.h b/src/Util.h index 630b1b52..e8e745b6 100644 --- a/src/Util.h +++ b/src/Util.h @@ -134,6 +134,10 @@ public: static bool isNumbersAndDotsNotation(const string& name); static void setGlobalSignalHandler(int signal, void (*handler)(int), int flags); + + static void indexRange(int32_t& startIndex, int32_t& endIndex, + int64_t offset, + uint32_t srcLength, uint32_t destLength); }; #endif // _D_UTIL_H_ diff --git a/src/Xml2MetalinkProcessor.cc b/src/Xml2MetalinkProcessor.cc index 45ae65d3..b7769a63 100644 --- a/src/Xml2MetalinkProcessor.cc +++ b/src/Xml2MetalinkProcessor.cc @@ -99,7 +99,7 @@ MetalinkEntryHandle Xml2MetalinkProcessor::getEntry(const string& xpath) { MetalinkEntryHandle entry(new MetalinkEntry()); entry->filename = filename; - + entry->size = STRTOLL(xpathContent(xpath+"/m:size").c_str()); entry->version = Util::trim(xpathContent(xpath+"/m:version")); entry->language = Util::trim(xpathContent(xpath+"/m:language")); entry->os = Util::trim(xpathContent(xpath+"/m:os")); @@ -116,6 +116,8 @@ MetalinkEntryHandle Xml2MetalinkProcessor::getEntry(const string& xpath) { entry->checksum.setDigestAlgo(DIGEST_ALGO_MD5); } } + entry->chunkChecksum = getPieceHash(xpath+"/m:verification/m:pieces[@type=\"sha1\"]", entry->size); + #endif // ENABLE_MESSAGE_DIGEST for(int index = 1; 1; index++) { MetalinkResourceHandle resource(getResource(xpath+"/m:resources/m:url["+Util::itos(index)+"]")); @@ -128,6 +130,33 @@ MetalinkEntryHandle Xml2MetalinkProcessor::getEntry(const string& xpath) { return entry; } +MetalinkChunkChecksumHandle Xml2MetalinkProcessor::getPieceHash(const string& xpath, + uint64_t totalSize) +{ + MetalinkChunkChecksumHandle chunkChecksum = new MetalinkChunkChecksum(); + chunkChecksum->digestAlgo = DIGEST_ALGO_SHA1; + + xmlXPathObjectPtr result = xpathEvaluation(xpath); + if(!result) { + return 0; + } + xmlNodeSetPtr nodeSet = result->nodesetval; + xmlNodePtr node = nodeSet->nodeTab[0]; + chunkChecksum->pieceLength = STRTOLL(Util::trim(xmlAttribute(node, "length")).c_str()); + xmlXPathFreeObject(result); + + uint64_t numPiece = + (totalSize+chunkChecksum->pieceLength-1)/chunkChecksum->pieceLength; + for(uint64_t i = 0; i < numPiece; i++) { + string pieceHash = Util::trim(xpathContent(xpath+"/m:hash[@piece=\""+Util::ullitos(i)+"\"]")); + if(pieceHash == "") { + throw new DlAbortEx("Piece hash missing. index=%u", i); + } + chunkChecksum->pieceHashes.push_back(pieceHash); + } + return chunkChecksum; +} + MetalinkResourceHandle Xml2MetalinkProcessor::getResource(const string& xpath) { xmlXPathObjectPtr result = xpathEvaluation(xpath); if(!result) { diff --git a/src/Xml2MetalinkProcessor.h b/src/Xml2MetalinkProcessor.h index 540414be..0e57e31b 100644 --- a/src/Xml2MetalinkProcessor.h +++ b/src/Xml2MetalinkProcessor.h @@ -46,6 +46,8 @@ private: MetalinkEntryHandle getEntry(const string& xpath); MetalinkResourceHandle getResource(const string& xpath); + MetalinkChunkChecksumHandle getPieceHash(const string& xpath, + uint64_t totalSize); xmlXPathObjectPtr xpathEvaluation(const string& xpath); string xpathContent(const string& xpath); diff --git a/src/main.cc b/src/main.cc index d018671d..2a241bef 100644 --- a/src/main.cc +++ b/src/main.cc @@ -344,6 +344,8 @@ int main(int argc, char* argv[]) { op->put(PREF_TRACKER_MAX_TRIES, "10"); op->put(PREF_FILE_ALLOCATION, V_NONE); op->put(PREF_FORCE_TRUNCATE, V_FALSE); + op->put(PREF_REALTIME_CHUNK_CHECKSUM, V_TRUE); + op->put(PREF_CHECK_INTEGRITY, V_TRUE); while(1) { int optIndex = 0; int lopt; diff --git a/src/messageDigest.h b/src/messageDigest.h index 7e05f9bb..4282537e 100644 --- a/src/messageDigest.h +++ b/src/messageDigest.h @@ -75,6 +75,10 @@ public: MessageDigestContext(DigestAlgo algo): algo(algo) {} + ~MessageDigestContext() + { + digestFree(); + } #ifdef HAVE_LIBSSL void digestInit() { EVP_MD_CTX_init(&ctx); diff --git a/src/prefs.h b/src/prefs.h index d6e8dea5..8be227ca 100644 --- a/src/prefs.h +++ b/src/prefs.h @@ -86,6 +86,10 @@ # define V_PREALLOC "prealloc" // value: true | false #define PREF_FORCE_TRUNCATE "force_truncate" +// value: true | false +#define PREF_REALTIME_CHUNK_CHECKSUM "realtime_chunk_checksum" +// value: true | false +#define PREF_CHECK_INTEGRITY "check_integrity" /** * FTP related preferences diff --git a/test/ChunkChecksumValidatorTest.cc b/test/ChunkChecksumValidatorTest.cc new file mode 100644 index 00000000..88aa6279 --- /dev/null +++ b/test/ChunkChecksumValidatorTest.cc @@ -0,0 +1,135 @@ +#include "ChunkChecksumValidator.h" +#include "DefaultDiskWriter.h" +#include + +using namespace std; + +class ChunkChecksumValidatorTest:public CppUnit::TestFixture { + + CPPUNIT_TEST_SUITE(ChunkChecksumValidatorTest); + CPPUNIT_TEST(testValidate); + CPPUNIT_TEST(testValidate2); + CPPUNIT_TEST(testValidate3); + CPPUNIT_TEST(testValidate4); + CPPUNIT_TEST_SUITE_END(); +private: + + static const char* csArray[];// = { "29b0e7878271645fffb7eec7db4a7473a1c00bc1", + // "4df75a661cb7eb2733d9cdaa7f772eae3a4e2976", + // "0a4ea2f7dd7c52ddf2099a444ab2184b4d341bdb" }; +public: + void setUp() { + } + + void testValidate(); + void testValidate2(); + void testValidate3(); + void testValidate4(); +}; + + +CPPUNIT_TEST_SUITE_REGISTRATION( ChunkChecksumValidatorTest ); + +const char* ChunkChecksumValidatorTest::csArray[] = { "29b0e7878271645fffb7eec7db4a7473a1c00bc1", + "4df75a661cb7eb2733d9cdaa7f772eae3a4e2976", + "0a4ea2f7dd7c52ddf2099a444ab2184b4d341bdb" }; + +void ChunkChecksumValidatorTest::testValidate() { + BitfieldMan bitfieldMan(100, 250); + bitfieldMan.setAllBit(); + Strings checksums(&csArray[0], &csArray[3]); + + DefaultDiskWriterHandle diskWriter = new DefaultDiskWriter(); + diskWriter->openExistingFile("chunkChecksumTestFile250.txt"); + + ChunkChecksumValidator validator; + validator.setDiskWriter(diskWriter); + + validator.validate(&bitfieldMan, checksums, 100); + + CPPUNIT_ASSERT(bitfieldMan.isAllBitSet()); + + checksums[1] = "ffffffffffffffffffffffffffffffffffffffff"; + + validator.validate(&bitfieldMan, checksums, 100); + + CPPUNIT_ASSERT(bitfieldMan.isBitSet(0)); + CPPUNIT_ASSERT(!bitfieldMan.isBitSet(1)); + CPPUNIT_ASSERT(bitfieldMan.isBitSet(2)); +} + +void ChunkChecksumValidatorTest::testValidate2() { + BitfieldMan bitfieldMan(50, 250); + bitfieldMan.setAllBit(); + Strings checksums(&csArray[0], &csArray[3]); + + DefaultDiskWriterHandle diskWriter = new DefaultDiskWriter(); + diskWriter->openExistingFile("chunkChecksumTestFile250.txt"); + + ChunkChecksumValidator validator; + validator.setDiskWriter(diskWriter); + + validator.validate(&bitfieldMan, checksums, 100); + + CPPUNIT_ASSERT(bitfieldMan.isAllBitSet()); + + checksums[1] = "ffffffffffffffffffffffffffffffffffffffff"; + validator.validate(&bitfieldMan, checksums, 100); + + CPPUNIT_ASSERT(bitfieldMan.isBitSet(0)); + CPPUNIT_ASSERT(bitfieldMan.isBitSet(1)); + CPPUNIT_ASSERT(!bitfieldMan.isBitSet(2)); + CPPUNIT_ASSERT(!bitfieldMan.isBitSet(3)); + CPPUNIT_ASSERT(bitfieldMan.isBitSet(4)); +} + +void ChunkChecksumValidatorTest::testValidate3() { + BitfieldMan bitfieldMan(50, 250); + bitfieldMan.setAllBit(); + Strings checksums; + checksums.push_back("898a81b8e0181280ae2ee1b81e269196d91e869a"); + + DefaultDiskWriterHandle diskWriter = new DefaultDiskWriter(); + diskWriter->openExistingFile("chunkChecksumTestFile250.txt"); + + ChunkChecksumValidator validator; + validator.setDiskWriter(diskWriter); + + validator.validate(&bitfieldMan, checksums, 250); + + CPPUNIT_ASSERT(bitfieldMan.isAllBitSet()); + + checksums[0] = "ffffffffffffffffffffffffffffffffffffffff"; + + validator.validate(&bitfieldMan, checksums, 250); + + CPPUNIT_ASSERT(!bitfieldMan.isBitSet(0)); + CPPUNIT_ASSERT(!bitfieldMan.isBitSet(1)); + CPPUNIT_ASSERT(!bitfieldMan.isBitSet(2)); + CPPUNIT_ASSERT(!bitfieldMan.isBitSet(3)); + CPPUNIT_ASSERT(!bitfieldMan.isBitSet(4)); +} + +void ChunkChecksumValidatorTest::testValidate4() { + BitfieldMan bitfieldMan(70, 250); + bitfieldMan.setAllBit(); + Strings checksums(&csArray[0], &csArray[3]); + + DefaultDiskWriterHandle diskWriter = new DefaultDiskWriter(); + diskWriter->openExistingFile("chunkChecksumTestFile250.txt"); + + ChunkChecksumValidator validator; + validator.setDiskWriter(diskWriter); + + validator.validate(&bitfieldMan, checksums, 100); + + CPPUNIT_ASSERT(bitfieldMan.isAllBitSet()); + + checksums[1] = "ffffffffffffffffffffffffffffffffffffffff"; + validator.validate(&bitfieldMan, checksums, 100); + + CPPUNIT_ASSERT(bitfieldMan.isBitSet(0)); + CPPUNIT_ASSERT(!bitfieldMan.isBitSet(1)); + CPPUNIT_ASSERT(!bitfieldMan.isBitSet(2)); + CPPUNIT_ASSERT(bitfieldMan.isBitSet(3)); +} diff --git a/test/ConsoleFileAllocationMonitorTest.cc b/test/ConsoleFileAllocationMonitorTest.cc index cef4e6b9..caaa87bc 100644 --- a/test/ConsoleFileAllocationMonitorTest.cc +++ b/test/ConsoleFileAllocationMonitorTest.cc @@ -31,7 +31,6 @@ void ConsoleFileAllocationMonitorTest::testShowProgress() { for(uint64_t i = monitor.getMinValue(); i <= monitor.getMaxValue(); i += 1234343) { monitor.setCurrentValue(i); monitor.showProgress(); - usleep(5); } monitor.setCurrentValue(monitor.getMaxValue()); monitor.showProgress(); diff --git a/test/DefaultDiskWriterTest.cc b/test/DefaultDiskWriterTest.cc index dc939918..c0588793 100644 --- a/test/DefaultDiskWriterTest.cc +++ b/test/DefaultDiskWriterTest.cc @@ -7,7 +7,7 @@ using namespace std; class DefaultDiskWriterTest:public CppUnit::TestFixture { CPPUNIT_TEST_SUITE(DefaultDiskWriterTest); - CPPUNIT_TEST(testSha1Sum); + CPPUNIT_TEST(testMessageDigest); CPPUNIT_TEST_SUITE_END(); private: @@ -15,21 +15,21 @@ public: void setUp() { } - void testSha1Sum(); + void testMessageDigest(); }; CPPUNIT_TEST_SUITE_REGISTRATION( DefaultDiskWriterTest ); -void DefaultDiskWriterTest::testSha1Sum() { +void DefaultDiskWriterTest::testMessageDigest() { DefaultDiskWriter dw; dw.openExistingFile("4096chunk.txt"); CPPUNIT_ASSERT_EQUAL(string("608cabc0f2fa18c260cafd974516865c772363d5"), - dw.sha1Sum(0, 4096)); + dw.messageDigest(0, 4096, DIGEST_ALGO_SHA1)); CPPUNIT_ASSERT_EQUAL(string("7a4a9ae537ebbbb826b1060e704490ad0f365ead"), - dw.sha1Sum(5, 100)); + dw.messageDigest(5, 100, DIGEST_ALGO_SHA1)); dw.closeFile(); } diff --git a/test/Makefile.am b/test/Makefile.am index 22570cda..cadd4fba 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -58,7 +58,8 @@ aria2c_SOURCES = AllTest.cc\ FixedNumberRandomizer.h\ MockBtMessageFactory.h\ MockBtMessage.h\ - ConsoleFileAllocationMonitorTest.cc + ConsoleFileAllocationMonitorTest.cc\ + ChunkChecksumValidatorTest.cc #aria2c_CXXFLAGS = ${CPPUNIT_CFLAGS} -I../src -I../lib -Wall -D_FILE_OFFSET_BITS=64 #aria2c_LDFLAGS = ${CPPUNIT_LIBS} diff --git a/test/Makefile.in b/test/Makefile.in index 649dee7e..d5feb1da 100644 --- a/test/Makefile.in +++ b/test/Makefile.in @@ -89,7 +89,8 @@ am_aria2c_OBJECTS = AllTest.$(OBJEXT) RequestTest.$(OBJEXT) \ BtSuggestPieceMessageTest.$(OBJEXT) \ BtUnchokeMessageTest.$(OBJEXT) \ BtHandshakeMessageTest.$(OBJEXT) \ - ConsoleFileAllocationMonitorTest.$(OBJEXT) + ConsoleFileAllocationMonitorTest.$(OBJEXT) \ + ChunkChecksumValidatorTest.$(OBJEXT) aria2c_OBJECTS = $(am_aria2c_OBJECTS) am__DEPENDENCIES_1 = aria2c_DEPENDENCIES = ../src/libaria2c.a $(am__DEPENDENCIES_1) @@ -311,7 +312,8 @@ aria2c_SOURCES = AllTest.cc\ FixedNumberRandomizer.h\ MockBtMessageFactory.h\ MockBtMessage.h\ - ConsoleFileAllocationMonitorTest.cc + ConsoleFileAllocationMonitorTest.cc\ + ChunkChecksumValidatorTest.cc #aria2c_CXXFLAGS = ${CPPUNIT_CFLAGS} -I../src -I../lib -Wall -D_FILE_OFFSET_BITS=64 #aria2c_LDFLAGS = ${CPPUNIT_LIBS} @@ -396,6 +398,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/BtRequestMessageTest.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/BtSuggestPieceMessageTest.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/BtUnchokeMessageTest.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ChunkChecksumValidatorTest.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ChunkedEncodingTest.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ConsoleFileAllocationMonitorTest.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/CookieBoxTest.Po@am__quote@ diff --git a/test/MockBtContext.h b/test/MockBtContext.h index fdc0f880..eaadabf6 100644 --- a/test/MockBtContext.h +++ b/test/MockBtContext.h @@ -43,6 +43,10 @@ public: return pieceHashes.at(index); } + virtual const Strings& getPieceHashes() const { + return pieceHashes; + } + void addPieceHash(const string pieceHash) { pieceHashes.push_back(pieceHash); } diff --git a/test/MockPieceStorage.h b/test/MockPieceStorage.h index 76929d3a..5c818bac 100644 --- a/test/MockPieceStorage.h +++ b/test/MockPieceStorage.h @@ -146,6 +146,10 @@ public: } virtual void removeAdvertisedPiece(int elapsed) {} + + virtual void markAllPiecesDone() {} + + virtual void checkIntegrity() {} }; typedef SharedHandle MockPieceStorageHandle; diff --git a/test/MultiDiskAdaptorTest.cc b/test/MultiDiskAdaptorTest.cc index 5f9437f7..f672c421 100644 --- a/test/MultiDiskAdaptorTest.cc +++ b/test/MultiDiskAdaptorTest.cc @@ -9,7 +9,7 @@ class MultiDiskAdaptorTest:public CppUnit::TestFixture { CPPUNIT_TEST_SUITE(MultiDiskAdaptorTest); CPPUNIT_TEST(testWriteData); CPPUNIT_TEST(testReadData); - CPPUNIT_TEST(testSha1Sum); + CPPUNIT_TEST(testMessageDigest); CPPUNIT_TEST_SUITE_END(); private: Option* option; @@ -30,7 +30,7 @@ public: void testWriteData(); void testReadData(); - void testSha1Sum(); + void testMessageDigest(); }; @@ -134,7 +134,7 @@ void MultiDiskAdaptorTest::testReadData() { CPPUNIT_ASSERT_EQUAL(string("1234567890ABCDEFGHIJKLMNO"), string((char*)buf)); } -void MultiDiskAdaptorTest::testSha1Sum() { +void MultiDiskAdaptorTest::testMessageDigest() { FileEntryHandle entry1(new FileEntry("file1r.txt", 15, 0)); FileEntryHandle entry2(new FileEntry("file2r.txt", 7, 15)); FileEntryHandle entry3(new FileEntry("file3r.txt", 3, 22)); @@ -146,11 +146,11 @@ void MultiDiskAdaptorTest::testSha1Sum() { adaptor->setFileEntries(entries); adaptor->openFile(); - string sha1sum = adaptor->sha1Sum(0, 25); + string sha1sum = adaptor->messageDigest(0, 25, DIGEST_ALGO_SHA1); CPPUNIT_ASSERT_EQUAL(string("76495faf71ca63df66dce99547d2c58da7266d9e"), sha1sum); - sha1sum = adaptor->sha1Sum(15, 7); + sha1sum = adaptor->messageDigest(15, 7, DIGEST_ALGO_SHA1); CPPUNIT_ASSERT_EQUAL(string("737660d816fb23c2d5bc74f62d9b01b852b2aaca"), sha1sum); - sha1sum = adaptor->sha1Sum(10, 14); + sha1sum = adaptor->messageDigest(10, 14, DIGEST_ALGO_SHA1); CPPUNIT_ASSERT_EQUAL(string("6238bf61dd8df8f77156b2378e9e39cd3939680c"), sha1sum); adaptor->closeFile(); } diff --git a/test/MultiDiskWriterTest.cc b/test/MultiDiskWriterTest.cc index 80e9972c..3415be2e 100644 --- a/test/MultiDiskWriterTest.cc +++ b/test/MultiDiskWriterTest.cc @@ -9,7 +9,7 @@ class MultiDiskWriterTest:public CppUnit::TestFixture { CPPUNIT_TEST_SUITE(MultiDiskWriterTest); CPPUNIT_TEST(testWriteData); CPPUNIT_TEST(testReadData); - CPPUNIT_TEST(testSha1Sum); + CPPUNIT_TEST(testMessageDigest); CPPUNIT_TEST_SUITE_END(); private: @@ -19,7 +19,7 @@ public: void testWriteData(); void testReadData(); - void testSha1Sum(); + void testMessageDigest(); }; @@ -120,7 +120,7 @@ void MultiDiskWriterTest::testReadData() { CPPUNIT_ASSERT_EQUAL(string("1234567890ABCDEFGHIJKLMNO"), string(buf)); } -void MultiDiskWriterTest::testSha1Sum() { +void MultiDiskWriterTest::testMessageDigest() { FileEntryHandle entry1(new FileEntry("file1r.txt", 15, 0)); FileEntryHandle entry2(new FileEntry("file2r.txt", 7, 15)); FileEntryHandle entry3(new FileEntry("file3r.txt", 3, 22)); @@ -132,11 +132,11 @@ void MultiDiskWriterTest::testSha1Sum() { dw.setFileEntries(entries); dw.openFile("."); - string sha1sum = dw.sha1Sum(0, 25); + string sha1sum = dw.messageDigest(0, 25, DIGEST_ALGO_SHA1); CPPUNIT_ASSERT_EQUAL(string("76495faf71ca63df66dce99547d2c58da7266d9e"), sha1sum); - sha1sum = dw.sha1Sum(15, 7); + sha1sum = dw.messageDigest(15, 7, DIGEST_ALGO_SHA1); CPPUNIT_ASSERT_EQUAL(string("737660d816fb23c2d5bc74f62d9b01b852b2aaca"), sha1sum); - sha1sum = dw.sha1Sum(10, 14); + sha1sum = dw.messageDigest(10, 14, DIGEST_ALGO_SHA1); CPPUNIT_ASSERT_EQUAL(string("6238bf61dd8df8f77156b2378e9e39cd3939680c"), sha1sum); dw.closeFile(); } diff --git a/test/Xml2MetalinkProcessorTest.cc b/test/Xml2MetalinkProcessorTest.cc index 31831ba9..c4b60c64 100644 --- a/test/Xml2MetalinkProcessorTest.cc +++ b/test/Xml2MetalinkProcessorTest.cc @@ -1,4 +1,5 @@ #include "Xml2MetalinkProcessor.h" +#include "Exception.h" #include using namespace std; @@ -26,6 +27,7 @@ CPPUNIT_TEST_SUITE_REGISTRATION( Xml2MetalinkProcessorTest ); void Xml2MetalinkProcessorTest::testParseFile() { Xml2MetalinkProcessor proc; + try { MetalinkerHandle metalinker = proc.parseFile("test.xml"); MetalinkEntries::iterator entryItr = metalinker->entries.begin(); @@ -55,10 +57,22 @@ void Xml2MetalinkProcessorTest::testParseFile() { entryItr++; MetalinkEntryHandle entry2 = *entryItr; + CPPUNIT_ASSERT_EQUAL((uint64_t)345689, entry2->size); CPPUNIT_ASSERT_EQUAL(string("0.5.1"), entry2->version); CPPUNIT_ASSERT_EQUAL(string("ja-JP"), entry2->language); CPPUNIT_ASSERT_EQUAL(string("Linux-m68k"), entry2->os); CPPUNIT_ASSERT_EQUAL(string("4c255b0ed130f5ea880f0aa061c3da0487e251cc"), entry2->checksum.getMessageDigest()); + CPPUNIT_ASSERT_EQUAL((size_t)2, entry2->chunkChecksum->pieceHashes.size()); + CPPUNIT_ASSERT_EQUAL((uint32_t)266144, entry2->chunkChecksum->pieceLength); + CPPUNIT_ASSERT_EQUAL(string("179463a88d79cbf0b1923991708aead914f26142"), + entry2->chunkChecksum->pieceHashes.at(0)); + CPPUNIT_ASSERT_EQUAL(string("fecf8bc9a1647505fe16746f94e97a477597dbf3"), + entry2->chunkChecksum->pieceHashes.at(1)); + CPPUNIT_ASSERT(DIGEST_ALGO_SHA1 == entry2->checksum.getDigestAlgo()); + } catch(Exception* e) { + cerr << e->getMsg() << endl; + delete e; + } } diff --git a/test/chunkChecksumTestFile250.txt b/test/chunkChecksumTestFile250.txt new file mode 100644 index 00000000..3ed5fc61 --- /dev/null +++ b/test/chunkChecksumTestFile250.txt @@ -0,0 +1 @@ +0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789ABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij \ No newline at end of file diff --git a/test/test.xml b/test/test.xml index e0473e16..58cb04d3 100644 --- a/test/test.xml +++ b/test/test.xml @@ -20,12 +20,17 @@ + 345689 0.5.1 ja-JP Linux-m68k 92296e19c406d77d21bda0bb944eac46 4c255b0ed130f5ea880f0aa061c3da0487e251cc + + 179463a88d79cbf0b1923991708aead914f26142 + fecf8bc9a1647505fe16746f94e97a477597dbf3 + ftp://ftphost/aria2-0.5.1.tar.bz2