diff --git a/ChangeLog b/ChangeLog index aecf3815..8c78635b 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,21 @@ +2009-01-24 Tatsuhiro Tsujikawa + + Made (piece) hash check sequential for each RequestGroup because + simultaneous hash check is slower than sequential execution. + * src/CheckIntegrityCommand.cc + * src/CheckIntegrityDispatcherCommand.cc + * src/CheckIntegrityDispatcherCommand.h + * src/CheckIntegrityMan.h + * src/ConsoleStatCalc.cc + * src/DownloadCommand.cc + * src/DownloadEngine.cc + * src/DownloadEngine.h + * src/DownloadEngineFactory.cc + * src/Makefile.am + * src/Makefile.in + * src/RequestGroup.cc + * src/StatCalc.h + 2009-01-24 Tatsuhiro Tsujikawa Moved setStatusRealtime() from FileAllocationDispatcherCommand to diff --git a/src/CheckIntegrityCommand.cc b/src/CheckIntegrityCommand.cc index d65056d2..4c2858e1 100644 --- a/src/CheckIntegrityCommand.cc +++ b/src/CheckIntegrityCommand.cc @@ -33,7 +33,6 @@ */ /* copyright --> */ #include "CheckIntegrityCommand.h" -#include "CheckIntegrityMan.h" #include "CheckIntegrityEntry.h" #include "DownloadEngine.h" #include "RequestGroup.h" @@ -43,17 +42,14 @@ namespace aria2 { -CheckIntegrityCommand::CheckIntegrityCommand(int32_t cuid, RequestGroup* requestGroup, DownloadEngine* e, const CheckIntegrityEntryHandle& entry): +CheckIntegrityCommand::CheckIntegrityCommand +(int32_t cuid, RequestGroup* requestGroup, DownloadEngine* e, + const CheckIntegrityEntryHandle& entry): RealtimeCommand(cuid, requestGroup, e), _entry(entry) -{ - _e->_checkIntegrityMan->addCheckIntegrityEntry(_entry); -} +{} -CheckIntegrityCommand::~CheckIntegrityCommand() -{ - _e->_checkIntegrityMan->removeCheckIntegrityEntry(_entry); -} +CheckIntegrityCommand::~CheckIntegrityCommand() {} bool CheckIntegrityCommand::executeInternal() { @@ -62,6 +58,8 @@ bool CheckIntegrityCommand::executeInternal() } _entry->validateChunk(); if(_entry->finished()) { + _e->_checkIntegrityMan->dropPickedEntry(); + if(_requestGroup->downloadFinished()) { logger->notice(MSG_VERIFICATION_SUCCESSFUL, _requestGroup->getFilePath().c_str()); @@ -85,8 +83,10 @@ bool CheckIntegrityCommand::executeInternal() bool CheckIntegrityCommand::handleException(Exception& e) { + _e->_checkIntegrityMan->dropPickedEntry(); logger->error(MSG_FILE_VALIDATION_FAILURE, e, cuid); - logger->error(MSG_DOWNLOAD_NOT_COMPLETE, cuid, _requestGroup->getFilePath().c_str()); + logger->error(MSG_DOWNLOAD_NOT_COMPLETE, + cuid, _requestGroup->getFilePath().c_str()); return true; } diff --git a/src/CheckIntegrityDispatcherCommand.cc b/src/CheckIntegrityDispatcherCommand.cc new file mode 100644 index 00000000..fff00638 --- /dev/null +++ b/src/CheckIntegrityDispatcherCommand.cc @@ -0,0 +1,62 @@ +/* */ +#include "CheckIntegrityDispatcherCommand.h" +#include "CheckIntegrityEntry.h" +#include "CheckIntegrityCommand.h" +#include "message.h" +#include "Logger.h" + +namespace aria2 { + +CheckIntegrityDispatcherCommand::CheckIntegrityDispatcherCommand +(int32_t cuid, + const SharedHandle& fileAllocMan, + DownloadEngine* e):SequentialDispatcherCommand + (cuid, fileAllocMan, e) +{ + setStatusRealtime(); +} + +Command* CheckIntegrityDispatcherCommand::createCommand +(const SharedHandle& entry) +{ + int32_t newCUID = _e->newCUID(); + logger->info("CUID#%d - Dispatching CheckIntegrityCommand CUID#%d.", + cuid, newCUID); + return new CheckIntegrityCommand + (newCUID, entry->getRequestGroup(), _e, entry); +} + +} // namespace aria2 diff --git a/src/CheckIntegrityDispatcherCommand.h b/src/CheckIntegrityDispatcherCommand.h new file mode 100644 index 00000000..480f6851 --- /dev/null +++ b/src/CheckIntegrityDispatcherCommand.h @@ -0,0 +1,59 @@ +/* */ +#ifndef _D_CHECK_INTEGRITY_DISPATCHER_COMMAND_H_ +#define _D_CHECK_INTEGRITY_DISPATCHER_COMMAND_H_ + +#include "SequentialDispatcherCommand.h" +#include "CheckIntegrityMan.h" + +namespace aria2 { + +class CheckIntegrityEntry; + +class CheckIntegrityDispatcherCommand : + public SequentialDispatcherCommand { +public: + CheckIntegrityDispatcherCommand + (int32_t cuid, + const SharedHandle& checkMan, + DownloadEngine* e); +protected: + virtual Command* createCommand + (const SharedHandle& entry); +}; + +} // namespace aria2 + +#endif // _D_CHECK_INTEGRITY_DISPATCHER_COMMAND_H_ diff --git a/src/CheckIntegrityMan.h b/src/CheckIntegrityMan.h index d47a6aa1..4f5d0278 100644 --- a/src/CheckIntegrityMan.h +++ b/src/CheckIntegrityMan.h @@ -36,31 +36,13 @@ #define _D_CHECK_INTEGRITY_MAN_H_ #include "common.h" -#include "SharedHandle.h" -#include +#include "SequentialPicker.h" namespace aria2 { class CheckIntegrityEntry; -class CheckIntegrityMan { -private: - std::deque > _checkIntegrityEntries; -public: - CheckIntegrityMan(); - - ~CheckIntegrityMan(); - - void addCheckIntegrityEntry(const SharedHandle& entry); - - bool removeCheckIntegrityEntry(const SharedHandle& entry); - - SharedHandle getFirstCheckIntegrityEntry() const; - - size_t countCheckIntegrityEntry() const; -}; - -typedef SharedHandle CheckIntegrityManHandle; +typedef SequentialPicker CheckIntegrityMan; } // namespace aria2 diff --git a/src/ConsoleStatCalc.cc b/src/ConsoleStatCalc.cc index d40bed8e..89ba1d4e 100644 --- a/src/ConsoleStatCalc.cc +++ b/src/ConsoleStatCalc.cc @@ -165,7 +165,7 @@ void ConsoleStatCalc::calculateStat (const RequestGroupManHandle& requestGroupMan, const SharedHandle& fileAllocationMan, - const CheckIntegrityManHandle& checkIntegrityMan) + const SharedHandle& checkIntegrityMan) { if(!_cp.elapsed(1)) { return; @@ -239,7 +239,7 @@ ConsoleStatCalc::calculateStat } #ifdef ENABLE_MESSAGE_DIGEST { - CheckIntegrityEntryHandle entry = checkIntegrityMan->getFirstCheckIntegrityEntry(); + CheckIntegrityEntryHandle entry = checkIntegrityMan->getPickedEntry(); if(!entry.isNull()) { o << " " << "[Checksum:" @@ -253,10 +253,10 @@ ConsoleStatCalc::calculateStat << 100*entry->getCurrentLength()/entry->getTotalLength() << "%)" << "]"; - if(checkIntegrityMan->countCheckIntegrityEntry() > 1) { + if(checkIntegrityMan->hasNext()) { o << "(" - << checkIntegrityMan->countCheckIntegrityEntry()-1 - << "more...)"; + << checkIntegrityMan->countEntryInQueue() + << "waiting...)"; } } } diff --git a/src/DownloadCommand.cc b/src/DownloadCommand.cc index 56c358da..19a473ad 100644 --- a/src/DownloadCommand.cc +++ b/src/DownloadCommand.cc @@ -254,9 +254,8 @@ bool DownloadCommand::prepareForNextSegment() { CheckIntegrityEntryHandle entry(new ChecksumCheckIntegrityEntry(_requestGroup)); if(entry->isValidationReady()) { entry->initValidator(); - CheckIntegrityCommand* command = - new CheckIntegrityCommand(e->newCUID(), _requestGroup, e, entry); - e->commands.push_back(command); + // TODO do we need cuttrailinggarbage here? + e->_checkIntegrityMan->pushEntry(entry); } e->setNoWait(true); e->setRefreshInterval(0); diff --git a/src/DownloadEngine.cc b/src/DownloadEngine.cc index 3da08503..d4ace481 100644 --- a/src/DownloadEngine.cc +++ b/src/DownloadEngine.cc @@ -44,7 +44,6 @@ #include "StatCalc.h" #include "RequestGroup.h" #include "RequestGroupMan.h" -#include "CheckIntegrityMan.h" #include "DownloadResult.h" #include "StatCalc.h" #include "LogFactory.h" @@ -65,6 +64,7 @@ #include "EventPoll.h" #include "Command.h" #include "FileAllocationEntry.h" +#include "CheckIntegrityEntry.h" #include "BtRegistry.h" #include "BtContext.h" diff --git a/src/DownloadEngine.h b/src/DownloadEngine.h index f6d79a8c..59e10efc 100644 --- a/src/DownloadEngine.h +++ b/src/DownloadEngine.h @@ -50,6 +50,7 @@ #endif // ENABLE_ASYNC_DNS #include "CUIDCounter.h" #include "FileAllocationMan.h" +#include "CheckIntegrityMan.h" namespace aria2 { @@ -57,7 +58,6 @@ class Logger; class Option; class RequestGroupMan; class StatCalc; -class CheckIntegrityMan; class SocketCore; class CookieStorage; class BtRegistry; diff --git a/src/DownloadEngineFactory.cc b/src/DownloadEngineFactory.cc index 206e9e06..04c15fc3 100644 --- a/src/DownloadEngineFactory.cc +++ b/src/DownloadEngineFactory.cc @@ -45,6 +45,8 @@ #include "FileAllocationMan.h" #ifdef ENABLE_MESSAGE_DIGEST # include "CheckIntegrityMan.h" +# include "CheckIntegrityEntry.h" +# include "CheckIntegrityDispatcherCommand.h" #endif // ENABLE_MESSAGE_DIGEST #include "prefs.h" #include "FillRequestGroupCommand.h" @@ -118,6 +120,11 @@ DownloadEngineFactory::newDownloadEngine(Option* op, e->addRoutineCommand(new FillRequestGroupCommand(e->newCUID(), e.get(), 1)); e->addRoutineCommand(new FileAllocationDispatcherCommand (e->newCUID(), e->_fileAllocationMan, e.get())); +#ifdef ENABLE_MESSAGE_DIGEST + e->addRoutineCommand(new CheckIntegrityDispatcherCommand + (e->newCUID(), e->_checkIntegrityMan, e.get())); +#endif // ENABLE_MESSAGE_DIGEST + if(op->getAsInt(PREF_AUTO_SAVE_INTERVAL) > 0) { e->addRoutineCommand (new AutoSaveCommand(e->newCUID(), e.get(), diff --git a/src/Makefile.am b/src/Makefile.am index 073c670b..616009a8 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -135,7 +135,7 @@ SRCS = Socket.h\ MemoryBufferPreDownloadHandler.cc MemoryBufferPreDownloadHandler.h\ HaveEraseCommand.cc HaveEraseCommand.h\ Piece.cc Piece.h\ - CheckIntegrityMan.cc CheckIntegrityMan.h\ + CheckIntegrityMan.h\ CheckIntegrityEntry.cc CheckIntegrityEntry.h\ PieceHashCheckIntegrityEntry.cc PieceHashCheckIntegrityEntry.h\ StreamCheckIntegrityEntry.cc StreamCheckIntegrityEntry.h\ @@ -233,6 +233,7 @@ endif # ENABLE_ASYNC_DNS if ENABLE_MESSAGE_DIGEST SRCS += IteratableChunkChecksumValidator.cc IteratableChunkChecksumValidator.h\ IteratableChecksumValidator.cc IteratableChecksumValidator.h\ + CheckIntegrityDispatcherCommand.cc CheckIntegrityDispatcherCommand.h\ CheckIntegrityCommand.cc CheckIntegrityCommand.h\ ChecksumCheckIntegrityEntry.cc ChecksumCheckIntegrityEntry.h\ messageDigest.cc messageDigest.h\ diff --git a/src/Makefile.in b/src/Makefile.in index 3a215eaa..8c1ecc15 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -44,6 +44,7 @@ bin_PROGRAMS = aria2c$(EXEEXT) @ENABLE_ASYNC_DNS_TRUE@am__append_7 = AsyncNameResolver.cc AsyncNameResolver.h @ENABLE_MESSAGE_DIGEST_TRUE@am__append_8 = IteratableChunkChecksumValidator.cc IteratableChunkChecksumValidator.h\ @ENABLE_MESSAGE_DIGEST_TRUE@ IteratableChecksumValidator.cc IteratableChecksumValidator.h\ +@ENABLE_MESSAGE_DIGEST_TRUE@ CheckIntegrityDispatcherCommand.cc CheckIntegrityDispatcherCommand.h\ @ENABLE_MESSAGE_DIGEST_TRUE@ CheckIntegrityCommand.cc CheckIntegrityCommand.h\ @ENABLE_MESSAGE_DIGEST_TRUE@ ChecksumCheckIntegrityEntry.cc ChecksumCheckIntegrityEntry.h\ @ENABLE_MESSAGE_DIGEST_TRUE@ messageDigest.cc messageDigest.h\ @@ -375,17 +376,16 @@ am__libaria2c_a_SOURCES_DIST = Socket.h SocketCore.cc SocketCore.h \ DownloadHandlerConstants.h DownloadHandlerFactory.cc \ DownloadHandlerFactory.h MemoryBufferPreDownloadHandler.cc \ MemoryBufferPreDownloadHandler.h HaveEraseCommand.cc \ - HaveEraseCommand.h Piece.cc Piece.h CheckIntegrityMan.cc \ - CheckIntegrityMan.h CheckIntegrityEntry.cc \ - CheckIntegrityEntry.h PieceHashCheckIntegrityEntry.cc \ - PieceHashCheckIntegrityEntry.h StreamCheckIntegrityEntry.cc \ - StreamCheckIntegrityEntry.h IteratableValidator.h \ - DiskAdaptor.cc DiskAdaptor.h AbstractSingleDiskAdaptor.cc \ - AbstractSingleDiskAdaptor.h CopyDiskAdaptor.cc \ - CopyDiskAdaptor.h DirectDiskAdaptor.cc DirectDiskAdaptor.h \ - MultiDiskAdaptor.cc MultiDiskAdaptor.h Peer.cc \ - PeerSessionResource.cc PeerSessionResource.h BtRegistry.cc \ - BtRegistry.h MultiFileAllocationIterator.cc \ + HaveEraseCommand.h Piece.cc Piece.h CheckIntegrityMan.h \ + CheckIntegrityEntry.cc CheckIntegrityEntry.h \ + PieceHashCheckIntegrityEntry.cc PieceHashCheckIntegrityEntry.h \ + StreamCheckIntegrityEntry.cc StreamCheckIntegrityEntry.h \ + IteratableValidator.h DiskAdaptor.cc DiskAdaptor.h \ + AbstractSingleDiskAdaptor.cc AbstractSingleDiskAdaptor.h \ + CopyDiskAdaptor.cc CopyDiskAdaptor.h DirectDiskAdaptor.cc \ + DirectDiskAdaptor.h MultiDiskAdaptor.cc MultiDiskAdaptor.h \ + Peer.cc PeerSessionResource.cc PeerSessionResource.h \ + BtRegistry.cc BtRegistry.h MultiFileAllocationIterator.cc \ MultiFileAllocationIterator.h PeerConnection.cc \ PeerConnection.h ByteArrayDiskWriter.cc ByteArrayDiskWriter.h \ ByteArrayDiskWriterFactory.cc ByteArrayDiskWriterFactory.h \ @@ -422,12 +422,14 @@ am__libaria2c_a_SOURCES_DIST = Socket.h SocketCore.cc SocketCore.h \ AsyncNameResolver.h IteratableChunkChecksumValidator.cc \ IteratableChunkChecksumValidator.h \ IteratableChecksumValidator.cc IteratableChecksumValidator.h \ - CheckIntegrityCommand.cc CheckIntegrityCommand.h \ - ChecksumCheckIntegrityEntry.cc ChecksumCheckIntegrityEntry.h \ - messageDigest.cc messageDigest.h MessageDigestHelper.cc \ - MessageDigestHelper.h Checksum.h ChunkChecksum.h \ - PeerMessageUtil.cc PeerMessageUtil.h PeerAbstractCommand.cc \ - PeerAbstractCommand.h PeerInitiateConnectionCommand.cc \ + CheckIntegrityDispatcherCommand.cc \ + CheckIntegrityDispatcherCommand.h CheckIntegrityCommand.cc \ + CheckIntegrityCommand.h ChecksumCheckIntegrityEntry.cc \ + ChecksumCheckIntegrityEntry.h messageDigest.cc messageDigest.h \ + MessageDigestHelper.cc MessageDigestHelper.h Checksum.h \ + ChunkChecksum.h PeerMessageUtil.cc PeerMessageUtil.h \ + PeerAbstractCommand.cc PeerAbstractCommand.h \ + PeerInitiateConnectionCommand.cc \ PeerInitiateConnectionCommand.h PeerInteractionCommand.cc \ PeerInteractionCommand.h Peer.h PeerListenCommand.cc \ PeerListenCommand.h RequestSlot.cc RequestSlot.h \ @@ -581,6 +583,7 @@ am__objects_2 = @ENABLE_ASYNC_DNS_TRUE@am__objects_7 = AsyncNameResolver.$(OBJEXT) @ENABLE_MESSAGE_DIGEST_TRUE@am__objects_8 = IteratableChunkChecksumValidator.$(OBJEXT) \ @ENABLE_MESSAGE_DIGEST_TRUE@ IteratableChecksumValidator.$(OBJEXT) \ +@ENABLE_MESSAGE_DIGEST_TRUE@ CheckIntegrityDispatcherCommand.$(OBJEXT) \ @ENABLE_MESSAGE_DIGEST_TRUE@ CheckIntegrityCommand.$(OBJEXT) \ @ENABLE_MESSAGE_DIGEST_TRUE@ ChecksumCheckIntegrityEntry.$(OBJEXT) \ @ENABLE_MESSAGE_DIGEST_TRUE@ messageDigest.$(OBJEXT) \ @@ -785,7 +788,7 @@ am__objects_22 = SocketCore.$(OBJEXT) Command.$(OBJEXT) \ DownloadHandlerFactory.$(OBJEXT) \ MemoryBufferPreDownloadHandler.$(OBJEXT) \ HaveEraseCommand.$(OBJEXT) Piece.$(OBJEXT) \ - CheckIntegrityMan.$(OBJEXT) CheckIntegrityEntry.$(OBJEXT) \ + CheckIntegrityEntry.$(OBJEXT) \ PieceHashCheckIntegrityEntry.$(OBJEXT) \ StreamCheckIntegrityEntry.$(OBJEXT) DiskAdaptor.$(OBJEXT) \ AbstractSingleDiskAdaptor.$(OBJEXT) CopyDiskAdaptor.$(OBJEXT) \ @@ -1102,17 +1105,16 @@ SRCS = Socket.h SocketCore.cc SocketCore.h BinaryStream.h Command.cc \ DownloadHandlerConstants.h DownloadHandlerFactory.cc \ DownloadHandlerFactory.h MemoryBufferPreDownloadHandler.cc \ MemoryBufferPreDownloadHandler.h HaveEraseCommand.cc \ - HaveEraseCommand.h Piece.cc Piece.h CheckIntegrityMan.cc \ - CheckIntegrityMan.h CheckIntegrityEntry.cc \ - CheckIntegrityEntry.h PieceHashCheckIntegrityEntry.cc \ - PieceHashCheckIntegrityEntry.h StreamCheckIntegrityEntry.cc \ - StreamCheckIntegrityEntry.h IteratableValidator.h \ - DiskAdaptor.cc DiskAdaptor.h AbstractSingleDiskAdaptor.cc \ - AbstractSingleDiskAdaptor.h CopyDiskAdaptor.cc \ - CopyDiskAdaptor.h DirectDiskAdaptor.cc DirectDiskAdaptor.h \ - MultiDiskAdaptor.cc MultiDiskAdaptor.h Peer.cc \ - PeerSessionResource.cc PeerSessionResource.h BtRegistry.cc \ - BtRegistry.h MultiFileAllocationIterator.cc \ + HaveEraseCommand.h Piece.cc Piece.h CheckIntegrityMan.h \ + CheckIntegrityEntry.cc CheckIntegrityEntry.h \ + PieceHashCheckIntegrityEntry.cc PieceHashCheckIntegrityEntry.h \ + StreamCheckIntegrityEntry.cc StreamCheckIntegrityEntry.h \ + IteratableValidator.h DiskAdaptor.cc DiskAdaptor.h \ + AbstractSingleDiskAdaptor.cc AbstractSingleDiskAdaptor.h \ + CopyDiskAdaptor.cc CopyDiskAdaptor.h DirectDiskAdaptor.cc \ + DirectDiskAdaptor.h MultiDiskAdaptor.cc MultiDiskAdaptor.h \ + Peer.cc PeerSessionResource.cc PeerSessionResource.h \ + BtRegistry.cc BtRegistry.h MultiFileAllocationIterator.cc \ MultiFileAllocationIterator.h PeerConnection.cc \ PeerConnection.h ByteArrayDiskWriter.cc ByteArrayDiskWriter.h \ ByteArrayDiskWriterFactory.cc ByteArrayDiskWriterFactory.h \ @@ -1287,8 +1289,8 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ByteArrayDiskWriter.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ByteArrayDiskWriterFactory.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/CheckIntegrityCommand.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/CheckIntegrityDispatcherCommand.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/CheckIntegrityEntry.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/CheckIntegrityMan.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ChecksumCheckIntegrityEntry.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ChunkedDecoder.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Command.Po@am__quote@ diff --git a/src/RequestGroup.cc b/src/RequestGroup.cc index 1d8cbab3..30c9572e 100644 --- a/src/RequestGroup.cc +++ b/src/RequestGroup.cc @@ -347,10 +347,8 @@ void RequestGroup::processCheckIntegrityEntry(std::deque& commands, if(e->option->getAsBool(PREF_CHECK_INTEGRITY) && entry->isValidationReady()) { entry->initValidator(); - entry->cutTrailingGarbage(); - CheckIntegrityCommand* command = - new CheckIntegrityCommand(e->newCUID(), this, e, entry); - commands.push_back(command); + entry->cutTrailingGarbage(); + e->_checkIntegrityMan->pushEntry(entry); } else #endif // ENABLE_MESSAGE_DIGEST { diff --git a/src/StatCalc.h b/src/StatCalc.h index 405a9bb7..635366db 100644 --- a/src/StatCalc.h +++ b/src/StatCalc.h @@ -38,11 +38,11 @@ #include "common.h" #include "SharedHandle.h" #include "FileAllocationMan.h" +#include "CheckIntegrityMan.h" namespace aria2 { class RequestGroupMan; -class CheckIntegrityMan; class StatCalc { public: