From c1c5e7369f77a1125b12ac58b8b30378858d9317 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 20 Apr 2008 05:42:15 +0000 Subject: [PATCH] 2008-04-20 Tatsuhiro Tsujikawa Eliminates the time lag between sequential downloads and commands in the same RequestGroup. In old implementation, aria2 occasionally waits 1 seconds before executing next command or RequestGroup. This is really a waste of time, and new implementation eliminates this unnecessary time lag. * src/AbstractCommand.cc * src/AutoSaveCommand.cc * src/CheckIntegrityCommand.cc * src/Command.{cc, h} * src/DownloadEngine.{cc, h} * src/DownloadEngineFactory.cc * src/FileAllocationCommand.cc * src/FileAllocationDispatcherCommand.cc * src/FillRequestGroupCommand.cc * src/FtpInitiateConnectionCommand.cc * src/HaveEraseCommand.cc * src/HttpInitiateConnectionCommand.cc * src/HttpResponseCommand.cc * src/RealtimeCommand.cc * src/RequestGroup.cc * src/RequestGroupMan.cc * src/StreamFileAllocationEntry.cc * src/TimeBasedCommand.{cc, h} * src/TimedHaltCommand.cc --- ChangeLog | 28 ++++++++++++++++++++++++++ src/AbstractCommand.cc | 2 ++ src/AutoSaveCommand.cc | 2 +- src/CheckIntegrityCommand.cc | 1 + src/Command.cc | 5 +++++ src/Command.h | 5 ++++- src/DownloadEngine.cc | 25 ++++++++++++++++------- src/DownloadEngine.h | 11 ++++++++-- src/DownloadEngineFactory.cc | 10 ++++----- src/FileAllocationCommand.cc | 2 +- src/FileAllocationDispatcherCommand.cc | 4 ++-- src/FillRequestGroupCommand.cc | 2 +- src/FtpInitiateConnectionCommand.cc | 2 +- src/HaveEraseCommand.cc | 2 +- src/HttpInitiateConnectionCommand.cc | 2 +- src/HttpResponseCommand.cc | 2 -- src/RealtimeCommand.cc | 2 +- src/RequestGroup.cc | 2 ++ src/RequestGroupMan.cc | 1 + src/StreamFileAllocationEntry.cc | 2 ++ src/TimeBasedCommand.cc | 13 +++++++++--- src/TimeBasedCommand.h | 5 ++++- src/TimedHaltCommand.cc | 2 +- 23 files changed, 101 insertions(+), 31 deletions(-) diff --git a/ChangeLog b/ChangeLog index cff39669..38d5a0ad 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,31 @@ +2008-04-20 Tatsuhiro Tsujikawa + + Eliminates the time lag between sequential downloads and commands in + the same RequestGroup. + In old implementation, aria2 occasionally waits 1 seconds before + executing next command or RequestGroup. + This is really a waste of time, and new implementation eliminates + this unnecessary time lag. + * src/AbstractCommand.cc + * src/AutoSaveCommand.cc + * src/CheckIntegrityCommand.cc + * src/Command.{cc, h} + * src/DownloadEngine.{cc, h} + * src/DownloadEngineFactory.cc + * src/FileAllocationCommand.cc + * src/FileAllocationDispatcherCommand.cc + * src/FillRequestGroupCommand.cc + * src/FtpInitiateConnectionCommand.cc + * src/HaveEraseCommand.cc + * src/HttpInitiateConnectionCommand.cc + * src/HttpResponseCommand.cc + * src/RealtimeCommand.cc + * src/RequestGroup.cc + * src/RequestGroupMan.cc + * src/StreamFileAllocationEntry.cc + * src/TimeBasedCommand.{cc, h} + * src/TimedHaltCommand.cc + 2008-04-20 Tatsuhiro Tsujikawa Rewritten SharedHandle. Now copy constructor taking raw pointer has diff --git a/src/AbstractCommand.cc b/src/AbstractCommand.cc index 7482c6a8..e808e0db 100644 --- a/src/AbstractCommand.cc +++ b/src/AbstractCommand.cc @@ -190,6 +190,7 @@ bool AbstractCommand::prepareForRetry(time_t wait) { } Command* command = InitiateConnectionCommandFactory::createInitiateConnectionCommand(cuid, req, _requestGroup, e); if(wait == 0) { + e->setNoWait(true); e->commands.push_back(command); } else { SleepCommand* scom = new SleepCommand(cuid, e, command, wait); @@ -318,6 +319,7 @@ void AbstractCommand::prepareForNextAction(Command* nextCommand) { CheckIntegrityEntryHandle entry(new StreamCheckIntegrityEntry(req, _requestGroup, nextCommand)); e->addCommand(_requestGroup->processCheckIntegrityEntry(entry, e)); + e->setNoWait(true); } } // namespace aria2 diff --git a/src/AutoSaveCommand.cc b/src/AutoSaveCommand.cc index e6c6d622..ad5001bc 100644 --- a/src/AutoSaveCommand.cc +++ b/src/AutoSaveCommand.cc @@ -39,7 +39,7 @@ namespace aria2 { AutoSaveCommand::AutoSaveCommand(int32_t cuid, DownloadEngine* e, time_t interval): - TimeBasedCommand(cuid, e, interval) {} + TimeBasedCommand(cuid, e, interval, true) {} AutoSaveCommand::~AutoSaveCommand() {} diff --git a/src/CheckIntegrityCommand.cc b/src/CheckIntegrityCommand.cc index e5c9fa11..4d6cbf23 100644 --- a/src/CheckIntegrityCommand.cc +++ b/src/CheckIntegrityCommand.cc @@ -71,6 +71,7 @@ bool CheckIntegrityCommand::executeInternal() _requestGroup->getFilePath().c_str()); _e->addCommand(_entry->onDownloadIncomplete(_e)); } + _e->setNoWait(true); return true; } else { _e->commands.push_back(this); diff --git a/src/Command.cc b/src/Command.cc index c7b22f7f..fa733e47 100644 --- a/src/Command.cc +++ b/src/Command.cc @@ -55,4 +55,9 @@ void Command::transitStatus() } } +void Command::setStatus(STATUS status) +{ + this->status = status; +} + } // namespace aria2 diff --git a/src/Command.h b/src/Command.h index 5e2e4e13..f2f24282 100644 --- a/src/Command.h +++ b/src/Command.h @@ -51,7 +51,8 @@ public: STATUS_ALL, STATUS_INACTIVE, STATUS_ACTIVE, - STATUS_REALTIME + STATUS_REALTIME, + STATUS_ONESHOT_REALTIME }; private: CommandUuid uuid; @@ -77,6 +78,8 @@ public: void setStatusRealtime() { this->status = STATUS_REALTIME; } + void setStatus(STATUS status); + bool statusMatch(Command::STATUS statusFilter) const { return statusFilter <= status; diff --git a/src/DownloadEngine.cc b/src/DownloadEngine.cc index 85ece318..46844246 100644 --- a/src/DownloadEngine.cc +++ b/src/DownloadEngine.cc @@ -85,7 +85,7 @@ bool NameResolverEntry::operator==(const NameResolverEntry& entry) DownloadEngine::DownloadEngine():logger(LogFactory::getInstance()), _haltRequested(false), - noWait(false) + _noWait(false) {} DownloadEngine::~DownloadEngine() { @@ -105,7 +105,8 @@ void DownloadEngine::cleanQueue() { commands.clear(); } -void DownloadEngine::executeCommand(Command::STATUS statusFilter) +static void executeCommand(std::deque& commands, + Command::STATUS statusFilter) { size_t max = commands.size(); for(size_t i = 0; i < max; i++) { @@ -126,19 +127,19 @@ void DownloadEngine::executeCommand(Command::STATUS statusFilter) void DownloadEngine::run() { Time cp; cp.setTimeInSec(0); - Commands activeCommands; while(!commands.empty()) { if(cp.elapsed(1)) { cp.reset(); - executeCommand(Command::STATUS_ALL); + executeCommand(commands, Command::STATUS_ALL); } else { - executeCommand(Command::STATUS_ACTIVE); + executeCommand(commands, Command::STATUS_ACTIVE); } + executeCommand(_routineCommands, Command::STATUS_ALL); afterEachIteration(); if(!commands.empty()) { waitData(); } - noWait = false; + _noWait = false; calculateStatistics(); } onEndOfRun(); @@ -173,7 +174,7 @@ void DownloadEngine::waitData() { } #endif // ENABLE_ASYNC_DNS - tv.tv_sec = noWait ? 0 : 1; + tv.tv_sec = _noWait ? 0 : 1; tv.tv_usec = 0; int retval = select(fdmax+1, &rfds, &wfds, NULL, &tv); if(retval > 0) { @@ -350,4 +351,14 @@ bool DownloadEngine::deleteNameResolverCheck(const NameResolverHandle& resolver, } #endif // ENABLE_ASYNC_DNS +void DownloadEngine::setNoWait(bool b) +{ + _noWait = b; +} + +void DownloadEngine::addRoutineCommand(Command* command) +{ + _routineCommands.push_back(command); +} + } // namespace aria2 diff --git a/src/DownloadEngine.h b/src/DownloadEngine.h index 18c9b646..f134299a 100644 --- a/src/DownloadEngine.h +++ b/src/DownloadEngine.h @@ -107,7 +107,6 @@ private: void shortSleep() const; bool addSocket(const SocketEntry& socketEntry); bool deleteSocket(const SocketEntry& socketEntry); - void executeCommand(Command::STATUS statusFilter); /** * Delegates to StatCalc @@ -118,8 +117,12 @@ private: void afterEachIteration(); +private: + bool _noWait; + + std::deque _routineCommands; + public: - bool noWait; std::deque commands; SharedHandle _requestGroupMan; SharedHandle _fileAllocationMan; @@ -163,6 +166,10 @@ public: } void requestHalt(); + + void setNoWait(bool b); + + void addRoutineCommand(Command* command); }; typedef SharedHandle DownloadEngineHandle; diff --git a/src/DownloadEngineFactory.cc b/src/DownloadEngineFactory.cc index 39875e90..09d7c9d2 100644 --- a/src/DownloadEngineFactory.cc +++ b/src/DownloadEngineFactory.cc @@ -83,14 +83,14 @@ DownloadEngineFactory::newDownloadEngine(Option* op, #ifdef ENABLE_MESSAGE_DIGEST e->_checkIntegrityMan.reset(new CheckIntegrityMan()); #endif // ENABLE_MESSAGE_DIGEST - e->commands.push_back(new FillRequestGroupCommand(CUIDCounterSingletonHolder::instance()->newID(), e.get(), 1)); - e->commands.push_back(new FileAllocationDispatcherCommand(CUIDCounterSingletonHolder::instance()->newID(), e.get())); - e->commands.push_back(new AutoSaveCommand(CUIDCounterSingletonHolder::instance()->newID(), e.get(), op->getAsInt(PREF_AUTO_SAVE_INTERVAL))); - e->commands.push_back(new HaveEraseCommand(CUIDCounterSingletonHolder::instance()->newID(), e.get(), 10)); + e->addRoutineCommand(new FillRequestGroupCommand(CUIDCounterSingletonHolder::instance()->newID(), e.get(), 1)); + e->addRoutineCommand(new FileAllocationDispatcherCommand(CUIDCounterSingletonHolder::instance()->newID(), e.get())); + e->addRoutineCommand(new AutoSaveCommand(CUIDCounterSingletonHolder::instance()->newID(), e.get(), op->getAsInt(PREF_AUTO_SAVE_INTERVAL))); + e->addRoutineCommand(new HaveEraseCommand(CUIDCounterSingletonHolder::instance()->newID(), e.get(), 10)); { time_t stopSec = op->getAsInt(PREF_STOP); if(stopSec > 0) { - e->commands.push_back(new TimedHaltCommand(CUIDCounterSingletonHolder::instance()->newID(), e.get(), stopSec)); + e->addRoutineCommand(new TimedHaltCommand(CUIDCounterSingletonHolder::instance()->newID(), e.get(), stopSec)); } } return e; diff --git a/src/FileAllocationCommand.cc b/src/FileAllocationCommand.cc index 839af620..79e1d059 100644 --- a/src/FileAllocationCommand.cc +++ b/src/FileAllocationCommand.cc @@ -63,7 +63,7 @@ bool FileAllocationCommand::executeInternal() _e->_fileAllocationMan->markCurrentFileAllocationEntryDone(); _e->addCommand(_fileAllocationEntry->prepareForNextAction(_e)); - + _e->setNoWait(true); return true; } else { _e->commands.push_back(this); diff --git a/src/FileAllocationDispatcherCommand.cc b/src/FileAllocationDispatcherCommand.cc index 489d6b2a..ae73b7e9 100644 --- a/src/FileAllocationDispatcherCommand.cc +++ b/src/FileAllocationDispatcherCommand.cc @@ -68,10 +68,10 @@ bool FileAllocationDispatcherCommand::execute() _e, entry); _e->commands.push_back(command); - _e->noWait = true; + _e->setNoWait(true); } - _e->commands.push_back(this); + _e->addRoutineCommand(this); return false; } diff --git a/src/FillRequestGroupCommand.cc b/src/FillRequestGroupCommand.cc index 72b0270a..3ba7b7ad 100644 --- a/src/FillRequestGroupCommand.cc +++ b/src/FillRequestGroupCommand.cc @@ -68,7 +68,7 @@ bool FillRequestGroupCommand::execute() if(_e->_requestGroupMan->downloadFinished()) { return true; } - _e->commands.push_back(this); + _e->addRoutineCommand(this); return false; } diff --git a/src/FtpInitiateConnectionCommand.cc b/src/FtpInitiateConnectionCommand.cc index 54ae0a9c..9abb0574 100644 --- a/src/FtpInitiateConnectionCommand.cc +++ b/src/FtpInitiateConnectionCommand.cc @@ -61,7 +61,7 @@ FtpInitiateConnectionCommand::FtpInitiateConnectionCommand(int cuid, nameResolver(new NameResolver()) { setTimeout(e->option->getAsInt(PREF_DNS_TIMEOUT)); - setStatusActive(); + setStatus(Command::STATUS_ONESHOT_REALTIME); disableReadCheckSocket(); disableWriteCheckSocket(); } diff --git a/src/HaveEraseCommand.cc b/src/HaveEraseCommand.cc index 04844615..4ad39505 100644 --- a/src/HaveEraseCommand.cc +++ b/src/HaveEraseCommand.cc @@ -41,7 +41,7 @@ namespace aria2 { HaveEraseCommand::HaveEraseCommand(int32_t cuid, DownloadEngine* e, time_t interval) - :TimeBasedCommand(cuid, e, interval) {} + :TimeBasedCommand(cuid, e, interval, true) {} HaveEraseCommand::~HaveEraseCommand() {} diff --git a/src/HttpInitiateConnectionCommand.cc b/src/HttpInitiateConnectionCommand.cc index 2ef6b812..f0e6c6d2 100644 --- a/src/HttpInitiateConnectionCommand.cc +++ b/src/HttpInitiateConnectionCommand.cc @@ -59,7 +59,7 @@ HttpInitiateConnectionCommand::HttpInitiateConnectionCommand(int cuid, nameResolver(new NameResolver()) { setTimeout(e->option->getAsInt(PREF_DNS_TIMEOUT)); - setStatusActive(); + setStatus(Command::STATUS_ONESHOT_REALTIME); disableReadCheckSocket(); disableWriteCheckSocket(); } diff --git a/src/HttpResponseCommand.cc b/src/HttpResponseCommand.cc index b92ef5ce..4e0f2870 100644 --- a/src/HttpResponseCommand.cc +++ b/src/HttpResponseCommand.cc @@ -91,7 +91,6 @@ bool HttpResponseCommand::executeInternal() if(httpResponse->isRedirect()) { httpResponse->processRedirect(); logger->info(MSG_REDIRECT, cuid, httpResponse->getRedirectURI().c_str()); - e->noWait = true; return prepareForRetry(0); } if(!_requestGroup->isSingleHostMultiConnectionEnabled()) { @@ -152,7 +151,6 @@ bool HttpResponseCommand::handleDefaultEncoding(const HttpResponseHandle& httpRe _requestGroup->getSegmentMan()->cancelSegment(cuid); } prepareForNextAction(command); - e->noWait = true; } catch(Exception* e) { delete command; throw; diff --git a/src/RealtimeCommand.cc b/src/RealtimeCommand.cc index 2cdecbe4..932154d4 100644 --- a/src/RealtimeCommand.cc +++ b/src/RealtimeCommand.cc @@ -41,7 +41,7 @@ namespace aria2 { bool RealtimeCommand::execute() { setStatusRealtime(); - _e->noWait = true; + _e->setNoWait(true); try { return executeInternal(); } catch(Exception* e) { diff --git a/src/RequestGroup.cc b/src/RequestGroup.cc index bdfc6736..7f8c33bf 100644 --- a/src/RequestGroup.cc +++ b/src/RequestGroup.cc @@ -460,6 +460,8 @@ Commands RequestGroup::createNextCommand(DownloadEngine* e, unsigned int numComm Command* command = InitiateConnectionCommandFactory::createInitiateConnectionCommand(CUIDCounterSingletonHolder::instance()->newID(), req, this, e); ServerHostHandle sv(new ServerHost(command->getCuid(), req->getHost())); registerServerHost(sv); + // give a chance to be executed in the next loop in DownloadEngine + command->setStatus(Command::STATUS_ONESHOT_REALTIME); commands.push_back(command); } else { pendingURIs.push_front(uri); diff --git a/src/RequestGroupMan.cc b/src/RequestGroupMan.cc index cd713fb3..8bcd7bca 100644 --- a/src/RequestGroupMan.cc +++ b/src/RequestGroupMan.cc @@ -166,6 +166,7 @@ void RequestGroupMan::fillRequestGroupFromReserver(DownloadEngine* e) } std::copy(temp.begin(), temp.end(), std::front_inserter(_reservedGroups)); if(count > 0) { + e->setNoWait(true); _logger->debug("%d RequestGroup(s) added.", count); } } diff --git a/src/StreamFileAllocationEntry.cc b/src/StreamFileAllocationEntry.cc index c7db4a83..4e213fb9 100644 --- a/src/StreamFileAllocationEntry.cc +++ b/src/StreamFileAllocationEntry.cc @@ -57,6 +57,8 @@ Commands StreamFileAllocationEntry::prepareForNextAction(DownloadEngine* e) { Commands commands; if(_nextCommand) { + // give _nextCommand a chance to execute in the next execution loop. + _nextCommand->setStatus(Command::STATUS_ONESHOT_REALTIME); commands.push_back(popNextCommand()); // try remaining uris Commands streamCommands = _requestGroup->createNextCommandWithAdj(e, -1); diff --git a/src/TimeBasedCommand.cc b/src/TimeBasedCommand.cc index 5628aa74..95f01e9b 100644 --- a/src/TimeBasedCommand.cc +++ b/src/TimeBasedCommand.cc @@ -37,8 +37,11 @@ namespace aria2 { -TimeBasedCommand::TimeBasedCommand(int32_t cuid, DownloadEngine* e, time_t interval): - Command(cuid), _e(e),_exit(false), _interval(interval) {} +TimeBasedCommand::TimeBasedCommand(int32_t cuid, DownloadEngine* e, + time_t interval, + bool routineCommand): + Command(cuid), _e(e),_exit(false), _interval(interval), + _routineCommand(routineCommand) {} TimeBasedCommand::~TimeBasedCommand() {} @@ -59,7 +62,11 @@ bool TimeBasedCommand::execute() if(_exit) { return true; } - _e->commands.push_back(this); + if(_routineCommand) { + _e->addRoutineCommand(this); + } else { + _e->commands.push_back(this); + } return false; } diff --git a/src/TimeBasedCommand.h b/src/TimeBasedCommand.h index ad105e4b..ed049170 100644 --- a/src/TimeBasedCommand.h +++ b/src/TimeBasedCommand.h @@ -57,6 +57,8 @@ protected: bool _exit; time_t _interval; // unit: sec + + bool _routineCommand; private: Time _checkPoint; public: @@ -77,7 +79,8 @@ public: virtual void postProcess() {}; public: - TimeBasedCommand(int32_t cuid, DownloadEngine* e, time_t interval); + TimeBasedCommand(int32_t cuid, DownloadEngine* e, time_t interval, + bool routineCommand = false); virtual ~TimeBasedCommand(); diff --git a/src/TimedHaltCommand.cc b/src/TimedHaltCommand.cc index 1d35766e..48e1170f 100644 --- a/src/TimedHaltCommand.cc +++ b/src/TimedHaltCommand.cc @@ -42,7 +42,7 @@ namespace aria2 { TimedHaltCommand::TimedHaltCommand(int32_t cuid, DownloadEngine* e, time_t secondsToHalt): - TimeBasedCommand(cuid, e, secondsToHalt) {} + TimeBasedCommand(cuid, e, secondsToHalt, true) {} TimedHaltCommand::~TimedHaltCommand() {}