2008-04-20 Tatsuhiro Tsujikawa <tujikawa at rednoah dot com>

Eliminates the time lag between sequential downloads and 
commands in
	the same RequestGroup.
	In old implementation, aria2 occasionally waits 1 seconds before
	executing next command or RequestGroup.
	This is really a waste of time, and new implementation 
eliminates
	this unnecessary time lag.
	* src/AbstractCommand.cc
	* src/AutoSaveCommand.cc
	* src/CheckIntegrityCommand.cc
	* src/Command.{cc, h}
	* src/DownloadEngine.{cc, h}
	* src/DownloadEngineFactory.cc
	* src/FileAllocationCommand.cc
	* src/FileAllocationDispatcherCommand.cc
	* src/FillRequestGroupCommand.cc
	* src/FtpInitiateConnectionCommand.cc
	* src/HaveEraseCommand.cc
	* src/HttpInitiateConnectionCommand.cc
	* src/HttpResponseCommand.cc
	* src/RealtimeCommand.cc
	* src/RequestGroup.cc
	* src/RequestGroupMan.cc
	* src/StreamFileAllocationEntry.cc
	* src/TimeBasedCommand.{cc, h}
	* src/TimedHaltCommand.cc

ChangeLog

@@ -1,3 +1,31 @@
+2008-04-20  Tatsuhiro Tsujikawa  <tujikawa at rednoah dot com>
+
+	Eliminates the time lag between sequential downloads and commands in
+	the same RequestGroup.
+	In old implementation, aria2 occasionally waits 1 seconds before
+	executing next command or RequestGroup.
+	This is really a waste of time, and new implementation eliminates
+	this unnecessary time lag.
+	* src/AbstractCommand.cc
+	* src/AutoSaveCommand.cc
+	* src/CheckIntegrityCommand.cc
+	* src/Command.{cc, h}
+	* src/DownloadEngine.{cc, h}
+	* src/DownloadEngineFactory.cc
+	* src/FileAllocationCommand.cc
+	* src/FileAllocationDispatcherCommand.cc
+	* src/FillRequestGroupCommand.cc
+	* src/FtpInitiateConnectionCommand.cc
+	* src/HaveEraseCommand.cc
+	* src/HttpInitiateConnectionCommand.cc
+	* src/HttpResponseCommand.cc
+	* src/RealtimeCommand.cc
+	* src/RequestGroup.cc
+	* src/RequestGroupMan.cc
+	* src/StreamFileAllocationEntry.cc
+	* src/TimeBasedCommand.{cc, h}
+	* src/TimedHaltCommand.cc
+
 2008-04-20  Tatsuhiro Tsujikawa  <tujikawa at rednoah dot com>
 
 	Rewritten SharedHandle. Now copy constructor taking raw pointer has

src/AbstractCommand.cc

@@ -190,6 +190,7 @@ bool AbstractCommand::prepareForRetry(time_t wait) {
   }
   Command* command = InitiateConnectionCommandFactory::createInitiateConnectionCommand(cuid, req, _requestGroup, e);
   if(wait == 0) {
+    e->setNoWait(true);
     e->commands.push_back(command);
   } else {
     SleepCommand* scom = new SleepCommand(cuid, e, command, wait);
@@ -318,6 +319,7 @@ void AbstractCommand::prepareForNextAction(Command* nextCommand)
 {
   CheckIntegrityEntryHandle entry(new StreamCheckIntegrityEntry(req, _requestGroup, nextCommand));
   e->addCommand(_requestGroup->processCheckIntegrityEntry(entry, e));
+  e->setNoWait(true);
 }
 
 } // namespace aria2

src/AutoSaveCommand.cc

@@ -39,7 +39,7 @@
 namespace aria2 {
 
 AutoSaveCommand::AutoSaveCommand(int32_t cuid, DownloadEngine* e, time_t interval):
-  TimeBasedCommand(cuid, e, interval) {}
+  TimeBasedCommand(cuid, e, interval, true) {}
 
 AutoSaveCommand::~AutoSaveCommand() {}
 

src/CheckIntegrityCommand.cc

@@ -71,6 +71,7 @@ bool CheckIntegrityCommand::executeInternal()
 		    _requestGroup->getFilePath().c_str());
       _e->addCommand(_entry->onDownloadIncomplete(_e));
     }
+    _e->setNoWait(true);
     return true;
   } else {
     _e->commands.push_back(this);

src/Command.cc

@@ -55,4 +55,9 @@ void Command::transitStatus()
     }
 }
 
+void Command::setStatus(STATUS status)
+{
+  this->status = status;
+}
+
 } // namespace aria2

src/Command.h

@@ -51,7 +51,8 @@ public:
     STATUS_ALL,
     STATUS_INACTIVE,
     STATUS_ACTIVE,
-    STATUS_REALTIME
+    STATUS_REALTIME,
+    STATUS_ONESHOT_REALTIME
   };
 private:
   CommandUuid uuid;
@@ -77,6 +78,8 @@ public:
 
   void setStatusRealtime() { this->status = STATUS_REALTIME; }
 
+  void setStatus(STATUS status);
+
   bool statusMatch(Command::STATUS statusFilter) const
   {
     return statusFilter <= status;

src/DownloadEngine.cc

@@ -85,7 +85,7 @@ bool NameResolverEntry::operator==(const NameResolverEntry& entry)
 
 DownloadEngine::DownloadEngine():logger(LogFactory::getInstance()),
 				 _haltRequested(false),
-				 noWait(false)
+				 _noWait(false)
 {}
 
 DownloadEngine::~DownloadEngine() {
@@ -105,7 +105,8 @@ void DownloadEngine::cleanQueue() {
   commands.clear();
 }
 
-void DownloadEngine::executeCommand(Command::STATUS statusFilter)
+static void executeCommand(std::deque<Command*>& commands,
+			   Command::STATUS statusFilter)
 {
   size_t max = commands.size();
   for(size_t i = 0; i < max; i++) {
@@ -126,19 +127,19 @@ void DownloadEngine::executeCommand(Command::STATUS statusFilter)
 void DownloadEngine::run() {
   Time cp;
   cp.setTimeInSec(0);
-  Commands activeCommands;
   while(!commands.empty()) {
     if(cp.elapsed(1)) {
       cp.reset();
-      executeCommand(Command::STATUS_ALL);
+      executeCommand(commands, Command::STATUS_ALL);
     } else {
-      executeCommand(Command::STATUS_ACTIVE);
+      executeCommand(commands, Command::STATUS_ACTIVE);
     }
+    executeCommand(_routineCommands, Command::STATUS_ALL);
     afterEachIteration();
     if(!commands.empty()) {
       waitData();
     }
-    noWait = false;
+    _noWait = false;
     calculateStatistics();
   }
   onEndOfRun();
@@ -173,7 +174,7 @@ void DownloadEngine::waitData() {
   }
 #endif // ENABLE_ASYNC_DNS
 
-  tv.tv_sec = noWait ? 0 : 1;
+  tv.tv_sec = _noWait ? 0 : 1;
   tv.tv_usec = 0;
   int retval = select(fdmax+1, &rfds, &wfds, NULL, &tv);
   if(retval > 0) {
@@ -350,4 +351,14 @@ bool DownloadEngine::deleteNameResolverCheck(const NameResolverHandle& resolver,
 }
 #endif // ENABLE_ASYNC_DNS
 
+void DownloadEngine::setNoWait(bool b)
+{
+  _noWait = b;
+}
+
+void DownloadEngine::addRoutineCommand(Command* command)
+{
+  _routineCommands.push_back(command);
+}
+
 } // namespace aria2

src/DownloadEngine.h

@@ -107,7 +107,6 @@ private:
   void shortSleep() const;
   bool addSocket(const SocketEntry& socketEntry);
   bool deleteSocket(const SocketEntry& socketEntry);
-  void executeCommand(Command::STATUS statusFilter);
 
   /**
    * Delegates to StatCalc
@@ -118,8 +117,12 @@ private:
 
   void afterEachIteration();
 
+private:
+  bool _noWait;
+
+  std::deque<Command*> _routineCommands;
+
 public:
-  bool noWait;
   std::deque<Command*> commands;
   SharedHandle<RequestGroupMan> _requestGroupMan;
   SharedHandle<FileAllocationMan> _fileAllocationMan;
@@ -163,6 +166,10 @@ public:
   }
 
   void requestHalt();
+
+  void setNoWait(bool b);
+
+  void addRoutineCommand(Command* command);
 };
 
 typedef SharedHandle<DownloadEngine> DownloadEngineHandle;

src/DownloadEngineFactory.cc

@@ -83,14 +83,14 @@ DownloadEngineFactory::newDownloadEngine(Option* op,
 #ifdef ENABLE_MESSAGE_DIGEST
   e->_checkIntegrityMan.reset(new CheckIntegrityMan());
 #endif // ENABLE_MESSAGE_DIGEST
-  e->commands.push_back(new FillRequestGroupCommand(CUIDCounterSingletonHolder::instance()->newID(), e.get(), 1));
+  e->addRoutineCommand(new FillRequestGroupCommand(CUIDCounterSingletonHolder::instance()->newID(), e.get(), 1));
-  e->commands.push_back(new FileAllocationDispatcherCommand(CUIDCounterSingletonHolder::instance()->newID(), e.get()));
+  e->addRoutineCommand(new FileAllocationDispatcherCommand(CUIDCounterSingletonHolder::instance()->newID(), e.get()));
-  e->commands.push_back(new AutoSaveCommand(CUIDCounterSingletonHolder::instance()->newID(), e.get(), op->getAsInt(PREF_AUTO_SAVE_INTERVAL)));
+  e->addRoutineCommand(new AutoSaveCommand(CUIDCounterSingletonHolder::instance()->newID(), e.get(), op->getAsInt(PREF_AUTO_SAVE_INTERVAL)));
-  e->commands.push_back(new HaveEraseCommand(CUIDCounterSingletonHolder::instance()->newID(), e.get(), 10));
+  e->addRoutineCommand(new HaveEraseCommand(CUIDCounterSingletonHolder::instance()->newID(), e.get(), 10));
   {
     time_t stopSec = op->getAsInt(PREF_STOP);
     if(stopSec > 0) {
-      e->commands.push_back(new TimedHaltCommand(CUIDCounterSingletonHolder::instance()->newID(), e.get(), stopSec));
+      e->addRoutineCommand(new TimedHaltCommand(CUIDCounterSingletonHolder::instance()->newID(), e.get(), stopSec));
     }
   }
   return e;

src/FileAllocationCommand.cc

@@ -63,7 +63,7 @@ bool FileAllocationCommand::executeInternal()
     _e->_fileAllocationMan->markCurrentFileAllocationEntryDone();
     
     _e->addCommand(_fileAllocationEntry->prepareForNextAction(_e));
-    
+    _e->setNoWait(true);
     return true;
   } else {
     _e->commands.push_back(this);

src/FileAllocationDispatcherCommand.cc

@@ -68,10 +68,10 @@ bool FileAllocationDispatcherCommand::execute()
 				_e,
 				entry);
     _e->commands.push_back(command);
-    _e->noWait = true;
+    _e->setNoWait(true);
   }
 
-  _e->commands.push_back(this);
+  _e->addRoutineCommand(this);
 
   return false;
 }

src/FillRequestGroupCommand.cc

@@ -68,7 +68,7 @@ bool FillRequestGroupCommand::execute()
   if(_e->_requestGroupMan->downloadFinished()) {
     return true;
   }
-  _e->commands.push_back(this);
+  _e->addRoutineCommand(this);
   return false;
 }
 

src/FtpInitiateConnectionCommand.cc

@@ -61,7 +61,7 @@ FtpInitiateConnectionCommand::FtpInitiateConnectionCommand(int cuid,
    nameResolver(new NameResolver())
 {
   setTimeout(e->option->getAsInt(PREF_DNS_TIMEOUT));
-  setStatusActive();
+  setStatus(Command::STATUS_ONESHOT_REALTIME);
   disableReadCheckSocket();
   disableWriteCheckSocket();
 }

src/HaveEraseCommand.cc

@@ -41,7 +41,7 @@
 namespace aria2 {
 
 HaveEraseCommand::HaveEraseCommand(int32_t cuid, DownloadEngine* e, time_t interval)
-  :TimeBasedCommand(cuid, e, interval) {}
+  :TimeBasedCommand(cuid, e, interval, true) {}
 
 HaveEraseCommand::~HaveEraseCommand() {}
 

src/HttpInitiateConnectionCommand.cc

@@ -59,7 +59,7 @@ HttpInitiateConnectionCommand::HttpInitiateConnectionCommand(int cuid,
   nameResolver(new NameResolver())
 {
   setTimeout(e->option->getAsInt(PREF_DNS_TIMEOUT));
-  setStatusActive();
+  setStatus(Command::STATUS_ONESHOT_REALTIME);
   disableReadCheckSocket();
   disableWriteCheckSocket();
 }

src/HttpResponseCommand.cc

@@ -91,7 +91,6 @@ bool HttpResponseCommand::executeInternal()
   if(httpResponse->isRedirect()) {
     httpResponse->processRedirect();
     logger->info(MSG_REDIRECT, cuid, httpResponse->getRedirectURI().c_str());
-    e->noWait = true;
     return prepareForRetry(0);
   }
   if(!_requestGroup->isSingleHostMultiConnectionEnabled()) {
@@ -152,7 +151,6 @@ bool HttpResponseCommand::handleDefaultEncoding(const HttpResponseHandle& httpRe
       _requestGroup->getSegmentMan()->cancelSegment(cuid);
     }
     prepareForNextAction(command);
-    e->noWait = true;
   } catch(Exception* e) {
     delete command;
     throw;

src/RealtimeCommand.cc

@@ -41,7 +41,7 @@ namespace aria2 {
 bool RealtimeCommand::execute()
 {
   setStatusRealtime();
-  _e->noWait = true;
+  _e->setNoWait(true);
   try {
     return executeInternal();
   } catch(Exception* e) {

src/RequestGroup.cc

@@ -460,6 +460,8 @@ Commands RequestGroup::createNextCommand(DownloadEngine* e, unsigned int numComm
 	Command* command = InitiateConnectionCommandFactory::createInitiateConnectionCommand(CUIDCounterSingletonHolder::instance()->newID(), req, this, e);
 	ServerHostHandle sv(new ServerHost(command->getCuid(), req->getHost()));
 	registerServerHost(sv);
+	// give a chance to be executed in the next loop in DownloadEngine
+	command->setStatus(Command::STATUS_ONESHOT_REALTIME);
 	commands.push_back(command);
       } else {
 	pendingURIs.push_front(uri);

src/RequestGroupMan.cc

@@ -166,6 +166,7 @@ void RequestGroupMan::fillRequestGroupFromReserver(DownloadEngine* e)
   }
   std::copy(temp.begin(), temp.end(), std::front_inserter(_reservedGroups));
   if(count > 0) {
+    e->setNoWait(true);
     _logger->debug("%d RequestGroup(s) added.", count);
   }
 }

src/StreamFileAllocationEntry.cc

@@ -57,6 +57,8 @@ Commands StreamFileAllocationEntry::prepareForNextAction(DownloadEngine* e)
 {
   Commands commands;
   if(_nextCommand) {
+    // give _nextCommand a chance to execute in the next execution loop.
+    _nextCommand->setStatus(Command::STATUS_ONESHOT_REALTIME);
     commands.push_back(popNextCommand());
     // try remaining uris
     Commands streamCommands = _requestGroup->createNextCommandWithAdj(e, -1);

src/TimeBasedCommand.cc

@@ -37,8 +37,11 @@
 
 namespace aria2 {
 
-TimeBasedCommand::TimeBasedCommand(int32_t cuid, DownloadEngine* e, time_t interval):
+TimeBasedCommand::TimeBasedCommand(int32_t cuid, DownloadEngine* e,
-  Command(cuid), _e(e),_exit(false), _interval(interval) {}
+				   time_t interval,
+				   bool routineCommand):
+  Command(cuid), _e(e),_exit(false), _interval(interval),
+  _routineCommand(routineCommand) {}
 
 TimeBasedCommand::~TimeBasedCommand() {}
 
@@ -59,7 +62,11 @@ bool TimeBasedCommand::execute()
   if(_exit) {
     return true;
   }
-  _e->commands.push_back(this);
+  if(_routineCommand) {
+    _e->addRoutineCommand(this);
+  } else {
+    _e->commands.push_back(this);
+  }
   return false;
 }
 

src/TimeBasedCommand.h

@@ -57,6 +57,8 @@ protected:
   bool _exit;
 
   time_t _interval; // unit: sec
+
+  bool _routineCommand;
 private:
   Time _checkPoint;
 public:
@@ -77,7 +79,8 @@ public:
   virtual void postProcess() {};
 
 public:
-  TimeBasedCommand(int32_t cuid, DownloadEngine* e, time_t interval);
+  TimeBasedCommand(int32_t cuid, DownloadEngine* e, time_t interval,
+		   bool routineCommand = false);
 
   virtual ~TimeBasedCommand();
 

src/TimedHaltCommand.cc

@@ -42,7 +42,7 @@ namespace aria2 {
 
 TimedHaltCommand::TimedHaltCommand(int32_t cuid, DownloadEngine* e,
 				   time_t secondsToHalt):
-  TimeBasedCommand(cuid, e, secondsToHalt) {}
+  TimeBasedCommand(cuid, e, secondsToHalt, true) {}
 
 TimedHaltCommand::~TimedHaltCommand() {}