2007-06-03 Tatsuhiro Tsujikawa <tujikawa at rednoah dot com>

RequestGroup::getNextCommand() was renamed to 
createNextCommand().
	Added its overloaded method.
	* src/RequestGroup.h 
	(_numConcurrentCommand): New variable.
	(setNumConcurrentCommand): New function.
	* src/RequestGroup.cc
	
	Abort download if same file is being downloaded concurrently.
	* src/RequestGroup.h, src/RequestGroupMan.cc
	(isSameFileBeingDownloaded): New function.
	* src/HttpResponseCommand.cc (executeInternal)
	* src/FtpNegotiateCommand.cc (recvSize)
	* src/message.h (EX_DUPLICATE_FILE_DOWNLOAD): New definition.
	
	* main.cc: Added help message for -i option.

ChangeLog

@@ -1,3 +1,21 @@
+2007-06-03  Tatsuhiro Tsujikawa  <tujikawa at rednoah dot com>
+
+	RequestGroup::getNextCommand() was renamed to createNextCommand().
+	Added its overloaded method.
+	* src/RequestGroup.h 
+	(_numConcurrentCommand): New variable.
+	(setNumConcurrentCommand): New function.
+	* src/RequestGroup.cc
+	
+	Abort download if same file is being downloaded concurrently.
+	* src/RequestGroup.h, src/RequestGroupMan.cc
+	(isSameFileBeingDownloaded): New function.
+	* src/HttpResponseCommand.cc (executeInternal)
+	* src/FtpNegotiateCommand.cc (recvSize)
+	* src/message.h (EX_DUPLICATE_FILE_DOWNLOAD): New definition.
+	
+	* main.cc: Added help message for -i option.
+	
 2007-06-01  Tatsuhiro Tsujikawa  <tujikawa at rednoah dot com>
 
 	* src/FileAllocationCommand.cc: Derived from RealtimeCommand.

TODO

@@ -31,13 +31,6 @@
 100K/300M(10%)(3cn)(3more) 100KB/s [FileAlloc:35MB/40MB(90%)][Checksum:10MB/20MB(50%)]
 * exit status: all downloads have been successful-> EXIT_SUCCESS,
 		some of downloads have been failed -> EXIT_FAILURE
-* use hintFilename and hintTotalLength if these are provided.
-	-> test against ftp downloads
-* make sure that the same file name is not used at the same time.
 * Do not use ufilename in multi-simultaneous download mode.
-* Replace numCommandToGenerate to the value of PREF_METALINK_SERVERS
-* Do not send range header if the position of starting byte is 0 and
-the position of ending byte is not specified.
 * Create download command directly when 1connection download.
 Consider timeout when file allocation/check integrity is enabled.
-* Test DefaultPeerStorage

src/AbstractCommand.cc

@@ -141,7 +141,7 @@ bool AbstractCommand::execute() {
 }
 
 void AbstractCommand::tryReserved() {
-  Commands commands = _requestGroup->getNextCommand(e, 1);
+  Commands commands = _requestGroup->createNextCommand(e, 1);
   e->addCommand(commands);
 }
 

src/CheckIntegrityCommand.cc

@@ -69,8 +69,7 @@ bool CheckIntegrityCommand::executeInternal()
 	_e->commands.push_back(_nextDownloadCommand);
 	_nextDownloadCommand = 0;
       } else {
-	int32_t numCommandsToGenerate = 15;
+	Commands commands = _requestGroup->createNextCommand(_e);
-	Commands commands = _requestGroup->getNextCommand(_e, numCommandsToGenerate);
 	Command* command = InitiateConnectionCommandFactory::createInitiateConnectionCommand(cuid, _req, _requestGroup, _e);
 	commands.push_front(command);
 	_e->addCommand(commands);

src/FileAllocationCommand.cc

@@ -52,9 +52,7 @@ bool FileAllocationCommand::executeInternal()
       _e->commands.push_back(_fileAllocationEntry->getNextDownloadCommand());
       _fileAllocationEntry->setNextDownloadCommand(0);
     } else {
-      int32_t numCommandsToGenerate = 15;
+      Commands commands = _requestGroup->createNextCommand(_e);
-      Commands commands = _requestGroup->getNextCommand(_e, numCommandsToGenerate);
-      
       Command* command = InitiateConnectionCommandFactory::createInitiateConnectionCommand(cuid, _req, _requestGroup, _e);
       
       commands.push_front(command);

src/FtpNegotiationCommand.cc

@@ -197,6 +197,10 @@ bool FtpNegotiationCommand::recvSize() {
     // TODO validate filename and totalsize against hintFilename and hintTotalSize if these are provided.
     _requestGroup->validateTotalLengthByHint(size);
 
+    if(e->_requestGroupMan->isSameFileBeingDownloaded(_requestGroup)) {
+      throw new FatalException(EX_DUPLICATE_FILE_DOWNLOAD, _requestGroup->getFilePath().c_str());
+    }
+
     if(req->getMethod() == Request::METHOD_HEAD) {
       _requestGroup->getSegmentMan()->isSplittable = false; // TODO because we don't want segment file to be saved.
       sequence = SEQ_HEAD_OK;

src/HttpResponseCommand.cc

@@ -41,6 +41,7 @@
 #include "prefs.h"
 #include "File.h"
 #include "InitiateConnectionCommandFactory.h"
+#include "FatalException.h"
 #include <sys/types.h>
 #include <unistd.h>
 
@@ -98,6 +99,10 @@ bool HttpResponseCommand::executeInternal()
     _requestGroup->validateFilenameByHint(httpResponse->determinFilename());
     _requestGroup->validateTotalLengthByHint(httpResponse->getEntityLength());
 
+    if(e->_requestGroupMan->isSameFileBeingDownloaded(_requestGroup)) {
+      throw new FatalException(EX_DUPLICATE_FILE_DOWNLOAD, _requestGroup->getFilePath().c_str());
+    }
+
     if(httpResponse->isTransferEncodingSpecified()) {
       return handleOtherEncoding(httpResponse);
     } else {

src/MetalinkRequestInfo.cc

@@ -105,16 +105,12 @@ RequestInfos MetalinkRequestInfo::execute() {
 		entry->resources.end(),
 		FindBitTorrentUrl());
       Strings urls;
-      int maxConnection = 0;
       ChecksumHandle checksum = 0;
       if(itr == entry->resources.end()) {
 	entry->reorderResourcesByPreference();
 	
 	for_each(entry->resources.begin(), entry->resources.end(),
 		 AccumulateNonP2PUrl(&urls, op->getAsInt(PREF_SPLIT)));
-	maxConnection =
-	  op->getAsInt(PREF_METALINK_SERVERS)*op->getAsInt(PREF_SPLIT);
-	
 	// TODO
 	// set checksum
 	checksum = entry->checksum;
@@ -125,6 +121,7 @@ RequestInfos MetalinkRequestInfo::execute() {
       RequestGroupHandle rg = new RequestGroup(urls, op);
       rg->setHintFilename(entry->filename);
       rg->setHintTotalLength(entry->size);
+      rg->setNumConcurrentCommand(op->getAsInt(PREF_METALINK_SERVERS));
 
 #ifdef ENABLE_MESSAGE_DIGEST
       if(entry->chunkChecksum.isNull()) {

src/RequestGroup.cc

@@ -61,10 +61,16 @@ SegmentManHandle RequestGroup::initSegmentMan()
   return _segmentMan;
 }
 
-Commands RequestGroup::getNextCommand(DownloadEngine* e, int32_t maxNum, const string& method)
+Commands RequestGroup::createNextCommand(DownloadEngine* e, const string& method)
+{
+  int32_t numCommand = _numConcurrentCommand == 0 ? _uris.size() : _numConcurrentCommand;
+  return createNextCommand(e, numCommand, method);
+}
+
+Commands RequestGroup::createNextCommand(DownloadEngine* e, int32_t numCommand, const string& method)
 {
   Commands commands;
-  for(;!_uris.empty() && commands.size() < (size_t)maxNum; _uris.pop_front()) {
+  for(;!_uris.empty() && commands.size() < (size_t)numCommand; _uris.pop_front()) {
     string uri = _uris.front();
     _spentUris.push_back(uri);
     RequestHandle req = RequestFactorySingletonHolder::instance()->createRequest();
@@ -207,8 +213,7 @@ void RequestGroup::prepareForNextAction(int cuid, const RequestHandle& req, Down
     if(downloadCommand) {
       e->commands.push_back(downloadCommand);
     } else {
-      int32_t numCommandsToGenerate = 15;
+      Commands commands = createNextCommand(e);
-      Commands commands = getNextCommand(e, numCommandsToGenerate);
       Command* command = InitiateConnectionCommandFactory::createInitiateConnectionCommand(cuid, req, this, e);
       commands.push_front(command);
       e->addCommand(commands);

src/RequestGroup.h

@@ -60,6 +60,7 @@ private:
   const Logger* logger;
   ChunkChecksumHandle _chunkChecksum;
   ChecksumHandle _checksum;
+  int32_t _numConcurrentCommand;
 
   void validateFilename(const string& expectedFilename,
 			const string& actualFilename) const;
@@ -82,6 +83,7 @@ public:
     logger(LogFactory::getInstance()),
     _chunkChecksum(0),
     _checksum(0),
+    _numConcurrentCommand(0),
     numConnection(0),
     isTorrent(false) {}
 
@@ -92,6 +94,7 @@ public:
     _option(option),
     logger(LogFactory::getInstance()),
     _chunkChecksum(0),
+    _numConcurrentCommand(0),
     numConnection(0),
     isTorrent(false)
   {
@@ -109,7 +112,9 @@ public:
     return _segmentMan;
   }
 
-  Commands getNextCommand(DownloadEngine* e, int32_t maxNum, const string& method = "GET");
+  Commands createNextCommand(DownloadEngine* e, const string& method = "GET");
+
+  Commands createNextCommand(DownloadEngine* e, int32_t numCommand, const string& method = "GET");
   
   void addURI(const string& uri)
   {
@@ -260,6 +265,11 @@ public:
   {
     _segmentManFactory = segmentManFactory;
   }
+
+  void setNumConcurrentCommand(int32_t num)
+  {
+    _numConcurrentCommand = num;
+  }
 };
 
 typedef SharedHandle<RequestGroup> RequestGroupHandle;

src/RequestGroupMan.cc

@@ -74,7 +74,7 @@ void RequestGroupMan::fillRequestGroupFromReserver(DownloadEngine* e)
     
     _requestGroups.push_back(groupToAdd);
     groupToAdd->initSegmentMan();
-    Commands commands = groupToAdd->getNextCommand(e, 1);
+    Commands commands = groupToAdd->createNextCommand(e, 1);
     count += commands.size();
     e->addCommand(commands);
   }
@@ -89,7 +89,7 @@ Commands RequestGroupMan::getInitialCommands(DownloadEngine* e) const
   for(RequestGroups::const_iterator itr = _requestGroups.begin();
 	itr != _requestGroups.end(); ++itr) {
     (*itr)->initSegmentMan();
-    commands.push_back((*itr)->getNextCommand(e, 1).front());
+    commands.push_back((*itr)->createNextCommand(e, 1).front());
   }
   return commands;
 }
@@ -129,3 +129,15 @@ void RequestGroupMan::showDownloadResults(ostream& o) const
     o << "\n";
   }
 }
+
+bool RequestGroupMan::isSameFileBeingDownloaded(RequestGroup* requestGroup) const
+{
+  for(RequestGroups::const_iterator itr = _requestGroups.begin();
+      itr != _requestGroups.end(); ++itr) {
+    if((*itr).get() != requestGroup &&
+       (*itr)->getFilePath() == requestGroup->getFilePath()) {
+      return true;
+    }
+  }
+  return false;
+}

src/RequestGroupMan.h

@@ -148,6 +148,8 @@ public:
     }
     return errors;
   }
+
+  bool isSameFileBeingDownloaded(RequestGroup* requestGroup) const;
 };
 
 typedef SharedHandle<RequestGroupMan> RequestGroupManHandle;

src/main.cc

@@ -206,6 +206,9 @@ void showUsage() {
 	    "                              http(s)/ftp downloads.") << endl;
   cout << _(" -U, --user-agent=USER_AGENT  Set user agent for http(s) downloads.") << endl;
   cout << _(" -n, --no-netrc               Disables netrc support.") << endl;
+  cout << _(" -i, --input-file=FILE        Downloads URIs found in FILE. You can specify\n"
+	    "                              multiple URIs for a single entity: deliminate\n"
+	    "                              URIs by Tab in a single line.") << endl;
 #ifdef ENABLE_BITTORRENT
   cout << _(" -T, --torrent-file=TORRENT_FILE  The file path to .torrent file.") << endl;
   cout << _(" --follow-torrent=true|false  Setting this option to false prevents aria2 to\n"
@@ -616,7 +619,8 @@ int main(int argc, char* argv[]) {
   if(op->defined(PREF_HTTP_PROXY_USER)) {
     op->put(PREF_HTTP_PROXY_AUTH_ENABLED, V_TRUE);
   }
-  if(!op->defined(PREF_TORRENT_FILE) && !op->defined(PREF_METALINK_FILE)) {
+  if(!op->defined(PREF_TORRENT_FILE) && !op->defined(PREF_METALINK_FILE) &&
+     !op->defined(PREF_INPUT_FILE)) {
     if(optind == argc) {
       cerr << _("specify at least one URL") << endl;
       exit(EXIT_FAILURE);

src/message.h

@@ -121,4 +121,5 @@
 #define EX_INVALID_BT_MESSAGE_ID _("Invalid ID=%d for %s. It should be %d.")
 #define EX_INVALID_CHUNK_CHECKSUM _("Chunk checksum validation failed. checksumIndex=%d, offset=%lld, expectedHash=%s, actualHash=%s")
 #define EX_DOWNLOAD_ABORTED _("Download aborted.")
+#define EX_DUPLICATE_FILE_DOWNLOAD _("File %s is being downloaded by other command.")
 #endif // _D_MESSAGE_H_

test/DefaultPeerStorageTest.cc

@@ -19,6 +19,7 @@ class DefaultPeerStorageTest:public CppUnit::TestFixture {
   CPPUNIT_TEST(testAddIncomingPeer);
   CPPUNIT_TEST(testReturnPeer);
   CPPUNIT_TEST(testOnErasingPeer);
+  CPPUNIT_TEST(testReturnPeer);
   CPPUNIT_TEST_SUITE_END();
 private:
   BtContextHandle btContext;

test/Makefile.am

@@ -1,6 +1,7 @@
 TESTS = aria2c
 check_PROGRAMS = $(TESTS)
 aria2c_SOURCES = AllTest.cc\
+	RequestGroupManTest.cc\
 	IteratableChecksumValidatorTest.cc\
 	IteratableChunkChecksumValidatorTest.cc\
 	UriFileListParserTest.cc\

test/Makefile.in

@@ -57,7 +57,7 @@ mkinstalldirs = $(SHELL) $(top_srcdir)/mkinstalldirs
 CONFIG_HEADER = $(top_builddir)/config.h
 CONFIG_CLEAN_FILES =
 am__EXEEXT_1 = aria2c$(EXEEXT)
-am_aria2c_OBJECTS = AllTest.$(OBJEXT) \
+am_aria2c_OBJECTS = AllTest.$(OBJEXT) RequestGroupManTest.$(OBJEXT) \
 	IteratableChecksumValidatorTest.$(OBJEXT) \
 	IteratableChunkChecksumValidatorTest.$(OBJEXT) \
 	UriFileListParserTest.$(OBJEXT) PeerTest.$(OBJEXT) \
@@ -265,6 +265,7 @@ sysconfdir = @sysconfdir@
 target_alias = @target_alias@
 TESTS = aria2c
 aria2c_SOURCES = AllTest.cc\
+	RequestGroupManTest.cc\
 	IteratableChecksumValidatorTest.cc\
 	IteratableChunkChecksumValidatorTest.cc\
 	UriFileListParserTest.cc\
@@ -457,6 +458,7 @@ distclean-compile:
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/PeerMessageUtilTest.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/PeerTest.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/RequestFactoryTest.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/RequestGroupManTest.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/RequestTest.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/SegmentManTest.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ShaVisitorTest.Po@am__quote@

test/RequestGroupManTest.cc

@@ -0,0 +1,47 @@
+#include "RequestGroupMan.h"
+#include <cppunit/extensions/HelperMacros.h>
+
+using namespace std;
+
+class RequestGroupManTest : public CppUnit::TestFixture {
+
+  CPPUNIT_TEST_SUITE(RequestGroupManTest);
+  CPPUNIT_TEST(testIsSameFileBeingDownloaded);
+  CPPUNIT_TEST_SUITE_END();
+private:
+
+public:
+  void setUp() {}
+
+  void testIsSameFileBeingDownloaded();
+};
+
+
+CPPUNIT_TEST_SUITE_REGISTRATION( RequestGroupManTest );
+
+void RequestGroupManTest::testIsSameFileBeingDownloaded()
+{
+  Option option;
+  RequestGroupMan gm;
+
+  RequestGroupHandle rg1 = new RequestGroup("http://localhost/aria2.tar.bz2",
+					    &option);
+  RequestGroupHandle rg2 = new RequestGroup("http://localhost/aria2.tar.bz2",
+					    &option);
+
+  gm.addRequestGroup(rg1);
+  gm.addRequestGroup(rg2);
+
+  rg1->initSegmentMan();
+  rg2->initSegmentMan();
+
+  rg1->getSegmentMan()->filename = "aria2.tar.bz2";
+  rg2->getSegmentMan()->filename = "aria2.tar.bz2";
+
+  CPPUNIT_ASSERT(gm.isSameFileBeingDownloaded(rg1.get()));
+
+  rg2->getSegmentMan()->filename = "aria2-0.10.2.tar.bz2";
+
+  CPPUNIT_ASSERT(!gm.isSameFileBeingDownloaded(rg1.get()));
+
+}