From 453e2f10dddf3b158de39810c1130203fd8d67eb Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 3 Jun 2007 14:24:37 +0000 Subject: [PATCH] 2007-06-03 Tatsuhiro Tsujikawa RequestGroup::getNextCommand() was renamed to createNextCommand(). Added its overloaded method. * src/RequestGroup.h (_numConcurrentCommand): New variable. (setNumConcurrentCommand): New function. * src/RequestGroup.cc Abort download if same file is being downloaded concurrently. * src/RequestGroup.h, src/RequestGroupMan.cc (isSameFileBeingDownloaded): New function. * src/HttpResponseCommand.cc (executeInternal) * src/FtpNegotiateCommand.cc (recvSize) * src/message.h (EX_DUPLICATE_FILE_DOWNLOAD): New definition. * main.cc: Added help message for -i option. --- ChangeLog | 18 +++++++++++++ TODO | 7 ----- src/AbstractCommand.cc | 2 +- src/CheckIntegrityCommand.cc | 3 +-- src/FileAllocationCommand.cc | 4 +-- src/FtpNegotiationCommand.cc | 4 +++ src/HttpResponseCommand.cc | 5 ++++ src/MetalinkRequestInfo.cc | 5 +--- src/RequestGroup.cc | 13 +++++++--- src/RequestGroup.h | 12 ++++++++- src/RequestGroupMan.cc | 16 ++++++++++-- src/RequestGroupMan.h | 2 ++ src/main.cc | 6 ++++- src/message.h | 1 + test/DefaultPeerStorageTest.cc | 1 + test/Makefile.am | 1 + test/Makefile.in | 4 ++- test/RequestGroupManTest.cc | 47 ++++++++++++++++++++++++++++++++++ 18 files changed, 125 insertions(+), 26 deletions(-) create mode 100644 test/RequestGroupManTest.cc diff --git a/ChangeLog b/ChangeLog index 1df130f1..23236260 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,21 @@ +2007-06-03 Tatsuhiro Tsujikawa + + RequestGroup::getNextCommand() was renamed to createNextCommand(). + Added its overloaded method. + * src/RequestGroup.h + (_numConcurrentCommand): New variable. + (setNumConcurrentCommand): New function. + * src/RequestGroup.cc + + Abort download if same file is being downloaded concurrently. + * src/RequestGroup.h, src/RequestGroupMan.cc + (isSameFileBeingDownloaded): New function. + * src/HttpResponseCommand.cc (executeInternal) + * src/FtpNegotiateCommand.cc (recvSize) + * src/message.h (EX_DUPLICATE_FILE_DOWNLOAD): New definition. + + * main.cc: Added help message for -i option. + 2007-06-01 Tatsuhiro Tsujikawa * src/FileAllocationCommand.cc: Derived from RealtimeCommand. diff --git a/TODO b/TODO index 673ca9b5..f89dda27 100644 --- a/TODO +++ b/TODO @@ -31,13 +31,6 @@ 100K/300M(10%)(3cn)(3more) 100KB/s [FileAlloc:35MB/40MB(90%)][Checksum:10MB/20MB(50%)] * exit status: all downloads have been successful-> EXIT_SUCCESS, some of downloads have been failed -> EXIT_FAILURE -* use hintFilename and hintTotalLength if these are provided. - -> test against ftp downloads -* make sure that the same file name is not used at the same time. * Do not use ufilename in multi-simultaneous download mode. -* Replace numCommandToGenerate to the value of PREF_METALINK_SERVERS -* Do not send range header if the position of starting byte is 0 and -the position of ending byte is not specified. * Create download command directly when 1connection download. Consider timeout when file allocation/check integrity is enabled. -* Test DefaultPeerStorage \ No newline at end of file diff --git a/src/AbstractCommand.cc b/src/AbstractCommand.cc index 51051ac8..250cc230 100644 --- a/src/AbstractCommand.cc +++ b/src/AbstractCommand.cc @@ -141,7 +141,7 @@ bool AbstractCommand::execute() { } void AbstractCommand::tryReserved() { - Commands commands = _requestGroup->getNextCommand(e, 1); + Commands commands = _requestGroup->createNextCommand(e, 1); e->addCommand(commands); } diff --git a/src/CheckIntegrityCommand.cc b/src/CheckIntegrityCommand.cc index 51df08d3..3df57295 100644 --- a/src/CheckIntegrityCommand.cc +++ b/src/CheckIntegrityCommand.cc @@ -69,8 +69,7 @@ bool CheckIntegrityCommand::executeInternal() _e->commands.push_back(_nextDownloadCommand); _nextDownloadCommand = 0; } else { - int32_t numCommandsToGenerate = 15; - Commands commands = _requestGroup->getNextCommand(_e, numCommandsToGenerate); + Commands commands = _requestGroup->createNextCommand(_e); Command* command = InitiateConnectionCommandFactory::createInitiateConnectionCommand(cuid, _req, _requestGroup, _e); commands.push_front(command); _e->addCommand(commands); diff --git a/src/FileAllocationCommand.cc b/src/FileAllocationCommand.cc index ae9750cf..c334b0c5 100644 --- a/src/FileAllocationCommand.cc +++ b/src/FileAllocationCommand.cc @@ -52,9 +52,7 @@ bool FileAllocationCommand::executeInternal() _e->commands.push_back(_fileAllocationEntry->getNextDownloadCommand()); _fileAllocationEntry->setNextDownloadCommand(0); } else { - int32_t numCommandsToGenerate = 15; - Commands commands = _requestGroup->getNextCommand(_e, numCommandsToGenerate); - + Commands commands = _requestGroup->createNextCommand(_e); Command* command = InitiateConnectionCommandFactory::createInitiateConnectionCommand(cuid, _req, _requestGroup, _e); commands.push_front(command); diff --git a/src/FtpNegotiationCommand.cc b/src/FtpNegotiationCommand.cc index d9f8d741..ef39ce35 100644 --- a/src/FtpNegotiationCommand.cc +++ b/src/FtpNegotiationCommand.cc @@ -197,6 +197,10 @@ bool FtpNegotiationCommand::recvSize() { // TODO validate filename and totalsize against hintFilename and hintTotalSize if these are provided. _requestGroup->validateTotalLengthByHint(size); + if(e->_requestGroupMan->isSameFileBeingDownloaded(_requestGroup)) { + throw new FatalException(EX_DUPLICATE_FILE_DOWNLOAD, _requestGroup->getFilePath().c_str()); + } + if(req->getMethod() == Request::METHOD_HEAD) { _requestGroup->getSegmentMan()->isSplittable = false; // TODO because we don't want segment file to be saved. sequence = SEQ_HEAD_OK; diff --git a/src/HttpResponseCommand.cc b/src/HttpResponseCommand.cc index f3dcfe09..99b37971 100644 --- a/src/HttpResponseCommand.cc +++ b/src/HttpResponseCommand.cc @@ -41,6 +41,7 @@ #include "prefs.h" #include "File.h" #include "InitiateConnectionCommandFactory.h" +#include "FatalException.h" #include #include @@ -98,6 +99,10 @@ bool HttpResponseCommand::executeInternal() _requestGroup->validateFilenameByHint(httpResponse->determinFilename()); _requestGroup->validateTotalLengthByHint(httpResponse->getEntityLength()); + if(e->_requestGroupMan->isSameFileBeingDownloaded(_requestGroup)) { + throw new FatalException(EX_DUPLICATE_FILE_DOWNLOAD, _requestGroup->getFilePath().c_str()); + } + if(httpResponse->isTransferEncodingSpecified()) { return handleOtherEncoding(httpResponse); } else { diff --git a/src/MetalinkRequestInfo.cc b/src/MetalinkRequestInfo.cc index 69e28c2a..2af1188d 100644 --- a/src/MetalinkRequestInfo.cc +++ b/src/MetalinkRequestInfo.cc @@ -105,16 +105,12 @@ RequestInfos MetalinkRequestInfo::execute() { entry->resources.end(), FindBitTorrentUrl()); Strings urls; - int maxConnection = 0; ChecksumHandle checksum = 0; if(itr == entry->resources.end()) { entry->reorderResourcesByPreference(); for_each(entry->resources.begin(), entry->resources.end(), AccumulateNonP2PUrl(&urls, op->getAsInt(PREF_SPLIT))); - maxConnection = - op->getAsInt(PREF_METALINK_SERVERS)*op->getAsInt(PREF_SPLIT); - // TODO // set checksum checksum = entry->checksum; @@ -125,6 +121,7 @@ RequestInfos MetalinkRequestInfo::execute() { RequestGroupHandle rg = new RequestGroup(urls, op); rg->setHintFilename(entry->filename); rg->setHintTotalLength(entry->size); + rg->setNumConcurrentCommand(op->getAsInt(PREF_METALINK_SERVERS)); #ifdef ENABLE_MESSAGE_DIGEST if(entry->chunkChecksum.isNull()) { diff --git a/src/RequestGroup.cc b/src/RequestGroup.cc index 63e8ac7b..5e537dfa 100644 --- a/src/RequestGroup.cc +++ b/src/RequestGroup.cc @@ -61,10 +61,16 @@ SegmentManHandle RequestGroup::initSegmentMan() return _segmentMan; } -Commands RequestGroup::getNextCommand(DownloadEngine* e, int32_t maxNum, const string& method) +Commands RequestGroup::createNextCommand(DownloadEngine* e, const string& method) +{ + int32_t numCommand = _numConcurrentCommand == 0 ? _uris.size() : _numConcurrentCommand; + return createNextCommand(e, numCommand, method); +} + +Commands RequestGroup::createNextCommand(DownloadEngine* e, int32_t numCommand, const string& method) { Commands commands; - for(;!_uris.empty() && commands.size() < (size_t)maxNum; _uris.pop_front()) { + for(;!_uris.empty() && commands.size() < (size_t)numCommand; _uris.pop_front()) { string uri = _uris.front(); _spentUris.push_back(uri); RequestHandle req = RequestFactorySingletonHolder::instance()->createRequest(); @@ -207,8 +213,7 @@ void RequestGroup::prepareForNextAction(int cuid, const RequestHandle& req, Down if(downloadCommand) { e->commands.push_back(downloadCommand); } else { - int32_t numCommandsToGenerate = 15; - Commands commands = getNextCommand(e, numCommandsToGenerate); + Commands commands = createNextCommand(e); Command* command = InitiateConnectionCommandFactory::createInitiateConnectionCommand(cuid, req, this, e); commands.push_front(command); e->addCommand(commands); diff --git a/src/RequestGroup.h b/src/RequestGroup.h index d1e496b7..144c4e8f 100644 --- a/src/RequestGroup.h +++ b/src/RequestGroup.h @@ -60,6 +60,7 @@ private: const Logger* logger; ChunkChecksumHandle _chunkChecksum; ChecksumHandle _checksum; + int32_t _numConcurrentCommand; void validateFilename(const string& expectedFilename, const string& actualFilename) const; @@ -82,6 +83,7 @@ public: logger(LogFactory::getInstance()), _chunkChecksum(0), _checksum(0), + _numConcurrentCommand(0), numConnection(0), isTorrent(false) {} @@ -92,6 +94,7 @@ public: _option(option), logger(LogFactory::getInstance()), _chunkChecksum(0), + _numConcurrentCommand(0), numConnection(0), isTorrent(false) { @@ -109,7 +112,9 @@ public: return _segmentMan; } - Commands getNextCommand(DownloadEngine* e, int32_t maxNum, const string& method = "GET"); + Commands createNextCommand(DownloadEngine* e, const string& method = "GET"); + + Commands createNextCommand(DownloadEngine* e, int32_t numCommand, const string& method = "GET"); void addURI(const string& uri) { @@ -260,6 +265,11 @@ public: { _segmentManFactory = segmentManFactory; } + + void setNumConcurrentCommand(int32_t num) + { + _numConcurrentCommand = num; + } }; typedef SharedHandle RequestGroupHandle; diff --git a/src/RequestGroupMan.cc b/src/RequestGroupMan.cc index 72f2717e..06861a12 100644 --- a/src/RequestGroupMan.cc +++ b/src/RequestGroupMan.cc @@ -74,7 +74,7 @@ void RequestGroupMan::fillRequestGroupFromReserver(DownloadEngine* e) _requestGroups.push_back(groupToAdd); groupToAdd->initSegmentMan(); - Commands commands = groupToAdd->getNextCommand(e, 1); + Commands commands = groupToAdd->createNextCommand(e, 1); count += commands.size(); e->addCommand(commands); } @@ -89,7 +89,7 @@ Commands RequestGroupMan::getInitialCommands(DownloadEngine* e) const for(RequestGroups::const_iterator itr = _requestGroups.begin(); itr != _requestGroups.end(); ++itr) { (*itr)->initSegmentMan(); - commands.push_back((*itr)->getNextCommand(e, 1).front()); + commands.push_back((*itr)->createNextCommand(e, 1).front()); } return commands; } @@ -129,3 +129,15 @@ void RequestGroupMan::showDownloadResults(ostream& o) const o << "\n"; } } + +bool RequestGroupMan::isSameFileBeingDownloaded(RequestGroup* requestGroup) const +{ + for(RequestGroups::const_iterator itr = _requestGroups.begin(); + itr != _requestGroups.end(); ++itr) { + if((*itr).get() != requestGroup && + (*itr)->getFilePath() == requestGroup->getFilePath()) { + return true; + } + } + return false; +} diff --git a/src/RequestGroupMan.h b/src/RequestGroupMan.h index 77152053..3d2058b2 100644 --- a/src/RequestGroupMan.h +++ b/src/RequestGroupMan.h @@ -148,6 +148,8 @@ public: } return errors; } + + bool isSameFileBeingDownloaded(RequestGroup* requestGroup) const; }; typedef SharedHandle RequestGroupManHandle; diff --git a/src/main.cc b/src/main.cc index 73caa868..2f6a139e 100644 --- a/src/main.cc +++ b/src/main.cc @@ -206,6 +206,9 @@ void showUsage() { " http(s)/ftp downloads.") << endl; cout << _(" -U, --user-agent=USER_AGENT Set user agent for http(s) downloads.") << endl; cout << _(" -n, --no-netrc Disables netrc support.") << endl; + cout << _(" -i, --input-file=FILE Downloads URIs found in FILE. You can specify\n" + " multiple URIs for a single entity: deliminate\n" + " URIs by Tab in a single line.") << endl; #ifdef ENABLE_BITTORRENT cout << _(" -T, --torrent-file=TORRENT_FILE The file path to .torrent file.") << endl; cout << _(" --follow-torrent=true|false Setting this option to false prevents aria2 to\n" @@ -616,7 +619,8 @@ int main(int argc, char* argv[]) { if(op->defined(PREF_HTTP_PROXY_USER)) { op->put(PREF_HTTP_PROXY_AUTH_ENABLED, V_TRUE); } - if(!op->defined(PREF_TORRENT_FILE) && !op->defined(PREF_METALINK_FILE)) { + if(!op->defined(PREF_TORRENT_FILE) && !op->defined(PREF_METALINK_FILE) && + !op->defined(PREF_INPUT_FILE)) { if(optind == argc) { cerr << _("specify at least one URL") << endl; exit(EXIT_FAILURE); diff --git a/src/message.h b/src/message.h index e45de293..61b381ad 100644 --- a/src/message.h +++ b/src/message.h @@ -121,4 +121,5 @@ #define EX_INVALID_BT_MESSAGE_ID _("Invalid ID=%d for %s. It should be %d.") #define EX_INVALID_CHUNK_CHECKSUM _("Chunk checksum validation failed. checksumIndex=%d, offset=%lld, expectedHash=%s, actualHash=%s") #define EX_DOWNLOAD_ABORTED _("Download aborted.") +#define EX_DUPLICATE_FILE_DOWNLOAD _("File %s is being downloaded by other command.") #endif // _D_MESSAGE_H_ diff --git a/test/DefaultPeerStorageTest.cc b/test/DefaultPeerStorageTest.cc index 967077a6..e2864818 100644 --- a/test/DefaultPeerStorageTest.cc +++ b/test/DefaultPeerStorageTest.cc @@ -19,6 +19,7 @@ class DefaultPeerStorageTest:public CppUnit::TestFixture { CPPUNIT_TEST(testAddIncomingPeer); CPPUNIT_TEST(testReturnPeer); CPPUNIT_TEST(testOnErasingPeer); + CPPUNIT_TEST(testReturnPeer); CPPUNIT_TEST_SUITE_END(); private: BtContextHandle btContext; diff --git a/test/Makefile.am b/test/Makefile.am index 9eab8d70..95ed3f99 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -1,6 +1,7 @@ TESTS = aria2c check_PROGRAMS = $(TESTS) aria2c_SOURCES = AllTest.cc\ + RequestGroupManTest.cc\ IteratableChecksumValidatorTest.cc\ IteratableChunkChecksumValidatorTest.cc\ UriFileListParserTest.cc\ diff --git a/test/Makefile.in b/test/Makefile.in index 349f8c20..9f41f7a5 100644 --- a/test/Makefile.in +++ b/test/Makefile.in @@ -57,7 +57,7 @@ mkinstalldirs = $(SHELL) $(top_srcdir)/mkinstalldirs CONFIG_HEADER = $(top_builddir)/config.h CONFIG_CLEAN_FILES = am__EXEEXT_1 = aria2c$(EXEEXT) -am_aria2c_OBJECTS = AllTest.$(OBJEXT) \ +am_aria2c_OBJECTS = AllTest.$(OBJEXT) RequestGroupManTest.$(OBJEXT) \ IteratableChecksumValidatorTest.$(OBJEXT) \ IteratableChunkChecksumValidatorTest.$(OBJEXT) \ UriFileListParserTest.$(OBJEXT) PeerTest.$(OBJEXT) \ @@ -265,6 +265,7 @@ sysconfdir = @sysconfdir@ target_alias = @target_alias@ TESTS = aria2c aria2c_SOURCES = AllTest.cc\ + RequestGroupManTest.cc\ IteratableChecksumValidatorTest.cc\ IteratableChunkChecksumValidatorTest.cc\ UriFileListParserTest.cc\ @@ -457,6 +458,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/PeerMessageUtilTest.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/PeerTest.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/RequestFactoryTest.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/RequestGroupManTest.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/RequestTest.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/SegmentManTest.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ShaVisitorTest.Po@am__quote@ diff --git a/test/RequestGroupManTest.cc b/test/RequestGroupManTest.cc new file mode 100644 index 00000000..72ccee93 --- /dev/null +++ b/test/RequestGroupManTest.cc @@ -0,0 +1,47 @@ +#include "RequestGroupMan.h" +#include + +using namespace std; + +class RequestGroupManTest : public CppUnit::TestFixture { + + CPPUNIT_TEST_SUITE(RequestGroupManTest); + CPPUNIT_TEST(testIsSameFileBeingDownloaded); + CPPUNIT_TEST_SUITE_END(); +private: + +public: + void setUp() {} + + void testIsSameFileBeingDownloaded(); +}; + + +CPPUNIT_TEST_SUITE_REGISTRATION( RequestGroupManTest ); + +void RequestGroupManTest::testIsSameFileBeingDownloaded() +{ + Option option; + RequestGroupMan gm; + + RequestGroupHandle rg1 = new RequestGroup("http://localhost/aria2.tar.bz2", + &option); + RequestGroupHandle rg2 = new RequestGroup("http://localhost/aria2.tar.bz2", + &option); + + gm.addRequestGroup(rg1); + gm.addRequestGroup(rg2); + + rg1->initSegmentMan(); + rg2->initSegmentMan(); + + rg1->getSegmentMan()->filename = "aria2.tar.bz2"; + rg2->getSegmentMan()->filename = "aria2.tar.bz2"; + + CPPUNIT_ASSERT(gm.isSameFileBeingDownloaded(rg1.get())); + + rg2->getSegmentMan()->filename = "aria2-0.10.2.tar.bz2"; + + CPPUNIT_ASSERT(!gm.isSameFileBeingDownloaded(rg1.get())); + +}