From c99960aa330d9666682a9871b1194eaa38ab4dc7 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 14 Jul 2010 11:39:05 +0000 Subject: [PATCH] 2010-07-14 Tatsuhiro Tsujikawa Added --max-connection-per-server=NUM option. The default value of NUM is 1. This option limits the number of connections allowed to one server for each download. This means when NUM is 2 and 1 URI is provided, even if you specified -s 5, aria2 establishes 2 connections. Default value of -j option is changed from 5 to 2. * src/BtDependency.cc * src/CreateRequestCommand.cc * src/FileEntry.cc * src/FileEntry.h * src/FtpNegotiationCommand.cc * src/HttpResponseCommand.cc * src/Metalink2RequestGroup.cc * src/OptionHandlerFactory.cc * src/TrackerWatcherCommand.cc * src/download_helper.cc * src/prefs.cc * src/prefs.h * src/usage_text.h * test/DownloadHelperTest.cc * test/FileEntryTest.cc --- ChangeLog | 23 ++++++++ src/BtDependency.cc | 5 +- src/CreateRequestCommand.cc | 6 +- src/FileEntry.cc | 104 ++++++++++++++++++++++------------- src/FileEntry.h | 36 ++++++++---- src/FtpNegotiationCommand.cc | 2 +- src/HttpResponseCommand.cc | 2 +- src/Metalink2RequestGroup.cc | 6 +- src/OptionHandlerFactory.cc | 11 +++- src/TrackerWatcherCommand.cc | 1 + src/download_helper.cc | 27 +++++---- src/prefs.cc | 2 + src/prefs.h | 2 + src/usage_text.h | 3 + test/DownloadHelperTest.cc | 79 ++++++++++++-------------- test/FileEntryTest.cc | 80 +++++++++++++++++++++++---- 16 files changed, 260 insertions(+), 129 deletions(-) diff --git a/ChangeLog b/ChangeLog index f2b2dcb7..63ff3680 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,26 @@ +2010-07-14 Tatsuhiro Tsujikawa + + Added --max-connection-per-server=NUM option. The default value of + NUM is 1. This option limits the number of connections allowed to + one server for each download. This means when NUM is 2 and 1 URI + is provided, even if you specified -s 5, aria2 establishes 2 + connections. Default value of -j option is changed from 5 to 2. + * src/BtDependency.cc + * src/CreateRequestCommand.cc + * src/FileEntry.cc + * src/FileEntry.h + * src/FtpNegotiationCommand.cc + * src/HttpResponseCommand.cc + * src/Metalink2RequestGroup.cc + * src/OptionHandlerFactory.cc + * src/TrackerWatcherCommand.cc + * src/download_helper.cc + * src/prefs.cc + * src/prefs.h + * src/usage_text.h + * test/DownloadHelperTest.cc + * test/FileEntryTest.cc + 2010-07-12 Tatsuhiro Tsujikawa Set end byte in Range header if start byte > 0 to get more chance diff --git a/src/BtDependency.cc b/src/BtDependency.cc index 61719ffd..75babd74 100644 --- a/src/BtDependency.cc +++ b/src/BtDependency.cc @@ -66,9 +66,8 @@ static void copyValues(const SharedHandle& d, d->setPath(s->getPath()); d->addUris(s->getRemainingUris().begin(), s->getRemainingUris().end()); - if(!s->isSingleHostMultiConnectionEnabled()) { - d->disableSingleHostMultiConnection(); - } + d->setMaxConnectionPerServer(s->getMaxConnectionPerServer()); + d->setUniqueProtocol(s->isUniqueProtocol()); } bool BtDependency::resolve() diff --git a/src/CreateRequestCommand.cc b/src/CreateRequestCommand.cc index e519c017..0de8abe1 100644 --- a/src/CreateRequestCommand.cc +++ b/src/CreateRequestCommand.cc @@ -74,13 +74,9 @@ bool CreateRequestCommand::executeInternal() setFileEntry(getDownloadContext()->findFileEntryByOffset (getSegments().front()->getPositionToWrite())); } - if(getFileEntry()->getRemainingUris().empty() && - getOption()->getAsBool(PREF_REUSE_URI) && - getFileEntry()->countPooledRequest() == 0) { - getFileEntry()->reuseUri(getRequestGroup()->getNumConcurrentCommand()); - } setRequest (getFileEntry()->getRequest(getRequestGroup()->getURISelector(), + getOption()->getAsBool(PREF_REUSE_URI), getOption()->get(PREF_REFERER), // Don't use HEAD request when file // size is known. diff --git a/src/FileEntry.cc b/src/FileEntry.cc index b65ea5dd..fc746cdf 100644 --- a/src/FileEntry.cc +++ b/src/FileEntry.cc @@ -51,13 +51,15 @@ FileEntry::FileEntry(const std::string& path, path_(path), uris_(uris.begin(), uris.end()), length_(length), offset_(offset), requested_(true), - singleHostMultiConnection_(true), + uniqueProtocol_(false), + maxConnectionPerServer_(1), lastFasterReplace_(0), logger_(LogFactory::getInstance()) {} FileEntry::FileEntry(): length_(0), offset_(0), requested_(false), - singleHostMultiConnection_(true), + uniqueProtocol_(false), + maxConnectionPerServer_(1), logger_(LogFactory::getInstance()) {} FileEntry::~FileEntry() {} @@ -106,53 +108,65 @@ std::string FileEntry::selectUri(const SharedHandle& uriSelector) } template -static bool inFlightHost(InputIterator first, InputIterator last, - const std::string& hostname) +static size_t countInFlightHost(InputIterator first, InputIterator last, + const std::string& hostname) { // TODO redirection should be considered here. We need to parse // original URI to get hostname. + size_t count = 0; for(; first != last; ++first) { if((*first)->getHost() == hostname) { - return true; + ++count; } } - return false; + return count; } SharedHandle FileEntry::getRequest (const SharedHandle& selector, + bool uriReuse, const std::string& referer, const std::string& method) { SharedHandle req; if(requestPool_.empty()) { - std::vector pending; - while(1) { - std::string uri = selector->select(this); - if(uri.empty()) { - return req; - } - req.reset(new Request()); - if(req->setUri(uri)) { - if(!singleHostMultiConnection_) { - if(inFlightHost(inFlightRequests_.begin(), inFlightRequests_.end(), - req->getHost())) { + for(int g = 0; g < 2; ++g) { + std::vector pending; + std::vector ignoreHost; + while(1) { + std::string uri = selector->select(this); + if(uri.empty()) { + break; + } + req.reset(new Request()); + if(req->setUri(uri)) { + if(countInFlightHost(inFlightRequests_.begin(), + inFlightRequests_.end(), + req->getHost()) >= maxConnectionPerServer_) { pending.push_back(uri); + ignoreHost.push_back(req->getHost()); req.reset(); continue; } + req->setReferer(referer); + req->setMethod(method); + spentUris_.push_back(uri); + inFlightRequests_.push_back(req); + break; + } else { + req.reset(); } - req->setReferer(referer); - req->setMethod(method); - spentUris_.push_back(uri); - inFlightRequests_.push_back(req); - break; + } + uris_.insert(uris_.begin(), pending.begin(), pending.end()); + // TODO UriReuse is performed only when PREF_REUSE_URI is true. + if(g == 0 && uriReuse && req.isNull() && uris_.size() == pending.size()) { + // Reuse URIs other than ones in pending + reuseUri(ignoreHost); } else { - req.reset(); + break; } } - uris_.insert(uris_.begin(), pending.begin(), pending.end()); } else { req = requestPool_.front(); requestPool_.pop_front(); @@ -290,8 +304,14 @@ void FileEntry::extractURIResult uriResults_.erase(uriResults_.begin(), i); } -void FileEntry::reuseUri(size_t num) +void FileEntry::reuseUri(const std::vector& ignore) { + if(logger_->debug()) { + for(std::vector::const_iterator i = ignore.begin(), + eoi = ignore.end(); i != eoi; ++i) { + logger_->debug("ignore host=%s", (*i).c_str()); + } + } std::deque uris = spentUris_; std::sort(uris.begin(), uris.end()); uris.erase(std::unique(uris.begin(), uris.end()), uris.end()); @@ -302,11 +322,29 @@ void FileEntry::reuseUri(size_t num) std::sort(errorUris.begin(), errorUris.end()); errorUris.erase(std::unique(errorUris.begin(), errorUris.end()), errorUris.end()); - + if(logger_->debug()) { + for(std::vector::const_iterator i = errorUris.begin(), + eoi = errorUris.end(); i != eoi; ++i) { + logger_->debug("error URI=%s", (*i).c_str()); + } + } std::vector reusableURIs; std::set_difference(uris.begin(), uris.end(), errorUris.begin(), errorUris.end(), std::back_inserter(reusableURIs)); + std::vector::iterator insertionPoint = reusableURIs.begin(); + Request req; + for(std::vector::iterator i = reusableURIs.begin(), + eoi = reusableURIs.end(); i != eoi; ++i) { + req.setUri(*i); + if(std::find(ignore.begin(), ignore.end(), req.getHost()) == ignore.end()) { + if(i != insertionPoint) { + *insertionPoint = *i; + } + ++insertionPoint; + } + } + reusableURIs.erase(insertionPoint, reusableURIs.end()); size_t ininum = reusableURIs.size(); if(logger_->debug()) { logger_->debug("Found %u reusable URIs", static_cast(ininum)); @@ -315,19 +353,7 @@ void FileEntry::reuseUri(size_t num) logger_->debug("URI=%s", (*i).c_str()); } } - // Reuse at least num URIs here to avoid to - // run this process repeatedly. - if(ininum > 0) { - for(size_t i = 0; i < num/ininum; ++i) { - uris_.insert(uris_.end(), reusableURIs.begin(), reusableURIs.end()); - } - uris_.insert(uris_.end(), reusableURIs.begin(), - reusableURIs.begin()+(num%ininum)); - if(logger_->debug()) { - logger_->debug("Duplication complete: now %u URIs for reuse", - static_cast(uris_.size())); - } - } + uris_.insert(uris_.end(), reusableURIs.begin(), reusableURIs.end()); } void FileEntry::releaseRuntimeResource() diff --git a/src/FileEntry.h b/src/FileEntry.h index fc62cd05..db022295 100644 --- a/src/FileEntry.h +++ b/src/FileEntry.h @@ -70,7 +70,8 @@ private: // URIResult is stored in the ascending order of the time when its result is // available. std::deque uriResults_; - bool singleHostMultiConnection_; + bool uniqueProtocol_; + size_t maxConnectionPerServer_; std::string originalName_; Timer lastFasterReplace_; Logger* logger_; @@ -162,9 +163,13 @@ public: // newly created Request. If Request object is retrieved from the // pool, referer is ignored. If method is given, it is set to newly // created Request. If Request object is retrieved from the pool, - // method is ignored. + // method is ignored. If uriReuse is true and selector does not + // returns Request object either because uris_ is empty or all URI + // are not be usable because maxConnectionPerServer_ limit, then + // reuse used URIs and do selection again. SharedHandle getRequest (const SharedHandle& selector, + bool uriReuse = true, const std::string& referer = A2STR::NIL, const std::string& method = Request::METHOD_GET); @@ -215,21 +220,20 @@ public: void extractURIResult (std::deque& res, downloadresultcode::RESULT r); - void disableSingleHostMultiConnection() + void setMaxConnectionPerServer(size_t n) { - singleHostMultiConnection_ = false; + maxConnectionPerServer_ = n; } - bool isSingleHostMultiConnectionEnabled() const + size_t getMaxConnectionPerServer() const { - return singleHostMultiConnection_; + return maxConnectionPerServer_; } - // Reuse URIs which have not emitted error so far. Thie method - // tries to reuse at least num URIs. If less than num URIs found to - // reuse, those URIs are used more than once so that num URIs total - // are available to reuse. - void reuseUri(size_t num); + // Reuse URIs which have not emitted error so far and whose host + // component is not included in ignore. The reusable URIs are + // appended to uris_ maxConnectionPerServer_ times. + void reuseUri(const std::vector& ignore); void releaseRuntimeResource(); @@ -249,6 +253,16 @@ public: { return uris_.empty() && inFlightRequests_.empty() && requestPool_.empty(); } + + void setUniqueProtocol(bool f) + { + uniqueProtocol_ = f; + } + + bool isUniqueProtocol() const + { + return uniqueProtocol_; + } }; // Returns the first FileEntry which isRequested() method returns diff --git a/src/FtpNegotiationCommand.cc b/src/FtpNegotiationCommand.cc index 558d4944..7a43b273 100644 --- a/src/FtpNegotiationCommand.cc +++ b/src/FtpNegotiationCommand.cc @@ -114,7 +114,7 @@ bool FtpNegotiationCommand::executeInternal() { command->setStartupIdleTime(getOption()->getAsInt(PREF_STARTUP_IDLE_TIME)); command->setLowestDownloadSpeedLimit (getOption()->getAsInt(PREF_LOWEST_SPEED_LIMIT)); - if(!getFileEntry()->isSingleHostMultiConnectionEnabled()) { + if(getFileEntry()->isUniqueProtocol()) { getFileEntry()->removeURIWhoseHostnameIs(getRequest()->getHost()); } getRequestGroup()->getURISelector()->tuneDownloadCommand diff --git a/src/HttpResponseCommand.cc b/src/HttpResponseCommand.cc index c1b94848..b523699a 100644 --- a/src/HttpResponseCommand.cc +++ b/src/HttpResponseCommand.cc @@ -125,7 +125,7 @@ bool HttpResponseCommand::executeInternal() } return skipResponseBody(httpResponse); } - if(!getFileEntry()->isSingleHostMultiConnectionEnabled()) { + if(getFileEntry()->isUniqueProtocol()) { // TODO redirection should be considered here. We need to parse // original URI to get hostname. getFileEntry()->removeURIWhoseHostnameIs(getRequest()->getHost()); diff --git a/src/Metalink2RequestGroup.cc b/src/Metalink2RequestGroup.cc index 46caac4b..f041dee0 100644 --- a/src/Metalink2RequestGroup.cc +++ b/src/Metalink2RequestGroup.cc @@ -252,8 +252,9 @@ Metalink2RequestGroup::createRequestGroup util::applyDir(option->get(PREF_DIR), entry->file->getPath()))); dctx->getFirstFileEntry()->setUris(uris); + dctx->getFirstFileEntry()->setMaxConnectionPerServer(1); if(option->getAsBool(PREF_METALINK_ENABLE_UNIQUE_PROTOCOL)) { - dctx->getFirstFileEntry()->disableSingleHostMultiConnection(); + dctx->getFirstFileEntry()->setUniqueProtocol(true); } #ifdef ENABLE_MESSAGE_DIGEST if(!entry->checksum.isNull()) { @@ -291,8 +292,9 @@ Metalink2RequestGroup::createRequestGroup (new FileEntry (util::applyDir(option->get(PREF_DIR), (*i)->file->getPath()), (*i)->file->getLength(), offset, uris)); + fe->setMaxConnectionPerServer(1); if(option->getAsBool(PREF_METALINK_ENABLE_UNIQUE_PROTOCOL)) { - fe->disableSingleHostMultiConnection(); + fe->setUniqueProtocol(true); } fe->setOriginalName((*i)->metaurls[0]->name); fileEntries.push_back(fe); diff --git a/src/OptionHandlerFactory.cc b/src/OptionHandlerFactory.cc index 8a42f905..4884cd84 100644 --- a/src/OptionHandlerFactory.cc +++ b/src/OptionHandlerFactory.cc @@ -318,12 +318,21 @@ OptionHandlers OptionHandlerFactory::createOptionHandlers() SharedHandle op(new NumberOptionHandler (PREF_MAX_CONCURRENT_DOWNLOADS, TEXT_MAX_CONCURRENT_DOWNLOADS, - "5", + "2", 1, -1, 'j')); op->addTag(TAG_BASIC); handlers.push_back(op); } + { + SharedHandle op(new NumberOptionHandler + (PREF_MAX_CONNECTION_PER_SERVER, + TEXT_MAX_CONNECTION_PER_SERVER, + "1", + 1, 4)); + op->addTag(TAG_ADVANCED); + handlers.push_back(op); + } { SharedHandle op(new UnitNumberOptionHandler (PREF_MAX_DOWNLOAD_LIMIT, diff --git a/src/TrackerWatcherCommand.cc b/src/TrackerWatcherCommand.cc index c9496ed8..34fa582d 100644 --- a/src/TrackerWatcherCommand.cc +++ b/src/TrackerWatcherCommand.cc @@ -230,6 +230,7 @@ TrackerWatcherCommand::createRequestGroup(const std::string& uri) getLogger()->debug("This is single-tracker announce."); } } + rg->setNumConcurrentCommand(1); // If backup tracker is available, try 2 times for each tracker // and if they all fails, then try next one. rg->getOption()->put(PREF_MAX_TRIES, "2"); diff --git a/src/download_helper.cc b/src/download_helper.cc index accbf00c..627fec05 100644 --- a/src/download_helper.cc +++ b/src/download_helper.cc @@ -183,16 +183,20 @@ template static void splitURI(std::vector& result, InputIterator begin, InputIterator end, - size_t numSplit) + size_t numSplit, + size_t maxIter) { size_t numURIs = std::distance(begin, end); if(numURIs >= numSplit) { result.insert(result.end(), begin, end); } else if(numURIs > 0) { - for(size_t i = 0; i < numSplit/numURIs; ++i) { + size_t num = std::min(numSplit/numURIs, maxIter); + for(size_t i = 0; i < num; ++i) { result.insert(result.end(), begin, end); } - result.insert(result.end(), begin, begin+(numSplit%numURIs)); + if(num < maxIter) { + result.insert(result.end(), begin, begin+(numSplit%numURIs)); + } } } @@ -209,6 +213,8 @@ static SharedHandle createRequestGroup util::applyDir(option->get(PREF_DIR), option->get(PREF_OUT)):A2STR::NIL)); dctx->setDir(option->get(PREF_DIR)); dctx->getFirstFileEntry()->setUris(uris); + dctx->getFirstFileEntry()->setMaxConnectionPerServer + (option->getAsInt(PREF_MAX_CONNECTION_PER_SERVER)); rg->setDownloadContext(dctx); return rg; } @@ -304,10 +310,8 @@ void createRequestGroupForBitTorrent } // we ignore -Z option here size_t numSplit = option->getAsInt(PREF_SPLIT); - std::vector auxUris; - splitURI(auxUris, nargs.begin(), nargs.end(), numSplit); SharedHandle rg = - createBtRequestGroup(option->get(PREF_TORRENT_FILE), option, auxUris, + createBtRequestGroup(option->get(PREF_TORRENT_FILE), option, nargs, torrentData); rg->setNumConcurrentCommand(numSplit); result.push_back(rg); @@ -351,12 +355,13 @@ public: { if(detector_.isStreamProtocol(uri)) { std::vector streamURIs; + size_t numIter = option_->getAsInt(PREF_MAX_CONNECTION_PER_SERVER); size_t numSplit = option_->getAsInt(PREF_SPLIT); - for(size_t i = 0; i < numSplit; ++i) { + size_t num = std::min(numIter, numSplit); + for(size_t i = 0; i < num; ++i) { streamURIs.push_back(uri); } - SharedHandle rg = - createRequestGroup(option_, streamURIs); + SharedHandle rg = createRequestGroup(option_, streamURIs); rg->setNumConcurrentCommand(numSplit); requestGroups_.push_back(rg); } @@ -429,10 +434,10 @@ void createRequestGroupForUri std::stable_partition(nargs.begin(), nargs.end(), StreamProtocolFilter()); // let's process http/ftp protocols first. if(nargs.begin() != strmProtoEnd) { + size_t numIter = option->getAsInt(PREF_MAX_CONNECTION_PER_SERVER); size_t numSplit = option->getAsInt(PREF_SPLIT); std::vector streamURIs; - splitURI(streamURIs, nargs.begin(), strmProtoEnd, - numSplit); + splitURI(streamURIs, nargs.begin(), strmProtoEnd, numSplit, numIter); SharedHandle rg = createRequestGroup(option, streamURIs, true); rg->setNumConcurrentCommand(numSplit); diff --git a/src/prefs.cc b/src/prefs.cc index 637d7a51..95afc73d 100644 --- a/src/prefs.cc +++ b/src/prefs.cc @@ -190,6 +190,8 @@ const std::string PREF_ALWAYS_RESUME("always-resume"); const std::string PREF_MAX_RESUME_FAILURE_TRIES("max-resume-failure-tries"); // value: string that your file system recognizes as a file name. const std::string PREF_SAVE_SESSION("save-session"); +// value: 1*digit +const std::string PREF_MAX_CONNECTION_PER_SERVER("max-connection-per-server"); /** * FTP related preferences diff --git a/src/prefs.h b/src/prefs.h index 2dfbf89d..2a59453a 100644 --- a/src/prefs.h +++ b/src/prefs.h @@ -194,6 +194,8 @@ extern const std::string PREF_ALWAYS_RESUME; extern const std::string PREF_MAX_RESUME_FAILURE_TRIES; // value: string that your file system recognizes as a file name. extern const std::string PREF_SAVE_SESSION; +// value: 1*digit +extern const std::string PREF_MAX_CONNECTION_PER_SERVER; /** * FTP related preferences diff --git a/src/usage_text.h b/src/usage_text.h index 94ff826c..8e9b3517 100644 --- a/src/usage_text.h +++ b/src/usage_text.h @@ -683,3 +683,6 @@ " option on restart. Please note that downloads\n" \ " added by aria2.addTorrent and aria2.addMetalink\n" \ " XML-RPC method are not saved.") +#define TEXT_MAX_CONNECTION_PER_SERVER \ + _(" --max-connection-per-server=NUM The maximum number of connections to one server\n"\ + " for each download.") diff --git a/test/DownloadHelperTest.cc b/test/DownloadHelperTest.cc index c310df7e..368415a9 100644 --- a/test/DownloadHelperTest.cc +++ b/test/DownloadHelperTest.cc @@ -70,23 +70,22 @@ void DownloadHelperTest::testCreateRequestGroupForUri() "http://charlie/file" }; std::vector uris(vbegin(array), vend(array)); - option_->put(PREF_SPLIT, "3"); + option_->put(PREF_SPLIT, "7"); + option_->put(PREF_MAX_CONNECTION_PER_SERVER, "2"); option_->put(PREF_DIR, "/tmp"); option_->put(PREF_OUT, "file.out"); { std::vector > result; - createRequestGroupForUri(result, option_, uris); - CPPUNIT_ASSERT_EQUAL((size_t)1, result.size()); SharedHandle group = result[0]; - std::vector uris; - group->getDownloadContext()->getFirstFileEntry()->getUris(uris); - CPPUNIT_ASSERT_EQUAL((size_t)3, uris.size()); - for(size_t i = 0; i < A2_ARRAY_LEN(array); ++i) { - CPPUNIT_ASSERT_EQUAL(array[i], uris[i]); + std::vector xuris; + group->getDownloadContext()->getFirstFileEntry()->getUris(xuris); + CPPUNIT_ASSERT_EQUAL((size_t)6, xuris.size()); + for(size_t i = 0; i < 6; ++i) { + CPPUNIT_ASSERT_EQUAL(array[i%3], xuris[i]); } - CPPUNIT_ASSERT_EQUAL((unsigned int)3, group->getNumConcurrentCommand()); + CPPUNIT_ASSERT_EQUAL((unsigned int)7, group->getNumConcurrentCommand()); SharedHandle ctx = group->getDownloadContext(); CPPUNIT_ASSERT_EQUAL(std::string("/tmp"), ctx->getDir()); CPPUNIT_ASSERT_EQUAL(std::string("/tmp/file.out"), ctx->getBasePath()); @@ -94,53 +93,41 @@ void DownloadHelperTest::testCreateRequestGroupForUri() option_->put(PREF_SPLIT, "5"); { std::vector > result; - createRequestGroupForUri(result, option_, uris); - CPPUNIT_ASSERT_EQUAL((size_t)1, result.size()); SharedHandle group = result[0]; - std::vector uris; - group->getDownloadContext()->getFirstFileEntry()->getUris(uris); - CPPUNIT_ASSERT_EQUAL((size_t)5, uris.size()); - for(size_t i = 0; i < A2_ARRAY_LEN(array); ++i) { - CPPUNIT_ASSERT_EQUAL(array[i], uris[i]); + std::vector xuris; + group->getDownloadContext()->getFirstFileEntry()->getUris(xuris); + CPPUNIT_ASSERT_EQUAL((size_t)5, xuris.size()); + for(size_t i = 0; i < 5; ++i) { + CPPUNIT_ASSERT_EQUAL(array[i%3], xuris[i]); } - for(size_t i = 0; i < 5-A2_ARRAY_LEN(array); ++i) { - CPPUNIT_ASSERT_EQUAL(array[i], uris[i+A2_ARRAY_LEN(array)]); - } - CPPUNIT_ASSERT_EQUAL((unsigned int)5, group->getNumConcurrentCommand()); } option_->put(PREF_SPLIT, "2"); { std::vector > result; - createRequestGroupForUri(result, option_, uris); - CPPUNIT_ASSERT_EQUAL((size_t)1, result.size()); SharedHandle group = result[0]; - std::vector uris; - group->getDownloadContext()->getFirstFileEntry()->getUris(uris); - CPPUNIT_ASSERT_EQUAL((size_t)3, uris.size()); - for(size_t i = 0; i < A2_ARRAY_LEN(array); ++i) { - CPPUNIT_ASSERT_EQUAL(array[i], uris[i]); + std::vector xuris; + group->getDownloadContext()->getFirstFileEntry()->getUris(xuris); + CPPUNIT_ASSERT_EQUAL((size_t)3, xuris.size()); + for(size_t i = 0; i < 3; ++i) { + CPPUNIT_ASSERT_EQUAL(array[i%3], xuris[i]); } - CPPUNIT_ASSERT_EQUAL((unsigned int)2, group->getNumConcurrentCommand()); } option_->put(PREF_FORCE_SEQUENTIAL, V_TRUE); { std::vector > result; - createRequestGroupForUri(result, option_, uris); - CPPUNIT_ASSERT_EQUAL((size_t)3, result.size()); - // for alpha server SharedHandle alphaGroup = result[0]; std::vector alphaURIs; alphaGroup->getDownloadContext()->getFirstFileEntry()->getUris(alphaURIs); CPPUNIT_ASSERT_EQUAL((size_t)2, alphaURIs.size()); for(size_t i = 0; i < 2; ++i) { - CPPUNIT_ASSERT_EQUAL(array[0], uris[0]); + CPPUNIT_ASSERT_EQUAL(array[0], alphaURIs[i]); } CPPUNIT_ASSERT_EQUAL((unsigned int)2, alphaGroup->getNumConcurrentCommand()); @@ -158,6 +145,7 @@ void DownloadHelperTest::testCreateRequestGroupForUri_parameterized() "http://charlie/file" }; std::vector uris(vbegin(array), vend(array)); + option_->put(PREF_MAX_CONNECTION_PER_SERVER, "1"); option_->put(PREF_SPLIT, "3"); option_->put(PREF_DIR, "/tmp"); option_->put(PREF_OUT, "file.out"); @@ -194,6 +182,7 @@ void DownloadHelperTest::testCreateRequestGroupForUri_BitTorrent() "http://charlie/file" }; std::vector uris(vbegin(array), vend(array)); + option_->put(PREF_MAX_CONNECTION_PER_SERVER, "1"); option_->put(PREF_SPLIT, "3"); option_->put(PREF_DIR, "/tmp"); option_->put(PREF_OUT, "file.out"); @@ -204,13 +193,13 @@ void DownloadHelperTest::testCreateRequestGroupForUri_BitTorrent() CPPUNIT_ASSERT_EQUAL((size_t)2, result.size()); SharedHandle group = result[0]; - std::vector uris; - group->getDownloadContext()->getFirstFileEntry()->getUris(uris); - CPPUNIT_ASSERT_EQUAL((size_t)3, uris.size()); + std::vector xuris; + group->getDownloadContext()->getFirstFileEntry()->getUris(xuris); + CPPUNIT_ASSERT_EQUAL((size_t)3, xuris.size()); - CPPUNIT_ASSERT_EQUAL(array[0], uris[0]); - CPPUNIT_ASSERT_EQUAL(array[2], uris[1]); - CPPUNIT_ASSERT_EQUAL(array[3], uris[2]); + CPPUNIT_ASSERT_EQUAL(array[0], xuris[0]); + CPPUNIT_ASSERT_EQUAL(array[2], xuris[1]); + CPPUNIT_ASSERT_EQUAL(array[3], xuris[2]); CPPUNIT_ASSERT_EQUAL((unsigned int)3, group->getNumConcurrentCommand()); SharedHandle ctx = group->getDownloadContext(); @@ -242,6 +231,7 @@ void DownloadHelperTest::testCreateRequestGroupForUri_Metalink() "test.xml" }; std::vector uris(vbegin(array), vend(array)); + option_->put(PREF_MAX_CONNECTION_PER_SERVER, "1"); option_->put(PREF_SPLIT, "3"); option_->put(PREF_METALINK_SERVERS, "2"); option_->put(PREF_DIR, "/tmp"); @@ -260,11 +250,11 @@ void DownloadHelperTest::testCreateRequestGroupForUri_Metalink() #endif // !ENABLE_BITTORRENT SharedHandle group = result[0]; - std::vector uris; - group->getDownloadContext()->getFirstFileEntry()->getUris(uris); - CPPUNIT_ASSERT_EQUAL((size_t)3, uris.size()); + std::vector xuris; + group->getDownloadContext()->getFirstFileEntry()->getUris(xuris); + CPPUNIT_ASSERT_EQUAL((size_t)3, xuris.size()); for(size_t i = 0; i < 3; ++i) { - CPPUNIT_ASSERT_EQUAL(array[i], uris[i]); + CPPUNIT_ASSERT_EQUAL(array[i], xuris[i]); } CPPUNIT_ASSERT_EQUAL((unsigned int)3, group->getNumConcurrentCommand()); SharedHandle ctx = group->getDownloadContext(); @@ -290,6 +280,7 @@ void DownloadHelperTest::testCreateRequestGroupForUri_Metalink() void DownloadHelperTest::testCreateRequestGroupForUriList() { + option_->put(PREF_MAX_CONNECTION_PER_SERVER, "3"); option_->put(PREF_SPLIT, "3"); option_->put(PREF_INPUT_FILE, "input_uris.txt"); option_->put(PREF_DIR, "/tmp"); @@ -330,6 +321,7 @@ void DownloadHelperTest::testCreateRequestGroupForBitTorrent() }; std::vector auxURIs(vbegin(array), vend(array)); + option_->put(PREF_MAX_CONNECTION_PER_SERVER, "2"); option_->put(PREF_SPLIT, "5"); option_->put(PREF_TORRENT_FILE, "test.torrent"); option_->put(PREF_DIR, "/tmp"); @@ -344,7 +336,8 @@ void DownloadHelperTest::testCreateRequestGroupForBitTorrent() SharedHandle group = result[0]; std::vector uris; group->getDownloadContext()->getFirstFileEntry()->getUris(uris); - // See -s option is ignored + // See -s option is ignored. See processRootDictionary() in + // bittorrent_helper.cc CPPUNIT_ASSERT_EQUAL((size_t)3, uris.size()); for(size_t i = 0; i < A2_ARRAY_LEN(array); ++i) { CPPUNIT_ASSERT_EQUAL(array[i]+"/aria2-test/aria2/src/aria2c", uris[i]); diff --git a/test/FileEntryTest.cc b/test/FileEntryTest.cc index daf0a3ad..34aa863d 100644 --- a/test/FileEntryTest.cc +++ b/test/FileEntryTest.cc @@ -3,6 +3,7 @@ #include #include "InOrderURISelector.h" +#include "util.h" namespace aria2 { @@ -13,7 +14,8 @@ class FileEntryTest : public CppUnit::TestFixture { CPPUNIT_TEST(testRemoveURIWhoseHostnameIs); CPPUNIT_TEST(testExtractURIResult); CPPUNIT_TEST(testGetRequest); - CPPUNIT_TEST(testGetRequest_disableSingleHostMultiConnection); + CPPUNIT_TEST(testGetRequest_withoutUriReuse); + CPPUNIT_TEST(testGetRequest_withUniqueProtocol); CPPUNIT_TEST(testReuseUri); CPPUNIT_TEST(testAddUri); CPPUNIT_TEST(testAddUris); @@ -27,7 +29,8 @@ public: void testRemoveURIWhoseHostnameIs(); void testExtractURIResult(); void testGetRequest(); - void testGetRequest_disableSingleHostMultiConnection(); + void testGetRequest_withoutUriReuse(); + void testGetRequest_withUniqueProtocol(); void testReuseUri(); void testAddUri(); void testAddUris(); @@ -109,7 +112,6 @@ void FileEntryTest::testGetRequest() SharedHandle req = fileEntry->getRequest(selector); CPPUNIT_ASSERT_EQUAL(std::string("localhost"), req->getHost()); CPPUNIT_ASSERT_EQUAL(std::string("http"), req->getProtocol()); - fileEntry->poolRequest(req); SharedHandle req2nd = fileEntry->getRequest(selector); @@ -117,14 +119,51 @@ void FileEntryTest::testGetRequest() CPPUNIT_ASSERT_EQUAL(std::string("http"), req2nd->getProtocol()); SharedHandle req3rd = fileEntry->getRequest(selector); - CPPUNIT_ASSERT_EQUAL(std::string("localhost"), req3rd->getHost()); - CPPUNIT_ASSERT_EQUAL(std::string("ftp"), req3rd->getProtocol()); + CPPUNIT_ASSERT_EQUAL(std::string("mirror"), req3rd->getHost()); + CPPUNIT_ASSERT_EQUAL(std::string("http"), req3rd->getProtocol()); + + SharedHandle req4th = fileEntry->getRequest(selector); + CPPUNIT_ASSERT(req4th.isNull()); + + fileEntry->setMaxConnectionPerServer(2); + + SharedHandle req5th = fileEntry->getRequest(selector); + CPPUNIT_ASSERT_EQUAL(std::string("localhost"), req5th->getHost()); + CPPUNIT_ASSERT_EQUAL(std::string("ftp"), req5th->getProtocol()); + + SharedHandle req6th = fileEntry->getRequest(selector); + CPPUNIT_ASSERT_EQUAL(std::string("mirror"), req6th->getHost()); + CPPUNIT_ASSERT_EQUAL(std::string("http"), req6th->getProtocol()); + + SharedHandle req7th = fileEntry->getRequest(selector); + CPPUNIT_ASSERT(req7th.isNull()); } -void FileEntryTest::testGetRequest_disableSingleHostMultiConnection() +void FileEntryTest::testGetRequest_withoutUriReuse() { SharedHandle fileEntry = createFileEntry(); - fileEntry->disableSingleHostMultiConnection(); + fileEntry->setMaxConnectionPerServer(2); + SharedHandle selector(new InOrderURISelector()); + SharedHandle req = fileEntry->getRequest(selector, false); + CPPUNIT_ASSERT_EQUAL(std::string("localhost"), req->getHost()); + CPPUNIT_ASSERT_EQUAL(std::string("http"), req->getProtocol()); + + SharedHandle req2nd = fileEntry->getRequest(selector, false); + CPPUNIT_ASSERT_EQUAL(std::string("localhost"), req2nd->getHost()); + CPPUNIT_ASSERT_EQUAL(std::string("ftp"), req2nd->getProtocol()); + + SharedHandle req3rd = fileEntry->getRequest(selector, false); + CPPUNIT_ASSERT_EQUAL(std::string("mirror"), req3rd->getHost()); + CPPUNIT_ASSERT_EQUAL(std::string("http"), req3rd->getProtocol()); + + SharedHandle req4th = fileEntry->getRequest(selector, false); + CPPUNIT_ASSERT(req4th.isNull()); +} + +void FileEntryTest::testGetRequest_withUniqueProtocol() +{ + SharedHandle fileEntry = createFileEntry(); + fileEntry->setUniqueProtocol(true); SharedHandle selector(new InOrderURISelector()); SharedHandle req = fileEntry->getRequest(selector); CPPUNIT_ASSERT_EQUAL(std::string("localhost"), req->getHost()); @@ -136,25 +175,42 @@ void FileEntryTest::testGetRequest_disableSingleHostMultiConnection() SharedHandle req3rd = fileEntry->getRequest(selector); CPPUNIT_ASSERT(req3rd.isNull()); + + CPPUNIT_ASSERT_EQUAL((size_t)2, fileEntry->getRemainingUris().size()); + CPPUNIT_ASSERT_EQUAL(std::string("ftp://localhost/aria2.zip"), + fileEntry->getRemainingUris()[0]); + CPPUNIT_ASSERT_EQUAL(std::string("http://mirror/aria2.zip"), + fileEntry->getRemainingUris()[1]); } void FileEntryTest::testReuseUri() { SharedHandle selector(new InOrderURISelector()); SharedHandle fileEntry = createFileEntry(); + fileEntry->setMaxConnectionPerServer(3); size_t numUris = fileEntry->getRemainingUris().size(); for(size_t i = 0; i < numUris; ++i) { - fileEntry->getRequest(selector); + fileEntry->getRequest(selector, false); } CPPUNIT_ASSERT_EQUAL((size_t)0, fileEntry->getRemainingUris().size()); fileEntry->addURIResult("http://localhost/aria2.zip", downloadresultcode::UNKNOWN_ERROR); - fileEntry->reuseUri(3); - CPPUNIT_ASSERT_EQUAL((size_t)3, fileEntry->getRemainingUris().size()); - const std::deque& uris = fileEntry->getRemainingUris(); + std::vector ignore; + fileEntry->reuseUri(ignore); + CPPUNIT_ASSERT_EQUAL((size_t)2, fileEntry->getRemainingUris().size()); + std::deque uris = fileEntry->getRemainingUris(); CPPUNIT_ASSERT_EQUAL(std::string("ftp://localhost/aria2.zip"), uris[0]); CPPUNIT_ASSERT_EQUAL(std::string("http://mirror/aria2.zip"), uris[1]); - CPPUNIT_ASSERT_EQUAL(std::string("ftp://localhost/aria2.zip"), uris[2]); + for(size_t i = 0; i < 2; ++i) { + fileEntry->getRequest(selector, false); + } + CPPUNIT_ASSERT_EQUAL((size_t)0, fileEntry->getRemainingUris().size()); + ignore.clear(); + ignore.push_back("mirror"); + fileEntry->reuseUri(ignore); + CPPUNIT_ASSERT_EQUAL((size_t)1, fileEntry->getRemainingUris().size()); + uris = fileEntry->getRemainingUris(); + CPPUNIT_ASSERT_EQUAL(std::string("ftp://localhost/aria2.zip"), uris[0]); } void FileEntryTest::testAddUri()