From d64c8e75f6c3b6895f51036e5d16ad696dfe0f66 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Mon, 4 Aug 2008 17:06:47 +0000 Subject: [PATCH] 2008-08-05 Tatsuhiro Tsujikawa Implemented download speed based URI selection algorithm. Introduced new option --uri-selector. If --uri-selector=feedback is given, aria2 uses download speed observed in the previous downloads and chooses fastest server in the URI list. Currently at most 10 URIs are considered to introduce randomeness for finding better servers. The speed is average download speed in the downloads. On the other hand, if --uri-selector=inorder is given, which is default, URI is tried in order in URI list. The usage text for the new option has not been written yet. * src/AbstractCommand.cc * src/DownloadCommand.cc * src/DownloadEngine.cc * src/DownloadEngineFactory.cc * src/InOrderURISelector.cc * src/InOrderURISelector.h * src/OptionHandlerFactory.cc * src/PeerStat.h * src/RequestGroup.cc * src/RequestGroup.h * src/RequestGroupMan.cc * src/RequestGroupMan.h * src/SegmentMan.cc * src/SegmentMan.h * src/ServerStat.cc * src/ServerStat.h * src/ServerStatMan.cc * src/ServerStatMan.h * src/ServerStatURISelector.cc * src/ServerStatURISelector.h * src/URISelector.h * src/option_processing.cc * src/prefs.cc * src/prefs.h * test/InOrderURISelectorTest.cc * test/RequestGroupManTest.cc * test/ServerStatManTest.cc * test/ServerStatURISelectorTest.cc --- ChangeLog | 41 +++++++++ src/AbstractCommand.cc | 12 +++ src/DownloadCommand.cc | 2 +- src/DownloadEngine.cc | 2 + src/DownloadEngineFactory.cc | 4 +- src/InOrderURISelector.cc | 55 ++++++++++++ src/InOrderURISelector.h | 51 ++++++++++++ src/Makefile.am | 7 +- src/Makefile.in | 41 +++++---- src/OptionHandlerFactory.cc | 7 ++ src/PeerStat.h | 22 +++++ src/RequestGroup.cc | 12 ++- src/RequestGroup.h | 5 ++ src/RequestGroupMan.cc | 85 ++++++++++++++++++- src/RequestGroupMan.h | 19 ++++- src/SegmentMan.cc | 5 ++ src/SegmentMan.h | 3 + src/ServerStat.cc | 133 ++++++++++++++++++++++++++++++ src/ServerStat.h | 108 ++++++++++++++++++++++++ src/ServerStatMan.cc | 83 +++++++++++++++++++ src/ServerStatMan.h | 69 ++++++++++++++++ src/ServerStatURISelector.cc | 108 ++++++++++++++++++++++++ src/ServerStatURISelector.h | 58 +++++++++++++ src/URISelector.h | 52 ++++++++++++ src/option_processing.cc | 5 ++ src/prefs.cc | 4 + src/prefs.h | 4 + test/InOrderURISelectorTest.cc | 50 +++++++++++ test/Makefile.am | 5 +- test/Makefile.in | 19 +++-- test/RequestGroupManTest.cc | 3 +- test/ServerStatManTest.cc | 50 +++++++++++ test/ServerStatURISelectorTest.cc | 97 ++++++++++++++++++++++ 33 files changed, 1192 insertions(+), 29 deletions(-) create mode 100644 src/InOrderURISelector.cc create mode 100644 src/InOrderURISelector.h create mode 100644 src/ServerStat.cc create mode 100644 src/ServerStat.h create mode 100644 src/ServerStatMan.cc create mode 100644 src/ServerStatMan.h create mode 100644 src/ServerStatURISelector.cc create mode 100644 src/ServerStatURISelector.h create mode 100644 src/URISelector.h create mode 100644 test/InOrderURISelectorTest.cc create mode 100644 test/ServerStatManTest.cc create mode 100644 test/ServerStatURISelectorTest.cc diff --git a/ChangeLog b/ChangeLog index 2c95d58a..bc20f970 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,44 @@ +2008-08-05 Tatsuhiro Tsujikawa + + Implemented download speed based URI selection algorithm. + Introduced new option --uri-selector. + If --uri-selector=feedback is given, aria2 uses download speed observed + in the previous downloads and chooses fastest server in the URI list. + Currently at most 10 URIs are considered to introduce randomeness for + finding better servers. The speed is average download speed in the + downloads. + On the other hand, if --uri-selector=inorder is given, which is default, + URI is tried in order in URI list. + The usage text for the new option has not been written yet. + * src/AbstractCommand.cc + * src/DownloadCommand.cc + * src/DownloadEngine.cc + * src/DownloadEngineFactory.cc + * src/InOrderURISelector.cc + * src/InOrderURISelector.h + * src/OptionHandlerFactory.cc + * src/PeerStat.h + * src/RequestGroup.cc + * src/RequestGroup.h + * src/RequestGroupMan.cc + * src/RequestGroupMan.h + * src/SegmentMan.cc + * src/SegmentMan.h + * src/ServerStat.cc + * src/ServerStat.h + * src/ServerStatMan.cc + * src/ServerStatMan.h + * src/ServerStatURISelector.cc + * src/ServerStatURISelector.h + * src/URISelector.h + * src/option_processing.cc + * src/prefs.cc + * src/prefs.h + * test/InOrderURISelectorTest.cc + * test/RequestGroupManTest.cc + * test/ServerStatManTest.cc + * test/ServerStatURISelectorTest.cc + 2008-08-05 Tatsuhiro Tsujikawa * Release 0.15.1+2 diff --git a/src/AbstractCommand.cc b/src/AbstractCommand.cc index eb2f2aee..59671a77 100644 --- a/src/AbstractCommand.cc +++ b/src/AbstractCommand.cc @@ -55,6 +55,8 @@ #include "message.h" #include "prefs.h" #include "StringFormat.h" +#include "ServerStat.h" +#include "RequestGroupMan.h" namespace aria2 { @@ -136,6 +138,16 @@ bool AbstractCommand::execute() { return executeInternal(); } else { if(checkPoint.elapsed(timeout)) { + // timeout triggers ServerStat error state. + SharedHandle ss = + e->_requestGroupMan->findServerStat(req->getHost(), + req->getProtocol()); + if(ss.isNull()) { + ss.reset(new ServerStat(req->getHost(), req->getProtocol())); + e->_requestGroupMan->addServerStat(ss); + } + ss->setError(); + throw DlRetryEx(EX_TIME_OUT); } e->commands.push_back(this); diff --git a/src/DownloadCommand.cc b/src/DownloadCommand.cc index e6585e72..deed4401 100644 --- a/src/DownloadCommand.cc +++ b/src/DownloadCommand.cc @@ -88,7 +88,7 @@ DownloadCommand::DownloadCommand(int cuid, #endif // ENABLE_MESSAGE_DIGEST peerStat = _requestGroup->getSegmentMan()->getPeerStat(cuid); if(peerStat.isNull()) { - peerStat.reset(new PeerStat(cuid)); + peerStat.reset(new PeerStat(cuid, req->getHost(), req->getProtocol())); _requestGroup->getSegmentMan()->registerPeerStat(peerStat); } peerStat->downloadStart(); diff --git a/src/DownloadEngine.cc b/src/DownloadEngine.cc index 7332bcc3..177347f4 100644 --- a/src/DownloadEngine.cc +++ b/src/DownloadEngine.cc @@ -51,6 +51,7 @@ #include "Util.h" #include "a2functional.h" #include "DlAbortEx.h" +#include "ServerStatMan.h" #include #include #include @@ -783,6 +784,7 @@ void DownloadEngine::calculateStatistics() void DownloadEngine::onEndOfRun() { + _requestGroupMan->updateServerStat(); _requestGroupMan->closeFile(); _requestGroupMan->save(); } diff --git a/src/DownloadEngineFactory.cc b/src/DownloadEngineFactory.cc index fa717a78..8e3d6383 100644 --- a/src/DownloadEngineFactory.cc +++ b/src/DownloadEngineFactory.cc @@ -51,6 +51,7 @@ #include "HaveEraseCommand.h" #include "TimedHaltCommand.h" #include "DownloadResult.h" +#include "ServerStatMan.h" #include namespace aria2 { @@ -80,7 +81,8 @@ DownloadEngineFactory::newDownloadEngine(Option* op, DownloadEngineHandle e(new DownloadEngine()); e->option = op; RequestGroupManHandle - requestGroupMan(new RequestGroupMan(workingSet, MAX_CONCURRENT_DOWNLOADS)); + requestGroupMan(new RequestGroupMan(workingSet, MAX_CONCURRENT_DOWNLOADS, + op)); requestGroupMan->addReservedGroup(reservedSet); e->_requestGroupMan = requestGroupMan; e->_fileAllocationMan.reset(new FileAllocationMan()); diff --git a/src/InOrderURISelector.cc b/src/InOrderURISelector.cc new file mode 100644 index 00000000..0f89e863 --- /dev/null +++ b/src/InOrderURISelector.cc @@ -0,0 +1,55 @@ +/* */ +#include "InOrderURISelector.h" +#include "A2STR.h" + +namespace aria2 { + +InOrderURISelector::InOrderURISelector() {} + +InOrderURISelector::~InOrderURISelector() {} + +std::string InOrderURISelector::select(std::deque& uris) +{ + if(uris.empty()) { + return A2STR::NIL; + } else { + std::string nextURI = uris.front(); + uris.pop_front(); + return nextURI; + } +} + +} // namespace aria2 diff --git a/src/InOrderURISelector.h b/src/InOrderURISelector.h new file mode 100644 index 00000000..43496f8f --- /dev/null +++ b/src/InOrderURISelector.h @@ -0,0 +1,51 @@ +/* */ +#ifndef _D_IN_ORDER_URI_SELECTOR_H_ +#define _D_IN_ORDER_URI_SELECTOR_H_ +#include "URISelector.h" + +namespace aria2 { + +class InOrderURISelector:public URISelector { +public: + InOrderURISelector(); + + virtual ~InOrderURISelector(); + + virtual std::string select(std::deque& uris); +}; + +} // namespace aria2 +#endif // _D_IN_ORDER_URI_SELECTOR_H_ diff --git a/src/Makefile.am b/src/Makefile.am index cfabaf61..0442c125 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -189,7 +189,12 @@ SRCS = Socket.h\ RarestPieceSelector.cc RarestPieceSelector.h\ Decoder.h\ ChunkedDecoder.cc ChunkedDecoder.h\ - Signature.cc Signature.h + Signature.cc Signature.h\ + ServerStat.cc ServerStat.h\ + ServerStatMan.cc ServerStatMan.h\ + URISelector.h\ + InOrderURISelector.cc InOrderURISelector.h\ + ServerStatURISelector.cc ServerStatURISelector.h if HAVE_LIBZ SRCS += GZipDecoder.cc GZipDecoder.h diff --git a/src/Makefile.in b/src/Makefile.in index c194fb1e..c25f9466 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -409,8 +409,12 @@ am__libaria2c_a_SOURCES_DIST = Socket.h SocketCore.cc SocketCore.h \ FtpFinishDownloadCommand.cc FtpFinishDownloadCommand.h \ A2STR.cc A2STR.h RarestPieceSelector.cc RarestPieceSelector.h \ Decoder.h ChunkedDecoder.cc ChunkedDecoder.h Signature.cc \ - Signature.h GZipDecoder.cc GZipDecoder.h AsyncNameResolver.cc \ - AsyncNameResolver.h IteratableChunkChecksumValidator.cc \ + Signature.h ServerStat.cc ServerStat.h ServerStatMan.cc \ + ServerStatMan.h URISelector.h InOrderURISelector.cc \ + InOrderURISelector.h ServerStatURISelector.cc \ + ServerStatURISelector.h GZipDecoder.cc GZipDecoder.h \ + AsyncNameResolver.cc AsyncNameResolver.h \ + IteratableChunkChecksumValidator.cc \ IteratableChunkChecksumValidator.h \ IteratableChecksumValidator.cc IteratableChecksumValidator.h \ CheckIntegrityCommand.cc CheckIntegrityCommand.h \ @@ -798,12 +802,14 @@ am__objects_17 = SocketCore.$(OBJEXT) Command.$(OBJEXT) \ InitiateConnectionCommand.$(OBJEXT) \ FtpFinishDownloadCommand.$(OBJEXT) A2STR.$(OBJEXT) \ RarestPieceSelector.$(OBJEXT) ChunkedDecoder.$(OBJEXT) \ - Signature.$(OBJEXT) $(am__objects_1) $(am__objects_2) \ - $(am__objects_3) $(am__objects_4) $(am__objects_5) \ - $(am__objects_6) $(am__objects_7) $(am__objects_8) \ - $(am__objects_9) $(am__objects_10) $(am__objects_11) \ - $(am__objects_12) $(am__objects_13) $(am__objects_14) \ - $(am__objects_15) $(am__objects_16) + Signature.$(OBJEXT) ServerStat.$(OBJEXT) \ + ServerStatMan.$(OBJEXT) InOrderURISelector.$(OBJEXT) \ + ServerStatURISelector.$(OBJEXT) $(am__objects_1) \ + $(am__objects_2) $(am__objects_3) $(am__objects_4) \ + $(am__objects_5) $(am__objects_6) $(am__objects_7) \ + $(am__objects_8) $(am__objects_9) $(am__objects_10) \ + $(am__objects_11) $(am__objects_12) $(am__objects_13) \ + $(am__objects_14) $(am__objects_15) $(am__objects_16) am_libaria2c_a_OBJECTS = $(am__objects_17) libaria2c_a_OBJECTS = $(am_libaria2c_a_OBJECTS) am__installdirs = "$(DESTDIR)$(bindir)" @@ -1122,12 +1128,15 @@ SRCS = Socket.h SocketCore.cc SocketCore.h BinaryStream.h Command.cc \ FtpFinishDownloadCommand.cc FtpFinishDownloadCommand.h \ A2STR.cc A2STR.h RarestPieceSelector.cc RarestPieceSelector.h \ Decoder.h ChunkedDecoder.cc ChunkedDecoder.h Signature.cc \ - Signature.h $(am__append_1) $(am__append_2) $(am__append_3) \ - $(am__append_4) $(am__append_5) $(am__append_6) \ - $(am__append_7) $(am__append_8) $(am__append_9) \ - $(am__append_10) $(am__append_11) $(am__append_12) \ - $(am__append_13) $(am__append_14) $(am__append_15) \ - $(am__append_16) + Signature.h ServerStat.cc ServerStat.h ServerStatMan.cc \ + ServerStatMan.h URISelector.h InOrderURISelector.cc \ + InOrderURISelector.h ServerStatURISelector.cc \ + ServerStatURISelector.h $(am__append_1) $(am__append_2) \ + $(am__append_3) $(am__append_4) $(am__append_5) \ + $(am__append_6) $(am__append_7) $(am__append_8) \ + $(am__append_9) $(am__append_10) $(am__append_11) \ + $(am__append_12) $(am__append_13) $(am__append_14) \ + $(am__append_15) $(am__append_16) noinst_LIBRARIES = libaria2c.a libaria2c_a_SOURCES = $(SRCS) aria2c_LDADD = libaria2c.a @LIBINTL@ @ALLOCA@ @LIBGNUTLS_LIBS@\ @@ -1395,6 +1404,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/HttpResponse.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/HttpResponseCommand.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/HttpSkipResponseCommand.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/InOrderURISelector.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/InitialMetalinkParserState.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/InitiateConnectionCommand.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/InitiateConnectionCommandFactory.Po@am__quote@ @@ -1461,6 +1471,9 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/SeedCheckCommand.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/SegmentMan.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ServerHost.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ServerStat.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ServerStatMan.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ServerStatURISelector.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Signature.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/SignatureMetalinkParserState.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/SimpleBtMessage.Po@am__quote@ diff --git a/src/OptionHandlerFactory.cc b/src/OptionHandlerFactory.cc index f7d873de..8153f6eb 100644 --- a/src/OptionHandlerFactory.cc +++ b/src/OptionHandlerFactory.cc @@ -143,6 +143,13 @@ OptionHandlers OptionHandlerFactory::createOptionHandlers() ¶ms[arrayLength(params)])))); } handlers.push_back(SH(new BooleanOptionHandler(PREF_BT_SEED_UNVERIFIED))); + { + const std::string params[] = { V_INORDER, V_FEEDBACK }; + handlers.push_back(SH(new ParameterOptionHandler + (PREF_URI_SELECTOR, + std::deque + (¶ms[0], ¶ms[arrayLength(params)])))); + } return handlers; } diff --git a/src/PeerStat.h b/src/PeerStat.h index eb118b71..fa66eab0 100644 --- a/src/PeerStat.h +++ b/src/PeerStat.h @@ -38,6 +38,7 @@ #include "common.h" #include "SpeedCalc.h" #include "SharedHandle.h" +#include namespace aria2 { @@ -50,6 +51,8 @@ public: }; private: int32_t cuid; + std::string _hostname; + std::string _protocol; SpeedCalc downloadSpeed; SpeedCalc uploadSpeed; Time downloadStartTime; @@ -58,6 +61,15 @@ private: unsigned int _avgUploadSpeed; public: + PeerStat(int32_t cuid, const std::string& hostname, + const::std::string& protocol): + cuid(cuid), + _hostname(hostname), + _protocol(protocol), + status(PeerStat::IDLE), + _avgDownloadSpeed(0), + _avgUploadSpeed(0) {} + PeerStat(int32_t cuid = 0):cuid(cuid), status(PeerStat::IDLE), _avgDownloadSpeed(0), _avgUploadSpeed(0) {} @@ -150,6 +162,16 @@ public: int32_t getCuid() const { return cuid; } + + const std::string& getHostname() const + { + return _hostname; + } + + const std::string& getProtocol() const + { + return _protocol; + } }; typedef SharedHandle PeerStatHandle; diff --git a/src/RequestGroup.cc b/src/RequestGroup.cc index 79a48012..f8a5173c 100644 --- a/src/RequestGroup.cc +++ b/src/RequestGroup.cc @@ -72,6 +72,8 @@ #include "FileAllocationIterator.h" #include "StringFormat.h" #include "A2STR.h" +#include "URISelector.h" +#include "InOrderURISelector.h" #ifdef ENABLE_MESSAGE_DIGEST # include "CheckIntegrityCommand.h" #endif // ENABLE_MESSAGE_DIGEST @@ -120,6 +122,7 @@ RequestGroup::RequestGroup(const Option* option, _haltRequested(false), _forceHaltRequested(false), _singleHostMultiConnectionEnabled(true), + _uriSelector(new InOrderURISelector()), _option(option), _logger(LogFactory::getInstance()) { @@ -498,8 +501,8 @@ void RequestGroup::createNextCommand(std::deque& commands, const std::string& method) { std::deque pendingURIs; - for(;!_uris.empty() && numCommand--; _uris.pop_front()) { - std::string uri = _uris.front(); + for(; !_uris.empty() && numCommand--; ) { + std::string uri = _uriSelector->select(_uris); RequestHandle req(new Request()); if(req->setUrl(uri)) { ServerHostHandle sv; @@ -1007,4 +1010,9 @@ void RequestGroup::removeAcceptType(const std::string& type) _acceptTypes.end()); } +void RequestGroup::setURISelector(const SharedHandle& uriSelector) +{ + _uriSelector = uriSelector; +} + } // namespace aria2 diff --git a/src/RequestGroup.h b/src/RequestGroup.h index 6eeaf741..47543ad6 100644 --- a/src/RequestGroup.h +++ b/src/RequestGroup.h @@ -60,6 +60,7 @@ class RequestGroup; class CheckIntegrityEntry; class DownloadResult; class ServerHost; +class URISelector; class RequestGroup { private: @@ -112,6 +113,8 @@ private: std::deque _acceptTypes; + SharedHandle _uriSelector; + const Option* _option; Logger* _logger; @@ -354,6 +357,8 @@ public: void removeAcceptType(const std::string& type); static const std::string ACCEPT_METALINK; + + void setURISelector(const SharedHandle& uriSelector); }; typedef SharedHandle RequestGroupHandle; diff --git a/src/RequestGroupMan.cc b/src/RequestGroupMan.cc index d493cdf1..530cf2e3 100644 --- a/src/RequestGroupMan.cc +++ b/src/RequestGroupMan.cc @@ -43,6 +43,14 @@ #include "a2functional.h" #include "DownloadResult.h" #include "DownloadContext.h" +#include "ServerStatMan.h" +#include "ServerStat.h" +#include "PeerStat.h" +#include "SegmentMan.h" +#include "ServerStatURISelector.h" +#include "InOrderURISelector.h" +#include "Option.h" +#include "prefs.h" #include #include #include @@ -52,11 +60,14 @@ namespace aria2 { RequestGroupMan::RequestGroupMan(const RequestGroups& requestGroups, - unsigned int maxSimultaneousDownloads): + unsigned int maxSimultaneousDownloads, + const Option* option): _requestGroups(requestGroups), _logger(LogFactory::getInstance()), _maxSimultaneousDownloads(maxSimultaneousDownloads), - _gidCounter(0) {} + _gidCounter(0), + _option(option), + _serverStatMan(new ServerStatMan()) {} bool RequestGroupMan::downloadFinished() { @@ -168,6 +179,41 @@ public: } }; +class CollectServerStat { +private: + RequestGroupMan* _requestGroupMan; +public: + CollectServerStat(RequestGroupMan* requestGroupMan): + _requestGroupMan(requestGroupMan) {} + + void operator()(const SharedHandle& group) + { + if(group->getNumCommand() == 0) { + // Collect statistics during download in PeerStats and update/register + // ServerStatMan + if(!group->getSegmentMan().isNull()) { + const std::deque >& peerStats = + group->getSegmentMan()->getPeerStats(); + for(std::deque >::const_iterator i = + peerStats.begin(); i != peerStats.end(); ++i) { + if((*i)->getHostname().empty() || (*i)->getProtocol().empty()) { + continue; + } + SharedHandle ss = + _requestGroupMan->findServerStat((*i)->getHostname(), + (*i)->getProtocol()); + if(ss.isNull()) { + ss.reset(new ServerStat((*i)->getHostname(), + (*i)->getProtocol())); + _requestGroupMan->addServerStat(ss); + } + ss->updateDownloadSpeed((*i)->getAvgDownloadSpeed()); + } + } + } + } +}; + class FindStoppedRequestGroup { public: bool operator()(const SharedHandle& group) { @@ -175,10 +221,18 @@ public: } }; +void RequestGroupMan::updateServerStat() +{ + std::for_each(_requestGroups.begin(), _requestGroups.end(), + CollectServerStat(this)); +} + void RequestGroupMan::removeStoppedGroup() { size_t numPrev = _requestGroups.size(); + updateServerStat(); + std::for_each(_requestGroups.begin(), _requestGroups.end(), ProcessStoppedRequestGroup(_reservedGroups, _downloadResults)); @@ -193,6 +247,19 @@ void RequestGroupMan::removeStoppedGroup() } } +void RequestGroupMan::configureRequestGroup +(const SharedHandle& requestGroup) const +{ + const std::string& uriSelectorValue = _option->get(PREF_URI_SELECTOR); + if(uriSelectorValue == V_FEEDBACK) { + requestGroup->setURISelector + (SharedHandle(new ServerStatURISelector(_serverStatMan))); + } else if(uriSelectorValue == V_INORDER) { + requestGroup->setURISelector + (SharedHandle(new InOrderURISelector())); + } +} + void RequestGroupMan::fillRequestGroupFromReserver(DownloadEngine* e) { RequestGroups temp; @@ -207,6 +274,7 @@ void RequestGroupMan::fillRequestGroupFromReserver(DownloadEngine* e) temp.push_back(groupToAdd); continue; } + configureRequestGroup(groupToAdd); Commands commands; groupToAdd->createInitialCommand(commands, e); _requestGroups.push_back(groupToAdd); @@ -231,6 +299,7 @@ void RequestGroupMan::getInitialCommands(std::deque& commands, itr != _requestGroups.end();) { try { if((*itr)->isDependencyResolved()) { + configureRequestGroup(*itr); (*itr)->createInitialCommand(commands, e); ++itr; } else { @@ -396,4 +465,16 @@ RequestGroupMan::getDownloadResults() const return _downloadResults; } +SharedHandle +RequestGroupMan::findServerStat(const std::string& hostname, + const std::string& protocol) const +{ + return _serverStatMan->find(hostname, protocol); +} + +bool RequestGroupMan::addServerStat(const SharedHandle& serverStat) +{ + return _serverStatMan->add(serverStat); +} + } // namespace aria2 diff --git a/src/RequestGroupMan.h b/src/RequestGroupMan.h index 88a8908d..700b71a4 100644 --- a/src/RequestGroupMan.h +++ b/src/RequestGroupMan.h @@ -49,6 +49,9 @@ class RequestGroup; class Command; class Logger; class DownloadResult; +class ServerStatMan; +class ServerStat; +class Option; class RequestGroupMan { private: @@ -59,13 +62,20 @@ private: unsigned int _maxSimultaneousDownloads; int32_t _gidCounter; + const Option* _option; + + SharedHandle _serverStatMan; + std::string formatDownloadResult(const std::string& status, const SharedHandle& downloadResult) const; + void configureRequestGroup + (const SharedHandle& requestGroup) const; public: RequestGroupMan(const std::deque >& requestGroups, - unsigned int maxSimultaneousDownloads = 1); + unsigned int maxSimultaneousDownloads, + const Option* option); bool downloadFinished(); @@ -126,6 +136,13 @@ public: const std::deque >& getDownloadResults() const; + SharedHandle findServerStat(const std::string& hostname, + const std::string& protocol) const; + + bool addServerStat(const SharedHandle& serverStat); + + + void updateServerStat(); }; typedef SharedHandle RequestGroupManHandle; diff --git a/src/SegmentMan.cc b/src/SegmentMan.cc index a2f4442b..0d3f8a15 100644 --- a/src/SegmentMan.cc +++ b/src/SegmentMan.cc @@ -284,6 +284,11 @@ PeerStatHandle SegmentMan::getPeerStat(int32_t cuid) const } } +const std::deque >& SegmentMan::getPeerStats() const +{ + return peerStats; +} + unsigned int SegmentMan::calculateDownloadSpeed() const { unsigned int speed = 0; for(std::deque >::const_iterator itr = peerStats.begin(); itr != peerStats.end(); itr++) { diff --git a/src/SegmentMan.h b/src/SegmentMan.h index 8d8be720..1b3b6993 100644 --- a/src/SegmentMan.h +++ b/src/SegmentMan.h @@ -171,6 +171,9 @@ public: */ SharedHandle getPeerStat(int32_t cuid) const; + + const std::deque >& getPeerStats() const; + /** * Returns current download speed in bytes per sec. */ diff --git a/src/ServerStat.cc b/src/ServerStat.cc new file mode 100644 index 00000000..d1782c61 --- /dev/null +++ b/src/ServerStat.cc @@ -0,0 +1,133 @@ +/* */ +#include "ServerStat.h" +#include + +namespace aria2 { + +ServerStat::ServerStat(const std::string& hostname, const std::string& protocol) + : + _hostname(hostname), + _protocol(protocol), + _downloadSpeed(0), + _status(OK) {} + +ServerStat::~ServerStat() {} + +const std::string& ServerStat::getHostname() const +{ + return _hostname; +} + +const std::string& ServerStat::getProtocol() const +{ + return _protocol; +} + +Time ServerStat::getLastUpdated() const +{ + return _lastUpdated; +} + +unsigned int ServerStat::getDownloadSpeed() const +{ + return _downloadSpeed; +} + +void ServerStat::updateDownloadSpeed(unsigned int downloadSpeed) +{ + _downloadSpeed = downloadSpeed; + if(downloadSpeed > 0) { + _status = OK; + } + _lastUpdated.reset(); +} + +void ServerStat::setStatus(STATUS status) +{ + _status = status; + _lastUpdated.reset(); +} + +ServerStat::STATUS ServerStat::getStatus() const +{ + return _status; +} + +bool ServerStat::isOK() const +{ + return _status == OK; +} + +void ServerStat::setOK() +{ + setStatus(OK); +} + +bool ServerStat::isError() const +{ + return _status == ERROR; +} + +void ServerStat::setError() +{ + setStatus(ERROR); +} + +bool ServerStat::operator<(const ServerStat& serverStat) const +{ + int c = _hostname.compare(serverStat._hostname); + if(c == 0) { + return _protocol < serverStat._protocol; + } else { + return c < 0; + } +} + +bool ServerStat::operator==(const ServerStat& serverStat) const +{ + return _hostname == serverStat._hostname && _protocol == serverStat._protocol; +} + +std::ostream& operator<<(std::ostream& o, const ServerStat& serverStat) +{ + o << "host=" << serverStat.getHostname() << ", " + << "protocol=" << serverStat.getProtocol() << ", " + << "dl_speed=" << serverStat.getDownloadSpeed() << ", " + << "status=" << serverStat.getStatus() << "\n"; + return o; +} + +} // namespace aria2 diff --git a/src/ServerStat.h b/src/ServerStat.h new file mode 100644 index 00000000..7f31ea93 --- /dev/null +++ b/src/ServerStat.h @@ -0,0 +1,108 @@ +/* */ +#ifndef _D_SERVER_STAT_H_ +#define _D_SERVER_STAT_H_ +#include "common.h" +#include "TimeA2.h" +#include +#include + +namespace aria2 { + +// ServerStatMan: has many ServerStat +// URISelector: interface +// ServerStatURISelector: Has a reference of ServerStatMan +// InOrderURISelector: this is default. +class ServerStat { +public: + enum STATUS { + OK, + ERROR + }; + + ServerStat(const std::string& hostname, const std::string& protocol); + + ~ServerStat(); + + const std::string& getHostname() const; + + const std::string& getProtocol() const; + + Time getLastUpdated() const; + + void setLastUpdated(const Time& time); + + unsigned int getDownloadSpeed() const; + + // update download speed and update _lastUpdated + void updateDownloadSpeed(unsigned int downloadSpeed); + + // set download speed. This method doesn't update _lastUpdate. + void setDownloadSpeed(unsigned int downloadSpeed); + + void setStatus(STATUS status); + + STATUS getStatus() const; + + bool isOK() const; + + // set status OK and update _lastUpdated + void setOK(); + + bool isError() const; + + // set status ERROR and update _lastUpdated + void setError(); + + bool operator<(const ServerStat& serverStat) const; + + bool operator==(const ServerStat& serverStat) const; +private: + std::string _hostname; + + std::string _protocol; + + unsigned int _downloadSpeed; + + STATUS _status; + + Time _lastUpdated; +}; + +std::ostream& operator<<(std::ostream& o, const ServerStat& serverStat); + +} // namespace aria2 + +#endif // _D_SERVER_STAT_H_ diff --git a/src/ServerStatMan.cc b/src/ServerStatMan.cc new file mode 100644 index 00000000..a4719029 --- /dev/null +++ b/src/ServerStatMan.cc @@ -0,0 +1,83 @@ +/* */ +#include "ServerStatMan.h" +#include "ServerStat.h" +#include +#include + +namespace aria2 { + +ServerStatMan::ServerStatMan() {} + +ServerStatMan::~ServerStatMan() {} + +SharedHandle ServerStatMan::find(const std::string& hostname, + const std::string& protocol) const +{ + SharedHandle ss(new ServerStat(hostname, protocol)); + std::deque >::const_iterator i = + std::lower_bound(_serverStats.begin(), _serverStats.end(), ss); + if(i != _serverStats.end() && + (*i)->getHostname() == hostname && (*i)->getProtocol() == protocol) { + return *i; + } else { + return SharedHandle(); + } +} + +bool ServerStatMan::add(const SharedHandle& serverStat) +{ + std::deque >::iterator i = + std::lower_bound(_serverStats.begin(), _serverStats.end(), serverStat); + + if(i != _serverStats.end() && (*i) == serverStat) { + return false; + } else { + _serverStats.insert(i, serverStat); + return true; + } +} + +//bool save(const std::string& filepath) const; + +void ServerStatMan::print(std::ostream& o) const +{ + for(std::deque >::const_iterator i = + _serverStats.begin(); i != _serverStats.end(); ++i) { + o << *i; + } +} + +} // namespace aria2 diff --git a/src/ServerStatMan.h b/src/ServerStatMan.h new file mode 100644 index 00000000..b2301ccc --- /dev/null +++ b/src/ServerStatMan.h @@ -0,0 +1,69 @@ +/* */ +#ifndef _D_SERVER_STAT_MAN_H_ +#define _D_SERVER_STAT_MAN_H_ +#include "common.h" +#include "SharedHandle.h" +#include +#include +#include + +namespace aria2 { + +class ServerStat; + +class ServerStatMan { +public: + ServerStatMan(); + + ~ServerStatMan(); + + SharedHandle find(const std::string& hostname, + const std::string& protocol) const; + + bool add(const SharedHandle& serverStat); + + void load(std::istream& in); + + void save(std::ostream& out) const; + + void print(std::ostream& o) const; +private: + std::deque > _serverStats; +}; + +} // namespace aria2 + +#endif // _D_SERVER_STAT_MAN_H_ diff --git a/src/ServerStatURISelector.cc b/src/ServerStatURISelector.cc new file mode 100644 index 00000000..3e39bd00 --- /dev/null +++ b/src/ServerStatURISelector.cc @@ -0,0 +1,108 @@ +/* */ +#include "ServerStatURISelector.h" +#include "ServerStatMan.h" +#include "ServerStat.h" +#include "Request.h" +#include "A2STR.h" +#include + +namespace aria2 { + +ServerStatURISelector::ServerStatURISelector +(const SharedHandle& serverStatMan): + _serverStatMan(serverStatMan) {} + +ServerStatURISelector::~ServerStatURISelector() {} + +class ServerStatFaster { +public: + bool operator()(const std::pair, std::string> lhs, + const std::pair, std::string> rhs) + const + { + return lhs.first->getDownloadSpeed() > rhs.first->getDownloadSpeed(); + } +}; + +std::string ServerStatURISelector::select(std::deque& uris) +{ + if(uris.empty()) { + return A2STR::NIL; + } + // Use first 10 URIs to introduce some randomness. + const int NUM_URI = 10; + // Ignore low speed server + const unsigned int SPEED_THRESHOLD = 20*1024; + size_t max = std::min(uris.size(), static_cast(NUM_URI)); + std::deque::iterator urisLast = uris.begin()+max; + std::deque, std::string> > cands; + for(std::deque::iterator i = uris.begin(); + i != urisLast; ++i) { + Request r; + r.setUrl(*i); + SharedHandle ss = _serverStatMan->find(r.getHost(), + r.getProtocol()); + if(!ss.isNull() && ss->isOK() && ss->getDownloadSpeed() > SPEED_THRESHOLD) { + cands.push_back(std::pair, std::string>(ss, *i)); + } + } + if(cands.empty()) { + for(std::deque::iterator i = uris.begin(); + i != uris.end(); ++i) { + Request r; + r.setUrl(*i); + SharedHandle ss = _serverStatMan->find(r.getHost(), + r.getProtocol()); + // Skip ERROR state URI + if(ss.isNull() || ss->isOK()) { + std::string nextURI = *i; + uris.erase(uris.begin(), i+1); + return nextURI; + } + } + // All URIs are inspected but aria2 cannot find usable one. + // Return first URI anyway in this case. + std::string nextURI = uris.front(); + uris.pop_front(); + return nextURI; + } else { + std::sort(cands.begin(), cands.end(), ServerStatFaster()); + uris.erase(std::find(uris.begin(), uris.end(), cands.front().second)); + return cands.front().second; + } +} + +} // namespace aria2 diff --git a/src/ServerStatURISelector.h b/src/ServerStatURISelector.h new file mode 100644 index 00000000..e6b8ec32 --- /dev/null +++ b/src/ServerStatURISelector.h @@ -0,0 +1,58 @@ +/* */ +#ifndef _D_SERVER_STAT_URI_SELECTOR_H_ +#define _D_SERVER_STAT_URI_SELECTOR_H_ +#include "URISelector.h" +#include "SharedHandle.h" + +namespace aria2 { + +class ServerStatMan; + +class ServerStatURISelector:public URISelector { +private: + SharedHandle _serverStatMan; + +public: + ServerStatURISelector(const SharedHandle& serverStatMan); + + virtual ~ServerStatURISelector(); + + virtual std::string select(std::deque& uris); +}; + +} // namespace aria2 + +#endif // _D_SERVER_STAT_URI_SELECTOR_H_ diff --git a/src/URISelector.h b/src/URISelector.h new file mode 100644 index 00000000..144bb03d --- /dev/null +++ b/src/URISelector.h @@ -0,0 +1,52 @@ +/* */ +#ifndef _D_URI_SELECTOR_H_ +#define _D_URI_SELECTOR_H_ +#include "common.h" +#include +#include + +namespace aria2 { + +class URISelector { +public: + virtual ~URISelector() {} + + virtual std::string select(std::deque& uris) = 0; +}; + +} // namespace aria2 + +#endif // _D_URI_SELECTOR_H_ diff --git a/src/option_processing.cc b/src/option_processing.cc index dafde204..9736ea7a 100644 --- a/src/option_processing.cc +++ b/src/option_processing.cc @@ -156,6 +156,7 @@ Option* createDefaultOption() op->put(PREF_FTP_REUSE_CONNECTION, V_TRUE); op->put(PREF_SUMMARY_INTERVAL, "60"); op->put(PREF_LOG_LEVEL, V_DEBUG); + op->put(PREF_URI_SELECTOR, V_INORDER); return op; } @@ -233,6 +234,7 @@ Option* option_processing(int argc, char* const argv[]) { PREF_FTP_REUSE_CONNECTION.c_str(), optional_argument, &lopt, 217 }, { PREF_SUMMARY_INTERVAL.c_str(), required_argument, &lopt, 218 }, { PREF_LOG_LEVEL.c_str(), required_argument, &lopt, 219 }, + { PREF_URI_SELECTOR.c_str(), required_argument, &lopt, 220 }, #if defined ENABLE_BITTORRENT || defined ENABLE_METALINK { PREF_SHOW_FILES.c_str(), no_argument, NULL, 'S' }, { PREF_SELECT_FILE.c_str(), required_argument, &lopt, 21 }, @@ -461,6 +463,9 @@ Option* option_processing(int argc, char* const argv[]) case 219: cmdstream << PREF_LOG_LEVEL << "=" << optarg << "\n"; break; + case 220: + cmdstream << PREF_URI_SELECTOR << "=" << optarg << "\n"; + break; } break; } diff --git a/src/prefs.cc b/src/prefs.cc index c1a85942..34608fb5 100644 --- a/src/prefs.cc +++ b/src/prefs.cc @@ -135,6 +135,10 @@ const std::string V_INFO("info"); const std::string V_NOTICE("notice"); const std::string V_WARN("warn"); const std::string V_ERROR("error"); +// value: inorder | feedback +const std::string PREF_URI_SELECTOR("uri-selector"); +const std::string V_INORDER("inorder"); +const std::string V_FEEDBACK("feedback"); /** * FTP related preferences diff --git a/src/prefs.h b/src/prefs.h index 2b35b767..e4f9c02d 100644 --- a/src/prefs.h +++ b/src/prefs.h @@ -139,6 +139,10 @@ extern const std::string V_INFO; extern const std::string V_NOTICE; extern const std::string V_WARN; extern const std::string V_ERROR; +// value: inorder | feedback +extern const std::string PREF_URI_SELECTOR; +extern const std::string V_INORDER; +extern const std::string V_FEEDBACK; /** * FTP related preferences diff --git a/test/InOrderURISelectorTest.cc b/test/InOrderURISelectorTest.cc new file mode 100644 index 00000000..c77b114c --- /dev/null +++ b/test/InOrderURISelectorTest.cc @@ -0,0 +1,50 @@ +#include "InOrderURISelector.h" +#include "Exception.h" +#include "Util.h" +#include "array_fun.h" +#include +#include + +namespace aria2 { + +class InOrderURISelectorTest:public CppUnit::TestFixture { + + CPPUNIT_TEST_SUITE(InOrderURISelectorTest); + CPPUNIT_TEST(testSelect); + CPPUNIT_TEST_SUITE_END(); +private: + + std::deque uris; + + SharedHandle sel; + +public: + void setUp() + { + static const char* urisSrc[] = { + "http://alpha/file", + "ftp://alpha/file", + "http://bravo/file" + }; + uris.assign(&urisSrc[0], &urisSrc[arrayLength(urisSrc)]); + + sel.reset(new InOrderURISelector()); + } + + void tearDown() {} + + void testSelect(); +}; + + +CPPUNIT_TEST_SUITE_REGISTRATION(InOrderURISelectorTest); + +void InOrderURISelectorTest::testSelect() +{ + CPPUNIT_ASSERT_EQUAL(std::string("http://alpha/file"), sel->select(uris)); + CPPUNIT_ASSERT_EQUAL(std::string("ftp://alpha/file"), sel->select(uris)); + CPPUNIT_ASSERT_EQUAL(std::string("http://bravo/file"), sel->select(uris)); + CPPUNIT_ASSERT_EQUAL(std::string(""), sel->select(uris)); +} + +} // namespace aria2 diff --git a/test/Makefile.am b/test/Makefile.am index f5d27299..4b66c706 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -53,7 +53,10 @@ aria2c_SOURCES = AllTest.cc\ ExceptionTest.cc\ DownloadHandlerFactoryTest.cc\ ChunkedDecoderTest.cc\ - SignatureTest.cc + SignatureTest.cc\ + ServerStatManTest.cc\ + ServerStatURISelectorTest.cc\ + InOrderURISelectorTest.cc if HAVE_LIBZ aria2c_SOURCES += GZipDecoderTest.cc diff --git a/test/Makefile.in b/test/Makefile.in index 8d29ccb5..0cdcb3ed 100644 --- a/test/Makefile.in +++ b/test/Makefile.in @@ -188,7 +188,9 @@ am__aria2c_SOURCES_DIST = AllTest.cc SocketCoreTest.cc \ MultiFileAllocationIteratorTest.cc FixedNumberRandomizer.h \ ProtocolDetectorTest.cc StringFormatTest.cc ExceptionTest.cc \ DownloadHandlerFactoryTest.cc ChunkedDecoderTest.cc \ - SignatureTest.cc GZipDecoderTest.cc MessageDigestHelperTest.cc \ + SignatureTest.cc ServerStatManTest.cc \ + ServerStatURISelectorTest.cc InOrderURISelectorTest.cc \ + GZipDecoderTest.cc MessageDigestHelperTest.cc \ IteratableChunkChecksumValidatorTest.cc \ IteratableChecksumValidatorTest.cc BtAllowedFastMessageTest.cc \ BtBitfieldMessageTest.cc BtCancelMessageTest.cc \ @@ -353,8 +355,10 @@ am_aria2c_OBJECTS = AllTest.$(OBJEXT) SocketCoreTest.$(OBJEXT) \ ProtocolDetectorTest.$(OBJEXT) StringFormatTest.$(OBJEXT) \ ExceptionTest.$(OBJEXT) DownloadHandlerFactoryTest.$(OBJEXT) \ ChunkedDecoderTest.$(OBJEXT) SignatureTest.$(OBJEXT) \ - $(am__objects_1) $(am__objects_2) $(am__objects_3) \ - $(am__objects_4) + ServerStatManTest.$(OBJEXT) \ + ServerStatURISelectorTest.$(OBJEXT) \ + InOrderURISelectorTest.$(OBJEXT) $(am__objects_1) \ + $(am__objects_2) $(am__objects_3) $(am__objects_4) aria2c_OBJECTS = $(am_aria2c_OBJECTS) am__DEPENDENCIES_1 = aria2c_DEPENDENCIES = ../src/libaria2c.a $(am__DEPENDENCIES_1) @@ -569,8 +573,10 @@ aria2c_SOURCES = AllTest.cc SocketCoreTest.cc array_funTest.cc \ MultiFileAllocationIteratorTest.cc FixedNumberRandomizer.h \ ProtocolDetectorTest.cc StringFormatTest.cc ExceptionTest.cc \ DownloadHandlerFactoryTest.cc ChunkedDecoderTest.cc \ - SignatureTest.cc $(am__append_1) $(am__append_2) \ - $(am__append_3) $(am__append_4) + SignatureTest.cc ServerStatManTest.cc \ + ServerStatURISelectorTest.cc InOrderURISelectorTest.cc \ + $(am__append_1) $(am__append_2) $(am__append_3) \ + $(am__append_4) #aria2c_CXXFLAGS = ${CPPUNIT_CFLAGS} -I../src -I../lib -Wall -D_FILE_OFFSET_BITS=64 #aria2c_LDFLAGS = ${CPPUNIT_LIBS} @@ -740,6 +746,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/HttpHeaderTest.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/HttpRequestTest.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/HttpResponseTest.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/InOrderURISelectorTest.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/IteratableChecksumValidatorTest.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/IteratableChunkChecksumValidatorTest.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ListTest.Po@am__quote@ @@ -773,6 +780,8 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/SegmentManTest.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/SegmentTest.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/SequenceTest.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ServerStatManTest.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ServerStatURISelectorTest.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ShareRatioSeedCriteriaTest.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/SharedHandleTest.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/SignatureTest.Po@am__quote@ diff --git a/test/RequestGroupManTest.cc b/test/RequestGroupManTest.cc index 5df3f3e7..8d08f34b 100644 --- a/test/RequestGroupManTest.cc +++ b/test/RequestGroupManTest.cc @@ -6,6 +6,7 @@ #include "Option.h" #include "DownloadResult.h" #include "FileEntry.h" +#include "ServerStatMan.h" #include namespace aria2 { @@ -53,7 +54,7 @@ void RequestGroupManTest::testIsSameFileBeingDownloaded() rgs.push_back(rg1); rgs.push_back(rg2); - RequestGroupMan gm(rgs, 1); + RequestGroupMan gm(rgs, 1, &option); CPPUNIT_ASSERT(gm.isSameFileBeingDownloaded(rg1.get())); diff --git a/test/ServerStatManTest.cc b/test/ServerStatManTest.cc new file mode 100644 index 00000000..df6cafa8 --- /dev/null +++ b/test/ServerStatManTest.cc @@ -0,0 +1,50 @@ +#include "ServerStatMan.h" +#include "ServerStat.h" +#include "Exception.h" +#include "Util.h" +#include +#include + +namespace aria2 { + +class ServerStatManTest:public CppUnit::TestFixture { + + CPPUNIT_TEST_SUITE(ServerStatManTest); + CPPUNIT_TEST(testAddAndFind); + CPPUNIT_TEST_SUITE_END(); +public: + void setUp() {} + + void tearDown() {} + + void testAddAndFind(); +}; + + +CPPUNIT_TEST_SUITE_REGISTRATION(ServerStatManTest); + +void ServerStatManTest::testAddAndFind() +{ + SharedHandle localhost_http(new ServerStat("localhost", "http")); + SharedHandle localhost_ftp(new ServerStat("localhost", "ftp")); + SharedHandle mirror(new ServerStat("mirror", "http")); + + ServerStatMan ssm; + CPPUNIT_ASSERT(ssm.add(localhost_http)); + CPPUNIT_ASSERT(!ssm.add(localhost_http)); + CPPUNIT_ASSERT(ssm.add(localhost_ftp)); + CPPUNIT_ASSERT(ssm.add(mirror)); + + { + SharedHandle r = ssm.find("localhost", "http"); + CPPUNIT_ASSERT(!r.isNull()); + CPPUNIT_ASSERT_EQUAL(std::string("localhost"), r->getHostname()); + CPPUNIT_ASSERT_EQUAL(std::string("http"), r->getProtocol()); + } + { + SharedHandle r = ssm.find("mirror", "ftp"); + CPPUNIT_ASSERT(r.isNull()); + } +} + +} // namespace aria2 diff --git a/test/ServerStatURISelectorTest.cc b/test/ServerStatURISelectorTest.cc new file mode 100644 index 00000000..888a590a --- /dev/null +++ b/test/ServerStatURISelectorTest.cc @@ -0,0 +1,97 @@ +#include "ServerStatURISelector.h" +#include "Exception.h" +#include "Util.h" +#include "array_fun.h" +#include "ServerStatMan.h" +#include "ServerStat.h" +#include +#include + +namespace aria2 { + +class ServerStatURISelectorTest:public CppUnit::TestFixture { + + CPPUNIT_TEST_SUITE(ServerStatURISelectorTest); + CPPUNIT_TEST(testSelect_withoutServerStat); + CPPUNIT_TEST(testSelect); + CPPUNIT_TEST(testSelect_skipErrorHost); + CPPUNIT_TEST_SUITE_END(); +private: + + std::deque uris; + + SharedHandle ssm; + + SharedHandle sel; + +public: + void setUp() + { + static const char* urisSrc[] = { + "http://alpha/file", + "ftp://alpha/file", + "http://bravo/file" + }; + uris.assign(&urisSrc[0], &urisSrc[arrayLength(urisSrc)]); + + ssm.reset(new ServerStatMan()); + sel.reset(new ServerStatURISelector(ssm)); + } + + void tearDown() {} + + void testSelect_withoutServerStat(); + + void testSelect(); + + void testSelect_skipErrorHost(); +}; + + +CPPUNIT_TEST_SUITE_REGISTRATION(ServerStatURISelectorTest); + +void ServerStatURISelectorTest::testSelect_withoutServerStat() +{ + // Without ServerStat, selector returns first URI + std::string uri = sel->select(uris); + CPPUNIT_ASSERT_EQUAL(std::string("http://alpha/file"), uri); + CPPUNIT_ASSERT_EQUAL((size_t)2, uris.size()); +} + +void ServerStatURISelectorTest::testSelect() +{ + SharedHandle bravo(new ServerStat("bravo", "http")); + bravo->updateDownloadSpeed(100000); + SharedHandle alphaFTP(new ServerStat("alpha", "ftp")); + alphaFTP->updateDownloadSpeed(80000); + SharedHandle alphaHTTP(new ServerStat("alpha", "http")); + alphaHTTP->updateDownloadSpeed(180000); + alphaHTTP->setError(); + + ssm->add(bravo); + ssm->add(alphaFTP); + ssm->add(alphaHTTP); + + CPPUNIT_ASSERT_EQUAL(std::string("http://bravo/file"), sel->select(uris)); + CPPUNIT_ASSERT_EQUAL((size_t)2, uris.size()); + + CPPUNIT_ASSERT_EQUAL(std::string("ftp://alpha/file"), sel->select(uris)); + CPPUNIT_ASSERT_EQUAL((size_t)1, uris.size()); +} + +void ServerStatURISelectorTest::testSelect_skipErrorHost() +{ + SharedHandle alphaHTTP(new ServerStat("alpha", "http")); + alphaHTTP->setError(); + SharedHandle alphaFTP(new ServerStat("alpha", "ftp")); + alphaFTP->setError(); + + ssm->add(alphaHTTP); + ssm->add(alphaFTP); + + // See error URIs are removed from URI List. + CPPUNIT_ASSERT_EQUAL(std::string("http://bravo/file"), sel->select(uris)); + CPPUNIT_ASSERT_EQUAL((size_t)0, uris.size()); +} + +} // namespace aria2