diff --git a/ChangeLog b/ChangeLog index 2e4b565c..146392b2 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,32 @@ +2009-01-06 Tatsuhiro Tsujikawa + + Applied AdaptiveURISelector patch from Aurelien Lefebvre. This + patch adds AdaptiveURISelector, which selects one of the bests + mirrors for first and reserved connections. For supplementary + ones, it returns mirrors which has not been tested yet, and if + each of them already tested, returns mirrors which has to be + tested again. Otherwise, it doesn't return anymore mirrors. + * src/AdaptiveURISelector.cc + * src/AdaptiveURISelector.h + * src/FtpNegotiationCommand.cc + * src/HttpResponseCommand.cc + * src/Makefile.am + * src/Makefile.in + * src/OptionHandlerFactory.cc + * src/RequestGroup.cc + * src/RequestGroup.h + * src/RequestGroupMan.cc + * src/ServerStat.cc + * src/ServerStat.h + * src/ServerStatMan.cc + * src/SpeedCalc.cc + * src/URISelector.h + * src/prefs.cc + * src/prefs.h + * test/RequestGroupManTest.cc + * test/ServerStatManTest.cc + * test/ServerStatTest.cc + 2009-01-04 Tatsuhiro Tsujikawa Fixed unit test error. diff --git a/src/AdaptiveURISelector.cc b/src/AdaptiveURISelector.cc new file mode 100644 index 00000000..2467a5aa --- /dev/null +++ b/src/AdaptiveURISelector.cc @@ -0,0 +1,308 @@ +/* */ +#include "AdaptiveURISelector.h" + +#include +#include +#include + +#include "DownloadCommand.h" +#include "DownloadContext.h" +#include "ServerStatMan.h" +#include "ServerStat.h" +#include "RequestGroup.h" +#include "LogFactory.h" +#include "A2STR.h" +#include "prefs.h" +#include "Option.h" +#include "SimpleRandomizer.h" +#include "SocketCore.h" + +namespace aria2 { + +/* In that URI Selector, select method returns one of the bests + * mirrors for first and reserved connections. For supplementary + * ones, it returns mirrors which has not been tested yet, and + * if each of them already tested, returns mirrors which has to + * be tested again. Otherwise, it doesn't return anymore mirrors. + */ + +AdaptiveURISelector::AdaptiveURISelector +(const SharedHandle& serverStatMan, + const SharedHandle& requestGroup): + _serverStatMan(serverStatMan), + _requestGroup(requestGroup), + _nbConnections(1), + _logger(LogFactory::getInstance()) +{ + const Option* op = _requestGroup->getOption(); + _nbServerToEvaluate = op->getAsInt(PREF_METALINK_SERVERS) - 1; +} + +AdaptiveURISelector::~AdaptiveURISelector() {} + +std::string AdaptiveURISelector::select(std::deque& uris) +{ + std::string selected = selectOne(uris); + + if(selected != A2STR::NIL) + uris.erase(std::find(uris.begin(), uris.end(), selected)); + + return selected; +} + +std::string AdaptiveURISelector::selectOne(const std::deque& uris) +{ + + if(uris.empty()) { + return A2STR::NIL; + } else { + const unsigned int numPieces = + _requestGroup->getDownloadContext()->getNumPieces(); + + bool reservedContext = numPieces > 0 && + _nbConnections > std::min(numPieces, + _requestGroup->getNumConcurrentCommand()); + bool selectBest = numPieces == 0 || reservedContext; + + if(numPieces > 0) + _nbConnections++; + + /* At least, 3 mirrors must be tested */ + if(getNbTestedServers(uris) < 3) { + std::string notTested = getFirstNotTestedUri(uris); + if(notTested != A2STR::NIL) { + _logger->debug("AdaptiveURISelector: choosing the first non tested" + " mirror: %s", notTested.c_str()); + --_nbServerToEvaluate; + return notTested; + } + } + + if(!selectBest && _nbConnections > 1 && _nbServerToEvaluate > 0) { + _nbServerToEvaluate--; + std::string notTested = getFirstNotTestedUri(uris); + if(notTested != A2STR::NIL) { + /* Here we return the first untested mirror */ + _logger->debug("AdaptiveURISelector: choosing non tested mirror %s for" + " connection #%d", notTested.c_str(), _nbConnections); + return notTested; + } else { + /* Here we return a mirror which need to be tested again */ + std::string toReTest = getFirstToTestUri(uris); + _logger->debug("AdaptiveURISelector: choosing mirror %s which has not" + " been tested recently for connection #%d", + toReTest.c_str(), _nbConnections); + return toReTest; + } + } + else { + /* Here we return one of the bests mirrors */ + unsigned int max = getMaxDownloadSpeed(uris); + unsigned int min = max-(int)(max*0.25); + std::deque bests = getUrisBySpeed(uris, min); + + if (bests.size() < 2) { + std::string uri = getMaxDownloadSpeedUri(uris); + _logger->debug("AdaptiveURISelector: choosing the best mirror :" + " %.2fKB/s %s (other mirrors are at least 25%% slower)", + (float) max/1024, uri.c_str()); + return uri; + } else { + std::string uri = selectRandomUri(bests); + _logger->debug("AdaptiveURISelector: choosing randomly one of the best" + " mirrors (range [%.2fKB/s, %.2fKB/s]): %s", + (float) min/1024, (float) max/1024, uri.c_str()); + return uri; + } + } + } +} + +void AdaptiveURISelector::resetCounters() +{ + const Option* op = _requestGroup->getOption(); + _nbConnections = 1; + _nbServerToEvaluate = op->getAsInt(PREF_METALINK_SERVERS) - 1; +} + +void AdaptiveURISelector::tuneDownloadCommand +(const std::deque& uris, DownloadCommand* command) +{ + adjustLowestSpeedLimit(uris, command); +} + +void AdaptiveURISelector::adjustLowestSpeedLimit +(const std::deque& uris, DownloadCommand* command) const +{ + const Option* op = _requestGroup->getOption(); + unsigned int lowest = op->getAsInt(PREF_LOWEST_SPEED_LIMIT); + if (lowest > 0) { + unsigned int low_lowest = 4 * 1024; + unsigned int max = getMaxDownloadSpeed(uris); + if (max > 0 && lowest > max / 4) { + _logger->notice("Lowering lowest-speed-limit since known max speed is too" + " near (new:%d was:%d max:%d)", max / 4, lowest, max); + command->setLowestDownloadSpeedLimit(max / 4); + } else if (max == 0 && lowest > low_lowest) { + _logger->notice("Lowering lowest-speed-limit since we have no clue about" + " available speed (now:%d was:%d)", low_lowest, lowest); + command->setLowestDownloadSpeedLimit(low_lowest); + } + } +} + +static unsigned int getUriMaxSpeed(SharedHandle ss) +{ + return std::max(ss->getSingleConnectionAvgSpeed(), + ss->getMultiConnectionAvgSpeed()); +} + +unsigned int AdaptiveURISelector::getMaxDownloadSpeed +(const std::deque& uris) const +{ + std::string uri = getMaxDownloadSpeedUri(uris); + if(uri == A2STR::NIL) + return 0; + return getUriMaxSpeed(getServerStats(uri)); +} + +std::string AdaptiveURISelector::getMaxDownloadSpeedUri +(const std::deque& uris) const +{ + int max = -1; + std::string uri = A2STR::NIL; + for(std::deque::const_iterator i = uris.begin(); + i != uris.end(); ++i) { + SharedHandle ss = getServerStats(*i); + if(ss.isNull()) + continue; + + if((int)ss->getSingleConnectionAvgSpeed() > max) { + max = ss->getSingleConnectionAvgSpeed(); + uri = (*i); + } + if((int)ss->getMultiConnectionAvgSpeed() > max) { + max = ss->getMultiConnectionAvgSpeed(); + uri = (*i); + } + } + return uri; +} + +std::deque AdaptiveURISelector::getUrisBySpeed +(const std::deque& uris, unsigned int min) const +{ + std::deque bests; + for(std::deque::const_iterator i = uris.begin(); + i != uris.end(); ++i) { + SharedHandle ss = getServerStats(*i); + if(ss.isNull()) + continue; + if(ss->getSingleConnectionAvgSpeed() > min || + ss->getMultiConnectionAvgSpeed() > min) { + bests.push_back(*i); + } + } + return bests; +} + +std::string AdaptiveURISelector::selectRandomUri +(const std::deque& uris) const +{ + int pos = SimpleRandomizer::getInstance()->getRandomNumber(uris.size()); + std::deque::const_iterator i = uris.begin(); + i = i+pos; + return *i; +} + +std::string AdaptiveURISelector::getFirstNotTestedUri +(const std::deque& uris) const +{ + for(std::deque::const_iterator i = uris.begin(); + i != uris.end(); ++i) { + SharedHandle ss = getServerStats(*i); + if(ss.isNull()) + return *i; + } + return A2STR::NIL; +} + +std::string AdaptiveURISelector::getFirstToTestUri +(const std::deque& uris) const +{ + unsigned int counter; + int power; + for(std::deque::const_iterator i = uris.begin(); + i != uris.end(); ++i) { + SharedHandle ss = getServerStats(*i); + if(ss.isNull()) + continue; + counter = ss->getCounter(); + if(counter > 8) + continue; + power = (int)pow(2.0, (float)counter); + /* We test the mirror another time if it has not been + * tested since 2^counter days */ + if(ss->getLastUpdated().difference() > power*24*60*60) { + return *i; + } + } + return A2STR::NIL; +} + +SharedHandle AdaptiveURISelector::getServerStats +(const std::string& uri) const +{ + Request r; + r.setUrl(uri); + return _serverStatMan->find(r.getHost(), r.getProtocol()); +} + +unsigned int AdaptiveURISelector::getNbTestedServers +(const std::deque& uris) const +{ + unsigned int counter = 0; + for(std::deque::const_iterator i = uris.begin(); + i != uris.end(); ++i) { + SharedHandle ss = getServerStats(*i); + if(ss.isNull()) + counter++; + } + return uris.size() - counter; +} + +} // namespace aria2 diff --git a/src/AdaptiveURISelector.h b/src/AdaptiveURISelector.h new file mode 100644 index 00000000..c35bc83f --- /dev/null +++ b/src/AdaptiveURISelector.h @@ -0,0 +1,83 @@ +/* */ +#ifndef _D_ADAPTIVE_URI_SELECTOR_H_ +#define _D_ADAPTIVE_URI_SELECTOR_H_ +#include "URISelector.h" +#include "SharedHandle.h" + +namespace aria2 { + +class ServerStatMan; +class RequestGroup; +class ServerStat; +class Logger; + +class AdaptiveURISelector:public URISelector { +private: + SharedHandle _serverStatMan; + SharedHandle _requestGroup; + unsigned int _nbServerToEvaluate; + unsigned int _nbConnections; + + Logger* _logger; + + std::string selectOne(const std::deque& uris); + void adjustLowestSpeedLimit(const std::deque& uris, + DownloadCommand* command) const; + unsigned int getMaxDownloadSpeed(const std::deque& uris) const; + std::string getMaxDownloadSpeedUri(const std::deque& uris) const; + std::deque getUrisBySpeed(const std::deque& uris, + unsigned int min) const; + std::string selectRandomUri(const std::deque& uris) const; + std::string getFirstNotTestedUri(const std::deque& uris) const; + std::string getFirstToTestUri(const std::deque& uris) const; + SharedHandle getServerStats(const std::string& uri) const; + unsigned int getNbTestedServers(const std::deque& uris) const; +public: + AdaptiveURISelector(const SharedHandle& serverStatMan, + const SharedHandle& requestGroup); + + virtual ~AdaptiveURISelector(); + + virtual std::string select(std::deque& uris); + virtual void tuneDownloadCommand(const std::deque& uris, + DownloadCommand* command); + + virtual void resetCounters(); +}; + +} // namespace aria2 +#endif // _D_ADAPTIVE_URI_SELECTOR_H_ diff --git a/src/FtpNegotiationCommand.cc b/src/FtpNegotiationCommand.cc index 71ebc50c..654bc7ed 100644 --- a/src/FtpNegotiationCommand.cc +++ b/src/FtpNegotiationCommand.cc @@ -106,6 +106,7 @@ bool FtpNegotiationCommand::executeInternal() { _requestGroup->removeURIWhoseHostnameIs(sv->getHostname()); } } + _requestGroup->tuneDownloadCommand(command); e->commands.push_back(command); return true; } else if(sequence == SEQ_HEAD_OK || sequence == SEQ_DOWNLOAD_ALREADY_COMPLETED) { diff --git a/src/HttpResponseCommand.cc b/src/HttpResponseCommand.cc index f0042644..994a3dd1 100644 --- a/src/HttpResponseCommand.cc +++ b/src/HttpResponseCommand.cc @@ -348,6 +348,8 @@ HttpDownloadCommand* HttpResponseCommand::createHttpDownloadCommand _requestGroup->setFileAllocationEnabled(false); } + _requestGroup->tuneDownloadCommand(command); + return command; } diff --git a/src/Makefile.am b/src/Makefile.am index 69bf32eb..53633eea 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -188,6 +188,7 @@ SRCS = Socket.h\ ServerStat.cc ServerStat.h\ ServerStatMan.cc ServerStatMan.h\ URISelector.h\ + AdaptiveURISelector.cc AdaptiveURISelector.h\ InOrderURISelector.cc InOrderURISelector.h\ ServerStatURISelector.cc ServerStatURISelector.h\ NsCookieParser.cc NsCookieParser.h\ diff --git a/src/Makefile.in b/src/Makefile.in index 5970960a..aff2e2ca 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -405,7 +405,8 @@ am__libaria2c_a_SOURCES_DIST = Socket.h SocketCore.cc SocketCore.h \ A2STR.cc A2STR.h RarestPieceSelector.cc RarestPieceSelector.h \ Decoder.h ChunkedDecoder.cc ChunkedDecoder.h Signature.cc \ Signature.h ServerStat.cc ServerStat.h ServerStatMan.cc \ - ServerStatMan.h URISelector.h InOrderURISelector.cc \ + ServerStatMan.h URISelector.h AdaptiveURISelector.cc \ + AdaptiveURISelector.h InOrderURISelector.cc \ InOrderURISelector.h ServerStatURISelector.cc \ ServerStatURISelector.h NsCookieParser.cc NsCookieParser.h \ CookieStorage.cc CookieStorage.h SocketBuffer.cc \ @@ -797,17 +798,17 @@ am__objects_21 = SocketCore.$(OBJEXT) Command.$(OBJEXT) \ FtpFinishDownloadCommand.$(OBJEXT) A2STR.$(OBJEXT) \ RarestPieceSelector.$(OBJEXT) ChunkedDecoder.$(OBJEXT) \ Signature.$(OBJEXT) ServerStat.$(OBJEXT) \ - ServerStatMan.$(OBJEXT) InOrderURISelector.$(OBJEXT) \ - ServerStatURISelector.$(OBJEXT) NsCookieParser.$(OBJEXT) \ - CookieStorage.$(OBJEXT) SocketBuffer.$(OBJEXT) \ - OptionHandlerException.$(OBJEXT) bencode.$(OBJEXT) \ - $(am__objects_1) $(am__objects_2) $(am__objects_3) \ - $(am__objects_4) $(am__objects_5) $(am__objects_6) \ - $(am__objects_7) $(am__objects_8) $(am__objects_9) \ - $(am__objects_10) $(am__objects_11) $(am__objects_12) \ - $(am__objects_13) $(am__objects_14) $(am__objects_15) \ - $(am__objects_16) $(am__objects_17) $(am__objects_18) \ - $(am__objects_19) $(am__objects_20) + ServerStatMan.$(OBJEXT) AdaptiveURISelector.$(OBJEXT) \ + InOrderURISelector.$(OBJEXT) ServerStatURISelector.$(OBJEXT) \ + NsCookieParser.$(OBJEXT) CookieStorage.$(OBJEXT) \ + SocketBuffer.$(OBJEXT) OptionHandlerException.$(OBJEXT) \ + bencode.$(OBJEXT) $(am__objects_1) $(am__objects_2) \ + $(am__objects_3) $(am__objects_4) $(am__objects_5) \ + $(am__objects_6) $(am__objects_7) $(am__objects_8) \ + $(am__objects_9) $(am__objects_10) $(am__objects_11) \ + $(am__objects_12) $(am__objects_13) $(am__objects_14) \ + $(am__objects_15) $(am__objects_16) $(am__objects_17) \ + $(am__objects_18) $(am__objects_19) $(am__objects_20) am_libaria2c_a_OBJECTS = $(am__objects_21) libaria2c_a_OBJECTS = $(am_libaria2c_a_OBJECTS) am__installdirs = "$(DESTDIR)$(bindir)" @@ -1126,7 +1127,8 @@ SRCS = Socket.h SocketCore.cc SocketCore.h BinaryStream.h Command.cc \ A2STR.cc A2STR.h RarestPieceSelector.cc RarestPieceSelector.h \ Decoder.h ChunkedDecoder.cc ChunkedDecoder.h Signature.cc \ Signature.h ServerStat.cc ServerStat.h ServerStatMan.cc \ - ServerStatMan.h URISelector.h InOrderURISelector.cc \ + ServerStatMan.h URISelector.h AdaptiveURISelector.cc \ + AdaptiveURISelector.h InOrderURISelector.cc \ InOrderURISelector.h ServerStatURISelector.cc \ ServerStatURISelector.h NsCookieParser.cc NsCookieParser.h \ CookieStorage.cc CookieStorage.h SocketBuffer.cc \ @@ -1237,6 +1239,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/AbstractProxyResponseCommand.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/AbstractSingleDiskAdaptor.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ActivePeerConnectionCommand.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/AdaptiveURISelector.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/AnnounceList.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/AsyncNameResolver.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/AuthConfig.Po@am__quote@ diff --git a/src/OptionHandlerFactory.cc b/src/OptionHandlerFactory.cc index a1cbe1a1..2fc2a760 100644 --- a/src/OptionHandlerFactory.cc +++ b/src/OptionHandlerFactory.cc @@ -417,7 +417,7 @@ OptionHandlers OptionHandlerFactory::createOptionHandlers() handlers.push_back(op); } { - const std::string params[] = { V_INORDER, V_FEEDBACK }; + const std::string params[] = { V_INORDER, V_FEEDBACK, V_ADAPTIVE }; SharedHandle op(new ParameterOptionHandler (PREF_URI_SELECTOR, TEXT_URI_SELECTOR, diff --git a/src/RequestGroup.cc b/src/RequestGroup.cc index 2ab92225..57b421fc 100644 --- a/src/RequestGroup.cc +++ b/src/RequestGroup.cc @@ -521,6 +521,8 @@ void RequestGroup::createNextCommand(std::deque& commands, std::deque pendingURIs; for(; !_uris.empty() && numCommand--; ) { std::string uri = _uriSelector->select(_uris); + if(uri.empty()) + continue; RequestHandle req(new Request()); if(req->setUrl(uri)) { ServerHostHandle sv; @@ -987,6 +989,7 @@ void RequestGroup::reportDownloadFinished() { _logger->notice(MSG_FILE_DOWNLOAD_COMPLETED, getFilePath().c_str()); + _uriSelector->resetCounters(); #ifdef ENABLE_BITTORRENT SharedHandle ctx = dynamic_pointer_cast(_downloadContext); if(!ctx.isNull()) { @@ -1089,4 +1092,9 @@ bool RequestGroup::inMemoryDownload() const return _inMemoryDownload; } +void RequestGroup::tuneDownloadCommand(DownloadCommand* command) +{ + _uriSelector->tuneDownloadCommand(_uris, command); +} + } // namespace aria2 diff --git a/src/RequestGroup.h b/src/RequestGroup.h index d54c40b4..e9d6558c 100644 --- a/src/RequestGroup.h +++ b/src/RequestGroup.h @@ -51,6 +51,7 @@ class DownloadEngine; class SegmentMan; class SegmentManFactory; class Command; +class DownloadCommand; class DownloadContext; class PieceStorage; class BtProgressInfoFile; @@ -405,6 +406,8 @@ public: // Returns inMemoryDownload flag. bool inMemoryDownload() const; + + void tuneDownloadCommand(DownloadCommand* command); }; typedef SharedHandle RequestGroupHandle; diff --git a/src/RequestGroupMan.cc b/src/RequestGroupMan.cc index f0c30e93..e64a07aa 100644 --- a/src/RequestGroupMan.cc +++ b/src/RequestGroupMan.cc @@ -57,6 +57,7 @@ #include "SegmentMan.h" #include "ServerStatURISelector.h" #include "InOrderURISelector.h" +#include "AdaptiveURISelector.h" #include "Option.h" #include "prefs.h" #include "File.h" @@ -205,15 +206,26 @@ public: if(!group->getSegmentMan().isNull()) { const std::deque >& peerStats = group->getSegmentMan()->getPeerStats(); + for(std::deque >::const_iterator i = peerStats.begin(); i != peerStats.end(); ++i) { if((*i)->getHostname().empty() || (*i)->getProtocol().empty()) { continue; } + int speed = (*i)->getAvgDownloadSpeed(); + if (speed == 0) continue; + SharedHandle ss = _requestGroupMan->getOrCreateServerStat((*i)->getHostname(), (*i)->getProtocol()); - ss->updateDownloadSpeed((*i)->getAvgDownloadSpeed()); + ss->increaseCounter(); + ss->updateDownloadSpeed(speed); + if(peerStats.size() == 1) { + ss->updateSingleConnectionAvgSpeed(speed); + } + else { + ss->updateMultiConnectionAvgSpeed(speed); + } } } } @@ -265,6 +277,10 @@ void RequestGroupMan::configureRequestGroup } else if(uriSelectorValue == V_INORDER) { requestGroup->setURISelector (SharedHandle(new InOrderURISelector())); + } else if(uriSelectorValue == V_ADAPTIVE) { + requestGroup->setURISelector + (SharedHandle(new AdaptiveURISelector(_serverStatMan, + requestGroup))); } } diff --git a/src/ServerStat.cc b/src/ServerStat.cc index 055648fb..d439ca75 100644 --- a/src/ServerStat.cc +++ b/src/ServerStat.cc @@ -33,10 +33,13 @@ */ /* copyright --> */ #include "ServerStat.h" -#include "array_fun.h" + #include #include +#include "array_fun.h" +#include "LogFactory.h" + namespace aria2 { const std::string ServerStat::STATUS_STRING[] = { @@ -49,7 +52,12 @@ ServerStat::ServerStat(const std::string& hostname, const std::string& protocol) _hostname(hostname), _protocol(protocol), _downloadSpeed(0), - _status(OK) {} + _singleConnectionAvgSpeed(0), + _multiConnectionAvgSpeed(0), + _counter(0), + _status(OK), + _logger(LogFactory::getInstance()) +{} ServerStat::~ServerStat() {} @@ -92,6 +100,94 @@ void ServerStat::updateDownloadSpeed(unsigned int downloadSpeed) _lastUpdated.reset(); } +unsigned int ServerStat::getSingleConnectionAvgSpeed() const +{ + return _singleConnectionAvgSpeed; +} + +void ServerStat::setSingleConnectionAvgSpeed +(unsigned int singleConnectionAvgSpeed) +{ + _singleConnectionAvgSpeed = singleConnectionAvgSpeed; +} + +void ServerStat::updateSingleConnectionAvgSpeed(unsigned int downloadSpeed) +{ + float avgDownloadSpeed; + if(_counter == 0) + return; + if(_counter < 5) { + avgDownloadSpeed = + ((((float)_counter-1)/(float)_counter)*(float)_singleConnectionAvgSpeed)+ + ((1.0/(float)_counter)*(float)downloadSpeed); + } + else { + avgDownloadSpeed = ((4.0/5.0)*(float)_singleConnectionAvgSpeed) + + ((1.0/5.0)*(float)downloadSpeed); + } + if(avgDownloadSpeed < (int)(0.80*_singleConnectionAvgSpeed)) { + _logger->debug("ServerStat:%s: resetting counter since single connection" + " speed dropped", getHostname().c_str()); + _counter = 0; + } + _logger->debug("ServerStat:%s: _singleConnectionAvgSpeed old:%.2fKB/s" + " new:%.2fKB/s last:%.2fKB/s", + getHostname().c_str(), + (float) _singleConnectionAvgSpeed/1024, + (float) avgDownloadSpeed/1024, + (float) downloadSpeed / 1024); + _singleConnectionAvgSpeed = (int)avgDownloadSpeed; +} + +unsigned int ServerStat::getMultiConnectionAvgSpeed() const +{ + return _multiConnectionAvgSpeed; +} + +void ServerStat::setMultiConnectionAvgSpeed +(unsigned int multiConnectionAvgSpeed) +{ + _multiConnectionAvgSpeed = multiConnectionAvgSpeed; +} + +void ServerStat::updateMultiConnectionAvgSpeed(unsigned int downloadSpeed) +{ + float avgDownloadSpeed; + if(_counter == 0) + return; + if(_counter < 5) { + avgDownloadSpeed = + ((((float)_counter-1)/(float)_counter)*(float)_multiConnectionAvgSpeed) + + ((1.0/(float)_counter)*(float)downloadSpeed); + } + else { + avgDownloadSpeed = ((4.0/5.0)*(float)_multiConnectionAvgSpeed) + + ((1.0/5.0)*(float)downloadSpeed); + } + _logger->debug("ServerStat:%s: _multiConnectionAvgSpeed old:%.2fKB/s" + " new:%.2fKB/s last:%.2fKB/s", + getHostname().c_str(), + (float) _multiConnectionAvgSpeed/1024, + (float) avgDownloadSpeed/1024, + (float) downloadSpeed / 1024); + _multiConnectionAvgSpeed = (int)avgDownloadSpeed; +} + +unsigned int ServerStat::getCounter() const +{ + return _counter; +} + +void ServerStat::increaseCounter() +{ + _counter++; +} + +void ServerStat::setCounter(unsigned int value) +{ + _counter = value; +} + void ServerStat::setStatus(STATUS status) { _status = status; @@ -160,7 +256,10 @@ std::ostream& operator<<(std::ostream& o, const ServerStat& serverStat) o << "host=" << serverStat.getHostname() << ", " << "protocol=" << serverStat.getProtocol() << ", " << "dl_speed=" << serverStat.getDownloadSpeed() << ", " + << "sc_avg_speed=" << serverStat.getSingleConnectionAvgSpeed() << ", " + << "mc_avg_speed=" << serverStat.getMultiConnectionAvgSpeed() << ", " << "last_updated=" << serverStat.getLastUpdated().getTime() << ", " + << "counter=" << serverStat.getCounter() << ", " << "status=" << ServerStat::STATUS_STRING[serverStat.getStatus()]; return o; } diff --git a/src/ServerStat.h b/src/ServerStat.h index 7b2cd6f7..c0895c42 100644 --- a/src/ServerStat.h +++ b/src/ServerStat.h @@ -35,12 +35,16 @@ #ifndef _D_SERVER_STAT_H_ #define _D_SERVER_STAT_H_ #include "common.h" -#include "TimeA2.h" + #include #include +#include "TimeA2.h" + namespace aria2 { +class Logger; + // ServerStatMan: has many ServerStat // URISelector: interface // ServerStatURISelector: Has a reference of ServerStatMan @@ -75,6 +79,18 @@ public: // set download speed. This method doesn't update _lastUpdate. void setDownloadSpeed(unsigned int downloadSpeed); + unsigned int getSingleConnectionAvgSpeed() const; + void updateSingleConnectionAvgSpeed(unsigned int downloadSpeed); + void setSingleConnectionAvgSpeed(unsigned int singleConnectionAvgSpeed); + + unsigned int getMultiConnectionAvgSpeed() const; + void updateMultiConnectionAvgSpeed(unsigned int downloadSpeed); + void setMultiConnectionAvgSpeed(unsigned int singleConnectionAvgSpeed); + + unsigned int getCounter() const; + void increaseCounter(); + void setCounter(unsigned int value); + // This method doesn't update _lastUpdate. void setStatus(STATUS status); @@ -104,6 +120,14 @@ private: std::string _protocol; unsigned int _downloadSpeed; + + unsigned int _singleConnectionAvgSpeed; + + unsigned int _multiConnectionAvgSpeed; + + unsigned int _counter; + + Logger* _logger; STATUS _status; diff --git a/src/ServerStatMan.cc b/src/ServerStatMan.cc index 77123ab3..ea329b44 100644 --- a/src/ServerStatMan.cc +++ b/src/ServerStatMan.cc @@ -86,7 +86,10 @@ bool ServerStatMan::load(std::istream& in) static const std::string S_HOST = "host"; static const std::string S_PROTOCOL = "protocol"; static const std::string S_DL_SPEED = "dl_speed"; + static const std::string S_SC_AVG_SPEED = "sc_avg_speed"; + static const std::string S_MC_AVG_SPEED = "mc_avg_speed"; static const std::string S_LAST_UPDATED = "last_updated"; + static const std::string S_COUNTER = "counter"; static const std::string S_STATUS = "status"; std::string line; @@ -111,6 +114,18 @@ bool ServerStatMan::load(std::istream& in) SharedHandle sstat(new ServerStat(m[S_HOST], m[S_PROTOCOL])); try { sstat->setDownloadSpeed(Util::parseUInt(m[S_DL_SPEED])); + // Old serverstat file doesn't contains SC_AVG_SPEED + if(m.find(S_SC_AVG_SPEED) != m.end()) { + sstat->setSingleConnectionAvgSpeed(Util::parseUInt(m[S_SC_AVG_SPEED])); + } + // Old serverstat file doesn't contains MC_AVG_SPEED + if(m.find(S_MC_AVG_SPEED) != m.end()) { + sstat->setMultiConnectionAvgSpeed(Util::parseUInt(m[S_MC_AVG_SPEED])); + } + // Old serverstat file doesn't contains COUNTER_SPEED + if(m.find(S_COUNTER) != m.end()) { + sstat->setCounter(Util::parseUInt(m[S_COUNTER])); + } sstat->setLastUpdated(Time(Util::parseInt(m[S_LAST_UPDATED]))); sstat->setStatus(m[S_STATUS]); add(sstat); diff --git a/src/SpeedCalc.cc b/src/SpeedCalc.cc index 94921c71..598e080c 100644 --- a/src/SpeedCalc.cc +++ b/src/SpeedCalc.cc @@ -115,7 +115,9 @@ void SpeedCalc::changeSw() { unsigned int SpeedCalc::calculateAvgSpeed() const { uint64_t milliElapsed = start.differenceInMillis(); - if(milliElapsed) { + + // if milliElapsed is too small, the average speed is rubish, better return 0 + if(milliElapsed > 4) { unsigned int speed = accumulatedLength*1000/milliElapsed; return speed; } else { diff --git a/src/URISelector.h b/src/URISelector.h index 144bb03d..d00aeb54 100644 --- a/src/URISelector.h +++ b/src/URISelector.h @@ -40,11 +40,18 @@ namespace aria2 { +class DownloadCommand; + class URISelector { public: virtual ~URISelector() {} virtual std::string select(std::deque& uris) = 0; + + virtual void tuneDownloadCommand(std::deque& uris, + DownloadCommand* command) {}; + + virtual void resetCounters() { return; }; }; } // namespace aria2 diff --git a/src/prefs.cc b/src/prefs.cc index e1e93924..1fdb318b 100644 --- a/src/prefs.cc +++ b/src/prefs.cc @@ -131,10 +131,11 @@ const std::string V_INFO("info"); const std::string V_NOTICE("notice"); const std::string V_WARN("warn"); const std::string V_ERROR("error"); -// value: inorder | feedback +// value: inorder | feedback | adaptive const std::string PREF_URI_SELECTOR("uri-selector"); const std::string V_INORDER("inorder"); const std::string V_FEEDBACK("feedback"); +const std::string V_ADAPTIVE("adaptive"); // value: 1*digit const std::string PREF_SERVER_STAT_TIMEOUT("server-stat-timeout"); // value: string that your file system recognizes as a file name. diff --git a/src/prefs.h b/src/prefs.h index 3ce4e553..a3ee16e6 100644 --- a/src/prefs.h +++ b/src/prefs.h @@ -135,10 +135,11 @@ extern const std::string V_INFO; extern const std::string V_NOTICE; extern const std::string V_WARN; extern const std::string V_ERROR; -// value: inorder | feedback +// value: inorder | feedback | adaptive extern const std::string PREF_URI_SELECTOR; extern const std::string V_INORDER; extern const std::string V_FEEDBACK; +extern const std::string V_ADAPTIVE; // value: 1*digit extern const std::string PREF_SERVER_STAT_TIMEOUT; // value: string that your file system recognizes as a file name. diff --git a/test/RequestGroupManTest.cc b/test/RequestGroupManTest.cc index d654011b..6f3182b5 100644 --- a/test/RequestGroupManTest.cc +++ b/test/RequestGroupManTest.cc @@ -1,4 +1,9 @@ #include "RequestGroupMan.h" + +#include + +#include + #include "prefs.h" #include "SingleFileDownloadContext.h" #include "RequestGroup.h" @@ -8,8 +13,6 @@ #include "ServerStatMan.h" #include "ServerStat.h" #include "File.h" -#include -#include namespace aria2 { @@ -97,6 +100,7 @@ void RequestGroupManTest::testLoadServerStat() Option option; RequestGroupMan rm(std::deque >(), 0, &option); + std::cerr << "testLoadServerStat" << std::endl; CPPUNIT_ASSERT(rm.loadServerStat(f.getPath())); SharedHandle ss_localhost = rm.findServerStat("localhost", "http"); diff --git a/test/ServerStatManTest.cc b/test/ServerStatManTest.cc index c000ff52..265c04e1 100644 --- a/test/ServerStatManTest.cc +++ b/test/ServerStatManTest.cc @@ -1,10 +1,13 @@ #include "ServerStatMan.h" + +#include +#include + +#include + #include "ServerStat.h" #include "Exception.h" #include "Util.h" -#include -#include -#include namespace aria2 { @@ -58,6 +61,9 @@ void ServerStatManTest::testSave() { SharedHandle localhost_http(new ServerStat("localhost", "http")); localhost_http->setDownloadSpeed(25000); + localhost_http->setSingleConnectionAvgSpeed(100); + localhost_http->setMultiConnectionAvgSpeed(101); + localhost_http->setCounter(5); localhost_http->setLastUpdated(Time(1210000000)); SharedHandle localhost_ftp(new ServerStat("localhost", "ftp")); localhost_ftp->setDownloadSpeed(30000); @@ -77,9 +83,29 @@ void ServerStatManTest::testSave() std::string out = ss.str(); CPPUNIT_ASSERT_EQUAL (std::string - ("host=localhost, protocol=ftp, dl_speed=30000, last_updated=1210000001, status=OK\n" - "host=localhost, protocol=http, dl_speed=25000, last_updated=1210000000, status=OK\n" - "host=mirror, protocol=http, dl_speed=0, last_updated=1210000002, status=ERROR\n"), + ("host=localhost, protocol=ftp," + " dl_speed=30000," + " sc_avg_speed=0," + " mc_avg_speed=0," + " last_updated=1210000001," + " counter=0," + " status=OK\n" + + "host=localhost, protocol=http," + " dl_speed=25000," + " sc_avg_speed=100," + " mc_avg_speed=101," + " last_updated=1210000000," + " counter=5," + " status=OK\n" + + "host=mirror, protocol=http," + " dl_speed=0," + " sc_avg_speed=0," + " mc_avg_speed=0," + " last_updated=1210000002," + " counter=0," + " status=ERROR\n"), out); } @@ -87,7 +113,7 @@ void ServerStatManTest::testLoad() { std::string in = "host=localhost, protocol=ftp, dl_speed=30000, last_updated=1210000001, status=OK\n" - "host=localhost, protocol=http, dl_speed=25000, last_updated=1210000000, status=OK\n" + "host=localhost, protocol=http, dl_speed=25000, sc_avg_speed=101, mc_avg_speed=102, last_updated=1210000000, counter=6, status=OK\n" "host=mirror, protocol=http, dl_speed=0, last_updated=1210000002, status=ERROR\n"; std::stringstream ss(in); @@ -101,6 +127,12 @@ void ServerStatManTest::testLoad() CPPUNIT_ASSERT_EQUAL(std::string("http"), localhost_http->getProtocol()); CPPUNIT_ASSERT_EQUAL(static_cast(25000), localhost_http->getDownloadSpeed()); + CPPUNIT_ASSERT_EQUAL(static_cast(101), + localhost_http->getSingleConnectionAvgSpeed()); + CPPUNIT_ASSERT_EQUAL(static_cast(102), + localhost_http->getMultiConnectionAvgSpeed()); + CPPUNIT_ASSERT_EQUAL(static_cast(6), + localhost_http->getCounter()); CPPUNIT_ASSERT_EQUAL(static_cast(1210000000), localhost_http->getLastUpdated().getTime()); CPPUNIT_ASSERT_EQUAL(ServerStat::OK, localhost_http->getStatus()); diff --git a/test/ServerStatTest.cc b/test/ServerStatTest.cc index bbd493b9..92903c7b 100644 --- a/test/ServerStatTest.cc +++ b/test/ServerStatTest.cc @@ -1,10 +1,13 @@ #include "ServerStat.h" -#include "Exception.h" -#include "Util.h" + #include #include + #include +#include "Exception.h" +#include "Util.h" + namespace aria2 { class ServerStatTest:public CppUnit::TestFixture { @@ -46,6 +49,9 @@ void ServerStatTest::testOperatorOstream() ServerStat localhost_http("localhost", "http"); localhost_http.setDownloadSpeed(90000); localhost_http.setLastUpdated(Time(1000)); + localhost_http.setSingleConnectionAvgSpeed(101); + localhost_http.setMultiConnectionAvgSpeed(102); + localhost_http.setCounter(5); std::stringstream ss; @@ -53,7 +59,9 @@ void ServerStatTest::testOperatorOstream() CPPUNIT_ASSERT_EQUAL (std::string - ("host=localhost, protocol=http, dl_speed=90000, last_updated=1000, status=OK"), + ("host=localhost, protocol=http, dl_speed=90000," + " sc_avg_speed=101, mc_avg_speed=102," + " last_updated=1000, counter=5, status=OK"), ss.str()); ss.str(""); @@ -67,7 +75,9 @@ void ServerStatTest::testOperatorOstream() CPPUNIT_ASSERT_EQUAL (std::string - ("host=localhost, protocol=ftp, dl_speed=10000, last_updated=1210000000, status=ERROR"), + ("host=localhost, protocol=ftp, dl_speed=10000," + " sc_avg_speed=0, mc_avg_speed=0," + " last_updated=1210000000, counter=0, status=ERROR"), ss.str()); }