From 0ecfa19925453e3e4bf8ca947dc72a597654e401 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 25 Oct 2012 23:33:45 +0900 Subject: [PATCH] Reworked download/upload statistics calculation The old implementation calculates download/upload statistics for a RequestGroup by summing up all PeerStat objects. For global statistics, those are summed together. This clearly incurs runtime penalty and we introduced some kind of caching which updates statistics every 250ms but it did not work right. This change removes all these aggregation code, and instead makes RequestGroup and RequestGroupMan objects hold NetStat object and download/upload bytes are directly calculated by thier own NetStat. This is far more simplar than the old way and less runtime penalty and brings more accuracy. --- src/AbstractCommand.cc | 2 +- src/BtPieceMessage.cc | 2 + src/BtSetup.cc | 2 +- src/ConsoleStatCalc.cc | 6 +- src/DefaultBtAnnounce.cc | 2 +- src/DefaultBtInteractive.cc | 2 - src/DefaultBtMessageDispatcher.cc | 3 - src/DefaultBtProgressInfoFile.cc | 4 +- src/DefaultPeerStorage.cc | 77 -------------- src/DefaultPeerStorage.h | 12 --- src/DownloadCommand.cc | 1 + src/DownloadContext.cc | 34 +++++-- src/DownloadContext.h | 16 ++- src/DownloadEngine.cc | 1 - src/Makefile.am | 1 + src/MultiUrlRequestInfo.cc | 1 + src/NetStat.cc | 126 +++++++++++++++++++++++ src/NetStat.h | 133 +++++++++++++++++++++++++ src/Peer.cc | 6 +- src/PeerSessionResource.cc | 9 +- src/PeerSessionResource.h | 8 +- src/PeerStat.cc | 87 +++++++++------- src/PeerStat.h | 72 ++++--------- src/PeerStorage.h | 9 -- src/RequestGroup.cc | 18 +--- src/RequestGroup.h | 5 + src/RequestGroupMan.cc | 13 +-- src/RequestGroupMan.h | 8 ++ src/RpcMethodImpl.cc | 7 +- src/SegmentMan.cc | 32 +----- src/SegmentMan.h | 10 -- src/ShareRatioSeedCriteria.cc | 13 +-- src/ShareRatioSeedCriteria.h | 6 +- test/DefaultBtAnnounceTest.cc | 7 +- test/DefaultBtMessageDispatcherTest.cc | 8 -- test/DefaultBtProgressInfoFileTest.cc | 5 +- test/MockPeerStorage.h | 16 --- test/ShareRatioSeedCriteriaTest.cc | 10 +- 38 files changed, 434 insertions(+), 340 deletions(-) create mode 100644 src/NetStat.cc create mode 100644 src/NetStat.h diff --git a/src/AbstractCommand.cc b/src/AbstractCommand.cc index 35d1b0d1..dc18d97f 100644 --- a/src/AbstractCommand.cc +++ b/src/AbstractCommand.cc @@ -431,7 +431,7 @@ void AbstractCommand::onAbort() { // limitation of current implementation. if(!getOption()->getAsBool(PREF_ALWAYS_RESUME) && fileEntry_ && - getSegmentMan()->calculateSessionDownloadLength() == 0 && + getDownloadContext()->getNetStat().getSessionDownloadLength() == 0 && !requestGroup_->p2pInvolved() && getDownloadContext()->getFileEntries().size() == 1) { const int maxTries = getOption()->getAsInt(PREF_MAX_RESUME_FAILURE_TRIES); diff --git a/src/BtPieceMessage.cc b/src/BtPieceMessage.cc index be2ced99..eb1a5c81 100644 --- a/src/BtPieceMessage.cc +++ b/src/BtPieceMessage.cc @@ -100,6 +100,7 @@ void BtPieceMessage::doReceivedAction() RequestSlot slot = getBtMessageDispatcher()->getOutstandingRequest (index_, begin_, blockLength_); getPeer()->updateDownloadLength(blockLength_); + downloadContext_->updateDownloadLength(blockLength_); if(!RequestSlot::isNull(slot)) { getPeer()->snubbing(false); SharedHandle piece = getPieceStorage()->getPiece(index_); @@ -188,6 +189,7 @@ void BtPieceMessage::send() } writtenLength = getPeerConnection()->sendPendingData(); getPeer()->updateUploadLength(writtenLength); + downloadContext_->updateUploadLength(writtenLength); setSendingInProgress(!getPeerConnection()->sendBufferIsEmpty()); } diff --git a/src/BtSetup.cc b/src/BtSetup.cc index 128a3510..1a8b6403 100644 --- a/src/BtSetup.cc +++ b/src/BtSetup.cc @@ -171,7 +171,7 @@ void BtSetup::setup(std::vector& commands, (new ShareRatioSeedCriteria(option->getAsDouble(PREF_SEED_RATIO), requestGroup->getDownloadContext())); cri->setPieceStorage(pieceStorage); - cri->setPeerStorage(peerStorage); + cri->setBtRuntime(btRuntime); unionCri->addSeedCriteria(cri); } diff --git a/src/ConsoleStatCalc.cc b/src/ConsoleStatCalc.cc index 95e569e5..3a3813ad 100644 --- a/src/ConsoleStatCalc.cc +++ b/src/ConsoleStatCalc.cc @@ -304,10 +304,8 @@ ConsoleStatCalc::calculateStat(const DownloadEngine* e) if(e->getRequestGroupMan()->countRequestGroup() > 1 && !e->getRequestGroupMan()->downloadFinished()) { - TransferStat stat = e->getRequestGroupMan()->calculateStat(); - o << " " - << "[TOTAL SPD:" - << sizeFormatter(stat.getDownloadSpeed()) << "Bs" << "]"; + int spd = e->getRequestGroupMan()->getNetStat().calculateDownloadSpeed(); + o << " [TOTAL SPD:" << sizeFormatter(spd) << "Bs" << "]"; } { diff --git a/src/DefaultBtAnnounce.cc b/src/DefaultBtAnnounce.cc index 4f3638f2..6a9cc713 100644 --- a/src/DefaultBtAnnounce.cc +++ b/src/DefaultBtAnnounce.cc @@ -141,7 +141,7 @@ std::string DefaultBtAnnounce::getAnnounceUrl() { if(!btRuntime_->lessThanMinPeers() || btRuntime_->isHalt()) { numWant = 0; } - TransferStat stat = peerStorage_->calculateStat(); + NetStat& stat = downloadContext_->getNetStat(); int64_t left = pieceStorage_->getTotalLength()-pieceStorage_->getCompletedLength(); // Use last 8 bytes of peer ID as a key diff --git a/src/DefaultBtInteractive.cc b/src/DefaultBtInteractive.cc index ad1119b9..af851923 100644 --- a/src/DefaultBtInteractive.cc +++ b/src/DefaultBtInteractive.cc @@ -305,8 +305,6 @@ size_t DefaultBtInteractive::receiveMessages() { } break; case BtPieceMessage::ID: - peerStorage_->updateTransferStatFor(peer_); - // pass through case BtRequestMessage::ID: inactiveTimer_ = global::wallclock(); break; diff --git a/src/DefaultBtMessageDispatcher.cc b/src/DefaultBtMessageDispatcher.cc index cf686757..1d1dee50 100644 --- a/src/DefaultBtMessageDispatcher.cc +++ b/src/DefaultBtMessageDispatcher.cc @@ -100,9 +100,6 @@ void DefaultBtMessageDispatcher::sendMessages() { } } msg->send(); - if(msg->isUploading()) { - peerStorage_->updateTransferStatFor(peer_); - } if(msg->isSendingInProgress()) { messageQueue_.push_front(msg); break; diff --git a/src/DefaultBtProgressInfoFile.cc b/src/DefaultBtProgressInfoFile.cc index f064d3c7..44d9939a 100644 --- a/src/DefaultBtProgressInfoFile.cc +++ b/src/DefaultBtProgressInfoFile.cc @@ -159,8 +159,8 @@ void DefaultBtProgressInfoFile::save() uint64_t uploadLengthNL = 0; #ifdef ENABLE_BITTORRENT if(torrentDownload) { - TransferStat stat = peerStorage_->calculateStat(); - uploadLengthNL = hton64(stat.getAllTimeUploadLength()); + uploadLengthNL = hton64(btRuntime_->getUploadLengthAtStartup()+ + dctx_->getNetStat().getSessionUploadLength()); } #endif // ENABLE_BITTORRENT WRITE_CHECK(fp, &uploadLengthNL, sizeof(uploadLengthNL)); diff --git a/src/DefaultPeerStorage.cc b/src/DefaultPeerStorage.cc index dec61d6f..469f044e 100644 --- a/src/DefaultPeerStorage.cc +++ b/src/DefaultPeerStorage.cc @@ -60,8 +60,6 @@ const size_t MAX_PEER_LIST_UPDATE = 100; DefaultPeerStorage::DefaultPeerStorage() : maxPeerListSize_(MAX_PEER_LIST_SIZE), - removedPeerSessionDownloadLength_(0LL), - removedPeerSessionUploadLength_(0LL), seederStateChoke_(new BtSeederStateChoke()), leecherStateChoke_(new BtLeecherStateChoke()), lastTransferStatMapUpdated_(0) @@ -241,76 +239,6 @@ void DefaultPeerStorage::getActivePeers std::for_each(peers_.begin(), peers_.end(), CollectActivePeer(activePeers)); } -namespace { -TransferStat calculateStatFor(const SharedHandle& peer) -{ - TransferStat s; - s.downloadSpeed = peer->calculateDownloadSpeed(); - s.uploadSpeed = peer->calculateUploadSpeed(); - s.sessionDownloadLength = peer->getSessionDownloadLength(); - s.sessionUploadLength = peer->getSessionUploadLength(); - return s; -} -} // namespace - -TransferStat DefaultPeerStorage::calculateStat() -{ - TransferStat stat; - if(lastTransferStatMapUpdated_.differenceInMillis(global::wallclock())+ - A2_DELTA_MILLIS >= 250) { - A2_LOG_DEBUG("Updating TransferStat of PeerStorage"); - lastTransferStatMapUpdated_ = global::wallclock(); - peerTransferStatMap_.clear(); - std::vector > activePeers; - getActivePeers(activePeers); - for(std::vector >::const_iterator i = - activePeers.begin(), eoi = activePeers.end(); i != eoi; ++i) { - TransferStat s; - s.downloadSpeed = (*i)->calculateDownloadSpeed(); - s.uploadSpeed = (*i)->calculateUploadSpeed(); - s.sessionDownloadLength = (*i)->getSessionDownloadLength(); - s.sessionUploadLength = (*i)->getSessionUploadLength(); - - peerTransferStatMap_[(*i)->getID()] = calculateStatFor(*i); - stat += s; - } - cachedTransferStat_ = stat; - } else { - stat = cachedTransferStat_; - } - stat.sessionDownloadLength += removedPeerSessionDownloadLength_; - stat.sessionUploadLength += removedPeerSessionUploadLength_; - stat.setAllTimeUploadLength(btRuntime_->getUploadLengthAtStartup()+ - stat.getSessionUploadLength()); - return stat; -} - -void DefaultPeerStorage::updateTransferStatFor(const SharedHandle& peer) -{ - A2_LOG_DEBUG(fmt("Updating TransferStat for peer %s", peer->getID().c_str())); - std::map::iterator itr = - peerTransferStatMap_.find(peer->getID()); - if(itr == peerTransferStatMap_.end()) { - return; - } - cachedTransferStat_ -= (*itr).second; - TransferStat s = calculateStatFor(peer); - cachedTransferStat_ += s; - (*itr).second = s; -} - -TransferStat DefaultPeerStorage::getTransferStatFor -(const SharedHandle& peer) -{ - std::map::const_iterator itr = - peerTransferStatMap_.find(peer->getID()); - if(itr == peerTransferStatMap_.end()) { - return TransferStat(); - } else { - return (*itr).second; - } -} - bool DefaultPeerStorage::isBadPeer(const std::string& ipaddr) { std::map::iterator i = badPeers_.find(ipaddr); @@ -365,11 +293,6 @@ void DefaultPeerStorage::onErasingPeer(const SharedHandle& peer) {} void DefaultPeerStorage::onReturningPeer(const SharedHandle& peer) { if(peer->isActive()) { - TransferStat removedStat(calculateStatFor(peer)); - removedPeerSessionDownloadLength_ += removedStat.getSessionDownloadLength(); - removedPeerSessionUploadLength_ += removedStat.getSessionUploadLength(); - cachedTransferStat_ -= removedStat; - if(peer->isDisconnectedGracefully() && !peer->isIncomingPeer()) { peer->startBadCondition(); addDroppedPeer(peer); diff --git a/src/DefaultPeerStorage.h b/src/DefaultPeerStorage.h index 4e43d966..3b3dac20 100644 --- a/src/DefaultPeerStorage.h +++ b/src/DefaultPeerStorage.h @@ -56,18 +56,12 @@ private: size_t maxPeerListSize_; std::deque > peers_; std::deque > droppedPeers_; - int64_t removedPeerSessionDownloadLength_; - int64_t removedPeerSessionUploadLength_; BtSeederStateChoke* seederStateChoke_; BtLeecherStateChoke* leecherStateChoke_; - std::map peerTransferStatMap_; - Timer lastTransferStatMapUpdated_; - TransferStat cachedTransferStat_; - std::map badPeers_; Timer lastBadPeerCleaned_; @@ -97,12 +91,6 @@ public: virtual void getActivePeers(std::vector >& peers); - virtual TransferStat calculateStat(); - - virtual void updateTransferStatFor(const SharedHandle& peer); - - virtual TransferStat getTransferStatFor(const SharedHandle& peer); - virtual bool isBadPeer(const std::string& ipaddr); virtual void addBadPeer(const std::string& ipaddr); diff --git a/src/DownloadCommand.cc b/src/DownloadCommand.cc index 6ac7c640..574ed663 100644 --- a/src/DownloadCommand.cc +++ b/src/DownloadCommand.cc @@ -176,6 +176,7 @@ bool DownloadCommand::executeInternal() { } getSocketRecvBuffer()->shiftBuffer(bufSize); peerStat_->updateDownloadLength(bufSize); + getDownloadContext()->updateDownloadLength(bufSize); } bool segmentPartComplete = false; // Note that GrowSegment::complete() always returns false. diff --git a/src/DownloadContext.cc b/src/DownloadContext.cc index dbc355ea..1191259a 100644 --- a/src/DownloadContext.cc +++ b/src/DownloadContext.cc @@ -43,6 +43,7 @@ #include "DlAbortEx.h" #include "a2functional.h" #include "Signature.h" +#include "RequestGroupMan.h" namespace aria2 { @@ -52,8 +53,7 @@ DownloadContext::DownloadContext(): knowsTotalLength_(true), ownerRequestGroup_(0), attrs_(MAX_CTX_ATTR), - downloadStartTime_(0), - downloadStopTime_(downloadStartTime_), + downloadStopTime_(0), acceptMetalink_(true) {} DownloadContext::DownloadContext(int32_t pieceLength, @@ -64,7 +64,6 @@ DownloadContext::DownloadContext(int32_t pieceLength, knowsTotalLength_(true), ownerRequestGroup_(0), attrs_(MAX_CTX_ATTR), - downloadStartTime_(0), downloadStopTime_(0), acceptMetalink_(true) { @@ -76,23 +75,20 @@ DownloadContext::~DownloadContext() {} void DownloadContext::resetDownloadStartTime() { - downloadStartTime_ = global::wallclock(); downloadStopTime_.reset(0); + netStat_.downloadStart(); } void DownloadContext::resetDownloadStopTime() { downloadStopTime_ = global::wallclock(); + netStat_.downloadStop(); } int64_t DownloadContext::calculateSessionTime() const { - if(downloadStopTime_ > downloadStartTime_) { - return - downloadStartTime_.differenceInMillis(downloadStopTime_); - } else { - return 0; - } + const Timer& startTime = netStat_.getDownloadStartTime(); + return startTime.differenceInMillis(downloadStopTime_); } SharedHandle @@ -283,4 +279,22 @@ void DownloadContext::setSignature(const SharedHandle& signature) signature_ = signature; } +void DownloadContext::updateDownloadLength(size_t bytes) +{ + netStat_.updateDownloadLength(bytes); + RequestGroupMan* rgman = ownerRequestGroup_->getRequestGroupMan(); + if(rgman) { + rgman->getNetStat().updateDownloadLength(bytes); + } +} + +void DownloadContext::updateUploadLength(size_t bytes) +{ + netStat_.updateUploadLength(bytes); + RequestGroupMan* rgman = ownerRequestGroup_->getRequestGroupMan(); + if(rgman) { + rgman->getNetStat().updateUploadLength(bytes); + } +} + } // namespace aria2 diff --git a/src/DownloadContext.h b/src/DownloadContext.h index c05f4bcd..8ba146e3 100644 --- a/src/DownloadContext.h +++ b/src/DownloadContext.h @@ -47,6 +47,7 @@ #include "ValueBase.h" #include "SegList.h" #include "ContextAttribute.h" +#include "NetStat.h" namespace aria2 { @@ -79,7 +80,7 @@ private: std::vector > attrs_; - Timer downloadStartTime_; + NetStat netStat_; Timer downloadStopTime_; @@ -234,6 +235,19 @@ public: { return acceptMetalink_; } + + NetStat& getNetStat() + { + return netStat_; + } + + // This method also updates global download length held by + // RequestGroupMan via getOwnerRequestGroup(). + void updateDownloadLength(size_t bytes); + + // This method also updates global upload length held by + // RequestGroupMan via getOwnerRequestGroup(). + void updateUploadLength(size_t bytes); }; } // namespace aria2 diff --git a/src/DownloadEngine.cc b/src/DownloadEngine.cc index c81cdcbd..6703dd5b 100644 --- a/src/DownloadEngine.cc +++ b/src/DownloadEngine.cc @@ -220,7 +220,6 @@ void DownloadEngine::onEndOfRun() void DownloadEngine::afterEachIteration() { - requestGroupMan_->calculateStat(); if(global::globalHaltRequested == 1) { A2_LOG_NOTICE(_("Shutdown sequence commencing..." " Press Ctrl-C again for emergency shutdown.")); diff --git a/src/Makefile.am b/src/Makefile.am index e9e2444f..5e614926 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -58,6 +58,7 @@ SRCS = Socket.h\ DownloadEngineFactory.cc DownloadEngineFactory.h\ SpeedCalc.cc SpeedCalc.h\ PeerStat.cc PeerStat.h\ + NetStat.cc NetStat.h\ BitfieldMan.cc BitfieldMan.h\ Randomizer.h\ SimpleRandomizer.cc SimpleRandomizer.h\ diff --git a/src/MultiUrlRequestInfo.cc b/src/MultiUrlRequestInfo.cc index 284a08a7..48bfec4f 100644 --- a/src/MultiUrlRequestInfo.cc +++ b/src/MultiUrlRequestInfo.cc @@ -239,6 +239,7 @@ error_code::Value MultiUrlRequestInfo::execute() util::setGlobalSignalHandler(SIGINT, handler, 0); util::setGlobalSignalHandler(SIGTERM, handler, 0); + e->getRequestGroupMan()->getNetStat().downloadStart(); e->run(); if(!option_->blank(PREF_SAVE_COOKIES)) { diff --git a/src/NetStat.cc b/src/NetStat.cc new file mode 100644 index 00000000..fe0e6a45 --- /dev/null +++ b/src/NetStat.cc @@ -0,0 +1,126 @@ +/* */ +#include "NetStat.h" +#include "wallclock.h" + +namespace aria2 { + +NetStat::NetStat() + : status_(NetStat::IDLE), + avgDownloadSpeed_(0), + avgUploadSpeed_(0), + sessionDownloadLength_(0), + sessionUploadLength_(0) +{} + +NetStat::~NetStat() {} + +/** + * Returns current download speed in byte per sec. + */ +int NetStat::calculateDownloadSpeed() +{ + return downloadSpeed_.calculateSpeed(); +} + +int NetStat::calculateAvgDownloadSpeed() +{ + return avgDownloadSpeed_ = downloadSpeed_.calculateAvgSpeed(); +} + +int NetStat::calculateUploadSpeed() +{ + return uploadSpeed_.calculateSpeed(); +} + +int NetStat::calculateAvgUploadSpeed() +{ + return avgUploadSpeed_ = uploadSpeed_.calculateAvgSpeed(); +} + +void NetStat::updateDownloadLength(size_t bytes) +{ + downloadSpeed_.update(bytes); + sessionDownloadLength_ += bytes; +} + +void NetStat::updateUploadLength(size_t bytes) +{ + uploadSpeed_.update(bytes); + sessionUploadLength_ += bytes; +} + +int NetStat::getMaxDownloadSpeed() const +{ + return downloadSpeed_.getMaxSpeed(); +} + +int NetStat::getMaxUploadSpeed() const +{ + return uploadSpeed_.getMaxSpeed(); +} + +void NetStat::reset() +{ + downloadSpeed_.reset(); + uploadSpeed_.reset(); + downloadStartTime_ = global::wallclock(); + status_ = IDLE; +} + +void NetStat::downloadStart() +{ + reset(); + status_ = ACTIVE; +} + +void NetStat::downloadStop() +{ + calculateAvgDownloadSpeed(); + calculateAvgUploadSpeed(); + status_ = IDLE; +} + +TransferStat NetStat::toTransferStat() +{ + TransferStat stat; + stat.downloadSpeed = calculateDownloadSpeed(); + stat.uploadSpeed = calculateUploadSpeed(); + stat.sessionDownloadLength = getSessionDownloadLength(); + stat.sessionUploadLength = getSessionUploadLength(); + return stat; +} + +} // namespace aria2 diff --git a/src/NetStat.h b/src/NetStat.h new file mode 100644 index 00000000..f4efe101 --- /dev/null +++ b/src/NetStat.h @@ -0,0 +1,133 @@ +/* */ +#ifndef D_NET_STAT_H +#define D_NET_STAT_H + +#include "common.h" + +#include "SpeedCalc.h" +#include "TransferStat.h" + +namespace aria2 { + +class NetStat { +public: + enum STATUS { + IDLE, + ACTIVE, + }; + + NetStat(); + ~NetStat(); + + // Don't allow copying + NetStat(const NetStat&); + NetStat& operator=(const NetStat&); + + /** + * Returns current download speed in byte per sec. + */ + int calculateDownloadSpeed(); + + int calculateAvgDownloadSpeed(); + + int calculateUploadSpeed(); + + int calculateAvgUploadSpeed(); + + void updateDownloadLength(size_t bytes); + + void updateUploadLength(size_t bytes); + + int getMaxDownloadSpeed() const; + + int getMaxUploadSpeed() const; + + int getAvgDownloadSpeed() const + { + return avgDownloadSpeed_; + } + + int getAvgUploadSpeed() const + { + return avgUploadSpeed_; + } + + void reset(); + + void downloadStart(); + + void downloadStop(); + + const Timer& getDownloadStartTime() const + { + return downloadStartTime_; + } + + STATUS getStatus() const + { + return status_; + } + + uint64_t getSessionDownloadLength() const + { + return sessionDownloadLength_; + } + + uint64_t getSessionUploadLength() const + { + return sessionUploadLength_; + } + + void addSessionDownloadLength(uint64_t length) + { + sessionDownloadLength_ += length; + } + + TransferStat toTransferStat(); +private: + SpeedCalc downloadSpeed_; + SpeedCalc uploadSpeed_; + Timer downloadStartTime_; + STATUS status_; + int avgDownloadSpeed_; + int avgUploadSpeed_; + int64_t sessionDownloadLength_; + int64_t sessionUploadLength_; +}; + +} // namespace aria2 + +#endif // D_NET_STAT_H diff --git a/src/Peer.cc b/src/Peer.cc index caac3f43..b4fc13ac 100644 --- a/src/Peer.cc +++ b/src/Peer.cc @@ -77,7 +77,7 @@ void Peer::allocateSessionResource(int32_t pieceLength, int64_t totalLength) { delete res_; res_ = new PeerSessionResource(pieceLength, totalLength); - res_->getPeerStat().downloadStart(); + res_->getNetStat().downloadStart(); updateSeeder(); } @@ -219,13 +219,13 @@ void Peer::updateBitfield(size_t index, int operation) { int Peer::calculateUploadSpeed() { assert(res_); - return res_->getPeerStat().calculateUploadSpeed(); + return res_->getNetStat().calculateUploadSpeed(); } int Peer::calculateDownloadSpeed() { assert(res_); - return res_->getPeerStat().calculateDownloadSpeed(); + return res_->getNetStat().calculateDownloadSpeed(); } int64_t Peer::getSessionUploadLength() const diff --git a/src/PeerSessionResource.cc b/src/PeerSessionResource.cc index 57d360cb..2bb779d4 100644 --- a/src/PeerSessionResource.cc +++ b/src/PeerSessionResource.cc @@ -214,23 +214,22 @@ void PeerSessionResource::dhtEnabled(bool b) int64_t PeerSessionResource::uploadLength() const { - return peerStat_.getSessionUploadLength(); + return netStat_.getSessionUploadLength(); } void PeerSessionResource::updateUploadLength(int32_t bytes) { - peerStat_.updateUploadLength(bytes); + netStat_.updateUploadLength(bytes); } int64_t PeerSessionResource::downloadLength() const { - return peerStat_.getSessionDownloadLength(); + return netStat_.getSessionDownloadLength(); } void PeerSessionResource::updateDownloadLength(int32_t bytes) { - peerStat_.updateDownloadLength(bytes); - + netStat_.updateDownloadLength(bytes); lastDownloadUpdate_ = global::wallclock(); } diff --git a/src/PeerSessionResource.h b/src/PeerSessionResource.h index e1683d34..0bad7f15 100644 --- a/src/PeerSessionResource.h +++ b/src/PeerSessionResource.h @@ -41,7 +41,7 @@ #include #include "BtConstants.h" -#include "PeerStat.h" +#include "NetStat.h" #include "TimerA2.h" #include "ExtensionMessageRegistry.h" @@ -58,7 +58,7 @@ private: // fast index set which localhost has sent to a peer. std::set amAllowedIndexSet_; ExtensionMessageRegistry extreg_; - PeerStat peerStat_; + NetStat netStat_; Timer lastDownloadUpdate_; @@ -206,9 +206,9 @@ public: void dhtEnabled(bool b); - PeerStat& getPeerStat() + NetStat& getNetStat() { - return peerStat_; + return netStat_; } int64_t uploadLength() const; diff --git a/src/PeerStat.cc b/src/PeerStat.cc index 891f47f6..0aa8b4dd 100644 --- a/src/PeerStat.cc +++ b/src/PeerStat.cc @@ -33,7 +33,6 @@ */ /* copyright --> */ #include "PeerStat.h" -#include "SharedHandle.h" #include "wallclock.h" namespace aria2 { @@ -42,22 +41,11 @@ PeerStat::PeerStat (cuid_t cuid, const std::string& hostname, const::std::string& protocol) : cuid_(cuid), hostname_(hostname), - protocol_(protocol), - downloadStartTime_(global::wallclock()), - status_(PeerStat::IDLE), - avgDownloadSpeed_(0), - avgUploadSpeed_(0), - sessionDownloadLength_(0), - sessionUploadLength_(0) + protocol_(protocol) {} PeerStat::PeerStat(cuid_t cuid) - : cuid_(cuid), - status_(PeerStat::IDLE), - avgDownloadSpeed_(0), - avgUploadSpeed_(0), - sessionDownloadLength_(0), - sessionUploadLength_(0) + : cuid_(cuid) {} PeerStat::~PeerStat() {} @@ -67,72 +55,97 @@ PeerStat::~PeerStat() {} */ int PeerStat::calculateDownloadSpeed() { - return downloadSpeed_.calculateSpeed(); + return netStat_.calculateDownloadSpeed(); } int PeerStat::calculateAvgDownloadSpeed() { - avgDownloadSpeed_ = downloadSpeed_.calculateAvgSpeed(); - return avgDownloadSpeed_; + return netStat_.calculateAvgDownloadSpeed(); } int PeerStat::calculateUploadSpeed() { - return uploadSpeed_.calculateSpeed(); + return netStat_.calculateUploadSpeed(); } int PeerStat::calculateAvgUploadSpeed() { - avgUploadSpeed_ = uploadSpeed_.calculateAvgSpeed(); - return avgUploadSpeed_; + return netStat_.calculateAvgUploadSpeed(); } void PeerStat::updateDownloadLength(size_t bytes) { - downloadSpeed_.update(bytes); - sessionDownloadLength_ += bytes; + netStat_.updateDownloadLength(bytes); } void PeerStat::updateUploadLength(size_t bytes) { - uploadSpeed_.update(bytes); - sessionUploadLength_ += bytes; + netStat_.updateUploadLength(bytes); } int PeerStat::getMaxDownloadSpeed() const { - return downloadSpeed_.getMaxSpeed(); + return netStat_.getMaxDownloadSpeed(); } int PeerStat::getMaxUploadSpeed() const { - return uploadSpeed_.getMaxSpeed(); + return netStat_.getMaxUploadSpeed(); +} + +int PeerStat::getAvgDownloadSpeed() const +{ + return netStat_.getAvgDownloadSpeed(); +} + +int PeerStat::getAvgUploadSpeed() const +{ + return netStat_.getAvgUploadSpeed(); +} + +uint64_t PeerStat::getSessionDownloadLength() const +{ + return netStat_.getSessionDownloadLength(); +} + +uint64_t PeerStat::getSessionUploadLength() const +{ + return netStat_.getSessionUploadLength(); +} + +void PeerStat::addSessionDownloadLength(uint64_t length) +{ + netStat_.addSessionDownloadLength(length); +} + +const Timer& PeerStat::getDownloadStartTime() const +{ + return netStat_.getDownloadStartTime(); +} + +NetStat::STATUS PeerStat::getStatus() const +{ + return netStat_.getStatus(); } void PeerStat::reset() { - downloadSpeed_.reset(); - uploadSpeed_.reset(); - downloadStartTime_ = global::wallclock(); - status_ = PeerStat::IDLE; + netStat_.reset(); } void PeerStat::downloadStart() { - reset(); - status_ = PeerStat::ACTIVE; + netStat_.downloadStart(); } void PeerStat::downloadStop() { - calculateAvgDownloadSpeed(); - calculateAvgUploadSpeed(); - status_ = PeerStat::IDLE; + netStat_.downloadStop(); } -bool PeerStat::affectsOverallSpeed() const +TransferStat PeerStat::toTransferStat() { - return !downloadSpeed_.isIntervalOver(); + return netStat_.toTransferStat(); } } // namespace aria2 diff --git a/src/PeerStat.h b/src/PeerStat.h index c0b307b2..09f1ec53 100644 --- a/src/PeerStat.h +++ b/src/PeerStat.h @@ -39,30 +39,12 @@ #include -#include "SpeedCalc.h" -#include "SharedHandle.h" #include "Command.h" +#include "NetStat.h" namespace aria2 { class PeerStat { -public: - enum STATUS { - IDLE, - ACTIVE, - }; -private: - cuid_t cuid_; - std::string hostname_; - std::string protocol_; - SpeedCalc downloadSpeed_; - SpeedCalc uploadSpeed_; - Timer downloadStartTime_; - PeerStat::STATUS status_; - int avgDownloadSpeed_; - int avgUploadSpeed_; - int64_t sessionDownloadLength_; - int64_t sessionUploadLength_; public: PeerStat (cuid_t cuid, const std::string& hostname, const::std::string& protocol); @@ -94,15 +76,9 @@ public: int getMaxUploadSpeed() const; - int getAvgDownloadSpeed() const - { - return avgDownloadSpeed_; - } + int getAvgDownloadSpeed() const; - int getAvgUploadSpeed() const - { - return avgUploadSpeed_; - } + int getAvgUploadSpeed() const; void reset(); @@ -110,15 +86,17 @@ public: void downloadStop(); - const Timer& getDownloadStartTime() const - { - return downloadStartTime_; - } + const Timer& getDownloadStartTime() const; - PeerStat::STATUS getStatus() const - { - return status_; - } + NetStat::STATUS getStatus() const; + + uint64_t getSessionDownloadLength() const; + + uint64_t getSessionUploadLength() const; + + void addSessionDownloadLength(uint64_t length); + + TransferStat toTransferStat(); cuid_t getCuid() const { @@ -134,25 +112,11 @@ public: { return protocol_; } - - uint64_t getSessionDownloadLength() const - { - return sessionDownloadLength_; - } - - uint64_t getSessionUploadLength() const - { - return sessionUploadLength_; - } - - void addSessionDownloadLength(uint64_t length) - { - sessionDownloadLength_ += length; - } - - // Returns true if the download speed of this object still affects - // overall download speed statistics. - bool affectsOverallSpeed() const; +private: + cuid_t cuid_; + std::string hostname_; + std::string protocol_; + NetStat netStat_; }; } // namespace aria2 diff --git a/src/PeerStorage.h b/src/PeerStorage.h index 57d3b1ac..ab4cb2e2 100644 --- a/src/PeerStorage.h +++ b/src/PeerStorage.h @@ -94,15 +94,6 @@ public: */ virtual void getActivePeers(std::vector >& peers) = 0; - /** - * Calculates current download/upload statistics. - */ - virtual TransferStat calculateStat() = 0; - - virtual void updateTransferStatFor(const SharedHandle& peer) = 0; - - virtual TransferStat getTransferStatFor(const SharedHandle& peer) = 0; - /** * Returns true if peer with ipaddr should be ignored because, for * example, it sends bad data. diff --git a/src/RequestGroup.cc b/src/RequestGroup.cc index 4f3fca38..380d9b83 100644 --- a/src/RequestGroup.cc +++ b/src/RequestGroup.cc @@ -963,22 +963,15 @@ void RequestGroup::decreaseNumCommand() } } - TransferStat RequestGroup::calculateStat() const { - TransferStat stat; + TransferStat stat = downloadContext_->getNetStat().toTransferStat(); #ifdef ENABLE_BITTORRENT - if(peerStorage_) { - stat = peerStorage_->calculateStat(); + if(btRuntime_) { + stat.allTimeUploadLength = btRuntime_->getUploadLengthAtStartup()+ + stat.sessionUploadLength; } #endif // ENABLE_BITTORRENT - if(segmentMan_) { - stat.setDownloadSpeed - (stat.getDownloadSpeed()+segmentMan_->calculateDownloadSpeed()); - stat.setSessionDownloadLength - (stat.getSessionDownloadLength()+ - segmentMan_->calculateSessionDownloadLength()); - } return stat; } @@ -1242,8 +1235,7 @@ void RequestGroup::increaseAndValidateFileNotFoundCount() ++fileNotFoundCount_; const int maxCount = option_->getAsInt(PREF_MAX_FILE_NOT_FOUND); if(maxCount > 0 && fileNotFoundCount_ >= maxCount && - (!segmentMan_ || - segmentMan_->calculateSessionDownloadLength() == 0)) { + downloadContext_->getNetStat().getSessionDownloadLength() == 0) { throw DOWNLOAD_FAILURE_EXCEPTION2 (fmt("Reached max-file-not-found count=%d", maxCount), error_code::MAX_FILE_NOT_FOUND); diff --git a/src/RequestGroup.h b/src/RequestGroup.h index 7e822570..321d4d90 100644 --- a/src/RequestGroup.h +++ b/src/RequestGroup.h @@ -514,6 +514,11 @@ public: requestGroupMan_ = requestGroupMan; } + RequestGroupMan* getRequestGroupMan() + { + return requestGroupMan_; + } + int getResumeFailureCount() const { return resumeFailureCount_; diff --git a/src/RequestGroupMan.cc b/src/RequestGroupMan.cc index 4664dabe..be955827 100644 --- a/src/RequestGroupMan.cc +++ b/src/RequestGroupMan.cc @@ -54,7 +54,6 @@ #include "DownloadContext.h" #include "ServerStatMan.h" #include "ServerStat.h" -#include "PeerStat.h" #include "SegmentMan.h" #include "FeedbackURISelector.h" #include "InorderURISelector.h" @@ -918,12 +917,8 @@ void RequestGroupMan::forceHalt() TransferStat RequestGroupMan::calculateStat() { - TransferStat s; - for(std::deque >::const_iterator i = - requestGroups_.begin(), eoi = requestGroups_.end(); i != eoi; ++i) { - s += (*i)->calculateStat(); - } - return s; + // TODO Currently, all time upload length is not set. + return netStat_.toTransferStat(); } SharedHandle @@ -1033,13 +1028,13 @@ void RequestGroupMan::removeStaleServerStat(time_t timeout) bool RequestGroupMan::doesOverallDownloadSpeedExceed() { return maxOverallDownloadSpeedLimit_ > 0 && - maxOverallDownloadSpeedLimit_ < calculateStat().getDownloadSpeed(); + maxOverallDownloadSpeedLimit_ < netStat_.calculateDownloadSpeed(); } bool RequestGroupMan::doesOverallUploadSpeedExceed() { return maxOverallUploadSpeedLimit_ > 0 && - maxOverallUploadSpeedLimit_ < calculateStat().getUploadSpeed(); + maxOverallUploadSpeedLimit_ < netStat_.calculateUploadSpeed(); } void RequestGroupMan::getUsedHosts diff --git a/src/RequestGroupMan.h b/src/RequestGroupMan.h index 94ccc8ae..9eee73fe 100644 --- a/src/RequestGroupMan.h +++ b/src/RequestGroupMan.h @@ -46,6 +46,7 @@ #include "DownloadResult.h" #include "TransferStat.h" #include "RequestGroup.h" +#include "NetStat.h" namespace aria2 { @@ -75,6 +76,8 @@ private: int maxOverallUploadSpeedLimit_; + NetStat netStat_; + // true if JSON-RPC/XML-RPC is enabled. bool rpc_; @@ -333,6 +336,11 @@ public: } void setUriListParser(const SharedHandle& uriListParser); + + NetStat& getNetStat() + { + return netStat_; + } }; } // namespace aria2 diff --git a/src/RpcMethodImpl.cc b/src/RpcMethodImpl.cc index 196a8324..b9be9373 100644 --- a/src/RpcMethodImpl.cc +++ b/src/RpcMethodImpl.cc @@ -796,9 +796,10 @@ void gatherPeer util::toHex((*i)->getBitfield(), (*i)->getBitfieldLength())); peerEntry->put(KEY_AM_CHOKING, (*i)->amChoking()?VLB_TRUE:VLB_FALSE); peerEntry->put(KEY_PEER_CHOKING, (*i)->peerChoking()?VLB_TRUE:VLB_FALSE); - TransferStat stat = ps->getTransferStatFor(*i); - peerEntry->put(KEY_DOWNLOAD_SPEED, util::itos(stat.getDownloadSpeed())); - peerEntry->put(KEY_UPLOAD_SPEED, util::itos(stat.getUploadSpeed())); + peerEntry->put(KEY_DOWNLOAD_SPEED, + util::itos((*i)->calculateDownloadSpeed())); + peerEntry->put(KEY_UPLOAD_SPEED, + util::itos((*i)->calculateUploadSpeed())); peerEntry->put(KEY_SEEDER, (*i)->isSeeder()?VLB_TRUE:VLB_FALSE); peers->append(peerEntry); } diff --git a/src/SegmentMan.cc b/src/SegmentMan.cc index 87187f8e..840199d8 100644 --- a/src/SegmentMan.cc +++ b/src/SegmentMan.cc @@ -237,7 +237,7 @@ SharedHandle SegmentMan::getCleanSegmentIfOwnerIsIdle } cuid_t owner = segmentEntry->cuid; SharedHandle ps = getPeerStat(owner); - if(!ps || ps->getStatus() == PeerStat::IDLE) { + if(!ps || ps->getStatus() == NetStat::IDLE) { cancelSegment(owner); return getSegmentWithIndex(cuid, index); } else { @@ -395,36 +395,6 @@ void SegmentMan::updateFastestPeerStat(const SharedHandle& peerStat) } } -int SegmentMan::calculateDownloadSpeed() -{ - int speed = 0; - for(std::vector >::const_iterator i = - peerStats_.begin(), eoi = peerStats_.end(); i != eoi; ++i) { - // PeerStat which is IDLE but its last download speed calculation - // interval is not over must be added to the result. - if((*i)->getStatus() == PeerStat::ACTIVE || (*i)->affectsOverallSpeed()) { - speed += (*i)->calculateDownloadSpeed(); - } - } - return speed; -} - -namespace { -class PeerStatDownloadLengthOperator { -public: - int64_t operator()(int64_t total, const SharedHandle& ps) - { - return ps->getSessionDownloadLength()+total; - } -}; -} // namespace - -int64_t SegmentMan::calculateSessionDownloadLength() const -{ - return std::accumulate(fastestPeerStats_.begin(), fastestPeerStats_.end(), - 0LL, PeerStatDownloadLengthOperator()); -} - size_t SegmentMan::countFreePieceFrom(size_t index) const { size_t numPieces = downloadContext_->getNumPieces(); diff --git a/src/SegmentMan.h b/src/SegmentMan.h index 7c06214c..e89a5942 100644 --- a/src/SegmentMan.h +++ b/src/SegmentMan.h @@ -220,16 +220,6 @@ public: return fastestPeerStats_; } - /** - * Returns current download speed in bytes per sec. - */ - int calculateDownloadSpeed(); - - /** - * Returns the downloaded bytes in this session. - */ - int64_t calculateSessionDownloadLength() const; - size_t countFreePieceFrom(size_t index) const; // Excludes segments that fileEntry covers from segment selection. diff --git a/src/ShareRatioSeedCriteria.cc b/src/ShareRatioSeedCriteria.cc index 8c7105f7..6fb81dbb 100644 --- a/src/ShareRatioSeedCriteria.cc +++ b/src/ShareRatioSeedCriteria.cc @@ -34,7 +34,7 @@ /* copyright --> */ #include "ShareRatioSeedCriteria.h" #include "DownloadContext.h" -#include "PeerStorage.h" +#include "BtRuntime.h" #include "PieceStorage.h" namespace aria2 { @@ -55,15 +55,16 @@ bool ShareRatioSeedCriteria::evaluate() if(completedLength == 0) { return true; } - TransferStat stat = peerStorage_->calculateStat(); - return ratio_ <= 1.0*stat.getAllTimeUploadLength()/completedLength; + int64_t uploadLength = btRuntime_->getUploadLengthAtStartup()+ + downloadContext_->getNetStat().getSessionUploadLength(); + return ratio_ <= 1.0*uploadLength/completedLength; } -void ShareRatioSeedCriteria::setPeerStorage -(const SharedHandle& peerStorage) +void ShareRatioSeedCriteria::setBtRuntime +(const SharedHandle& btRuntime) { - peerStorage_ = peerStorage; + btRuntime_ = btRuntime; } void ShareRatioSeedCriteria::setPieceStorage diff --git a/src/ShareRatioSeedCriteria.h b/src/ShareRatioSeedCriteria.h index c4e2def6..9084d56d 100644 --- a/src/ShareRatioSeedCriteria.h +++ b/src/ShareRatioSeedCriteria.h @@ -40,14 +40,14 @@ namespace aria2 { class DownloadContext; -class PeerStorage; +class BtRuntime; class PieceStorage; class ShareRatioSeedCriteria : public SeedCriteria { private: double ratio_; SharedHandle downloadContext_; - SharedHandle peerStorage_; + SharedHandle btRuntime_; SharedHandle pieceStorage_; public: ShareRatioSeedCriteria @@ -67,7 +67,7 @@ public: return ratio_; } - void setPeerStorage(const SharedHandle& peerStorage); + void setBtRuntime(const SharedHandle& btRuntime); void setPieceStorage(const SharedHandle& pieceStorage); }; diff --git a/test/DefaultBtAnnounceTest.cc b/test/DefaultBtAnnounceTest.cc index 4390c541..6a5e49d3 100644 --- a/test/DefaultBtAnnounceTest.cc +++ b/test/DefaultBtAnnounceTest.cc @@ -60,6 +60,8 @@ public: SharedHandle torrentAttrs(new TorrentAttribute()); torrentAttrs->infoHash = std::string(vbegin(infoHash), vend(infoHash)); dctx_->setAttribute(CTX_ATTR_BT, torrentAttrs); + dctx_->getNetStat().updateDownloadLength(pieceLength*5); + dctx_->getNetStat().updateUploadLength(pieceLength*6); bittorrent::setStaticPeerId(peerId); pieceStorage_.reset(new MockPieceStorage()); @@ -67,11 +69,6 @@ public: pieceStorage_->setCompletedLength(pieceLength*10); peerStorage_.reset(new MockPeerStorage()); - TransferStat stat; - stat.setSessionDownloadLength(pieceLength*5); - stat.setSessionUploadLength(pieceLength*6); - peerStorage_->setStat(stat); - btRuntime_.reset(new BtRuntime()); } diff --git a/test/DefaultBtMessageDispatcherTest.cc b/test/DefaultBtMessageDispatcherTest.cc index a5f2a773..24065994 100644 --- a/test/DefaultBtMessageDispatcherTest.cc +++ b/test/DefaultBtMessageDispatcherTest.cc @@ -180,10 +180,6 @@ void DefaultBtMessageDispatcherTest::testAddMessage() { } void DefaultBtMessageDispatcherTest::testSendMessages() { - TransferStat stat; - stat.setUploadSpeed(0); - peerStorage->setStat(stat); - SharedHandle msg1(new MockBtMessage2()); msg1->setSendingInProgress(false); msg1->setUploading(false); @@ -199,10 +195,6 @@ void DefaultBtMessageDispatcherTest::testSendMessages() { } void DefaultBtMessageDispatcherTest::testSendMessages_underUploadLimit() { - TransferStat stat; - stat.setUploadSpeed(0); - peerStorage->setStat(stat); - SharedHandle msg1(new MockBtMessage2()); msg1->setSendingInProgress(false); msg1->setUploading(true); diff --git a/test/DefaultBtProgressInfoFileTest.cc b/test/DefaultBtProgressInfoFileTest.cc index fa34e294..48786c3c 100644 --- a/test/DefaultBtProgressInfoFileTest.cc +++ b/test/DefaultBtProgressInfoFileTest.cc @@ -211,12 +211,11 @@ void DefaultBtProgressInfoFileTest::testSave() initializeMembers(1024, 81920); dctx_->setBasePath(A2_TEST_OUT_DIR"/save-temp"); + dctx_->getNetStat().updateUploadLength(768); + btRuntime_->setUploadLengthAtStartup(256); bitfield_->setAllBit(); bitfield_->unsetBit(79); pieceStorage_->setCompletedLength(80896); - TransferStat stat; - stat.setAllTimeUploadLength(1024); - peerStorage_->setStat(stat); SharedHandle p1(new Piece(1, 1024)); SharedHandle p2(new Piece(2, 512)); diff --git a/test/MockPeerStorage.h b/test/MockPeerStorage.h index bcc4eb54..9060346e 100644 --- a/test/MockPeerStorage.h +++ b/test/MockPeerStorage.h @@ -11,7 +11,6 @@ namespace aria2 { class MockPeerStorage : public PeerStorage { private: - TransferStat stat; std::deque > peers; std::deque > droppedPeers; std::vector > activePeers; @@ -63,14 +62,6 @@ public: peers.insert(peers.end(), activePeers.begin(), activePeers.end()); } - virtual TransferStat calculateStat() { - return stat; - } - - void setStat(const TransferStat& stat) { - this->stat = stat; - } - virtual bool isBadPeer(const std::string& ipaddr) { return false; @@ -94,13 +85,6 @@ public: ++numChokeExecuted_; } - virtual void updateTransferStatFor(const SharedHandle& peer) {} - - virtual TransferStat getTransferStatFor(const SharedHandle& peer) - { - return TransferStat(); - } - int getNumChokeExecuted() const { return numChokeExecuted_; diff --git a/test/ShareRatioSeedCriteriaTest.cc b/test/ShareRatioSeedCriteriaTest.cc index 0230d354..03d1712b 100644 --- a/test/ShareRatioSeedCriteriaTest.cc +++ b/test/ShareRatioSeedCriteriaTest.cc @@ -3,7 +3,7 @@ #include #include "DownloadContext.h" -#include "MockPeerStorage.h" +#include "BtRuntime.h" #include "MockPieceStorage.h" #include "FileEntry.h" @@ -24,16 +24,14 @@ CPPUNIT_TEST_SUITE_REGISTRATION(ShareRatioSeedCriteriaTest); void ShareRatioSeedCriteriaTest::testEvaluate() { SharedHandle dctx(new DownloadContext(1024*1024, 1000000)); - SharedHandle peerStorage(new MockPeerStorage()); - TransferStat stat; - stat.setAllTimeUploadLength(1000000); - peerStorage->setStat(stat); + SharedHandle btRuntime(new BtRuntime()); + btRuntime->setUploadLengthAtStartup(1000000); SharedHandle pieceStorage(new MockPieceStorage()); pieceStorage->setCompletedLength(1000000); ShareRatioSeedCriteria cri(1.0, dctx); - cri.setPeerStorage(peerStorage); + cri.setBtRuntime(btRuntime); cri.setPieceStorage(pieceStorage); CPPUNIT_ASSERT(cri.evaluate());