From 07bb779eb05478864efb8b5f4d264945bb3f30ac Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 19 Dec 2012 23:03:48 +0900 Subject: [PATCH] Faster deletion of RequestGroup and DownloadResult lists --- src/ConsoleStatCalc.cc | 25 +-- src/HaveEraseCommand.cc | 11 +- src/IndexedList.h | 293 +++++++++++++++++++++++++ src/RequestGroupMan.cc | 417 +++++++++++------------------------- src/RequestGroupMan.h | 48 ++--- src/RpcMethodImpl.cc | 38 ++-- src/RpcMethodImpl.h | 22 +- src/SessionSerializer.cc | 34 +-- test/IndexedListTest.cc | 240 +++++++++++++++++++++ test/Makefile.am | 3 +- test/RequestGroupManTest.cc | 32 +-- test/RpcMethodTest.cc | 38 ++-- test/TestUtil.cc | 26 +++ test/TestUtil.h | 9 + 14 files changed, 813 insertions(+), 423 deletions(-) create mode 100644 src/IndexedList.h create mode 100644 test/IndexedListTest.cc diff --git a/src/ConsoleStatCalc.cc b/src/ConsoleStatCalc.cc index 8893fea6..ad820a8d 100644 --- a/src/ConsoleStatCalc.cc +++ b/src/ConsoleStatCalc.cc @@ -138,16 +138,16 @@ void printProgressCompact(std::ostream& o, const DownloadEngine* e, o << "[DL:" << sizeFormatter(dl) << "B UL:" << sizeFormatter(ul) << "B]"; } - const std::deque >& groups = + const RequestGroupList& groups = e->getRequestGroupMan()->getRequestGroups(); size_t cnt = 0; const size_t MAX_ITEM = 5; - for(std::deque >::const_iterator - i = groups.begin(), eoi = groups.end(); i != eoi && cnt < MAX_ITEM; - ++i, ++cnt) { - TransferStat stat = (*i)->calculateStat(); - o << "[#" << GroupId::toAbbrevHex((*i)->getGID()) << " "; - printSizeProgress(o, *i, stat, sizeFormatter); + for(RequestGroupList::SeqType::const_iterator i = groups.begin(), + eoi = groups.end(); i != eoi && cnt < MAX_ITEM; ++i, ++cnt) { + const SharedHandle& rg = (*i).second; + TransferStat stat = rg->calculateStat(); + o << "[#" << GroupId::toAbbrevHex(rg->getGID()) << " "; + printSizeProgress(o, rg, stat, sizeFormatter); o << "]"; } if(cnt < groups.size()) { @@ -210,8 +210,9 @@ public: const SizeFormatter& sizeFormatter): cols_(cols), e_(e), sizeFormatter_(sizeFormatter) {} - void operator()(const SharedHandle& rg) + void operator()(const RequestGroupList::SeqType::value_type& val) { + const SharedHandle& rg = val.second; const char SEP_CHAR = '-'; std::stringstream o; printProgress(o, rg, e_, sizeFormatter_); @@ -229,8 +230,7 @@ public: namespace { void printProgressSummary -(const std::deque >& groups, size_t cols, - const DownloadEngine* e, +(const RequestGroupList& groups, size_t cols, const DownloadEngine* e, const SizeFormatter& sizeFormatter) { const char SEP_CHAR = '='; @@ -318,8 +318,9 @@ ConsoleStatCalc::calculateStat(const DownloadEngine* e) } size_t numGroup = e->getRequestGroupMan()->countRequestGroup(); if(numGroup == 1) { - printProgress(o, e->getRequestGroupMan()->getRequestGroup(0), e, - sizeFormatter); + const SharedHandle& rg = + (*e->getRequestGroupMan()->getRequestGroups().begin()).second; + printProgress(o, rg, e, sizeFormatter); } else if(numGroup > 1) { // For more than 2 RequestGroups, use compact readout form printProgressCompact(o, e, sizeFormatter); diff --git a/src/HaveEraseCommand.cc b/src/HaveEraseCommand.cc index 5eff358d..29163860 100644 --- a/src/HaveEraseCommand.cc +++ b/src/HaveEraseCommand.cc @@ -55,12 +55,11 @@ void HaveEraseCommand::preProcess() void HaveEraseCommand::process() { - size_t numLoop = - getDownloadEngine()->getRequestGroupMan()->countRequestGroup(); - for(size_t i = 0; i < numLoop; ++i) { - SharedHandle ps = - getDownloadEngine()->getRequestGroupMan()->getRequestGroup(i)-> - getPieceStorage(); + const RequestGroupList& groups = + getDownloadEngine()->getRequestGroupMan()->getRequestGroups(); + for(RequestGroupList::SeqType::const_iterator i = groups.begin(), + eoi = groups.end(); i != eoi; ++i) { + const SharedHandle& ps = (*i).second->getPieceStorage(); if(ps) { ps->removeAdvertisedPiece(5); } diff --git a/src/IndexedList.h b/src/IndexedList.h new file mode 100644 index 00000000..fac92d28 --- /dev/null +++ b/src/IndexedList.h @@ -0,0 +1,293 @@ +/* */ +#ifndef D_INDEXED_LIST_H +#define D_INDEXED_LIST_H + +#include "common.h" + +#include +#include + +namespace aria2 { + +enum A2_HOW { + A2_POS_SET, + A2_POS_CUR, + A2_POS_END +}; + +// List with O(logN) look-up using std::map as an index. +template +class IndexedList { +public: + IndexedList() {} + ~IndexedList() {} + + typedef std::list > SeqType; + typedef std::map IndexType; + + // Inserts (|key|, |value|) to the end of the list. If the same key + // has been already added, this function fails. This function + // returns true if it succeeds. Complexity: O(logN) + bool push_back(KeyType key, ValuePtrType value) + { + typename IndexType::iterator i = index_.lower_bound(key); + if(i == index_.end() || (*i).first != key) { + seq_.push_back(std::make_pair(key, value)); + typename SeqType::iterator j = seq_.end(); + --j; + index_.insert(i, std::make_pair(key, j)); + return true; + } else { + return false; + } + } + + // Inserts (|key|, |value|) to the front of the list. If the same + // key has been already added, this function fails. This function + // returns true if it succeeds. Complexity: O(logN) + bool push_front(KeyType key, ValuePtrType value) + { + typename IndexType::iterator i = index_.lower_bound(key); + if(i == index_.end() || (*i).first != key) { + seq_.push_front(std::make_pair(key, value)); + typename SeqType::iterator j = seq_.begin(); + index_.insert(i, std::make_pair(key, j)); + return true; + } else { + return false; + } + } + + // Inserts (|key|, |value|) to the position |dest|. If the same key + // has been already added, this function fails. This function + // returns the iterator to the newly added element if it is + // succeeds, or end(). Complexity: O(N) + typename SeqType::iterator insert(size_t dest, KeyType key, + ValuePtrType value) + { + if(dest > size()) { + return seq_.end(); + } + typename IndexType::iterator i = index_.lower_bound(key); + if(i == index_.end() || (*i).first != key) { + typename SeqType::iterator j = seq_.begin(); + std::advance(j, dest); + j = seq_.insert(j, std::make_pair(key, value)); + index_.insert(i, std::make_pair(key, j)); + return j; + } else { + return seq_.end(); + } + } + + // Inserts (|key|, |value|) to the position |dest|. If the same key + // has been already added, this function fails. This function + // returns the iterator to the newly added element if it is + // succeeds, or end(). Complexity: O(logN) + typename SeqType::iterator insert(typename SeqType::iterator dest, + KeyType key, + ValuePtrType value) + { + typename IndexType::iterator i = index_.lower_bound(key); + if(i == index_.end() || (*i).first != key) { + dest = seq_.insert(dest, std::make_pair(key, value)); + index_.insert(i, std::make_pair(key, dest)); + return dest; + } else { + return seq_.end(); + } + } + + // Removes |key| from the list. If the element is not found, this + // function fails. This function returns true if it + // succeeds. Complexity: O(logN) + bool erase(KeyType key) + { + typename IndexType::iterator i = index_.find(key); + if(i == index_.end()) { + return false; + } + seq_.erase((*i).second); + index_.erase(i); + return true; + } + + // Removes element at the front of the list. If the list is empty, + // this function fails. This function returns true if it + // succeeds. Complexity: O(logN) + bool pop_front() + { + if(seq_.empty()) { + return false; + } + KeyType key = seq_.front().first; + index_.erase(key); + seq_.pop_front(); + return true; + } + + // Moves element with |key| to the specified position. If |how| is + // A2_POS_CUR, the element is moved to the position |offset| + // relative to the current position. If |how| is A2_POS_SET, the + // element is moved to the position |offset|. If |how| is + // A2_POS_END, the element is moved to the position |offset| + // relative to the end of the list. This function returns the + // position the elment is moved to if it succeeds, or -1 if no + // element with |key| is found or |how| is invalid. Complexity: + // O(N) + ssize_t move(KeyType key, ssize_t offset, A2_HOW how) + { + typename IndexType::iterator idxent = index_.find(key); + if(idxent == index_.end()) { + return -1; + } + ssize_t dest; + typename SeqType::iterator x = (*idxent).second; + typename SeqType::iterator d; + if(how == A2_POS_CUR) { + // Because aria2.changePosition() RPC method must return the + // absolute position after move, we have to calculate absolute + // position here. + if(offset > 0) { + d = x; + for(; offset >= 0 && d != seq_.end(); --offset, ++d); + dest = std::distance(seq_.begin(), d)-1; + } else { + d = x; + for(; offset < 0 && d != seq_.begin(); ++offset, --d); + dest = std::distance(seq_.begin(), d); + } + } else { + ssize_t size = index_.size(); + if(how == A2_POS_END) { + dest = std::min(size-1, size-1+offset); + } else if(how == A2_POS_SET) { + dest = std::min(size-1, offset); + } else { + return -1; + } + dest = std::max(dest, static_cast(0)); + d = seq_.begin(); + for(ssize_t i = 0; i < dest; ++i, ++d) { + if(d == x) { + ++d; + } + } + } + seq_.splice(d, seq_, x); + return dest; + } + + // Returns the value associated by |key|. If it is not found, + // returns ValuePtrType(). Complexity: O(logN) + ValuePtrType get(KeyType key) const + { + typename IndexType::const_iterator idxent = index_.find(key); + if(idxent == index_.end()) { + return ValuePtrType(); + } else { + return (*(*idxent).second).second; + } + } + + // Returns the iterator to the element associated by |key|. If it is + // not found, end() is returned. Complexity: O(logN) + typename SeqType::iterator find(KeyType key) + { + typename IndexType::iterator idxent = index_.find(key); + if(idxent == index_.end()) { + return seq_.end(); + } else { + return (*idxent).second; + } + } + + // Returns the iterator to the element associated by |key|. If it is + // not found, end() is returned. Complexity: O(logN) + typename SeqType::const_iterator find(KeyType key) const + { + typename IndexType::const_iterator idxent = index_.find(key); + if(idxent == index_.end()) { + return seq_.end(); + } else { + return (*idxent).second; + } + } + + size_t size() const + { + return index_.size(); + } + + size_t empty() const + { + return index_.empty(); + } + + typename SeqType::iterator begin() + { + return seq_.begin(); + } + + typename SeqType::iterator end() + { + return seq_.end(); + } + + typename SeqType::const_iterator begin() const + { + return seq_.begin(); + } + + typename SeqType::const_iterator end() const + { + return seq_.end(); + } + + // Removes all elements from the list. + void clear() + { + index_.clear(); + seq_.clear(); + } +private: + SeqType seq_; + IndexType index_; +}; + +} // namespace aria2 + +#endif // D_INDEXED_LIST_H diff --git a/src/RequestGroupMan.cc b/src/RequestGroupMan.cc index 046b43ad..d84f4973 100644 --- a/src/RequestGroupMan.cc +++ b/src/RequestGroupMan.cc @@ -85,12 +85,22 @@ namespace aria2 { +namespace { +template +void appendReservedGroup(RequestGroupList& list, + InputIterator first, InputIterator last) +{ + for(; first != last; ++first) { + list.push_back((*first)->getGID(), *first); + } +} +} // namespace + RequestGroupMan::RequestGroupMan (const std::vector >& requestGroups, int maxSimultaneousDownloads, const Option* option) - : reservedGroups_(requestGroups.begin(), requestGroups.end()), - maxSimultaneousDownloads_(maxSimultaneousDownloads), + : maxSimultaneousDownloads_(maxSimultaneousDownloads), option_(option), serverStatMan_(new ServerStatMan()), maxOverallDownloadSpeedLimit_ @@ -104,7 +114,8 @@ RequestGroupMan::RequestGroupMan maxDownloadResult_(option->getAsInt(PREF_MAX_DOWNLOAD_RESULT)), wrDiskCache_(0) { - addRequestGroupIndex(requestGroups); + appendReservedGroup(reservedGroups_, + requestGroups.begin(), requestGroups.end()); } RequestGroupMan::~RequestGroupMan() @@ -123,82 +134,42 @@ bool RequestGroupMan::downloadFinished() void RequestGroupMan::addRequestGroup (const SharedHandle& group) { - requestGroups_.push_back(group); + requestGroups_.push_back(group->getGID(), group); } void RequestGroupMan::addReservedGroup (const std::vector >& groups) { requestQueueCheck(); - addRequestGroupIndex(groups); - reservedGroups_.insert(reservedGroups_.end(), groups.begin(), groups.end()); + appendReservedGroup(reservedGroups_, groups.begin(), groups.end()); } void RequestGroupMan::addReservedGroup (const SharedHandle& group) { requestQueueCheck(); - addRequestGroupIndex(group); - reservedGroups_.push_back(group); + reservedGroups_.push_back(group->getGID(), group); } void RequestGroupMan::insertReservedGroup (size_t pos, const std::vector >& groups) { requestQueueCheck(); - addRequestGroupIndex(groups); - reservedGroups_.insert - (reservedGroups_.begin()+std::min(reservedGroups_.size(), pos), - groups.begin(), groups.end()); + pos = std::min(reservedGroups_.size(), pos); + RequestGroupList::SeqType::iterator dest = reservedGroups_.begin(); + std::advance(dest, pos); + for(std::vector >::const_iterator i = + groups.begin(), eoi = groups.end(); i != eoi; ++i, ++dest) { + dest = reservedGroups_.insert(dest, (*i)->getGID(), *i); + } } void RequestGroupMan::insertReservedGroup (size_t pos, const SharedHandle& group) { requestQueueCheck(); - addRequestGroupIndex(group); - reservedGroups_.insert - (reservedGroups_.begin()+std::min(reservedGroups_.size(), pos), group); -} - -void RequestGroupMan::addRequestGroupIndex -(const SharedHandle& group) -{ - assert(groupIndex_.count(group->getGID()) == 0); - groupIndex_[group->getGID()] = group; -} - -void RequestGroupMan::addRequestGroupIndex -(const std::vector >& groups) -{ - for(std::vector >::const_iterator i = - groups.begin(); i != groups.end(); ++i) { - addRequestGroupIndex(*i); - } -} - -namespace { -void removeRequestGroupIndex -(std::map >& groupIndex, - const SharedHandle& group) -{ - assert(groupIndex.count(group->getGID()) == 1); - groupIndex.erase(group->getGID()); -} -} // namespace - -void RequestGroupMan::addDownloadResultIndex -(const SharedHandle& dr) -{ - assert(drIndex_.count(dr->gid->getNumericId()) == 0); - drIndex_[dr->gid->getNumericId()] = dr; -} - -void RequestGroupMan::removeDownloadResultIndex -(const SharedHandle& dr) -{ - assert(drIndex_.count(dr->gid->getNumericId()) == 1); - drIndex_.erase(dr->gid->getNumericId()); + pos = std::min(reservedGroups_.size(), pos); + reservedGroups_.insert(pos, group->getGID(), group); } size_t RequestGroupMan::countRequestGroup() const @@ -206,123 +177,30 @@ size_t RequestGroupMan::countRequestGroup() const return requestGroups_.size(); } -SharedHandle RequestGroupMan::getRequestGroup(size_t index) const -{ - if(index < requestGroups_.size()) { - return requestGroups_[index]; - } else { - return SharedHandle(); - } -} - -namespace { -template -Iterator findByGID(Iterator first, Iterator last, a2_gid_t gid) -{ - for(; first != last; ++first) { - if((*first)->getGID() == gid) { - return first; - } - } - return first; -} -} // namespace - -SharedHandle -RequestGroupMan::findRequestGroup(a2_gid_t gid) const -{ - SharedHandle res = findGroup(gid); - if(res) { - if(res->getState() == RequestGroup::STATE_ACTIVE) { - return res; - } else { - return SharedHandle(); - } - } else { - return res; - } -} - -SharedHandle -RequestGroupMan::findReservedGroup(a2_gid_t gid) const -{ - SharedHandle res = findGroup(gid); - if(res) { - if(res->getState() == RequestGroup::STATE_WAITING) { - return res; - } else { - return SharedHandle(); - } - } else { - return res; - } -} - SharedHandle RequestGroupMan::findGroup(a2_gid_t gid) const { - std::map >::const_iterator i = - groupIndex_.find(gid); - if(i != groupIndex_.end()) { - return (*i).second; - } else { - return SharedHandle(); + SharedHandle rg = requestGroups_.get(gid); + if(!rg) { + rg = reservedGroups_.get(gid); } + return rg; } size_t RequestGroupMan::changeReservedGroupPosition -(a2_gid_t gid, int pos, HOW how) +(a2_gid_t gid, int pos, A2_HOW how) { - std::deque >::iterator i = - findByGID(reservedGroups_.begin(), reservedGroups_.end(), gid); - if(i == reservedGroups_.end()) { + ssize_t dest = reservedGroups_.move(gid, pos, how); + if(dest == -1) { throw DL_ABORT_EX(fmt("GID#%s not found in the waiting queue.", GroupId::toHex(gid).c_str())); - } - SharedHandle rg = *i; - const size_t maxPos = reservedGroups_.size()-1; - if(how == POS_SET) { - if(pos < 0) { - pos = 0; - } else if(pos > 0) { - pos = std::min(maxPos, (size_t)pos); - } - } else if(how == POS_CUR) { - size_t abspos = std::distance(reservedGroups_.begin(), i); - if(pos < 0) { - int dist = -std::distance(reservedGroups_.begin(), i); - pos = abspos+std::max(pos, dist); - } else if(pos > 0) { - int dist = std::distance(i, reservedGroups_.end())-1; - pos = abspos+std::min(pos, dist); - } else { - pos = abspos; - } - } else if(how == POS_END) { - if(pos >= 0) { - pos = maxPos; - } else { - pos = maxPos-std::min(maxPos, (size_t)-pos); - } - } - if(std::distance(reservedGroups_.begin(), i) < pos) { - std::rotate(i, i+1, reservedGroups_.begin()+pos+1); } else { - std::rotate(reservedGroups_.begin()+pos, i, i+1); + return dest; } - return pos; } bool RequestGroupMan::removeReservedGroup(a2_gid_t gid) { - std::deque >::iterator i = - findByGID(reservedGroups_.begin(), reservedGroups_.end(), gid); - if(i == reservedGroups_.end()) { - return false; - } else { - removeRequestGroupIndex(groupIndex_, *i); - reservedGroups_.erase(i); - return true; - } + return reservedGroups_.erase(gid); } namespace { @@ -368,9 +246,7 @@ namespace { class ProcessStoppedRequestGroup { private: DownloadEngine* e_; - std::deque >& downloadResults_; - std::deque >& reservedGroups_; - std::map >& groupIndex_; + RequestGroupList& reservedGroups_; void saveSignature(const SharedHandle& group) { @@ -390,17 +266,14 @@ private: public: ProcessStoppedRequestGroup (DownloadEngine* e, - std::deque >& downloadResults, - std::deque >& reservedGroups, - std::map >& groupIndex) + RequestGroupList& reservedGroups) : e_(e), - downloadResults_(downloadResults), - reservedGroups_(reservedGroups), - groupIndex_(groupIndex) + reservedGroups_(reservedGroups) {} - void operator()(const SharedHandle& group) + void operator()(const RequestGroupList::SeqType::value_type& val) { + const SharedHandle& group = val.second; if(group->getNumCommand() == 0) { const SharedHandle& dctx = group->getDownloadContext(); // DownloadContext::resetDownloadStopTime() is only called when @@ -474,7 +347,7 @@ public: } if(group->isPauseRequested()) { group->setState(RequestGroup::STATE_WAITING); - reservedGroups_.push_front(group); + reservedGroups_.push_front(group->getGID(), group); group->releaseRuntimeResource(e_); group->setForceHaltRequested(false); util::executeHookByOptName(group, e_->getOption(), @@ -487,7 +360,6 @@ public: e_->getRequestGroupMan()->addDownloadResult(dr); executeStopHook(group, e_->getOption(), dr->result); group->releaseRuntimeResource(e_); - removeRequestGroupIndex(groupIndex_, group); } } } @@ -502,8 +374,9 @@ public: CollectServerStat(RequestGroupMan* requestGroupMan): requestGroupMan_(requestGroupMan) {} - void operator()(const SharedHandle& group) + void operator()(const RequestGroupList::SeqType::value_type& val) { + const SharedHandle& group = val.second; if(group->getNumCommand() == 0) { // Collect statistics during download in PeerStats and update/register // ServerStatMan @@ -538,15 +411,6 @@ public: }; } // namespace -namespace { -class FindStoppedRequestGroup { -public: - bool operator()(const SharedHandle& group) { - return group->getNumCommand() == 0; - } -}; -} // namespace - void RequestGroupMan::updateServerStat() { std::for_each(requestGroups_.begin(), requestGroups_.end(), @@ -560,16 +424,18 @@ void RequestGroupMan::removeStoppedGroup(DownloadEngine* e) updateServerStat(); std::for_each(requestGroups_.begin(), requestGroups_.end(), - ProcessStoppedRequestGroup - (e, downloadResults_, reservedGroups_, groupIndex_)); - std::deque >::iterator i = - std::remove_if(requestGroups_.begin(), - requestGroups_.end(), - FindStoppedRequestGroup()); - if(i != requestGroups_.end()) { - requestGroups_.erase(i, requestGroups_.end()); + ProcessStoppedRequestGroup(e, reservedGroups_)); + for(RequestGroupList::SeqType::iterator i = requestGroups_.begin(), + eoi = requestGroups_.end(); i != eoi;) { + const SharedHandle& rg = (*i).second; + if(rg->getNumCommand() == 0) { + ++i; + requestGroups_.erase(rg->getGID()); + // rg is invalid here. + } else { + ++i; + } } - size_t numRemoved = numPrev-requestGroups_.size(); if(numRemoved > 0) { A2_LOG_DEBUG(fmt("%lu RequestGroup(s) deleted.", @@ -610,34 +476,39 @@ void RequestGroupMan::fillRequestGroupFromReserver(DownloadEngine* e) if(static_cast(maxSimultaneousDownloads_) <= requestGroups_.size()) { return; } - std::vector > temp; int count = 0; int num = maxSimultaneousDownloads_-requestGroups_.size(); - while(count < num && (uriListParser_ || !reservedGroups_.empty())) { - if(uriListParser_ && reservedGroups_.empty()) { + // In the following loop, the download which is not ready to start + // is kept in reservedGroups_. We use iterator to see if we + // evaluated them all. So don't use empty() for this. Compare to + // reservedGroups_.end() instead. + RequestGroupList::SeqType::iterator resitr = reservedGroups_.begin(); + while(count < num && (uriListParser_ || resitr != reservedGroups_.end())) { + if(uriListParser_ && resitr == reservedGroups_.end()) { std::vector > groups; bool ok = createRequestGroupFromUriListParser(groups, option_, uriListParser_.get()); if(ok) { - addRequestGroupIndex(groups); - reservedGroups_.insert(reservedGroups_.end(), groups.begin(), - groups.end()); + appendReservedGroup(reservedGroups_, groups.begin(), groups.end()); + resitr = reservedGroups_.end(); + std::advance(resitr, -static_cast(groups.size())); } else { uriListParser_.reset(); - if(reservedGroups_.empty()) { + if(resitr == reservedGroups_.end()) { break; } } } - SharedHandle groupToAdd = reservedGroups_.front(); - reservedGroups_.pop_front(); + SharedHandle groupToAdd = (*resitr).second; std::vector commands; try { if((rpc_ && groupToAdd->isPauseRequested()) || !groupToAdd->isDependencyResolved()) { - temp.push_back(groupToAdd); + ++resitr; continue; } + ++resitr; + reservedGroups_.erase(groupToAdd->getGID()); // Drop pieceStorage here because paused download holds its // reference. groupToAdd->dropPieceStorage(); @@ -648,7 +519,7 @@ void RequestGroupMan::fillRequestGroupFromReserver(DownloadEngine* e) requestQueueCheck(); } groupToAdd->setState(RequestGroup::STATE_ACTIVE); - requestGroups_.push_back(groupToAdd); + requestGroups_.push_back(groupToAdd->getGID(), groupToAdd); ++count; e->addCommand(commands); commands.clear(); @@ -662,16 +533,13 @@ void RequestGroupMan::fillRequestGroupFromReserver(DownloadEngine* e) // We add groupToAdd to e in order to it is processed in // removeStoppedGroup(). groupToAdd->setState(RequestGroup::STATE_ACTIVE); - requestGroups_.push_back(groupToAdd); + requestGroups_.push_back(groupToAdd->getGID(), groupToAdd); requestQueueCheck(); } util::executeHookByOptName(groupToAdd, e->getOption(), PREF_ON_DOWNLOAD_START); notifyDownloadEvent(Notifier::ON_DOWNLOAD_START, groupToAdd); } - if(!temp.empty()) { - reservedGroups_.insert(reservedGroups_.begin(), temp.begin(), temp.end()); - } if(count > 0) { e->setNoWait(true); e->setRefreshInterval(0); @@ -681,14 +549,15 @@ void RequestGroupMan::fillRequestGroupFromReserver(DownloadEngine* e) void RequestGroupMan::save() { - for(std::deque >::const_iterator itr = - requestGroups_.begin(), eoi = requestGroups_.end(); itr != eoi; ++itr) { - if((*itr)->allDownloadFinished() && - !(*itr)->getDownloadContext()->isChecksumVerificationNeeded()) { - (*itr)->removeControlFile(); + for(RequestGroupList::SeqType::iterator itr = requestGroups_.begin(), + eoi = requestGroups_.end(); itr != eoi; ++itr) { + const SharedHandle& rg = (*itr).second; + if(rg->allDownloadFinished() && + !rg->getDownloadContext()->isChecksumVerificationNeeded()) { + rg->removeControlFile(); } else { try { - (*itr)->saveControlFile(); + rg->saveControlFile(); } catch(RecoverableException& e) { A2_LOG_ERROR_EX(EX_EXCEPTION_CAUGHT, e); } @@ -698,9 +567,9 @@ void RequestGroupMan::save() void RequestGroupMan::closeFile() { - for(std::deque >::const_iterator itr = - requestGroups_.begin(), eoi = requestGroups_.end(); itr != eoi; ++itr) { - (*itr)->closeFile(); + for(RequestGroupList::SeqType::iterator itr = requestGroups_.begin(), + eoi = requestGroups_.end(); itr != eoi; ++itr) { + (*itr).second->closeFile(); } } @@ -711,21 +580,22 @@ RequestGroupMan::DownloadStat RequestGroupMan::getDownloadStat() const int inprogress = 0; int removed = 0; error_code::Value lastError = removedLastErrorResult_; - for(std::deque >::const_iterator itr = - downloadResults_.begin(), eoi = downloadResults_.end(); - itr != eoi; ++itr) { - if((*itr)->belongsTo != 0) { + for(DownloadResultList::SeqType::const_iterator itr = + downloadResults_.begin(), eoi = downloadResults_.end(); itr != eoi; + ++itr) { + const SharedHandle& dr = (*itr).second; + if(dr->belongsTo != 0) { continue; } - if((*itr)->result == error_code::FINISHED) { + if(dr->result == error_code::FINISHED) { ++finished; - } else if((*itr)->result == error_code::IN_PROGRESS) { + } else if(dr->result == error_code::IN_PROGRESS) { ++inprogress; - } else if((*itr)->result == error_code::REMOVED) { + } else if(dr->result == error_code::REMOVED) { ++removed; } else { ++error; - lastError = (*itr)->result; + lastError = dr->result; } } return DownloadStat(finished, error, inprogress, removed, @@ -799,30 +669,35 @@ void RequestGroupMan::showDownloadResults(OutputFile& o, bool full) const int err = 0; int inpr = 0; int rm = 0; - for(std::deque >::const_iterator itr = - downloadResults_.begin(), eoi = downloadResults_.end(); - itr != eoi; ++itr) { - if((*itr)->belongsTo != 0) { + for(DownloadResultList::SeqType::const_iterator itr = + downloadResults_.begin(), eoi = downloadResults_.end(); itr != eoi; + ++itr) { + const SharedHandle& dr = (*itr).second; + if(dr->belongsTo != 0) { continue; } const char* status; - if((*itr)->result == error_code::FINISHED) { + switch(dr->result) { + case error_code::FINISHED: status = getStatusStr(A2_STATUS_OK, useColor); ++ok; - } else if((*itr)->result == error_code::IN_PROGRESS) { + break; + case error_code::IN_PROGRESS: status = getStatusStr(A2_STATUS_INPR, useColor); ++inpr; - } else if((*itr)->result == error_code::REMOVED) { + break; + case error_code::REMOVED: status = getStatusStr(A2_STATUS_RM, useColor); ++rm; - } else { + break; + default: status = getStatusStr(A2_STATUS_ERR, useColor); ++err; } if(full) { - formatDownloadResultFull(o, status, *itr); + formatDownloadResultFull(o, status, dr); } else { - o.write(formatDownloadResult(status, *itr).c_str()); + o.write(formatDownloadResult(status, dr).c_str()); o.write("\n"); } } @@ -944,11 +819,12 @@ bool RequestGroupMan::isSameFileBeingDownloaded(RequestGroup* requestGroup) cons return false; } std::vector files; - for(std::deque >::const_iterator itr = - requestGroups_.begin(), eoi = requestGroups_.end(); itr != eoi; ++itr) { - if((*itr).get() != requestGroup) { + for(RequestGroupList::SeqType::const_iterator itr = requestGroups_.begin(), + eoi = requestGroups_.end(); itr != eoi; ++itr) { + const SharedHandle& rg = (*itr).second; + if(rg.get() != requestGroup) { const std::vector >& entries = - (*itr)->getDownloadContext()->getFileEntries(); + rg->getDownloadContext()->getFileEntries(); std::transform(entries.begin(), entries.end(), std::back_inserter(files), mem_fun_sh(&FileEntry::getPath)); @@ -963,17 +839,17 @@ bool RequestGroupMan::isSameFileBeingDownloaded(RequestGroup* requestGroup) cons void RequestGroupMan::halt() { - for(std::deque >::const_iterator i = - requestGroups_.begin(), eoi = requestGroups_.end(); i != eoi; ++i) { - (*i)->setHaltRequested(true); + for(RequestGroupList::SeqType::const_iterator i = requestGroups_.begin(), + eoi = requestGroups_.end(); i != eoi; ++i) { + (*i).second->setHaltRequested(true); } } void RequestGroupMan::forceHalt() { - for(std::deque >::const_iterator i = - requestGroups_.begin(), eoi = requestGroups_.end(); i != eoi; ++i) { - (*i)->setForceHaltRequested(true); + for(RequestGroupList::SeqType::const_iterator i = requestGroups_.begin(), + eoi = requestGroups_.end(); i != eoi; ++i) { + (*i).second->setForceHaltRequested(true); } } @@ -986,71 +862,31 @@ TransferStat RequestGroupMan::calculateStat() SharedHandle RequestGroupMan::findDownloadResult(a2_gid_t gid) const { - std::map >::const_iterator i = - drIndex_.find(gid); - if(i != drIndex_.end()) { - return (*i).second; - } else { - return SharedHandle(); - } + return downloadResults_.get(gid); } bool RequestGroupMan::removeDownloadResult(a2_gid_t gid) { - for(std::deque >::iterator i = - downloadResults_.begin(), eoi = downloadResults_.end(); i != eoi; ++i) { - if((*i)->gid->getNumericId() == gid) { - downloadResults_.erase(i); - removeDownloadResultIndex(*i); - return true; - } - } - return false; + return downloadResults_.erase(gid); } void RequestGroupMan::addDownloadResult(const SharedHandle& dr) { - if(maxDownloadResult_ == 0) { - if(!downloadResults_.empty()) { - for(std::deque >::iterator i = - downloadResults_.begin(), eoi = downloadResults_.end(); i != eoi; - ++i) { - if((*i)->belongsTo == 0 && (*i)->result != error_code::FINISHED) { - removedLastErrorResult_ = (*i)->result; - ++removedErrorResult_; - } - } - drIndex_.clear(); - downloadResults_.clear(); - } + bool rv = downloadResults_.push_back(dr->gid->getNumericId(), dr); + assert(rv); + while(downloadResults_.size() > maxDownloadResult_){ + DownloadResultList::SeqType::iterator i = downloadResults_.begin(); + const SharedHandle& dr = (*i).second; if(dr->belongsTo == 0 && dr->result != error_code::FINISHED) { removedLastErrorResult_ = dr->result; ++removedErrorResult_; } - } else { - int curSize = downloadResults_.size(); - if(curSize >= maxDownloadResult_) { - std::deque >::iterator last = - downloadResults_.begin()+curSize-maxDownloadResult_+1; - for(std::deque >::iterator i = - downloadResults_.begin(); i != last; ++i) { - if((*i)->belongsTo == 0 && (*i)->result != error_code::FINISHED) { - removedLastErrorResult_ = (*i)->result; - ++removedErrorResult_; - } - removeDownloadResultIndex(*i); - } - downloadResults_.erase(downloadResults_.begin(), last); - } - downloadResults_.push_back(dr); - addDownloadResultIndex(dr); } } void RequestGroupMan::purgeDownloadResult() { downloadResults_.clear(); - drIndex_.clear(); } SharedHandle @@ -1112,10 +948,11 @@ void RequestGroupMan::getUsedHosts // speed. We use -download speed so that we can sort them using // operator<(). std::vector > tempHosts; - for(std::deque >::const_iterator i = - requestGroups_.begin(), eoi = requestGroups_.end(); i != eoi; ++i) { + for(RequestGroupList::SeqType::const_iterator i = requestGroups_.begin(), + eoi = requestGroups_.end(); i != eoi; ++i) { + const SharedHandle& rg = (*i).second; const FileEntry::InFlightRequestSet& inFlightReqs = - (*i)->getDownloadContext()->getFirstFileEntry()->getInFlightRequests(); + rg->getDownloadContext()->getFirstFileEntry()->getInFlightRequests(); for(FileEntry::InFlightRequestSet::iterator j = inFlightReqs.begin(), eoj = inFlightReqs.end(); j != eoj; ++j) { uri_split_result us; diff --git a/src/RequestGroupMan.h b/src/RequestGroupMan.h index e09e44dd..05fe0e59 100644 --- a/src/RequestGroupMan.h +++ b/src/RequestGroupMan.h @@ -47,6 +47,7 @@ #include "TransferStat.h" #include "RequestGroup.h" #include "NetStat.h" +#include "IndexedList.h" namespace aria2 { @@ -60,15 +61,16 @@ class OutputFile; class UriListParser; class WrDiskCache; +typedef IndexedList > RequestGroupList; +typedef IndexedList > DownloadResultList; + class RequestGroupMan { private: - std::deque > requestGroups_; - std::deque > reservedGroups_; - // GID => RequestGroup index for faster retrieval. - std::map > groupIndex_; - std::deque > downloadResults_; - // GID => DownloadResult index for faster retrieval. - std::map > drIndex_; + RequestGroupList requestGroups_; + RequestGroupList reservedGroups_; + DownloadResultList downloadResults_; int maxSimultaneousDownloads_; @@ -94,7 +96,7 @@ private: // The last error of removed DownloadResult error_code::Value removedLastErrorResult_; - int maxDownloadResult_; + size_t maxDownloadResult_; // UriListParser for deferred input. SharedHandle uriListParser_; @@ -116,9 +118,6 @@ private: void addRequestGroupIndex(const SharedHandle& group); void addRequestGroupIndex (const std::vector >& groups); - - void addDownloadResultIndex(const SharedHandle& dr); - void removeDownloadResultIndex(const SharedHandle& dr); public: RequestGroupMan(const std::vector >& requestGroups, int maxSimultaneousDownloads, @@ -156,36 +155,20 @@ public: size_t countRequestGroup() const; - SharedHandle getRequestGroup(size_t index) const; - - const std::deque >& getRequestGroups() const + const RequestGroupList& getRequestGroups() const { return requestGroups_; } - // Note: Use only for unit testing. Use findGroup() and test - // RequestGroup::getState() instead. - SharedHandle findRequestGroup(a2_gid_t gid) const; - - const std::deque >& getReservedGroups() const + const RequestGroupList& getReservedGroups() const { return reservedGroups_; } - // Note: Use only for unit testing. Use findGroup() and test - // RequestGroup::getState() instead. - SharedHandle findReservedGroup(a2_gid_t gid) const; - // Returns RequestGroup object whose gid is gid. This method returns // RequestGroup either in requestGroups_ or reservedGroups_. SharedHandle findGroup(a2_gid_t gid) const; - enum HOW { - POS_SET, - POS_CUR, - POS_END - }; - // Changes the position of download denoted by gid. If how is // POS_SET, it moves the download to a position relative to the // beginning of the queue. If how is POS_CUR, it moves the download @@ -195,7 +178,8 @@ public: // beyond the end of the queue, it moves the download to the // beginning or the end of the queue respectively. Returns the // destination position. - size_t changeReservedGroupPosition(a2_gid_t gid, int pos, HOW how); + size_t changeReservedGroupPosition(a2_gid_t gid, int pos, + A2_HOW how); bool removeReservedGroup(a2_gid_t gid); @@ -246,7 +230,7 @@ public: DownloadStat getDownloadStat() const; - const std::deque >& getDownloadResults() const + const DownloadResultList& getDownloadResults() const { return downloadResults_; } @@ -339,7 +323,7 @@ public: return serverStatMan_; } - void setMaxDownloadResult(int v) + void setMaxDownloadResult(size_t v) { maxDownloadResult_ = v; } diff --git a/src/RpcMethodImpl.cc b/src/RpcMethodImpl.cc index d93e3c8e..60bed122 100644 --- a/src/RpcMethodImpl.cc +++ b/src/RpcMethodImpl.cc @@ -48,7 +48,6 @@ #include "RequestGroup.h" #include "download_helper.h" #include "util.h" -#include "RequestGroupMan.h" #include "fmt.h" #include "RpcRequest.h" #include "PieceStorage.h" @@ -486,7 +485,7 @@ void pauseRequestGroups (InputIterator first, InputIterator last, bool reserved, bool forcePause) { for(; first != last; ++first) { - pauseRequestGroup(*first, reserved, forcePause); + pauseRequestGroup((*first).second, reserved, forcePause); } } } // namespace @@ -495,10 +494,9 @@ namespace { SharedHandle pauseAllDownloads (const RpcRequest& req, DownloadEngine* e, bool forcePause) { - const std::deque >& groups = - e->getRequestGroupMan()->getRequestGroups(); + const RequestGroupList& groups = e->getRequestGroupMan()->getRequestGroups(); pauseRequestGroups(groups.begin(), groups.end(), false, forcePause); - const std::deque >& reservedGroups = + const RequestGroupList& reservedGroups = e->getRequestGroupMan()->getReservedGroups(); pauseRequestGroups(reservedGroups.begin(), reservedGroups.end(), true, forcePause); @@ -540,11 +538,12 @@ SharedHandle UnpauseRpcMethod::process SharedHandle UnpauseAllRpcMethod::process (const RpcRequest& req, DownloadEngine* e) { - const std::deque >& groups = + const RequestGroupList& groups = e->getRequestGroupMan()->getReservedGroups(); - std::for_each(groups.begin(), groups.end(), - std::bind2nd(mem_fun_sh(&RequestGroup::setPauseRequested), - false)); + for(RequestGroupList::SeqType::const_iterator i = groups.begin(), + eoi = groups.end(); i != eoi; ++i) { + (*i).second->setPauseRequested(false); + } e->getRequestGroupMan()->requestQueueCheck(); return VLB_OK; } @@ -1038,21 +1037,20 @@ SharedHandle TellActiveRpcMethod::process std::vector keys; toStringList(std::back_inserter(keys), keysParam); SharedHandle list = List::g(); - const std::deque >& groups = - e->getRequestGroupMan()->getRequestGroups(); - for(std::deque >::const_iterator i = - groups.begin(), eoi = groups.end(); i != eoi; ++i) { + const RequestGroupList& groups = e->getRequestGroupMan()->getRequestGroups(); + for(RequestGroupList::SeqType::const_iterator i = groups.begin(), + eoi = groups.end(); i != eoi; ++i) { SharedHandle entryDict = Dict::g(); if(requested_key(keys, KEY_STATUS)) { entryDict->put(KEY_STATUS, VLB_ACTIVE); } - gatherProgress(entryDict, *i, e, keys); + gatherProgress(entryDict, (*i).second, e, keys); list->append(entryDict); } return list; } -const std::deque >& +const RequestGroupList& TellWaitingRpcMethod::getItems(DownloadEngine* e) const { return e->getRequestGroupMan()->getReservedGroups(); @@ -1074,7 +1072,7 @@ void TellWaitingRpcMethod::createEntry gatherProgress(entryDict, item, e, keys); } -const std::deque >& +const DownloadResultList& TellStoppedRpcMethod::getItems(DownloadEngine* e) const { return e->getRequestGroupMan()->getDownloadResults(); @@ -1328,13 +1326,13 @@ SharedHandle ChangePositionRpcMethod::process a2_gid_t gid = str2Gid(gidParam); int pos = posParam->i(); const std::string& howStr = howParam->s(); - RequestGroupMan::HOW how; + A2_HOW how; if(howStr == "POS_SET") { - how = RequestGroupMan::POS_SET; + how = A2_POS_SET; } else if(howStr == "POS_CUR") { - how = RequestGroupMan::POS_CUR; + how = A2_POS_CUR; } else if(howStr == "POS_END") { - how = RequestGroupMan::POS_END; + how = A2_POS_END; } else { throw DL_ABORT_EX("Illegal argument."); } diff --git a/src/RpcMethodImpl.h b/src/RpcMethodImpl.h index f4db2846..caf27d51 100644 --- a/src/RpcMethodImpl.h +++ b/src/RpcMethodImpl.h @@ -46,6 +46,9 @@ #include "TorrentAttribute.h" #include "DlAbortEx.h" #include "fmt.h" +#include "IndexedList.h" +#include "GroupId.h" +#include "RequestGroupMan.h" namespace aria2 { @@ -371,6 +374,8 @@ private: return std::make_pair(first, last); } protected: + typedef IndexedList > ItemListType; + virtual SharedHandle process (const RpcRequest& req, DownloadEngine* e) { @@ -382,14 +387,14 @@ protected: int64_t num = numParam->i(); std::vector keys; toStringList(std::back_inserter(keys), keysParam); - const std::deque >& items = getItems(e); - std::pair >::const_iterator, - typename std::deque >::const_iterator> range = + const ItemListType& items = getItems(e); + std::pair range = getPaginationRange(offset, num, items.begin(), items.end()); SharedHandle list = List::g(); for(; range.first != range.second; ++range.first) { SharedHandle entryDict = Dict::g(); - createEntry(entryDict, *range.first, e, keys); + createEntry(entryDict, (*range.first).second, e, keys); list->append(entryDict); } if(offset < 0) { @@ -398,8 +403,7 @@ protected: return list; } - virtual const std::deque >& - getItems(DownloadEngine* e) const = 0; + virtual const ItemListType& getItems(DownloadEngine* e) const = 0; virtual void createEntry (const SharedHandle& entryDict, @@ -411,8 +415,7 @@ protected: class TellWaitingRpcMethod: public AbstractPaginationRpcMethod { protected: - virtual const std::deque >& - getItems(DownloadEngine* e) const; + virtual const RequestGroupList& getItems(DownloadEngine* e) const; virtual void createEntry (const SharedHandle& entryDict, @@ -429,8 +432,7 @@ public: class TellStoppedRpcMethod: public AbstractPaginationRpcMethod { protected: - virtual const std::deque >& - getItems(DownloadEngine* e) const; + virtual const DownloadResultList& getItems(DownloadEngine* e) const; virtual void createEntry (const SharedHandle& entryDict, diff --git a/src/SessionSerializer.cc b/src/SessionSerializer.cc index 185cda06..4cd0e170 100644 --- a/src/SessionSerializer.cc +++ b/src/SessionSerializer.cc @@ -184,46 +184,46 @@ bool writeDownloadResult bool SessionSerializer::save(BufferedFile& fp) const { std::set metainfoCache; - const std::deque >& results = - rgman_->getDownloadResults(); - for(std::deque >::const_iterator itr = - results.begin(), eoi = results.end(); itr != eoi; ++itr) { - if((*itr)->result == error_code::FINISHED || - (*itr)->result == error_code::REMOVED) { - if((*itr)->option->getAsBool(PREF_FORCE_SAVE)) { - if(!writeDownloadResult(fp, metainfoCache, *itr)) { + const DownloadResultList& results = rgman_->getDownloadResults(); + for(DownloadResultList::SeqType::const_iterator itr = results.begin(), + eoi = results.end(); itr != eoi; ++itr) { + const SharedHandle& dr = (*itr).second; + if(dr->result == error_code::FINISHED || + dr->result == error_code::REMOVED) { + if(dr->option->getAsBool(PREF_FORCE_SAVE)) { + if(!writeDownloadResult(fp, metainfoCache, dr)) { return false; } } else { continue; } - } else if((*itr)->result == error_code::IN_PROGRESS) { + } else if(dr->result == error_code::IN_PROGRESS) { if(saveInProgress_) { - if(!writeDownloadResult(fp, metainfoCache, *itr)) { + if(!writeDownloadResult(fp, metainfoCache, dr)) { return false; } } } else { // error download if(saveError_) { - if(!writeDownloadResult(fp, metainfoCache, *itr)) { + if(!writeDownloadResult(fp, metainfoCache, dr)) { return false; } } } } if(saveWaiting_) { - const std::deque >& groups = - rgman_->getReservedGroups(); - for(std::deque >::const_iterator itr = - groups.begin(), eoi = groups.end(); itr != eoi; ++itr) { - SharedHandle result = (*itr)->createDownloadResult(); + const RequestGroupList& groups = rgman_->getReservedGroups(); + for(RequestGroupList::SeqType::const_iterator itr = groups.begin(), + eoi = groups.end(); itr != eoi; ++itr) { + const SharedHandle& rg = (*itr).second; + SharedHandle result = rg->createDownloadResult(); if(!writeDownloadResult(fp, metainfoCache, result)) { return false; } // PREF_PAUSE was removed from option, so save it here looking // property separately. - if((*itr)->isPauseRequested()) { + if(rg->isPauseRequested()) { if(fp.printf(" %s=true\n", PREF_PAUSE->k) < 0) { return false; } diff --git a/test/IndexedListTest.cc b/test/IndexedListTest.cc new file mode 100644 index 00000000..8673709f --- /dev/null +++ b/test/IndexedListTest.cc @@ -0,0 +1,240 @@ +#include "IndexedList.h" + +#include +#include +#include + +#include + +#include "TestUtil.h" +#include "array_fun.h" +#include "TimerA2.h" + +namespace aria2 { + +class IndexedListTest:public CppUnit::TestFixture { + + CPPUNIT_TEST_SUITE(IndexedListTest); + CPPUNIT_TEST(testPushBack); + CPPUNIT_TEST(testPushFront); + CPPUNIT_TEST(testErase); + CPPUNIT_TEST(testPopFront); + CPPUNIT_TEST(testMove); + CPPUNIT_TEST(testGet); + CPPUNIT_TEST(testInsert); + CPPUNIT_TEST_SUITE_END(); +public: + void setUp() + {} + + void testPushBack(); + void testPushFront(); + void testErase(); + void testPopFront(); + void testMove(); + void testGet(); + void testInsert(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION( IndexedListTest ); + +void IndexedListTest::testPushBack() +{ + int a[] = {1,2,3,4,5}; + IndexedList list; + for(int i = 0; i < 5; ++i) { + CPPUNIT_ASSERT(list.push_back(i, &a[i])); + } + for(int i = 0; i < 5; ++i) { + CPPUNIT_ASSERT_EQUAL(a[i], *list.get(i)); + } + int ai = 0; + for(IndexedList::SeqType::iterator i = list.begin(); + i != list.end(); ++i) { + CPPUNIT_ASSERT_EQUAL(a[ai++], *((*i).second)); + } +} + +void IndexedListTest::testPushFront() +{ + int a[] = {1,2,3,4,5}; + IndexedList list; + for(int i = 0; i < 5; ++i) { + CPPUNIT_ASSERT(list.push_front(i, &a[i])); + } + for(int i = 0; i < 5; ++i) { + CPPUNIT_ASSERT_EQUAL(a[i], *list.get(i)); + } + int ai = 4; + for(IndexedList::SeqType::iterator i = list.begin(); + i != list.end(); ++i) { + CPPUNIT_ASSERT_EQUAL(a[ai--], *((*i).second)); + } +} + +void IndexedListTest::testErase() +{ + int a[] = {1,2,3,4,5}; + IndexedList list; + for(int i = 0; i < 5; ++i) { + list.push_back(i, &a[i]); + } + for(int i = 0; i < 5; ++i) { + CPPUNIT_ASSERT(list.erase(i)); + CPPUNIT_ASSERT_EQUAL((size_t)5-i-1, list.size()); + for(int j = i+1; j < 5; ++j) { + CPPUNIT_ASSERT_EQUAL(a[j], *list.get(j)); + } + } +} + +void IndexedListTest::testPopFront() +{ + int a[] = {1,2,3,4,5}; + IndexedList list; + for(int i = 0; i < 5; ++i) { + list.push_back(i, &a[i]); + } + for(int i = 0; i < 5; ++i) { + CPPUNIT_ASSERT(list.pop_front()); + CPPUNIT_ASSERT_EQUAL((size_t)5-i-1, list.size()); + for(int j = i+1; j < 5; ++j) { + CPPUNIT_ASSERT_EQUAL(a[j], *list.get(j)); + } + } +} + +#define LIST_CHECK(a, list) \ + { \ + int ai = 0; \ + for(IndexedList::SeqType::iterator i = list.begin(); \ + i != list.end(); ++i) { \ + CPPUNIT_ASSERT_EQUAL(a[ai++], *((*i).second)); \ + } \ + } + +void IndexedListTest::testMove() +{ + int a[] = {0,1,2,3,4}; + IndexedList list; + for(int i = 0; i < 5; ++i) { + list.push_back(i, &a[i]); + } + CPPUNIT_ASSERT_EQUAL((ssize_t)-1, list.move(100, 0, A2_POS_SET)); + int a0[] = {0,1,2,3,4}; + CPPUNIT_ASSERT_EQUAL((ssize_t)0, list.move(0, 0, A2_POS_SET)); + LIST_CHECK(a0, list); + + int a1[] = {0,2,3,4,1}; + CPPUNIT_ASSERT_EQUAL((ssize_t)4, list.move(1, 4, A2_POS_SET)); + LIST_CHECK(a1, list); + + int a2[] = {0,3,4,2,1}; + CPPUNIT_ASSERT_EQUAL((ssize_t)3, list.move(2, 3, A2_POS_SET)); + LIST_CHECK(a2, list); + + int a3[] = {0,2,3,4,1}; + CPPUNIT_ASSERT_EQUAL((ssize_t)1, list.move(2, 1, A2_POS_SET)); + LIST_CHECK(a3, list); + + int a4[] = {1,0,2,3,4}; + CPPUNIT_ASSERT_EQUAL((ssize_t)0, list.move(1, 0, A2_POS_SET)); + LIST_CHECK(a4, list); + + int a5[] = {1,0,3,2,4}; + CPPUNIT_ASSERT_EQUAL((ssize_t)2, list.move(3, 2, A2_POS_SET)); + LIST_CHECK(a5, list); + + int a6[] = {1,3,2,4,0}; + CPPUNIT_ASSERT_EQUAL((ssize_t)4, list.move(0, 5, A2_POS_SET)); + LIST_CHECK(a6, list); + + int a7[] = {3,1,2,4,0}; + CPPUNIT_ASSERT_EQUAL((ssize_t)1, list.move(1, 1, A2_POS_CUR)); + LIST_CHECK(a7, list); + + int a8[] = {3,2,4,1,0}; + CPPUNIT_ASSERT_EQUAL((ssize_t)3, list.move(1, 2, A2_POS_CUR)); + LIST_CHECK(a8, list); + + int a9[] = {3,2,1,4,0}; + CPPUNIT_ASSERT_EQUAL((ssize_t)2, list.move(1, -1, A2_POS_CUR)); + LIST_CHECK(a9, list); + + int a10[] = {1,3,2,4,0}; + CPPUNIT_ASSERT_EQUAL((ssize_t)0, list.move(1, -1233, A2_POS_CUR)); + LIST_CHECK(a10, list); + + int a11[] = {3,2,4,0,1}; + CPPUNIT_ASSERT_EQUAL((ssize_t)4, list.move(1, 8733, A2_POS_CUR)); + LIST_CHECK(a11, list); + + int a12[] = {3,2,4,0,1}; + CPPUNIT_ASSERT_EQUAL((ssize_t)3, list.move(0, -1, A2_POS_END)); + LIST_CHECK(a12, list); + + int a13[] = {3,2,0,4,1}; + CPPUNIT_ASSERT_EQUAL((ssize_t)2, list.move(0, -2, A2_POS_END)); + LIST_CHECK(a13, list); + + int a14[] = {0,3,2,4,1}; + CPPUNIT_ASSERT_EQUAL((ssize_t)0, list.move(0, -8733, A2_POS_END)); + LIST_CHECK(a14, list); + + int a15[] = {0,2,4,1,3}; + CPPUNIT_ASSERT_EQUAL((ssize_t)4, list.move(3, 0, A2_POS_END)); + LIST_CHECK(a15, list); + + int a16[] = {2,4,1,3,0}; + CPPUNIT_ASSERT_EQUAL((ssize_t)4, list.move(0, 1000, A2_POS_END)); + LIST_CHECK(a16, list); + + int a17[] = {2,1,4,3,0}; + CPPUNIT_ASSERT_EQUAL((ssize_t)2, list.move(4, 2, A2_POS_SET)); + LIST_CHECK(a17, list); +} + +void IndexedListTest::testGet() +{ + IndexedList list; + int a = 1000; + int b = 1; + list.push_back(123, &a); + list.push_back(1, &b); + CPPUNIT_ASSERT_EQUAL((int*)0, list.get(1000)); + CPPUNIT_ASSERT_EQUAL(&a, list.get(123)); + + CPPUNIT_ASSERT(list.begin() == list.find(123)); + CPPUNIT_ASSERT(list.end() == list.find(124)); +} + +void IndexedListTest::testInsert() +{ + int a[] = {0,1,2,3,4,5,6,7,8,9}; + IndexedList list; + IndexedList::SeqType::iterator itr; + CPPUNIT_ASSERT(list.end() == list.insert(1, 0, &a[5])); + itr = list.insert(0, 5, &a[5]); + CPPUNIT_ASSERT_EQUAL(5, *(*itr).second); + itr = list.insert(1, 3, &a[3]); + CPPUNIT_ASSERT_EQUAL(3, *(*itr).second); + itr = list.insert(1, 4, &a[4]); + CPPUNIT_ASSERT_EQUAL(4, *(*itr).second); + itr = list.insert(0, 9, &a[9]); + CPPUNIT_ASSERT_EQUAL(9, *(*itr).second); + int a1[] = { 9,5,4,3 }; + LIST_CHECK(a1, list); + + // use iterator to insert + itr = list.insert(itr, 2, &a[2]); + CPPUNIT_ASSERT_EQUAL(2, *(*itr).second); + itr = list.insert(list.end(), 1, &a[1]); + CPPUNIT_ASSERT_EQUAL(1, *(*itr).second); + int a2[] = { 2,9,5,4,3,1 }; + LIST_CHECK(a2, list); + + // 2 has been already added. + CPPUNIT_ASSERT(list.end() == list.insert(list.end(), 2, &a[2])); +} + +} // namespace aria2 diff --git a/test/Makefile.am b/test/Makefile.am index d15523fe..dacddc27 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -88,7 +88,8 @@ aria2c_SOURCES = AllTest.cc\ SinkStreamFilterTest.cc\ WrDiskCacheTest.cc\ WrDiskCacheEntryTest.cc\ - GroupIdTest.cc + GroupIdTest.cc\ + IndexedListTest.cc if ENABLE_XML_RPC aria2c_SOURCES += XmlRpcRequestParserControllerTest.cc diff --git a/test/RequestGroupManTest.cc b/test/RequestGroupManTest.cc index 5c4c3e7e..0e8c3ef4 100644 --- a/test/RequestGroupManTest.cc +++ b/test/RequestGroupManTest.cc @@ -133,57 +133,57 @@ void RequestGroupManTest::testChangeReservedGroupPosition() CPPUNIT_ASSERT_EQUAL ((size_t)0, rm.changeReservedGroupPosition(gs[0]->getGID(), - 0, RequestGroupMan::POS_SET)); + 0, A2_POS_SET)); CPPUNIT_ASSERT_EQUAL ((size_t)1, rm.changeReservedGroupPosition(gs[0]->getGID(), - 1, RequestGroupMan::POS_SET)); + 1, A2_POS_SET)); CPPUNIT_ASSERT_EQUAL ((size_t)3, rm.changeReservedGroupPosition(gs[0]->getGID(), - 10,RequestGroupMan::POS_SET)); + 10,A2_POS_SET)); CPPUNIT_ASSERT_EQUAL ((size_t)0, rm.changeReservedGroupPosition(gs[0]->getGID(), - -10, RequestGroupMan::POS_SET)); + -10, A2_POS_SET)); CPPUNIT_ASSERT_EQUAL ((size_t)1, rm.changeReservedGroupPosition(gs[1]->getGID(), - 0, RequestGroupMan::POS_CUR)); + 0, A2_POS_CUR)); CPPUNIT_ASSERT_EQUAL ((size_t)2, rm.changeReservedGroupPosition(gs[1]->getGID(), - 1, RequestGroupMan::POS_CUR)); + 1, A2_POS_CUR)); CPPUNIT_ASSERT_EQUAL ((size_t)1, rm.changeReservedGroupPosition(gs[1]->getGID(), - -1,RequestGroupMan::POS_CUR)); + -1,A2_POS_CUR)); CPPUNIT_ASSERT_EQUAL ((size_t)0, rm.changeReservedGroupPosition(gs[1]->getGID(), - -10, RequestGroupMan::POS_CUR)); + -10, A2_POS_CUR)); CPPUNIT_ASSERT_EQUAL ((size_t)1, rm.changeReservedGroupPosition(gs[1]->getGID(), - 1, RequestGroupMan::POS_CUR)); + 1, A2_POS_CUR)); CPPUNIT_ASSERT_EQUAL ((size_t)3, rm.changeReservedGroupPosition(gs[1]->getGID(), - 10, RequestGroupMan::POS_CUR)); + 10, A2_POS_CUR)); CPPUNIT_ASSERT_EQUAL ((size_t)1, rm.changeReservedGroupPosition(gs[1]->getGID(), - -2,RequestGroupMan::POS_CUR)); + -2,A2_POS_CUR)); CPPUNIT_ASSERT_EQUAL ((size_t)3, rm.changeReservedGroupPosition(gs[3]->getGID(), - 0, RequestGroupMan::POS_END)); + 0, A2_POS_END)); CPPUNIT_ASSERT_EQUAL ((size_t)2, rm.changeReservedGroupPosition(gs[3]->getGID(), - -1,RequestGroupMan::POS_END)); + -1,A2_POS_END)); CPPUNIT_ASSERT_EQUAL ((size_t)0, rm.changeReservedGroupPosition(gs[3]->getGID(), - -10, RequestGroupMan::POS_END)); + -10, A2_POS_END)); CPPUNIT_ASSERT_EQUAL ((size_t)3, rm.changeReservedGroupPosition(gs[3]->getGID(), - 10, RequestGroupMan::POS_END)); + 10, A2_POS_END)); CPPUNIT_ASSERT_EQUAL((size_t)4, rm.getReservedGroups().size()); try { rm.changeReservedGroupPosition(GroupId::create()->getNumericId(), - 0, RequestGroupMan::POS_CUR); + 0, A2_POS_CUR); CPPUNIT_FAIL("exception must be thrown."); } catch(RecoverableException& e) { // success diff --git a/test/RpcMethodTest.cc b/test/RpcMethodTest.cc index ae016d74..8f77800d 100644 --- a/test/RpcMethodTest.cc +++ b/test/RpcMethodTest.cc @@ -161,11 +161,11 @@ void RpcMethodTest::testAddUri() { RpcResponse res = m.execute(req, e_.get()); CPPUNIT_ASSERT_EQUAL(0, res.code); - const std::deque > rgs = + const RequestGroupList& rgs = e_->getRequestGroupMan()->getReservedGroups(); CPPUNIT_ASSERT_EQUAL((size_t)1, rgs.size()); CPPUNIT_ASSERT_EQUAL(std::string("http://localhost/"), - rgs.front()->getDownloadContext()-> + (*rgs.begin()).second->getDownloadContext()-> getFirstFileEntry()->getRemainingUris().front()); } // with options @@ -179,7 +179,7 @@ void RpcMethodTest::testAddUri() CPPUNIT_ASSERT_EQUAL(0, GroupId::toNumericId (gid, downcast(res.param)->s().c_str())); CPPUNIT_ASSERT_EQUAL(std::string("/sink"), - e_->getRequestGroupMan()->findReservedGroup(gid)-> + findReservedGroup(e_->getRequestGroupMan(), gid)-> getOption()->get(PREF_DIR)); } } @@ -236,7 +236,7 @@ void RpcMethodTest::testAddUri_withPosition() m.execute(req2, e_.get()); std::string uri = - e_->getRequestGroupMan()->getReservedGroups()[0]-> + getReservedGroup(e_->getRequestGroupMan(), 0)-> getDownloadContext()->getFirstFileEntry()->getRemainingUris()[0]; CPPUNIT_ASSERT_EQUAL(std::string("http://uri2"), uri); @@ -288,7 +288,7 @@ void RpcMethodTest::testAddTorrent() (gid, downcast(res.param)->s().c_str())); SharedHandle group = - e_->getRequestGroupMan()->findReservedGroup(gid); + findReservedGroup(e_->getRequestGroupMan(), gid); CPPUNIT_ASSERT(group); CPPUNIT_ASSERT_EQUAL(e_->getOption()->get(PREF_DIR)+"/aria2-0.8.2.tar.bz2", group->getFirstFilePath()); @@ -314,7 +314,7 @@ void RpcMethodTest::testAddTorrent() (gid, downcast(res.param)->s().c_str())); CPPUNIT_ASSERT_EQUAL (dir+"/aria2-0.8.2.tar.bz2", - e_->getRequestGroupMan()->findReservedGroup(gid)->getFirstFilePath()); + findReservedGroup(e_->getRequestGroupMan(), gid)->getFirstFilePath()); CPPUNIT_ASSERT (File(dir+"/0a3893293e27ac0490424c06de4d09242215f0a6.torrent").exists()); } @@ -355,7 +355,7 @@ void RpcMethodTest::testAddTorrent_withPosition() m.execute(req2, e_.get()); CPPUNIT_ASSERT_EQUAL((size_t)1, - e_->getRequestGroupMan()->getReservedGroups()[0]-> + getReservedGroup(e_->getRequestGroupMan(), 0)-> getDownloadContext()->getFileEntries().size()); } @@ -406,12 +406,12 @@ void RpcMethodTest::testAddMetalink() #endif // ENABLE_MESSAGE_DIGEST SharedHandle tar = - e_->getRequestGroupMan()->findReservedGroup(gid3); + findReservedGroup(e_->getRequestGroupMan(), gid3); CPPUNIT_ASSERT(tar); CPPUNIT_ASSERT_EQUAL(e_->getOption()->get(PREF_DIR)+"/aria2-5.0.0.tar.bz2", tar->getFirstFilePath()); SharedHandle deb = - e_->getRequestGroupMan()->findReservedGroup(gid4); + findReservedGroup(e_->getRequestGroupMan(), gid4); CPPUNIT_ASSERT(deb); CPPUNIT_ASSERT_EQUAL(e_->getOption()->get(PREF_DIR)+"/aria2-5.0.0.deb", deb->getFirstFilePath()); @@ -433,7 +433,7 @@ void RpcMethodTest::testAddMetalink() (0, GroupId::toNumericId (gid5, downcast(resParams->get(0))->s().c_str())); CPPUNIT_ASSERT_EQUAL(dir+"/aria2-5.0.0.tar.bz2", - e_->getRequestGroupMan()->findReservedGroup(gid5)-> + findReservedGroup(e_->getRequestGroupMan(), gid5)-> getFirstFilePath()); #ifdef ENABLE_MESSAGE_DIGEST CPPUNIT_ASSERT @@ -478,7 +478,7 @@ void RpcMethodTest::testAddMetalink_withPosition() CPPUNIT_ASSERT_EQUAL(0, res2.code); CPPUNIT_ASSERT_EQUAL(e_->getOption()->get(PREF_DIR)+"/aria2-5.0.0.tar.bz2", - e_->getRequestGroupMan()->getReservedGroups()[0]-> + getReservedGroup(e_->getRequestGroupMan(), 0)-> getFirstFilePath()); } @@ -684,9 +684,9 @@ void RpcMethodTest::testTellWaiting() CPPUNIT_ASSERT_EQUAL(0, res.code); const List* resParams = downcast(res.param); CPPUNIT_ASSERT_EQUAL((size_t)2, resParams->size()); - CPPUNIT_ASSERT_EQUAL(GroupId::toHex(rgman->getReservedGroups()[1]->getGID()), + CPPUNIT_ASSERT_EQUAL(GroupId::toHex(getReservedGroup(rgman, 1)->getGID()), getString(downcast(resParams->get(0)), "gid")); - CPPUNIT_ASSERT_EQUAL(GroupId::toHex(rgman->getReservedGroups()[2]->getGID()), + CPPUNIT_ASSERT_EQUAL(GroupId::toHex(getReservedGroup(rgman, 2)->getGID()), getString(downcast(resParams->get(1)), "gid")); // waiting.size() == offset+num req = RpcRequest(TellWaitingRpcMethod::getMethodName(), List::g()); @@ -742,9 +742,9 @@ void RpcMethodTest::testTellWaiting() CPPUNIT_ASSERT_EQUAL(0, res.code); resParams = downcast(res.param); CPPUNIT_ASSERT_EQUAL((size_t)2, resParams->size()); - CPPUNIT_ASSERT_EQUAL(GroupId::toHex(rgman->getReservedGroups()[3]->getGID()), + CPPUNIT_ASSERT_EQUAL(GroupId::toHex(getReservedGroup(rgman, 3)->getGID()), getString(downcast(resParams->get(0)), "gid")); - CPPUNIT_ASSERT_EQUAL(GroupId::toHex(rgman->getReservedGroups()[2]->getGID()), + CPPUNIT_ASSERT_EQUAL(GroupId::toHex(getReservedGroup(rgman, 2)->getGID()), getString(downcast(resParams->get(1)), "gid")); // negative offset and size < num req.params->set(1, Integer::g(100)); @@ -936,7 +936,7 @@ void RpcMethodTest::testChangePosition() (SharedHandle(new RequestGroup(GroupId::create(), util::copy(option_)))); - a2_gid_t gid = e_->getRequestGroupMan()->getReservedGroups()[0]->getGID(); + a2_gid_t gid = getReservedGroup(e_->getRequestGroupMan(), 0)->getGID(); ChangePositionRpcMethod m; RpcRequest req(ChangePositionRpcMethod::getMethodName(), List::g()); req.params->append(GroupId::toHex(gid)); @@ -946,7 +946,7 @@ void RpcMethodTest::testChangePosition() CPPUNIT_ASSERT_EQUAL(0, res.code); CPPUNIT_ASSERT_EQUAL((int64_t)1, downcast(res.param)->i()); CPPUNIT_ASSERT_EQUAL - (gid, e_->getRequestGroupMan()->getReservedGroups()[1]->getGID()); + (gid, getReservedGroup(e_->getRequestGroupMan(), 1)->getGID()); } void RpcMethodTest::testChangePosition_fail() @@ -1211,9 +1211,9 @@ void RpcMethodTest::testSystemMulticall() const List* resParams = downcast(res.param); CPPUNIT_ASSERT_EQUAL((size_t)7, resParams->size()); SharedHandle rgman = e_->getRequestGroupMan(); - CPPUNIT_ASSERT_EQUAL(GroupId::toHex(rgman->getReservedGroups()[0]->getGID()), + CPPUNIT_ASSERT_EQUAL(GroupId::toHex(getReservedGroup(rgman, 0)->getGID()), downcast(downcast(resParams->get(0))->get(0))->s()); - CPPUNIT_ASSERT_EQUAL(GroupId::toHex(rgman->getReservedGroups()[1]->getGID()), + CPPUNIT_ASSERT_EQUAL(GroupId::toHex(getReservedGroup(rgman, 1)->getGID()), downcast(downcast(resParams->get(1))->get(0))->s()); CPPUNIT_ASSERT_EQUAL((int64_t)1, downcast diff --git a/test/TestUtil.cc b/test/TestUtil.cc index 3ad3697f..7ed4ffcd 100644 --- a/test/TestUtil.cc +++ b/test/TestUtil.cc @@ -15,6 +15,8 @@ #include "DefaultDiskWriter.h" #include "fmt.h" #include "util.h" +#include "RequestGroupMan.h" +#include "RequestGroup.h" #ifdef ENABLE_MESSAGE_DIGEST # include "message_digest_helper.h" #endif // ENABLE_MESSAGE_DIGEST @@ -98,4 +100,28 @@ WrDiskCacheEntry::DataCell* createDataCell(int64_t goff, return cell; } +SharedHandle findReservedGroup +(const SharedHandle& rgman, a2_gid_t gid) +{ + SharedHandle rg = rgman->findGroup(gid); + if(rg) { + if(rg->getState() == RequestGroup::STATE_WAITING) { + return rg; + } else { + rg.reset(); + } + } + return rg; +} + +SharedHandle getReservedGroup +(const SharedHandle& rgman, size_t index) +{ + assert(rgman->getReservedGroups().size() > index); + RequestGroupList::SeqType::const_iterator i = + rgman->getReservedGroups().begin(); + std::advance(i, index); + return (*i).second; +} + } // namespace aria2 diff --git a/test/TestUtil.h b/test/TestUtil.h index 0eaf330c..05762a56 100644 --- a/test/TestUtil.h +++ b/test/TestUtil.h @@ -5,10 +5,13 @@ #include "SharedHandle.h" #include "Cookie.h" #include "WrDiskCacheEntry.h" +#include "GroupId.h" namespace aria2 { class MessageDigest; +class RequestGroupMan; +class RequestGroup; void createFile(const std::string& filename, size_t length); @@ -55,4 +58,10 @@ WrDiskCacheEntry::DataCell* createDataCell(int64_t goff, const char* data, size_t offset = 0); +SharedHandle findReservedGroup +(const SharedHandle& rgman, a2_gid_t gid); + +SharedHandle getReservedGroup +(const SharedHandle& rgman, size_t index); + } // namespace aria2