/* */ #include "FileEntry.h" #include #include #include "util.h" #include "URISelector.h" #include "LogFactory.h" #include "wallclock.h" namespace aria2 { FileEntry::FileEntry(const std::string& path, uint64_t length, off_t offset, const std::vector& uris): path_(path), uris_(uris.begin(), uris.end()), length_(length), offset_(offset), requested_(true), singleHostMultiConnection_(true), logger_(LogFactory::getInstance()) {} FileEntry::FileEntry(): length_(0), offset_(0), requested_(false), singleHostMultiConnection_(true), logger_(LogFactory::getInstance()) {} FileEntry::~FileEntry() {} void FileEntry::setupDir() { util::mkdirs(File(path_).getDirname()); } FileEntry& FileEntry::operator=(const FileEntry& entry) { if(this != &entry) { path_ = entry.path_; length_ = entry.length_; offset_ = entry.offset_; requested_ = entry.requested_; } return *this; } bool FileEntry::operator<(const FileEntry& fileEntry) const { return offset_ < fileEntry.offset_; } bool FileEntry::exists() const { return File(getPath()).exists(); } off_t FileEntry::gtoloff(off_t goff) const { assert(offset_ <= goff); return goff-offset_; } void FileEntry::getUris(std::vector& uris) const { uris.insert(uris.end(), spentUris_.begin(), spentUris_.end()); uris.insert(uris.end(), uris_.begin(), uris_.end()); } std::string FileEntry::selectUri(const SharedHandle& uriSelector) { return uriSelector->select(this); } template static bool inFlightHost(InputIterator first, InputIterator last, const std::string& hostname) { // TODO redirection should be considered here. We need to parse // original URI to get hostname. for(; first != last; ++first) { if((*first)->getHost() == hostname) { return true; } } return false; } SharedHandle FileEntry::getRequest (const SharedHandle& selector, const std::string& referer, const std::string& method) { SharedHandle req; if(requestPool_.empty()) { std::vector pending; while(1) { std::string uri = selector->select(this); if(uri.empty()) { return req; } req.reset(new Request()); if(req->setUri(uri)) { if(!singleHostMultiConnection_) { if(inFlightHost(inFlightRequests_.begin(), inFlightRequests_.end(), req->getHost())) { pending.push_back(uri); req.reset(); continue; } } req->setReferer(referer); req->setMethod(method); spentUris_.push_back(uri); inFlightRequests_.push_back(req); break; } else { req.reset(); } } uris_.insert(uris_.begin(), pending.begin(), pending.end()); } else { req = requestPool_.front(); requestPool_.pop_front(); inFlightRequests_.push_back(req); } return req; } SharedHandle FileEntry::findFasterRequest(const SharedHandle& base) { if(requestPool_.empty()) { return SharedHandle(); } const SharedHandle& fastest = requestPool_.front()->getPeerStat(); if(fastest.isNull()) { return SharedHandle(); } const SharedHandle& basestat = base->getPeerStat(); // TODO hard coded value. See PREF_STARTUP_IDLE_TIME const int startupIdleTime = 10; if(basestat.isNull() || (basestat->getDownloadStartTime(). difference(global::wallclock) >= startupIdleTime && fastest->getAvgDownloadSpeed()*0.8 > basestat->calculateDownloadSpeed())){ // TODO we should consider that "fastest" is very slow. SharedHandle fastestRequest = requestPool_.front(); requestPool_.pop_front(); inFlightRequests_.push_back(fastestRequest); return fastestRequest; } return SharedHandle(); } class RequestFaster { public: bool operator()(const SharedHandle& lhs, const SharedHandle& rhs) const { if(lhs->getPeerStat().isNull()) { return false; } if(rhs->getPeerStat().isNull()) { return true; } return lhs->getPeerStat()->getAvgDownloadSpeed() > rhs->getPeerStat()->getAvgDownloadSpeed(); } }; void FileEntry::storePool(const SharedHandle& request) { const SharedHandle& peerStat = request->getPeerStat(); if(!peerStat.isNull()) { // We need to calculate average download speed here in order to // store Request in the right position in the pool. peerStat->calculateAvgDownloadSpeed(); } std::deque >::iterator i = std::lower_bound(requestPool_.begin(), requestPool_.end(), request, RequestFaster()); requestPool_.insert(i, request); } void FileEntry::poolRequest(const SharedHandle& request) { removeRequest(request); if(!request->removalRequested()) { storePool(request); } } bool FileEntry::removeRequest(const SharedHandle& request) { for(std::deque >::iterator i = inFlightRequests_.begin(), eoi = inFlightRequests_.end(); i != eoi; ++i) { if((*i).get() == request.get()) { inFlightRequests_.erase(i); return true; } } return false; } void FileEntry::removeURIWhoseHostnameIs(const std::string& hostname) { std::deque newURIs; Request req; for(std::deque::const_iterator itr = uris_.begin(), eoi = uris_.end(); itr != eoi; ++itr) { if(((*itr).find(hostname) == std::string::npos) || (req.setUri(*itr) && (req.getHost() != hostname))) { newURIs.push_back(*itr); } } if(logger_->debug()) { logger_->debug("Removed %d duplicate hostname URIs for path=%s", uris_.size()-newURIs.size(), getPath().c_str()); } uris_ = newURIs; } void FileEntry::removeIdenticalURI(const std::string& uri) { uris_.erase(std::remove(uris_.begin(), uris_.end(), uri), uris_.end()); } void FileEntry::addURIResult(std::string uri, downloadresultcode::RESULT result) { uriResults_.push_back(URIResult(uri, result)); } class FindURIResultByResult { private: downloadresultcode::RESULT r_; public: FindURIResultByResult(downloadresultcode::RESULT r):r_(r) {} bool operator()(const URIResult& uriResult) const { return uriResult.getResult() == r_; } }; void FileEntry::extractURIResult (std::deque& res, downloadresultcode::RESULT r) { std::deque::iterator i = std::stable_partition(uriResults_.begin(), uriResults_.end(), FindURIResultByResult(r)); std::copy(uriResults_.begin(), i, std::back_inserter(res)); uriResults_.erase(uriResults_.begin(), i); } void FileEntry::reuseUri(size_t num) { std::deque uris = spentUris_; std::sort(uris.begin(), uris.end()); uris.erase(std::unique(uris.begin(), uris.end()), uris.end()); std::vector errorUris(uriResults_.size()); std::transform(uriResults_.begin(), uriResults_.end(), errorUris.begin(), std::mem_fun_ref(&URIResult::getURI)); std::sort(errorUris.begin(), errorUris.end()); errorUris.erase(std::unique(errorUris.begin(), errorUris.end()), errorUris.end()); std::vector reusableURIs; std::set_difference(uris.begin(), uris.end(), errorUris.begin(), errorUris.end(), std::back_inserter(reusableURIs)); size_t ininum = reusableURIs.size(); if(logger_->debug()) { logger_->debug("Found %u reusable URIs", static_cast(ininum)); for(std::vector::const_iterator i = reusableURIs.begin(), eoi = reusableURIs.end(); i != eoi; ++i) { logger_->debug("URI=%s", (*i).c_str()); } } // Reuse at least num URIs here to avoid to // run this process repeatedly. if(ininum > 0) { for(size_t i = 0; i < num/ininum; ++i) { uris_.insert(uris_.end(), reusableURIs.begin(), reusableURIs.end()); } uris_.insert(uris_.end(), reusableURIs.begin(), reusableURIs.begin()+(num%ininum)); if(logger_->debug()) { logger_->debug("Duplication complete: now %u URIs for reuse", static_cast(uris_.size())); } } } void FileEntry::releaseRuntimeResource() { requestPool_.clear(); inFlightRequests_.clear(); } template static InputIterator findRequestByUri (InputIterator first, InputIterator last, const T& uri) { for(; first != last; ++first) { if(!(*first)->removalRequested() && (*first)->getUri() == uri) { return first; } } return last; } bool FileEntry::removeUri(const std::string& uri) { std::deque::iterator itr = std::find(spentUris_.begin(), spentUris_.end(), uri); if(itr == spentUris_.end()) { itr = std::find(uris_.begin(), uris_.end(), uri); if(itr == uris_.end()) { return false; } else { uris_.erase(itr); return true; } } else { spentUris_.erase(itr); SharedHandle req; std::deque >::iterator riter = findRequestByUri(inFlightRequests_.begin(), inFlightRequests_.end(), uri); if(riter == inFlightRequests_.end()) { riter = findRequestByUri(requestPool_.begin(), requestPool_.end(), uri); if(riter == requestPool_.end()) { return true; } else { req = *riter; requestPool_.erase(riter); } } else { req = *riter; } req->requestRemoval(); return true; } } std::string FileEntry::getBasename() const { return File(path_).getBasename(); } std::string FileEntry::getDirname() const { return File(path_).getDirname(); } size_t FileEntry::setUris(const std::vector& uris) { uris_.clear(); return addUris(uris.begin(), uris.end()); } bool FileEntry::addUri(const std::string& uri) { if(Request().setUri(uri)) { uris_.push_back(uri); return true; } else { return false; } } bool FileEntry::insertUri(const std::string& uri, size_t pos) { if(Request().setUri(uri)) { pos = std::min(pos, uris_.size()); uris_.insert(uris_.begin()+pos, uri); return true; } else { return false; } } } // namespace aria2