mirror of https://github.com/aria2/aria2
381 lines
11 KiB
C++
381 lines
11 KiB
C++
/* <!-- copyright */
|
|
/*
|
|
* aria2 - The high speed download utility
|
|
*
|
|
* Copyright (C) 2006 Tatsuhiro Tsujikawa
|
|
*
|
|
* This program is free software; you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License as published by
|
|
* the Free Software Foundation; either version 2 of the License, or
|
|
* (at your option) any later version.
|
|
*
|
|
* This program is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU General Public License
|
|
* along with this program; if not, write to the Free Software
|
|
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|
*
|
|
* In addition, as a special exception, the copyright holders give
|
|
* permission to link the code of portions of this program with the
|
|
* OpenSSL library under certain conditions as described in each
|
|
* individual source file, and distribute linked combinations
|
|
* including the two.
|
|
* You must obey the GNU General Public License in all respects
|
|
* for all of the code used other than OpenSSL. If you modify
|
|
* file(s) with this exception, you may extend this exception to your
|
|
* version of the file(s), but you are not obligated to do so. If you
|
|
* do not wish to do so, delete this exception statement from your
|
|
* version. If you delete this exception statement from all source
|
|
* files in the program, then also delete it here.
|
|
*/
|
|
/* copyright --> */
|
|
#include "FileEntry.h"
|
|
|
|
#include <cassert>
|
|
#include <algorithm>
|
|
|
|
#include "util.h"
|
|
#include "URISelector.h"
|
|
#include "LogFactory.h"
|
|
#include "wallclock.h"
|
|
|
|
namespace aria2 {
|
|
|
|
FileEntry::FileEntry(const std::string& path,
|
|
uint64_t length,
|
|
off_t offset,
|
|
const std::vector<std::string>& uris):
|
|
path(path), _uris(uris.begin(), uris.end()), length(length), offset(offset),
|
|
requested(true),
|
|
_singleHostMultiConnection(true),
|
|
_logger(LogFactory::getInstance()) {}
|
|
|
|
FileEntry::FileEntry():
|
|
length(0), offset(0), requested(false),
|
|
_singleHostMultiConnection(true),
|
|
_logger(LogFactory::getInstance()) {}
|
|
|
|
FileEntry::~FileEntry() {}
|
|
|
|
void FileEntry::setupDir()
|
|
{
|
|
util::mkdirs(File(path).getDirname());
|
|
}
|
|
|
|
FileEntry& FileEntry::operator=(const FileEntry& entry)
|
|
{
|
|
if(this != &entry) {
|
|
path = entry.path;
|
|
length = entry.length;
|
|
offset = entry.offset;
|
|
requested = entry.requested;
|
|
}
|
|
return *this;
|
|
}
|
|
|
|
bool FileEntry::operator<(const FileEntry& fileEntry) const
|
|
{
|
|
return offset < fileEntry.offset;
|
|
}
|
|
|
|
bool FileEntry::exists() const
|
|
{
|
|
return File(getPath()).exists();
|
|
}
|
|
|
|
off_t FileEntry::gtoloff(off_t goff) const
|
|
{
|
|
assert(offset <= goff);
|
|
return goff-offset;
|
|
}
|
|
|
|
void FileEntry::getUris(std::vector<std::string>& uris) const
|
|
{
|
|
uris.insert(uris.end(), _spentUris.begin(), _spentUris.end());
|
|
uris.insert(uris.end(), _uris.begin(), _uris.end());
|
|
}
|
|
|
|
std::string FileEntry::selectUri(const SharedHandle<URISelector>& uriSelector)
|
|
{
|
|
return uriSelector->select(this);
|
|
}
|
|
|
|
template<typename InputIterator>
|
|
static bool inFlightHost(InputIterator first, InputIterator last,
|
|
const std::string& hostname)
|
|
{
|
|
// TODO redirection should be considered here. We need to parse
|
|
// original URI to get hostname.
|
|
for(; first != last; ++first) {
|
|
if((*first)->getHost() == hostname) {
|
|
return true;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
SharedHandle<Request>
|
|
FileEntry::getRequest
|
|
(const SharedHandle<URISelector>& selector,
|
|
const std::string& referer,
|
|
const std::string& method)
|
|
{
|
|
SharedHandle<Request> req;
|
|
if(_requestPool.empty()) {
|
|
std::vector<std::string> pending;
|
|
while(1) {
|
|
std::string uri = selector->select(this);
|
|
if(uri.empty()) {
|
|
return req;
|
|
}
|
|
req.reset(new Request());
|
|
if(req->setUrl(uri)) {
|
|
if(!_singleHostMultiConnection) {
|
|
if(inFlightHost(_inFlightRequests.begin(), _inFlightRequests.end(),
|
|
req->getHost())) {
|
|
pending.push_back(uri);
|
|
req.reset();
|
|
continue;
|
|
}
|
|
}
|
|
req->setReferer(referer);
|
|
req->setMethod(method);
|
|
_spentUris.push_back(uri);
|
|
_inFlightRequests.push_back(req);
|
|
break;
|
|
} else {
|
|
req.reset();
|
|
}
|
|
}
|
|
_uris.insert(_uris.begin(), pending.begin(), pending.end());
|
|
} else {
|
|
req = _requestPool.front();
|
|
_requestPool.pop_front();
|
|
_inFlightRequests.push_back(req);
|
|
}
|
|
return req;
|
|
}
|
|
|
|
SharedHandle<Request>
|
|
FileEntry::findFasterRequest(const SharedHandle<Request>& base)
|
|
{
|
|
if(_requestPool.empty()) {
|
|
return SharedHandle<Request>();
|
|
}
|
|
const SharedHandle<PeerStat>& fastest = _requestPool.front()->getPeerStat();
|
|
if(fastest.isNull()) {
|
|
return SharedHandle<Request>();
|
|
}
|
|
const SharedHandle<PeerStat>& basestat = base->getPeerStat();
|
|
// TODO hard coded value. See PREF_STARTUP_IDLE_TIME
|
|
const int startupIdleTime = 10;
|
|
if(basestat.isNull() ||
|
|
(basestat->getDownloadStartTime().
|
|
difference(global::wallclock) >= startupIdleTime &&
|
|
fastest->getAvgDownloadSpeed()*0.8 > basestat->calculateDownloadSpeed())){
|
|
// TODO we should consider that "fastest" is very slow.
|
|
SharedHandle<Request> fastestRequest = _requestPool.front();
|
|
_requestPool.pop_front();
|
|
_inFlightRequests.push_back(fastestRequest);
|
|
return fastestRequest;
|
|
}
|
|
return SharedHandle<Request>();
|
|
}
|
|
|
|
class RequestFaster {
|
|
public:
|
|
bool operator()(const SharedHandle<Request>& lhs,
|
|
const SharedHandle<Request>& rhs) const
|
|
{
|
|
if(lhs->getPeerStat().isNull()) {
|
|
return false;
|
|
}
|
|
if(rhs->getPeerStat().isNull()) {
|
|
return true;
|
|
}
|
|
return
|
|
lhs->getPeerStat()->getAvgDownloadSpeed() > rhs->getPeerStat()->getAvgDownloadSpeed();
|
|
}
|
|
};
|
|
|
|
void FileEntry::storePool(const SharedHandle<Request>& request)
|
|
{
|
|
const SharedHandle<PeerStat>& peerStat = request->getPeerStat();
|
|
if(!peerStat.isNull()) {
|
|
// We need to calculate average download speed here in order to
|
|
// store Request in the right position in the pool.
|
|
peerStat->calculateAvgDownloadSpeed();
|
|
}
|
|
std::deque<SharedHandle<Request> >::iterator i =
|
|
std::lower_bound(_requestPool.begin(), _requestPool.end(), request,
|
|
RequestFaster());
|
|
_requestPool.insert(i, request);
|
|
}
|
|
|
|
void FileEntry::poolRequest(const SharedHandle<Request>& request)
|
|
{
|
|
removeRequest(request);
|
|
if(!request->removalRequested()) {
|
|
storePool(request);
|
|
}
|
|
}
|
|
|
|
bool FileEntry::removeRequest(const SharedHandle<Request>& request)
|
|
{
|
|
for(std::deque<SharedHandle<Request> >::iterator i =
|
|
_inFlightRequests.begin(), eoi = _inFlightRequests.end();
|
|
i != eoi; ++i) {
|
|
if((*i).get() == request.get()) {
|
|
_inFlightRequests.erase(i);
|
|
return true;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
void FileEntry::removeURIWhoseHostnameIs(const std::string& hostname)
|
|
{
|
|
std::deque<std::string> newURIs;
|
|
Request req;
|
|
for(std::deque<std::string>::const_iterator itr = _uris.begin(),
|
|
eoi = _uris.end(); itr != eoi; ++itr) {
|
|
if(((*itr).find(hostname) == std::string::npos) ||
|
|
(req.setUrl(*itr) && (req.getHost() != hostname))) {
|
|
newURIs.push_back(*itr);
|
|
}
|
|
}
|
|
if(_logger->debug()) {
|
|
_logger->debug("Removed %d duplicate hostname URIs for path=%s",
|
|
_uris.size()-newURIs.size(), getPath().c_str());
|
|
}
|
|
_uris = newURIs;
|
|
}
|
|
|
|
void FileEntry::removeIdenticalURI(const std::string& uri)
|
|
{
|
|
_uris.erase(std::remove(_uris.begin(), _uris.end(), uri), _uris.end());
|
|
}
|
|
|
|
void FileEntry::addURIResult(std::string uri, downloadresultcode::RESULT result)
|
|
{
|
|
_uriResults.push_back(URIResult(uri, result));
|
|
}
|
|
|
|
class FindURIResultByResult {
|
|
private:
|
|
downloadresultcode::RESULT _r;
|
|
public:
|
|
FindURIResultByResult(downloadresultcode::RESULT r):_r(r) {}
|
|
|
|
bool operator()(const URIResult& uriResult) const
|
|
{
|
|
return uriResult.getResult() == _r;
|
|
}
|
|
};
|
|
|
|
void FileEntry::extractURIResult
|
|
(std::deque<URIResult>& res, downloadresultcode::RESULT r)
|
|
{
|
|
std::deque<URIResult>::iterator i =
|
|
std::stable_partition(_uriResults.begin(), _uriResults.end(),
|
|
FindURIResultByResult(r));
|
|
std::copy(_uriResults.begin(), i, std::back_inserter(res));
|
|
_uriResults.erase(_uriResults.begin(), i);
|
|
}
|
|
|
|
void FileEntry::reuseUri(size_t num)
|
|
{
|
|
std::deque<std::string> uris = _spentUris;
|
|
std::sort(uris.begin(), uris.end());
|
|
uris.erase(std::unique(uris.begin(), uris.end()), uris.end());
|
|
|
|
std::vector<std::string> errorUris(_uriResults.size());
|
|
std::transform(_uriResults.begin(), _uriResults.end(),
|
|
errorUris.begin(), std::mem_fun_ref(&URIResult::getURI));
|
|
std::sort(errorUris.begin(), errorUris.end());
|
|
errorUris.erase(std::unique(errorUris.begin(), errorUris.end()),
|
|
errorUris.end());
|
|
|
|
std::vector<std::string> reusableURIs;
|
|
std::set_difference(uris.begin(), uris.end(),
|
|
errorUris.begin(), errorUris.end(),
|
|
std::back_inserter(reusableURIs));
|
|
size_t ininum = reusableURIs.size();
|
|
if(_logger->debug()) {
|
|
_logger->debug("Found %u reusable URIs", static_cast<unsigned int>(ininum));
|
|
}
|
|
// Reuse at least num URIs here to avoid to
|
|
// run this process repeatedly.
|
|
if(ininum > 0 && ininum < num) {
|
|
if(_logger->debug()) {
|
|
_logger->debug("fewer than num=%u", num);
|
|
}
|
|
for(size_t i = 0; i < num/ininum; ++i) {
|
|
_uris.insert(_uris.end(), reusableURIs.begin(), reusableURIs.end());
|
|
}
|
|
_uris.insert(_uris.end(), reusableURIs.begin(),
|
|
reusableURIs.begin()+(num%ininum));
|
|
if(_logger->debug()) {
|
|
_logger->debug("Duplication complete: now %u URIs for reuse",
|
|
static_cast<unsigned int>(_uris.size()));
|
|
}
|
|
}
|
|
}
|
|
|
|
void FileEntry::releaseRuntimeResource()
|
|
{
|
|
_requestPool.clear();
|
|
_inFlightRequests.clear();
|
|
_uriResults.clear();
|
|
}
|
|
|
|
template<typename InputIterator, typename T>
|
|
static InputIterator findRequestByUri
|
|
(InputIterator first, InputIterator last, const T& uri)
|
|
{
|
|
for(; first != last; ++first) {
|
|
if(!(*first)->removalRequested() && (*first)->getUrl() == uri) {
|
|
return first;
|
|
}
|
|
}
|
|
return last;
|
|
}
|
|
|
|
bool FileEntry::removeUri(const std::string& uri)
|
|
{
|
|
std::deque<std::string>::iterator itr =
|
|
std::find(_spentUris.begin(), _spentUris.end(), uri);
|
|
if(itr == _spentUris.end()) {
|
|
itr = std::find(_uris.begin(), _uris.end(), uri);
|
|
if(itr == _uris.end()) {
|
|
return false;
|
|
} else {
|
|
_uris.erase(itr);
|
|
return true;
|
|
}
|
|
} else {
|
|
_spentUris.erase(itr);
|
|
SharedHandle<Request> req;
|
|
std::deque<SharedHandle<Request> >::iterator riter =
|
|
findRequestByUri(_inFlightRequests.begin(), _inFlightRequests.end(), uri);
|
|
if(riter == _inFlightRequests.end()) {
|
|
riter = findRequestByUri(_requestPool.begin(), _requestPool.end(), uri);
|
|
if(riter == _requestPool.end()) {
|
|
return true;
|
|
} else {
|
|
req = *riter;
|
|
_requestPool.erase(riter);
|
|
}
|
|
} else {
|
|
req = *riter;
|
|
}
|
|
req->requestRemoval();
|
|
return true;
|
|
}
|
|
}
|
|
|
|
} // namespace aria2
|