2010-08-31 Tatsuhiro Tsujikawa <t-tujikawa@users.sourceforge.net>

Fixed the bug which prevents HTTP redirection from working when
	downloading multiple files from same host.
	* src/AbstractCommand.cc
	* src/FileEntry.cc
pull/1/head
Tatsuhiro Tsujikawa 2010-08-31 11:17:38 +00:00
parent 3434be8110
commit 5cb09dabf5
3 changed files with 49 additions and 68 deletions

View File

@ -1,3 +1,10 @@
2010-08-31 Tatsuhiro Tsujikawa <t-tujikawa@users.sourceforge.net>
Fixed the bug which prevents HTTP redirection from working when
downloading multiple files from same host.
* src/AbstractCommand.cc
* src/FileEntry.cc
2010-08-29 Tatsuhiro Tsujikawa <t-tujikawa@users.sourceforge.net> 2010-08-29 Tatsuhiro Tsujikawa <t-tujikawa@users.sourceforge.net>
Release 1.10.1 Release 1.10.1

View File

@ -285,6 +285,7 @@ bool AbstractCommand::execute() {
} }
req_->addTryCount(); req_->addTryCount();
req_->resetRedirectCount(); req_->resetRedirectCount();
req_->resetUri();
const unsigned int maxTries = getOption()->getAsInt(PREF_MAX_TRIES); const unsigned int maxTries = getOption()->getAsInt(PREF_MAX_TRIES);
bool isAbort = maxTries != 0 && req_->getTryCount() >= maxTries; bool isAbort = maxTries != 0 && req_->getTryCount() >= maxTries;
if(isAbort) { if(isAbort) {

View File

@ -125,78 +125,51 @@ FileEntry::getRequest
const std::string& method) const std::string& method)
{ {
SharedHandle<Request> req; SharedHandle<Request> req;
Request r; if(requestPool_.empty()) {
std::vector<std::string> inFlightHosts; std::vector<std::string> inFlightHosts;
enumerateInFlightHosts(inFlightRequests_.begin(), inFlightRequests_.end(), enumerateInFlightHosts(inFlightRequests_.begin(), inFlightRequests_.end(),
std::back_inserter(inFlightHosts)); std::back_inserter(inFlightHosts));
for(int g = 0; g < 2; ++g) {
if(!requestPool_.empty()) { std::vector<std::string> pending;
for(std::deque<SharedHandle<Request> >::iterator i = requestPool_.begin(), std::vector<std::string> ignoreHost;
eoi = requestPool_.end(); i != eoi; ++i) { while(1) {
r.setUri((*i)->getUri()); std::string uri = selector->select(this, usedHosts);
if(findSecond(usedHosts.begin(), usedHosts.end(), r.getHost()) != if(uri.empty()) {
usedHosts.end()) { break;
continue;
}
if(std::count(inFlightHosts.begin(), inFlightHosts.end(), r.getHost())
>= static_cast<int>(maxConnectionPerServer_)) {
continue;
}
req = *i;
requestPool_.erase(i);
inFlightRequests_.push_back(req);
return req;
}
}
for(int g = 0; g < 2; ++g) {
std::vector<std::string> pending;
std::vector<std::string> ignoreHost;
while(1) {
std::string uri = selector->select(this, usedHosts);
if(uri.empty()) {
break;
}
req.reset(new Request());
if(req->setUri(uri)) {
if(std::count(inFlightHosts.begin(), inFlightHosts.end(),req->getHost())
>= static_cast<int>(maxConnectionPerServer_)) {
pending.push_back(uri);
ignoreHost.push_back(req->getHost());
req.reset();
continue;
} }
req->setReferer(referer); req.reset(new Request());
req->setMethod(method); if(req->setUri(uri)) {
spentUris_.push_back(uri); if(std::count(inFlightHosts.begin(),
inFlightRequests_.push_back(req); inFlightHosts.end(),req->getHost())
break; >= static_cast<int>(maxConnectionPerServer_)) {
pending.push_back(uri);
ignoreHost.push_back(req->getHost());
req.reset();
continue;
}
req->setReferer(referer);
req->setMethod(method);
spentUris_.push_back(uri);
inFlightRequests_.push_back(req);
break;
} else {
req.reset();
}
}
uris_.insert(uris_.begin(), pending.begin(), pending.end());
if(g == 0 && uriReuse && req.isNull() && uris_.size() == pending.size()) {
// Reuse URIs other than ones in pending
reuseUri(ignoreHost);
} else { } else {
req.reset(); break;
} }
} }
uris_.insert(uris_.begin(), pending.begin(), pending.end()); } else {
// TODO UriReuse is performed only when PREF_REUSE_URI is true. req = requestPool_.front();
if(g == 0 && uriReuse && req.isNull() && uris_.size() == pending.size()) { requestPool_.pop_front();
// Reuse URIs other than ones in pending inFlightRequests_.push_back(req);
reuseUri(ignoreHost); if(logger_->debug()) {
} else { logger_->debug("Picked up from pool: %s", req->getUri().c_str());
break;
}
}
if(req.isNull()) {
Request r;
for(std::deque<SharedHandle<Request> >::iterator i = requestPool_.begin(),
eoi = requestPool_.end(); i != eoi; ++i) {
r.setUri((*i)->getUri());
if(std::count(inFlightHosts.begin(), inFlightHosts.end(), r.getHost())
>= static_cast<int>(maxConnectionPerServer_)) {
continue;
}
req = *i;
requestPool_.erase(i);
inFlightRequests_.push_back(req);
return req;
} }
} }
return req; return req;