Rewritten DownloadHandlerConstants

DownloadHandlerConstants was simplified.  MIME type handling in Accept
header was also reworked.  DownloadContext's metalinkServerContacted_
is replaced with acceptMetalink_ and its boolean value is reverted.
RequestGroup and HttpRequest now do not hold vector of accepting
types.  HttpRequest has the flag acceptMetalink_ which will be set by
the same value of DownloadContext::accpetMetalink_ and if it is true,
Metalink MIME types are added to Accept header field.
pull/28/head
Tatsuhiro Tsujikawa 2012-09-22 23:19:41 +09:00
parent 9d294741fd
commit 461a542c5e
21 changed files with 118 additions and 196 deletions

View File

@ -59,10 +59,7 @@ BtPostDownloadHandler::BtPostDownloadHandler()
{
SharedHandle<RequestGroupCriteria> cri
(new ContentTypeRequestGroupCriteria
(DownloadHandlerConstants::getBtContentTypes().begin(),
DownloadHandlerConstants::getBtContentTypes().end(),
DownloadHandlerConstants::getBtExtensions().begin(),
DownloadHandlerConstants::getBtExtensions().end()));
(getBtContentTypes(), getBtExtensions()));
setCriteria(cri);
}

View File

@ -34,8 +34,6 @@
/* copyright --> */
#include "ContentTypeRequestGroupCriteria.h"
#include <algorithm>
#include "RequestGroup.h"
#include "util.h"
#include "FileEntry.h"
@ -43,19 +41,13 @@
namespace aria2 {
namespace {
template<typename InputIterator>
bool tailMatch
(InputIterator first, InputIterator last, const std::string& target)
{
for(; first != last; ++first) {
if(util::endsWith(target, *first)) {
return true;
}
}
return false;
}
} // namespace
ContentTypeRequestGroupCriteria::ContentTypeRequestGroupCriteria
(const char** contentTypes, const char** extensions)
: contentTypes_(contentTypes),
extensions_(extensions)
{}
ContentTypeRequestGroupCriteria::~ContentTypeRequestGroupCriteria() {}
bool ContentTypeRequestGroupCriteria::match
(const RequestGroup* requestGroup) const
@ -63,15 +55,18 @@ bool ContentTypeRequestGroupCriteria::match
if(requestGroup->getDownloadContext()->getFileEntries().size() != 1) {
return false;
}
if(tailMatch(extensions_.begin(), extensions_.end(),
requestGroup->getFirstFilePath())) {
return true;
} else {
return
std::find(contentTypes_.begin(), contentTypes_.end(),
requestGroup->getDownloadContext()->getFirstFileEntry()->
getContentType()) != contentTypes_.end();
for(size_t i = 0; extensions_[i]; ++i) {
if(util::iendsWith(requestGroup->getFirstFilePath(), extensions_[i])) {
return true;
}
}
for(size_t i = 0; contentTypes_[i]; ++i) {
if(util::strieq(requestGroup->getDownloadContext()->getFirstFileEntry()->
getContentType(), contentTypes_[i])) {
return true;
}
}
return false;
}
} // namespace aria2

View File

@ -44,16 +44,13 @@ namespace aria2 {
class ContentTypeRequestGroupCriteria:public RequestGroupCriteria
{
private:
std::vector<std::string> contentTypes_;
std::vector<std::string> extensions_;
const char** contentTypes_;
const char** extensions_;
public:
template<typename InputIterator>
ContentTypeRequestGroupCriteria(InputIterator contentTypeFirst,
InputIterator contentTypeLast,
InputIterator extensionFirst,
InputIterator extensionLast):
contentTypes_(contentTypeFirst, contentTypeLast),
extensions_(extensionFirst, extensionLast) {}
ContentTypeRequestGroupCriteria(const char** contentTypes,
const char** extensions);
virtual ~ContentTypeRequestGroupCriteria();
virtual bool match(const RequestGroup* requestGroup) const;
};

View File

@ -54,7 +54,7 @@ DownloadContext::DownloadContext():
attrs_(MAX_CTX_ATTR),
downloadStartTime_(0),
downloadStopTime_(downloadStartTime_),
metalinkServerContacted_(false) {}
acceptMetalink_(true) {}
DownloadContext::DownloadContext(int32_t pieceLength,
int64_t totalLength,
@ -66,7 +66,7 @@ DownloadContext::DownloadContext(int32_t pieceLength,
attrs_(MAX_CTX_ATTR),
downloadStartTime_(0),
downloadStopTime_(0),
metalinkServerContacted_(false)
acceptMetalink_(true)
{
SharedHandle<FileEntry> fileEntry(new FileEntry(path, totalLength, 0));
fileEntries_.push_back(fileEntry);

View File

@ -84,9 +84,9 @@ private:
Timer downloadStopTime_;
SharedHandle<Signature> signature_;
// This member variable is required to avoid to parse Metalink/HTTP
// Link header fields multiple times.
bool metalinkServerContacted_;
// This member variable is required to avoid to use parse Metalink
// (including both Metalink XML and Metalink/HTTP) twice.
bool acceptMetalink_;
public:
DownloadContext();
@ -226,13 +226,13 @@ public:
void releaseRuntimeResource();
void setMetalinkServerContacted(bool f)
void setAcceptMetalink(bool f)
{
metalinkServerContacted_ = f;
acceptMetalink_ = f;
}
bool getMetalinkServerContacted() const
bool getAcceptMetalink() const
{
return metalinkServerContacted_;
return acceptMetalink_;
}
};

View File

@ -2,7 +2,7 @@
/*
* aria2 - The high speed download utility
*
* Copyright (C) 2006 Tatsuhiro Tsujikawa
* Copyright (C) 2012 Tatsuhiro Tsujikawa
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@ -33,54 +33,57 @@
*/
/* copyright --> */
#include "DownloadHandlerConstants.h"
#include "array_fun.h"
namespace aria2 {
const char* DownloadHandlerConstants::METALINK_EXTENSIONS[] = {
namespace {
const char* METALINK_EXTENSIONS[] = {
".metalink", // Metalink3Spec
".meta4" // Metalink4Spec
".meta4", // Metalink4Spec
0
};
} // namespace
const char* DownloadHandlerConstants::METALINK_CONTENT_TYPES[] = {
const char** getMetalinkExtensions()
{
return METALINK_EXTENSIONS;
}
namespace {
const char* METALINK_CONTENT_TYPES[] = {
"application/metalink4+xml", // Metalink4Spec
"application/metalink+xml" // Metalink3Spec
"application/metalink+xml", // Metalink3Spec
0
};
} // namespace
const char* DownloadHandlerConstants::BT_EXTENSIONS[] = { ".torrent" };
const char** getMetalinkContentTypes()
{
return METALINK_CONTENT_TYPES;
}
const char* DownloadHandlerConstants::BT_CONTENT_TYPES[] = {
"application/x-bittorrent"
namespace {
const char* BT_EXTENSIONS[] = {
".torrent",
0
};
} // namespace
const std::vector<std::string>&
DownloadHandlerConstants::getMetalinkExtensions()
const char** getBtExtensions()
{
static const std::vector<std::string> l
(vbegin(METALINK_EXTENSIONS), vend(METALINK_EXTENSIONS));
return l;
return BT_EXTENSIONS;
}
const std::vector<std::string>&
DownloadHandlerConstants::getMetalinkContentTypes()
{
static const std::vector<std::string> l
(vbegin(METALINK_CONTENT_TYPES), vend(METALINK_CONTENT_TYPES));
return l;
}
namespace {
const char* BT_CONTENT_TYPES[] = {
"application/x-bittorrent",
0
};
} // namespace
const std::vector<std::string>& DownloadHandlerConstants::getBtExtensions()
const char** getBtContentTypes()
{
static const std::vector<std::string> l
(vbegin(BT_EXTENSIONS), vend(BT_EXTENSIONS));
return l;
}
const std::vector<std::string>& DownloadHandlerConstants::getBtContentTypes()
{
static const std::vector<std::string> l
(vbegin(BT_CONTENT_TYPES), vend(BT_CONTENT_TYPES));
return l;
return BT_CONTENT_TYPES;
}
} // namespace aria2

View File

@ -2,7 +2,7 @@
/*
* aria2 - The high speed download utility
*
* Copyright (C) 2006 Tatsuhiro Tsujikawa
* Copyright (C) 2012 Tatsuhiro Tsujikawa
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@ -36,30 +36,18 @@
#define D_DOWNLOAD_HANDLER_CONSTANTS_H
#include "common.h"
#include <string>
#include <vector>
namespace aria2 {
class DownloadHandlerConstants
{
public:
static const char* METALINK_EXTENSIONS[];
// These methods returns NULL-terminated list of c-strings.
static const std::vector<std::string>& getMetalinkExtensions();
const char** getMetalinkExtensions();
static const char* METALINK_CONTENT_TYPES[];
const char** getMetalinkContentTypes();
static const std::vector<std::string>& getMetalinkContentTypes();
const char** getBtExtensions();
static const char* BT_EXTENSIONS[];
static const std::vector<std::string>& getBtExtensions();
static const char* BT_CONTENT_TYPES[];
static const std::vector<std::string>& getBtContentTypes();
};
const char** getBtContentTypes();
} // namespace aria2

View File

@ -74,10 +74,7 @@ DownloadHandlerFactory::getMetalinkPreDownloadHandler()
RequestGroupCriteriaHandle criteria
(new ContentTypeRequestGroupCriteria
(DownloadHandlerConstants::getMetalinkContentTypes().begin(),
DownloadHandlerConstants::getMetalinkContentTypes().end(),
DownloadHandlerConstants::getMetalinkExtensions().begin(),
DownloadHandlerConstants::getMetalinkExtensions().end()));
(getMetalinkContentTypes(), getMetalinkExtensions()));
metalinkPreDownloadHandler_->setCriteria(criteria);
}
return metalinkPreDownloadHandler_;
@ -105,10 +102,7 @@ DownloadHandlerFactory::getBtPreDownloadHandler()
RequestGroupCriteriaHandle criteria
(new ContentTypeRequestGroupCriteria
(DownloadHandlerConstants::getBtContentTypes().begin(),
DownloadHandlerConstants::getBtContentTypes().end(),
DownloadHandlerConstants::getBtExtensions().begin(),
DownloadHandlerConstants::getBtExtensions().end()));
(getBtContentTypes(), getBtExtensions()));
btPreDownloadHandler_->setCriteria(criteria);
}
return btPreDownloadHandler_;

View File

@ -51,6 +51,7 @@
#include "TimeA2.h"
#include "array_fun.h"
#include "Request.h"
#include "DownloadHandlerConstants.h"
namespace aria2 {
@ -58,6 +59,7 @@ const std::string HttpRequest::USER_AGENT("aria2");
HttpRequest::HttpRequest():contentEncodingEnabled_(true),
userAgent_(USER_AGENT),
acceptMetalink_(false),
noCache_(true),
acceptGzip_(false),
endOffsetOverride_(0)
@ -164,10 +166,13 @@ std::string HttpRequest::createRequest()
builtinHds.reserve(20);
builtinHds.push_back(std::make_pair("User-Agent:", userAgent_));
std::string acceptTypes = "*/*";
for(std::vector<std::string>::const_iterator i = acceptTypes_.begin(),
eoi = acceptTypes_.end(); i != eoi; ++i) {
acceptTypes += ",";
acceptTypes += *i;
if(acceptMetalink_) {
// The mime types of Metalink are used for "transparent metalink".
const char** metalinkTypes = getMetalinkContentTypes();
for(size_t i = 0; metalinkTypes[i]; ++i) {
acceptTypes += ",";
acceptTypes += metalinkTypes[i];
}
}
builtinHds.push_back(std::make_pair("Accept:", acceptTypes));
if(contentEncodingEnabled_) {
@ -328,11 +333,6 @@ void HttpRequest::clearHeader()
headers_.clear();
}
void HttpRequest::addAcceptType(const std::string& type)
{
acceptTypes_.push_back(type);
}
void HttpRequest::setCookieStorage
(const SharedHandle<CookieStorage>& cookieStorage)
{

View File

@ -71,7 +71,8 @@ private:
std::vector<std::string> headers_;
std::vector<std::string> acceptTypes_;
// If true, metalink content types are sent in Accept header field.
bool acceptMetalink_;
SharedHandle<CookieStorage> cookieStorage_;
@ -172,10 +173,9 @@ public:
void addAcceptType(const std::string& type);
template<typename InputIterator>
void addAcceptType(InputIterator first, InputIterator last)
void setAcceptMetalink(bool f)
{
acceptTypes_.insert(acceptTypes_.end(), first, last);
acceptMetalink_ = f;
}
void setCookieStorage(const SharedHandle<CookieStorage>& cookieStorage);

View File

@ -103,8 +103,8 @@ createHttpRequest(const SharedHandle<Request>& req,
httpRequest->setCookieStorage(cookieStorage);
httpRequest->setAuthConfigFactory(authConfigFactory, option.get());
httpRequest->setProxyRequest(proxyRequest);
httpRequest->addAcceptType(rg->getAcceptTypes().begin(),
rg->getAcceptTypes().end());
httpRequest->setAcceptMetalink(rg->getDownloadContext()->
getAcceptMetalink());
if(option->getAsBool(PREF_HTTP_ACCEPT_GZIP)) {
httpRequest->enableAcceptGZip();
} else {

View File

@ -197,9 +197,9 @@ bool HttpResponseCommand::executeInternal()
}
if(!getPieceStorage()) {
// Metalink/HTTP
if(!getDownloadContext()->getMetalinkServerContacted()) {
if(getDownloadContext()->getAcceptMetalink()) {
if(httpHeader->defined(HttpHeader::LINK)) {
getDownloadContext()->setMetalinkServerContacted(true);
getDownloadContext()->setAcceptMetalink(false);
std::vector<MetalinkHttpEntry> entries;
httpResponse->getMetalinKHttpEntries(entries, getOption());
for(std::vector<MetalinkHttpEntry>::iterator i = entries.begin(),
@ -245,7 +245,7 @@ bool HttpResponseCommand::executeInternal()
}
}
if(!getPieceStorage()) {
util::removeMetalinkContentTypes(getRequestGroup());
getDownloadContext()->setAcceptMetalink(false);
int64_t totalLength = httpResponse->getEntityLength();
getFileEntry()->setLength(totalLength);
if(getFileEntry()->getPath().empty()) {

View File

@ -229,7 +229,7 @@ Metalink2RequestGroup::createRequestGroup
torrentRg->clearPostDownloadHandler();
// remove "metalink" from Accept Type list to avoid loop in
// tranparent metalink
util::removeMetalinkContentTypes(torrentRg);
torrentRg->getDownloadContext()->setAcceptMetalink(false);
// make it in-memory download
SharedHandle<PreDownloadHandler> preh
(new MemoryBufferPreDownloadHandler());
@ -328,7 +328,7 @@ Metalink2RequestGroup::createRequestGroup
removeOneshotOption(option);
// remove "metalink" from Accept Type list to avoid loop in
// tranparent metalink
util::removeMetalinkContentTypes(rg);
dctx->setAcceptMetalink(false);
#ifdef ENABLE_BITTORRENT
// Inject depenency between rg and torrentRg here if
// torrentRg is true

View File

@ -58,10 +58,7 @@ MetalinkPostDownloadHandler::MetalinkPostDownloadHandler()
{
SharedHandle<RequestGroupCriteria> cri
(new ContentTypeRequestGroupCriteria
(DownloadHandlerConstants::getMetalinkContentTypes().begin(),
DownloadHandlerConstants::getMetalinkContentTypes().end(),
DownloadHandlerConstants::getMetalinkExtensions().begin(),
DownloadHandlerConstants::getMetalinkExtensions().end()));
(getMetalinkContentTypes(), getMetalinkExtensions()));
setCriteria(cri);
}

View File

@ -154,12 +154,6 @@ RequestGroup::RequestGroup(const SharedHandle<Option>& option)
resumeFailureCount_(0)
{
fileAllocationEnabled_ = option_->get(PREF_FILE_ALLOCATION) != V_NONE;
// Add types to be sent as a Accept header value here.
// It would be good to put this value in Option so that user can tweak
// and add this list.
// The mime types of Metalink is used for `transparent metalink'.
addAcceptType(DownloadHandlerConstants::getMetalinkContentTypes().begin(),
DownloadHandlerConstants::getMetalinkContentTypes().end());
if(!option_->getAsBool(PREF_DRY_RUN)) {
initializePreDownloadHandler();
initializePostDownloadHandler();
@ -1213,19 +1207,6 @@ void RequestGroup::reportDownloadFinished()
#endif // ENABLE_BITTORRENT
}
void RequestGroup::addAcceptType(const std::string& type)
{
if(std::find(acceptTypes_.begin(), acceptTypes_.end(), type) == acceptTypes_.end()) {
acceptTypes_.push_back(type);
}
}
void RequestGroup::removeAcceptType(const std::string& type)
{
acceptTypes_.erase(std::remove(acceptTypes_.begin(), acceptTypes_.end(), type),
acceptTypes_.end());
}
void RequestGroup::setURISelector(const SharedHandle<URISelector>& uriSelector)
{
uriSelector_ = uriSelector;

View File

@ -138,8 +138,6 @@ private:
std::vector<SharedHandle<PostDownloadHandler> > postDownloadHandlers_;
std::vector<std::string> acceptTypes_;
SharedHandle<URISelector> uriSelector_;
Time lastModifiedTime_;
@ -410,26 +408,6 @@ public:
void reportDownloadFinished();
const std::vector<std::string>& getAcceptTypes() const
{
return acceptTypes_;
}
void addAcceptType(const std::string& type);
template<typename InputIterator>
void addAcceptType(InputIterator first, InputIterator last)
{
for(; first != last; ++first) {
if(std::find(acceptTypes_.begin(), acceptTypes_.end(), *first) ==
acceptTypes_.end()) {
acceptTypes_.push_back(*first);
}
}
}
void removeAcceptType(const std::string& type);
void setURISelector(const SharedHandle<URISelector>& uriSelector);
const SharedHandle<URISelector>& getURISelector() const

View File

@ -255,7 +255,7 @@ TrackerWatcherCommand::createRequestGroup(const std::string& uri)
// RequestGroup is not handled by RequestGroupMan.
rg->clearPreDownloadHandler();
rg->clearPostDownloadHandler();
util::removeMetalinkContentTypes(rg);
dctx->setAcceptMetalink(false);
A2_LOG_INFO(fmt("Creating tracker request group GID#%" PRId64 "", rg->getGID()));
return rg;
}

View File

@ -197,7 +197,7 @@ createBtRequestGroup(const std::string& metaInfoUri,
rg->setPauseRequested(option->getAsBool(PREF_PAUSE));
// Remove "metalink" from Accept Type list to avoid server from
// responding Metalink file for web-seeding URIs.
util::removeMetalinkContentTypes(rg);
dctx->setAcceptMetalink(false);
removeOneshotOption(option);
return rg;
}

View File

@ -1479,21 +1479,6 @@ bool inSameCidrBlock
return (s1[last] & mask) == (s2[last] & mask);
}
void removeMetalinkContentTypes(const SharedHandle<RequestGroup>& group)
{
removeMetalinkContentTypes(group.get());
}
void removeMetalinkContentTypes(RequestGroup* group)
{
for(std::vector<std::string>::const_iterator i =
DownloadHandlerConstants::getMetalinkContentTypes().begin(),
eoi = DownloadHandlerConstants::getMetalinkContentTypes().end();
i != eoi; ++i) {
group->removeAcceptType(*i);
}
}
namespace {
void executeHook
@ -1695,6 +1680,16 @@ bool tlsHostnameMatch(const std::string& pattern, const std::string& hostname)
ptWildcard+1, ptLeftLabelEnd);
}
bool strieq(const std::string& a, const char* b)
{
return strieq(a.begin(), a.end(), b);
}
bool strieq(const std::string& a, const std::string& b)
{
return strieq(a.begin(), a.end(), b.begin(), b.end());
}
bool startsWith(const std::string& a, const char* b)
{
return startsWith(a.begin(), a.end(), b);

View File

@ -625,6 +625,9 @@ bool strieq(InputIterator first, InputIterator last, const char* b)
return first == last && *b == '\0';
}
bool strieq(const std::string& a, const char* b);
bool strieq(const std::string& a, const std::string& b);
template<typename InputIterator1, typename InputIterator2>
bool startsWith
(InputIterator1 first1,
@ -762,9 +765,6 @@ std::string escapePath(const std::string& s);
bool inSameCidrBlock
(const std::string& ip1, const std::string& ip2, size_t bits);
void removeMetalinkContentTypes(const SharedHandle<RequestGroup>& group);
void removeMetalinkContentTypes(RequestGroup* group);
// No throw
void executeHookByOptName
(const SharedHandle<RequestGroup>& group, const Option* option,

View File

@ -35,7 +35,7 @@ class HttpRequestTest : public CppUnit::TestFixture {
CPPUNIT_TEST(testIsRangeSatisfied);
CPPUNIT_TEST(testUserAgent);
CPPUNIT_TEST(testAddHeader);
CPPUNIT_TEST(testAddAcceptType);
CPPUNIT_TEST(testAcceptMetalink);
CPPUNIT_TEST(testEnableAcceptEncoding);
CPPUNIT_TEST(testConditionalRequest);
CPPUNIT_TEST_SUITE_END();
@ -63,7 +63,7 @@ public:
void testIsRangeSatisfied();
void testUserAgent();
void testAddHeader();
void testAddAcceptType();
void testAcceptMetalink();
void testEnableAcceptEncoding();
void testConditionalRequest();
};
@ -758,11 +758,8 @@ void HttpRequestTest::testAddHeader()
CPPUNIT_ASSERT_EQUAL(expectedText, httpRequest.createRequest());
}
void HttpRequestTest::testAddAcceptType()
void HttpRequestTest::testAcceptMetalink()
{
std::string acceptTypes[] = { "cream/custard",
"muffin/chocolate" };
SharedHandle<Request> request(new Request());
request->setUri("http://localhost/archives/aria2-1.0.0.tar.bz2");
@ -770,12 +767,12 @@ void HttpRequestTest::testAddAcceptType()
httpRequest.disableContentEncoding();
httpRequest.setRequest(request);
httpRequest.setAuthConfigFactory(authConfigFactory_, option_.get());
httpRequest.addAcceptType(vbegin(acceptTypes), vend(acceptTypes));
httpRequest.setAcceptMetalink(true);
std::string expectedText =
"GET /archives/aria2-1.0.0.tar.bz2 HTTP/1.1\r\n"
"User-Agent: aria2\r\n"
"Accept: */*,cream/custard,muffin/chocolate\r\n"
"Accept: */*,application/metalink4+xml,application/metalink+xml\r\n"
"Host: localhost\r\n"
"Pragma: no-cache\r\n"
"Cache-Control: no-cache\r\n"