github
/
aria2
mirror of https://github.com/aria2/aria2
1
0
Fork 0
Code Issues Releases Wiki Activity

Support relative URI in Metalink file. 

If relative URI is found in Metalink file, aria2 resolves its full URI
contatenating the URI from which Metalink file is retrieved and
relative URI in Metalink file. This feature is not available if
Metalink file in local disk is specified in command line.
pull/1/head
Tatsuhiro Tsujikawa 2011-05-07 18:41:18 +09:00
parent e7d7233d54
commit ad5af56c17
14 changed files with 179 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/Metalink2RequestGroup.cc
View File

@ -110,10 +110,11 @@ void
Metalink2RequestGroup::generate
(std::vector<SharedHandle<RequestGroup> >& groups,
const std::string& metalinkFile,
const SharedHandle<Option>& option)
const SharedHandle<Option>& option,
const std::string& baseUri)
{
std::vector<SharedHandle<MetalinkEntry> > entries;
metalink::parseAndQuery(entries, metalinkFile, option.get());
metalink::parseAndQuery(entries, metalinkFile, option.get(), baseUri);
std::vector<SharedHandle<RequestGroup> > tempgroups;
createRequestGroup(tempgroups, entries, option);
SharedHandle<MetadataInfo> mi;
@ -130,10 +131,11 @@ void
Metalink2RequestGroup::generate
(std::vector<SharedHandle<RequestGroup> >& groups,
const SharedHandle<BinaryStream>& binaryStream,
const SharedHandle<Option>& option)
const SharedHandle<Option>& option,
const std::string& baseUri)
{
std::vector<SharedHandle<MetalinkEntry> > entries;
metalink::parseAndQuery(entries, binaryStream, option.get());
metalink::parseAndQuery(entries, binaryStream, option.get(), baseUri);
std::vector<SharedHandle<RequestGroup> > tempgroups;
createRequestGroup(tempgroups, entries, option);
SharedHandle<MetadataInfo> mi(new MetadataInfo());

10
src/Metalink2RequestGroup.h
View File

@ -36,10 +36,12 @@
#define D_METALINK_2_REQUEST_GROUP_H
#include "common.h"
#include "SharedHandle.h"
#include <string>
#include <vector>
#include "SharedHandle.h"
#include "A2STR.h"
namespace aria2 {
class Option;
@ -58,11 +60,13 @@ public:
void generate(std::vector<SharedHandle<RequestGroup> >& groups,
const std::string& metalinkFile,
const SharedHandle<Option>& option);
const SharedHandle<Option>& option,
const std::string& baseUri = A2STR::NIL);
void generate(std::vector<SharedHandle<RequestGroup> >& groups,
const SharedHandle<BinaryStream>& binaryStream,
const SharedHandle<Option>& option);
const SharedHandle<Option>& option,
const std::string& baseUri = A2STR::NIL);
};
} // namespace aria2

38
src/MetalinkParserController.cc
View File

@ -43,13 +43,17 @@
#include "FileEntry.h"
#include "a2functional.h"
#include "A2STR.h"
#include "uri.h"
#include "Signature.h"
#include "util.h"
#ifdef ENABLE_MESSAGE_DIGEST
# include "Checksum.h"
# include "ChunkChecksum.h"
# include "MessageDigest.h"
#endif // ENABLE_MESSAGE_DIGEST
#include "Signature.h"
#include "util.h"
#ifdef ENABLE_BITTORRENT
# include "magnet.h"
#endif // ENABLE_BITTORRENT
namespace aria2 {
@ -166,14 +170,15 @@ void MetalinkParserController::setURLOfResource(const std::string& url)
if(!tResource_) {
return;
}
tResource_->url = url;
// Metalink4Spec
if(tResource_->type == MetalinkResource::TYPE_UNKNOWN) {
// guess from URI sheme
std::string::size_type pos = url.find("://");
if(pos != std::string::npos) {
setTypeOfResource(url.substr(0, pos));
std::string u = uri::joinUri(baseUri_, url);
uri::UriStruct us;
if(uri::parse(us, u)) {
tResource_->url = u;
if(tResource_->type == MetalinkResource::TYPE_UNKNOWN) {
setTypeOfResource(us.protocol);
}
} else {
tResource_->url = url;
}
}
@ -563,7 +568,20 @@ void MetalinkParserController::setURLOfMetaurl(const std::string& url)
if(!tMetaurl_) {
return;
}
tMetaurl_->url = url;
#ifdef ENABLE_BITTORRENT
if(magnet::parse(url)) {
tMetaurl_->url = url;
} else
#endif // ENABLE_BITTORRENT
{
std::string u = uri::joinUri(baseUri_, url);
uri::UriStruct us;
if(uri::parse(us, u)) {
tMetaurl_->url = u;
} else {
tMetaurl_->url = url;
}
}
}
void MetalinkParserController::setMediatypeOfMetaurl

6
src/MetalinkParserController.h
View File

@ -81,6 +81,7 @@ private:
#endif // ENABLE_MESSAGE_DIGEST
SharedHandle<Signature> tSignature_;
std::string baseUri_;
public:
MetalinkParserController();
@ -190,6 +191,11 @@ public:
void commitMetaurlTransaction();
void cancelMetaurlTransaction();
void setBaseUri(const std::string& baseUri)
{
baseUri_ = baseUri;
}
};
} // namespace aria2

4
src/MetalinkParserStateMachine.cc
View File

@ -541,5 +541,9 @@ std::string MetalinkParserStateMachine::getErrorString() const
return error.str();
}
void MetalinkParserStateMachine::setBaseUri(const std::string& uri)
{
ctrl_->setBaseUri(uri);
}
} // namespace aria2

2
src/MetalinkParserStateMachine.h
View File

@ -266,6 +266,8 @@ public:
{
return ctrl_->getResult();
}
void setBaseUri(const std::string& uri);
};
} // namespace aria2

32
src/MetalinkPostDownloadHandler.cc
View File

@ -33,6 +33,9 @@
*/
/* copyright --> */
#include "MetalinkPostDownloadHandler.h"
#include <deque>
#include "RequestGroup.h"
#include "Metalink2RequestGroup.h"
#include "Logger.h"
@ -47,6 +50,7 @@
#include "DownloadContext.h"
#include "download_helper.h"
#include "fmt.h"
#include "FileEntry.h"
namespace aria2 {
@ -63,6 +67,31 @@ MetalinkPostDownloadHandler::MetalinkPostDownloadHandler()
MetalinkPostDownloadHandler::~MetalinkPostDownloadHandler() {}
namespace {
const std::string& getBaseUri(RequestGroup* requestGroup)
{
const SharedHandle<DownloadContext>& dctx =
requestGroup->getDownloadContext();
if(dctx->getFileEntries().empty()) {
return A2STR::NIL;
} else {
// TODO Check download result for each URI
const SharedHandle<FileEntry>& entry = dctx->getFirstFileEntry();
const std::deque<std::string>& spentUris = entry->getSpentUris();
if(spentUris.empty()) {
const std::deque<std::string>& remainingUris = entry->getRemainingUris();
if(remainingUris.empty()) {
return A2STR::NIL;
} else {
return remainingUris.front();
}
} else {
return spentUris.back();
}
}
}
} // namespace
void MetalinkPostDownloadHandler::getNextRequestGroups
(std::vector<SharedHandle<RequestGroup> >& groups,
RequestGroup* requestGroup)
@ -74,9 +103,10 @@ void MetalinkPostDownloadHandler::getNextRequestGroups
try {
diskAdaptor->openExistingFile();
//requestOption.put(PREF_DIR, requestGroup->getDownloadContext()->getDir());
const std::string& baseUri = getBaseUri(requestGroup);
std::vector<SharedHandle<RequestGroup> > newRgs;
Metalink2RequestGroup().generate(newRgs, diskAdaptor,
requestGroup->getOption());
requestGroup->getOption(), baseUri);
requestGroup->followedBy(newRgs.begin(), newRgs.end());
SharedHandle<MetadataInfo> mi =
createMetadataInfoFromFirstFileEntry(requestGroup->getDownloadContext());

10
src/XML2SAXMetalinkProcessor.cc
View File

@ -185,9 +185,12 @@ MetalinkProcessor::MetalinkProcessor() {}
MetalinkProcessor::~MetalinkProcessor() {}
SharedHandle<Metalinker>
MetalinkProcessor::parseFile(const std::string& filename)
MetalinkProcessor::parseFile
(const std::string& filename,
const std::string& baseUri)
{
stm_.reset(new MetalinkParserStateMachine());
stm_->setBaseUri(baseUri);
SharedHandle<SessionData> sessionData(new SessionData(stm_));
// Old libxml2(at least 2.7.6, Ubuntu 10.04LTS) does not read stdin
// when "/dev/stdin" is passed as filename while 2.7.7 does. So we
@ -216,9 +219,12 @@ MetalinkProcessor::parseFile(const std::string& filename)
}
SharedHandle<Metalinker>
MetalinkProcessor::parseFromBinaryStream(const SharedHandle<BinaryStream>& binaryStream)
MetalinkProcessor::parseFromBinaryStream
(const SharedHandle<BinaryStream>& binaryStream,
const std::string& baseUri)
{
stm_.reset(new MetalinkParserStateMachine());
stm_->setBaseUri(baseUri);
size_t bufSize = 4096;
unsigned char buf[bufSize];

8
src/XML2SAXMetalinkProcessor.h
View File

@ -43,6 +43,7 @@
#include <libxml/xpath.h>
#include "SharedHandle.h"
#include "A2STR.h"
namespace aria2 {
@ -58,10 +59,13 @@ public:
~MetalinkProcessor();
SharedHandle<Metalinker> parseFile(const std::string& filename);
SharedHandle<Metalinker> parseFile
(const std::string& filename,
const std::string& baseUri = A2STR::NIL);
SharedHandle<Metalinker> parseFromBinaryStream
(const SharedHandle<BinaryStream>& binaryStream);
(const SharedHandle<BinaryStream>& binaryStream,
const std::string& baseUri = A2STR::NIL);
};
} // namespace aria2

11
src/metalink_helper.cc
View File

@ -65,20 +65,23 @@ void query
void parseAndQuery
(std::vector<SharedHandle<MetalinkEntry> >& result,
const std::string& filename,
const Option* option)
const Option* option,
const std::string& baseUri)
{
MetalinkProcessor proc;
SharedHandle<Metalinker> metalinker = proc.parseFile(filename);
SharedHandle<Metalinker> metalinker = proc.parseFile(filename, baseUri);
query(result, metalinker, option);
}
void parseAndQuery
(std::vector<SharedHandle<MetalinkEntry> >& result,
const SharedHandle<BinaryStream>& binaryStream,
const Option* option)
const Option* option,
const std::string& baseUri)
{
MetalinkProcessor proc;
SharedHandle<Metalinker> metalinker =proc.parseFromBinaryStream(binaryStream);
SharedHandle<Metalinker> metalinker =
proc.parseFromBinaryStream(binaryStream, baseUri);
query(result, metalinker, option);
}

7
src/metalink_helper.h
View File

@ -41,6 +41,7 @@
#include <vector>
#include "SharedHandle.h"
#include "A2STR.h"
namespace aria2 {
@ -54,12 +55,14 @@ namespace metalink {
void parseAndQuery
(std::vector<SharedHandle<MetalinkEntry> >& result,
const std::string& filename,
const Option* option);
const Option* option,
const std::string& baseUri = A2STR::NIL);
void parseAndQuery
(std::vector<SharedHandle<MetalinkEntry> >& result,
const SharedHandle<BinaryStream>& binaryStream,
const Option* option);
const Option* option,
const std::string& baseUri = A2STR::NIL);
void groupEntryByMetaurlName
(std::vector<

42
test/MetalinkParserControllerTest.cc
View File

@ -20,6 +20,7 @@ class MetalinkParserControllerTest:public CppUnit::TestFixture {
CPPUNIT_TEST_SUITE(MetalinkParserControllerTest);
CPPUNIT_TEST(testEntryTransaction);
CPPUNIT_TEST(testResourceTransaction);
CPPUNIT_TEST(testResourceTransaction_withBaseUri);
CPPUNIT_TEST(testMetaurlTransaction);
#ifdef ENABLE_MESSAGE_DIGEST
CPPUNIT_TEST(testChecksumTransaction);
@ -38,6 +39,7 @@ public:
void testEntryTransaction();
void testResourceTransaction();
void testResourceTransaction_withBaseUri();
void testMetaurlTransaction();
#ifdef ENABLE_MESSAGE_DIGEST
void testChecksumTransaction();
@ -110,6 +112,46 @@ void MetalinkParserControllerTest::testResourceTransaction()
}
}
void MetalinkParserControllerTest::testResourceTransaction_withBaseUri()
{
MetalinkParserController ctrl;
ctrl.setBaseUri("http://base/dir/file");
ctrl.newEntryTransaction();
ctrl.newResourceTransaction();
ctrl.setURLOfResource("aria2.tar.bz2");
ctrl.commitResourceTransaction();
#ifdef ENABLE_BITTORRENT
ctrl.newMetaurlTransaction();
ctrl.setURLOfMetaurl("/meta/aria2.tar.bz2.torrent");
ctrl.setMediatypeOfMetaurl("torrent");
ctrl.commitMetaurlTransaction();
ctrl.newMetaurlTransaction();
ctrl.setURLOfMetaurl("magnet:?xt=urn:btih:248d0a1cd08284299de78d5c1ed359bb46717d8c");
ctrl.setMediatypeOfMetaurl("torrent");
ctrl.commitMetaurlTransaction();
#endif // ENABLE_BITTORRENT
ctrl.commitEntryTransaction();
{
SharedHandle<Metalinker> m = ctrl.getResult();
CPPUNIT_ASSERT_EQUAL((size_t)1, m->getEntries()[0]->resources.size());
SharedHandle<MetalinkResource> res = m->getEntries()[0]->resources[0];
CPPUNIT_ASSERT_EQUAL(std::string("http://base/dir/aria2.tar.bz2"),
res->url);
CPPUNIT_ASSERT_EQUAL(MetalinkResource::TYPE_HTTP, res->type);
#ifdef ENABLE_BITTORRENT
CPPUNIT_ASSERT_EQUAL((size_t)2, m->getEntries()[0]->metaurls.size());
SharedHandle<MetalinkMetaurl> metaurl = m->getEntries()[0]->metaurls[0];
CPPUNIT_ASSERT_EQUAL(std::string("http://base/meta/aria2.tar.bz2.torrent"),
metaurl->url);
metaurl = m->getEntries()[0]->metaurls[1];
CPPUNIT_ASSERT_EQUAL(std::string("magnet:?xt=urn:btih:248d0a1cd08284299de78d5c1ed359bb46717d8c"),
metaurl->url);
#endif // ENABLE_BITTORRENT
}
}
void MetalinkParserControllerTest::testMetaurlTransaction()
{
MetalinkParserController ctrl;

21
test/MetalinkPostDownloadHandlerTest.cc
View File

@ -17,6 +17,7 @@ class MetalinkPostDownloadHandlerTest:public CppUnit::TestFixture {
CPPUNIT_TEST(testCanHandle_extension);
CPPUNIT_TEST(testCanHandle_contentType);
CPPUNIT_TEST(testGetNextRequestGroups);
CPPUNIT_TEST(testGetNextRequestGroups_withBaseUri);
CPPUNIT_TEST_SUITE_END();
private:
SharedHandle<Option> option_;
@ -29,6 +30,7 @@ public:
void testCanHandle_extension();
void testCanHandle_contentType();
void testGetNextRequestGroups();
void testGetNextRequestGroups_withBaseUri();
};
@ -83,4 +85,23 @@ void MetalinkPostDownloadHandlerTest::testGetNextRequestGroups()
#endif // ENABLE_BITTORRENT
}
void MetalinkPostDownloadHandlerTest::testGetNextRequestGroups_withBaseUri()
{
SharedHandle<DownloadContext> dctx
(new DownloadContext(1024, 0, A2_TEST_DIR"/base_uri.xml"));
dctx->getFirstFileEntry()->addUri("http://base/dir/base_uri.xml");
RequestGroup rg(option_);
rg.setDownloadContext(dctx);
rg.initPieceStorage();
rg.getPieceStorage()->getDiskAdaptor()->enableReadOnly();
MetalinkPostDownloadHandler handler;
std::vector<SharedHandle<RequestGroup> > groups;
handler.getNextRequestGroups(groups, &rg);
CPPUNIT_ASSERT_EQUAL((size_t)1, groups.size());
CPPUNIT_ASSERT_EQUAL(std::string("http://base/dir/example.ext"),
groups[0]->getDownloadContext()->
getFirstFileEntry()->getRemainingUris()[0]);
}
} // namespace aria2

6
test/base_uri.xml Normal file
View File

@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<metalink xmlns="urn:ietf:params:xml:ns:metalink">
<file name="example.ext">
<url>example.ext</url>
</file>
</metalink>