2010-02-26 Tatsuhiro Tsujikawa <t-tujikawa@users.sourceforge.net>

Added MetalinkMetaurl class. It corresponds to metalink:metaurl
	element.
	* src/Makefile.am
	* src/Metalink2RequestGroup.cc
	* src/MetalinkEntry.cc
	* src/MetalinkEntry.h
	* src/MetalinkMetaurl.cc
	* src/MetalinkMetaurl.h
	* src/MetalinkParserController.cc
	* src/MetalinkParserController.h
	* src/MetalinkParserStateMachine.cc
	* src/MetalinkParserStateMachine.h
	* src/MetalinkParserStateV4Impl.cc
	* src/MetalinkParserStateV4Impl.h
	* test/MetalinkProcessorTest.cc
pull/1/head
Tatsuhiro Tsujikawa 2010-02-25 16:00:24 +00:00
parent 3880a5f71b
commit 15c1f64023
15 changed files with 333 additions and 28 deletions

View File

@ -1,3 +1,21 @@
2010-02-26 Tatsuhiro Tsujikawa <t-tujikawa@users.sourceforge.net>
Added MetalinkMetaurl class. It corresponds to metalink:metaurl
element.
* src/Makefile.am
* src/Metalink2RequestGroup.cc
* src/MetalinkEntry.cc
* src/MetalinkEntry.h
* src/MetalinkMetaurl.cc
* src/MetalinkMetaurl.h
* src/MetalinkParserController.cc
* src/MetalinkParserController.h
* src/MetalinkParserStateMachine.cc
* src/MetalinkParserStateMachine.h
* src/MetalinkParserStateV4Impl.cc
* src/MetalinkParserStateV4Impl.h
* test/MetalinkProcessorTest.cc
2010-02-25 Tatsuhiro Tsujikawa <t-tujikawa@users.sourceforge.net>
Added Metalink4 support. Currently, name attribute of

View File

@ -452,6 +452,7 @@ if ENABLE_METALINK
SRCS += Metalinker.cc Metalinker.h\
MetalinkEntry.cc MetalinkEntry.h\
MetalinkResource.cc MetalinkResource.h\
MetalinkMetaurl.cc MetalinkMetaurl.h\
MetalinkProcessor.h\
MetalinkParserController.cc MetalinkParserController.h\
MetalinkParserStateMachine.cc MetalinkParserStateMachine.h\

View File

@ -253,6 +253,7 @@ bin_PROGRAMS = aria2c$(EXEEXT)
@ENABLE_METALINK_TRUE@am__append_14 = Metalinker.cc Metalinker.h\
@ENABLE_METALINK_TRUE@ MetalinkEntry.cc MetalinkEntry.h\
@ENABLE_METALINK_TRUE@ MetalinkResource.cc MetalinkResource.h\
@ENABLE_METALINK_TRUE@ MetalinkMetaurl.cc MetalinkMetaurl.h\
@ENABLE_METALINK_TRUE@ MetalinkProcessor.h\
@ENABLE_METALINK_TRUE@ MetalinkParserController.cc MetalinkParserController.h\
@ENABLE_METALINK_TRUE@ MetalinkParserStateMachine.cc MetalinkParserStateMachine.h\
@ -583,7 +584,8 @@ am__libaria2c_a_SOURCES_DIST = Socket.h SocketCore.cc SocketCore.h \
LpdReceiveMessageCommand.cc LpdReceiveMessageCommand.h \
LpdDispatchMessageCommand.cc LpdDispatchMessageCommand.h \
Metalinker.cc Metalinker.h MetalinkEntry.cc MetalinkEntry.h \
MetalinkResource.cc MetalinkResource.h MetalinkProcessor.h \
MetalinkResource.cc MetalinkResource.h MetalinkMetaurl.cc \
MetalinkMetaurl.h MetalinkProcessor.h \
MetalinkParserController.cc MetalinkParserController.h \
MetalinkParserStateMachine.cc MetalinkParserStateMachine.h \
MetalinkParserState.h MetalinkParserStateImpl.cc \
@ -755,6 +757,7 @@ am__objects_6 =
@ENABLE_METALINK_TRUE@am__objects_14 = Metalinker.$(OBJEXT) \
@ENABLE_METALINK_TRUE@ MetalinkEntry.$(OBJEXT) \
@ENABLE_METALINK_TRUE@ MetalinkResource.$(OBJEXT) \
@ENABLE_METALINK_TRUE@ MetalinkMetaurl.$(OBJEXT) \
@ENABLE_METALINK_TRUE@ MetalinkParserController.$(OBJEXT) \
@ENABLE_METALINK_TRUE@ MetalinkParserStateMachine.$(OBJEXT) \
@ENABLE_METALINK_TRUE@ MetalinkParserStateImpl.$(OBJEXT) \
@ -1494,6 +1497,7 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Metalink2RequestGroup.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/MetalinkEntry.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/MetalinkHelper.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/MetalinkMetaurl.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/MetalinkParserController.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/MetalinkParserStateImpl.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/MetalinkParserStateMachine.Po@am__quote@

View File

@ -50,6 +50,7 @@
#include "TrueRequestGroupCriteria.h"
#include "MetalinkEntry.h"
#include "MetalinkResource.h"
#include "MetalinkMetaurl.h"
#include "FileEntry.h"
#include "A2STR.h"
#include "a2functional.h"
@ -175,19 +176,17 @@ Metalink2RequestGroup::createRequestGroup
}
}
entry->dropUnsupportedResource();
if(entry->resources.size() == 0) {
if(entry->resources.empty() && entry->metaurls.empty()) {
continue;
}
_logger->info(MSG_METALINK_QUEUEING, entry->getPath().c_str());
std::deque<SharedHandle<MetalinkResource> >::iterator itr =
std::find_if(entry->resources.begin(), entry->resources.end(), FindBitTorrentUrl());
#ifdef ENABLE_BITTORRENT
SharedHandle<RequestGroup> torrentRg;
// there is torrent entry
if(itr != entry->resources.end()) {
if(!entry->metaurls.empty()) {
entry->reorderMetaurlsByPriority();
// there is torrent entry
std::deque<std::string> uris;
uris.push_back((*itr)->url);
uris.push_back(entry->metaurls[0]->url);
{
std::deque<SharedHandle<RequestGroup> > result;
createRequestGroupForUri(result, option, uris,

View File

@ -37,6 +37,7 @@
#include <algorithm>
#include "MetalinkResource.h"
#include "MetalinkMetaurl.h"
#include "FileEntry.h"
#include "util.h"
#include "a2functional.h"
@ -133,10 +134,11 @@ void MetalinkEntry::setProtocolPriority(const std::string& protocol,
AddProtocolPriority(protocol, priorityToAdd));
}
class PrefOrder {
template<typename T>
class PriorityHigher {
public:
bool operator()(const SharedHandle<MetalinkResource>& res1,
const SharedHandle<MetalinkResource>& res2)
bool operator()(const SharedHandle<T>& res1,
const SharedHandle<T>& res2)
{
return res1->priority < res2->priority;
}
@ -145,7 +147,13 @@ public:
void MetalinkEntry::reorderResourcesByPriority() {
std::random_shuffle(resources.begin(), resources.end(),
*(SimpleRandomizer::getInstance().get()));
std::sort(resources.begin(), resources.end(), PrefOrder());
std::sort(resources.begin(), resources.end(),
PriorityHigher<MetalinkResource>());
}
void MetalinkEntry::reorderMetaurlsByPriority()
{
std::sort(metaurls.begin(), metaurls.end(),PriorityHigher<MetalinkMetaurl>());
}
class Supported:public std::unary_function<SharedHandle<MetalinkResource>, bool> {

View File

@ -47,6 +47,7 @@
namespace aria2 {
class MetalinkResource;
class MetalinkMetaurl;
class FileEntry;
#ifdef ENABLE_MESSAGE_DIGEST
class Checksum;
@ -61,6 +62,7 @@ public:
std::vector<std::string> languages;
std::vector<std::string> oses;
std::deque<SharedHandle<MetalinkResource> > resources;
std::vector<SharedHandle<MetalinkMetaurl> > metaurls;
int maxConnections; // Metalink3Spec
#ifdef ENABLE_MESSAGE_DIGEST
SharedHandle<Checksum> checksum;
@ -88,6 +90,8 @@ public:
void reorderResourcesByPriority();
void reorderMetaurlsByPriority();
bool containsLanguage(const std::string& lang) const
{
return

45
src/MetalinkMetaurl.cc Normal file
View File

@ -0,0 +1,45 @@
/* <!-- copyright */
/*
* aria2 - The high speed download utility
*
* Copyright (C) 2010 Tatsuhiro Tsujikawa
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*
* In addition, as a special exception, the copyright holders give
* permission to link the code of portions of this program with the
* OpenSSL library under certain conditions as described in each
* individual source file, and distribute linked combinations
* including the two.
* You must obey the GNU General Public License in all respects
* for all of the code used other than OpenSSL. If you modify
* file(s) with this exception, you may extend this exception to your
* version of the file(s), but you are not obligated to do so. If you
* do not wish to do so, delete this exception statement from your
* version. If you delete this exception statement from all source
* files in the program, then also delete it here.
*/
/* copyright --> */
#include "MetalinkMetaurl.h"
#include "MetalinkResource.h"
namespace aria2 {
const std::string MetalinkMetaurl::MEDIATYPE_TORRENT("torrent");
MetalinkMetaurl::MetalinkMetaurl():
priority(MetalinkResource::getLowestPriority()) {}
} // namespace aria2

57
src/MetalinkMetaurl.h Normal file
View File

@ -0,0 +1,57 @@
/* <!-- copyright */
/*
* aria2 - The high speed download utility
*
* Copyright (C) 2010 Tatsuhiro Tsujikawa
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*
* In addition, as a special exception, the copyright holders give
* permission to link the code of portions of this program with the
* OpenSSL library under certain conditions as described in each
* individual source file, and distribute linked combinations
* including the two.
* You must obey the GNU General Public License in all respects
* for all of the code used other than OpenSSL. If you modify
* file(s) with this exception, you may extend this exception to your
* version of the file(s), but you are not obligated to do so. If you
* do not wish to do so, delete this exception statement from your
* version. If you delete this exception statement from all source
* files in the program, then also delete it here.
*/
/* copyright --> */
#ifndef _D_METALINK_METAURL_H_
#define _D_METALINK_METAURL_H_
#include "common.h"
#include <string>
namespace aria2 {
class MetalinkMetaurl {
public:
std::string url;
std::string mediatype;
int priority;
MetalinkMetaurl();
static const std::string MEDIATYPE_TORRENT;
};
} // namespace aria2
#endif // _D_METALINK_METAURL_H_

View File

@ -40,6 +40,7 @@
#include "Metalinker.h"
#include "MetalinkEntry.h"
#include "MetalinkResource.h"
#include "MetalinkMetaurl.h"
#include "FileEntry.h"
#include "a2functional.h"
#include "A2STR.h"
@ -65,6 +66,7 @@ void MetalinkParserController::newEntryTransaction()
{
_tEntry.reset(new MetalinkEntry());
_tResource.reset();
_tMetaurl.reset();
#ifdef ENABLE_MESSAGE_DIGEST
_tChecksum.reset();
_tChunkChecksumV4.reset();
@ -229,7 +231,19 @@ void MetalinkParserController::commitResourceTransaction()
if(_tResource.isNull()) {
return;
}
#ifdef ENABLE_BITTORRENT
if(_tResource->type == MetalinkResource::TYPE_BITTORRENT) {
SharedHandle<MetalinkMetaurl> metaurl(new MetalinkMetaurl());
metaurl->url = _tResource->url;
metaurl->priority = _tResource->priority;
metaurl->mediatype = MetalinkMetaurl::MEDIATYPE_TORRENT;
_tEntry->metaurls.push_back(metaurl);
} else {
_tEntry->resources.push_back(_tResource);
}
#else // !ENABLE_BITTORRENT
_tEntry->resources.push_back(_tResource);
#endif // !ENABLE_BITTORRENT
_tResource.reset();
}
@ -520,4 +534,55 @@ void MetalinkParserController::cancelSignatureTransaction()
_tSignature.reset();
}
void MetalinkParserController::newMetaurlTransaction()
{
if(_tEntry.isNull()) {
return;
}
_tMetaurl.reset(new MetalinkMetaurl());
}
void MetalinkParserController::setURLOfMetaurl(const std::string& url)
{
if(_tMetaurl.isNull()) {
return;
}
_tMetaurl->url = url;
}
void MetalinkParserController::setMediatypeOfMetaurl
(const std::string& mediatype)
{
if(_tMetaurl.isNull()) {
return;
}
_tMetaurl->mediatype = mediatype;
}
void MetalinkParserController::setPriorityOfMetaurl(int priority)
{
if(_tMetaurl.isNull()) {
return;
}
_tMetaurl->priority = priority;
}
void MetalinkParserController::commitMetaurlTransaction()
{
if(_tMetaurl.isNull()) {
return;
}
#ifdef ENABLE_BITTORRENT
if(_tMetaurl->mediatype == MetalinkMetaurl::MEDIATYPE_TORRENT) {
_tEntry->metaurls.push_back(_tMetaurl);
}
#endif // ENABLE_BITTORRENT
_tMetaurl.reset();
}
void MetalinkParserController::cancelMetaurlTransaction()
{
_tMetaurl.reset();
}
} // namespace aria2

View File

@ -48,6 +48,7 @@ namespace aria2 {
class Metalinker;
class MetalinkEntry;
class MetalinkResource;
class MetalinkMetaurl;
class Signature;
#ifdef ENABLE_MESSAGE_DIGEST
@ -63,6 +64,7 @@ private:
SharedHandle<MetalinkResource> _tResource;
SharedHandle<MetalinkMetaurl> _tMetaurl;
#ifdef ENABLE_MESSAGE_DIGEST
SharedHandle<Checksum> _tChecksum;
@ -176,6 +178,18 @@ public:
void commitSignatureTransaction();
void cancelSignatureTransaction();
void newMetaurlTransaction();
void setURLOfMetaurl(const std::string& url);
void setMediatypeOfMetaurl(const std::string& mediatype);
void setPriorityOfMetaurl(int priority);
void commitMetaurlTransaction();
void cancelMetaurlTransaction();
};
} // namespace aria2

View File

@ -97,6 +97,8 @@ MetalinkParserState* MetalinkParserStateMachine::_signatureStateV4 =
new SignatureMetalinkParserStateV4();
MetalinkParserState* MetalinkParserStateMachine::_urlStateV4 =
new URLMetalinkParserStateV4();
MetalinkParserState* MetalinkParserStateMachine::_metaurlStateV4 =
new MetaurlMetalinkParserStateV4();
MetalinkParserStateMachine::MetalinkParserStateMachine():
_ctrl(new MetalinkParserController())
@ -229,6 +231,11 @@ void MetalinkParserStateMachine::setURLStateV4()
_stateStack.push(_urlStateV4);
}
void MetalinkParserStateMachine::setMetaurlStateV4()
{
_stateStack.push(_metaurlStateV4);
}
void MetalinkParserStateMachine::setSkipTagState()
{
_stateStack.push(_skipTagState);
@ -447,6 +454,37 @@ void MetalinkParserStateMachine::cancelSignatureTransaction()
_ctrl->cancelSignatureTransaction();
}
void MetalinkParserStateMachine::newMetaurlTransaction()
{
_ctrl->newMetaurlTransaction();
}
void MetalinkParserStateMachine::setURLOfMetaurl(const std::string& url)
{
_ctrl->setURLOfMetaurl(url);
}
void MetalinkParserStateMachine::setMediatypeOfMetaurl
(const std::string& mediatype)
{
_ctrl->setMediatypeOfMetaurl(mediatype);
}
void MetalinkParserStateMachine::setPriorityOfMetaurl(int priority)
{
_ctrl->setPriorityOfMetaurl(priority);
}
void MetalinkParserStateMachine::commitMetaurlTransaction()
{
_ctrl->commitMetaurlTransaction();
}
void MetalinkParserStateMachine::cancelMetaurlTransaction()
{
_ctrl->cancelMetaurlTransaction();
}
void MetalinkParserStateMachine::beginElement
(const std::string& localname,
const std::string& prefix,

View File

@ -85,6 +85,7 @@ private:
static MetalinkParserState* _pieceHashStateV4; // Metalink4Spec
static MetalinkParserState* _signatureStateV4;
static MetalinkParserState* _urlStateV4;
static MetalinkParserState* _metaurlStateV4;
public:
MetalinkParserStateMachine();
@ -130,6 +131,7 @@ public:
void setPieceHashStateV4(); // Metalink4Spec
void setSignatureStateV4();
void setURLStateV4();
void setMetaurlStateV4();
bool finished() const;
@ -227,6 +229,18 @@ public:
void cancelSignatureTransaction();
void newMetaurlTransaction();
void setURLOfMetaurl(const std::string& url);
void setMediatypeOfMetaurl(const std::string& mediatype);
void setPriorityOfMetaurl(int priority);
void commitMetaurlTransaction();
void cancelMetaurlTransaction();
bool needsCharactersBuffering() const;
const SharedHandle<Metalinker>& getResult() const

View File

@ -129,7 +129,7 @@ void FileMetalinkParserStateV4::beginElement
} else if(localname == OS) {
stm->setOSStateV4();
} else if(localname == METAURL) {
stm->setURLStateV4();
stm->setMetaurlStateV4();
// TODO currently NAME is ignored
int priority;
{
@ -156,9 +156,9 @@ void FileMetalinkParserStateV4::beginElement
mediatype = (*itr).value;
}
}
stm->newResourceTransaction();
stm->setPriorityOfResource(priority);
stm->setTypeOfResource(mediatype);
stm->newMetaurlTransaction();
stm->setPriorityOfMetaurl(priority);
stm->setMediatypeOfMetaurl(mediatype);
} else if(localname == URL) {
stm->setURLStateV4();
std::string location;
@ -363,4 +363,15 @@ void URLMetalinkParserStateV4::endElement
stm->commitResourceTransaction();
}
void MetaurlMetalinkParserStateV4::endElement
(MetalinkParserStateMachine* stm,
const std::string& localname,
const std::string& prefix,
const std::string& nsUri,
const std::string& characters)
{
stm->setURLOfMetaurl(characters);
stm->commitMetaurlTransaction();
}
} // namespace aria2

View File

@ -214,6 +214,21 @@ public:
}
};
class MetaurlMetalinkParserStateV4:public SkipTagMetalinkParserState
{
public:
virtual void endElement(MetalinkParserStateMachine* stm,
const std::string& localname,
const std::string& prefix,
const std::string& nsUri,
const std::string& characters);
virtual bool needsCharactersBuffering() const
{
return true;
}
};
} // namespace aria2
#endif // _D_METALINK_PARSER_STATE_IMPL_H_

View File

@ -11,6 +11,7 @@
#include "Metalinker.h"
#include "MetalinkEntry.h"
#include "MetalinkResource.h"
#include "MetalinkMetaurl.h"
#ifdef ENABLE_MESSAGE_DIGEST
# include "ChunkChecksum.h"
# include "Checksum.h"
@ -80,6 +81,7 @@ void MetalinkProcessorTest::testParseFileV4()
SharedHandle<MetalinkEntry> e;
SharedHandle<MetalinkResource> r;
SharedHandle<MetalinkMetaurl> mu;
CPPUNIT_ASSERT_EQUAL((size_t)1, m->entries.size());
e = m->entries[0];
@ -108,7 +110,7 @@ void MetalinkProcessorTest::testParseFileV4()
CPPUNIT_ASSERT_EQUAL(std::string("a signature"),
e->getSignature()->getBody());
CPPUNIT_ASSERT_EQUAL((size_t)3, e->resources.size());
CPPUNIT_ASSERT_EQUAL((size_t)2, e->resources.size());
r = e->resources[0];
CPPUNIT_ASSERT_EQUAL(std::string("ftp://ftp.example.com/example.ext"),
r->url);
@ -117,14 +119,16 @@ void MetalinkProcessorTest::testParseFileV4()
CPPUNIT_ASSERT_EQUAL(std::string("ftp"),
MetalinkResource::getTypeString(r->type));
CPPUNIT_ASSERT_EQUAL(-1, r->maxConnections);
r = e->resources[2];
#ifdef ENABLE_BITTORRENT
CPPUNIT_ASSERT_EQUAL((size_t)1, e->metaurls.size());
mu = e->metaurls[0];
CPPUNIT_ASSERT_EQUAL(std::string("http://example.com/example.ext.torrent"),
r->url);
CPPUNIT_ASSERT_EQUAL(2, r->priority);
CPPUNIT_ASSERT_EQUAL(std::string("bittorrent"),
MetalinkResource::getTypeString(r->type));
CPPUNIT_ASSERT_EQUAL(-1, r->maxConnections);
mu->url);
CPPUNIT_ASSERT_EQUAL(2, mu->priority);
CPPUNIT_ASSERT_EQUAL(std::string("torrent"), mu->mediatype);
#else // !ENABLE_BITTORRENT
CPPUNIT_ASSERT_EQUAL((size_t)0, e->metaurls.size());
#endif // !ENABLE_BITTORRENT
}
void MetalinkProcessorTest::testParseFileV4_dirtraversal()
@ -139,15 +143,23 @@ void MetalinkProcessorTest::testParseFileV4_attrs()
MetalinkProcessor proc;
SharedHandle<Metalinker> m = proc.parseFile("metalink4-attrs.xml");
CPPUNIT_ASSERT_EQUAL((size_t)1, m->entries.size());
CPPUNIT_ASSERT_EQUAL((size_t)6, m->entries[0]->resources.size());
std::deque<SharedHandle<MetalinkResource> > resources =
m->entries[0]->resources;
CPPUNIT_ASSERT_EQUAL((size_t)3, resources.size());
CPPUNIT_ASSERT_EQUAL(999999, resources[0]->priority);
CPPUNIT_ASSERT_EQUAL(999999, resources[1]->priority);
CPPUNIT_ASSERT_EQUAL(999999, resources[2]->priority);
CPPUNIT_ASSERT_EQUAL(999999, resources[3]->priority);
CPPUNIT_ASSERT_EQUAL(999999, resources[4]->priority);
CPPUNIT_ASSERT_EQUAL(999999, resources[5]->priority);
std::vector<SharedHandle<MetalinkMetaurl> > metaurls =
m->entries[0]->metaurls;
#ifdef ENABLE_BITTORRENT
CPPUNIT_ASSERT_EQUAL((size_t)3, metaurls.size());
CPPUNIT_ASSERT_EQUAL(999999, metaurls[0]->priority);
CPPUNIT_ASSERT_EQUAL(999999, metaurls[1]->priority);
CPPUNIT_ASSERT_EQUAL(999999, metaurls[2]->priority);
#else // !ENABLE_BITTORRENT
CPPUNIT_ASSERT_EQUAL((size_t)0, metaurls.size());
#endif // !ENABLE_BITTORRENT
}
void MetalinkProcessorTest::testParseFile()