github
/
aria2
mirror of https://github.com/aria2/aria2
1
0
Fork 0
Code Issues Releases Wiki Activity

2010-02-27 Tatsuhiro Tsujikawa <t-tujikawa@users.sourceforge.net> 

Supported name attribute of metalink::metaurl element and
	multi-file torrent with Metalink4.  Files with same metaurl are
	grouped and downloaded in one RequestGroup.
	* src/BtDependency.cc
	* src/FileEntry.h
	* src/Metalink2RequestGroup.cc
	* src/Metalink2RequestGroup.h
	* src/MetalinkEntry.cc
	* src/MetalinkEntry.h
	* src/MetalinkHelper.cc
	* src/MetalinkHelper.h
	* src/MetalinkMetaurl.cc
	* src/MetalinkMetaurl.h
	* src/MetalinkParserController.cc
	* src/RequestGroup.cc
	* src/RequestGroup.h
	* src/bittorrent_helper.cc
	* test/BittorrentHelperTest.cc
	* test/BtDependencyTest.cc
	* test/MetalinkHelperTest.cc
pull/1/head
Tatsuhiro Tsujikawa 2010-02-26 15:37:08 +00:00
parent 5032394c6a
commit 2a6775e80b
18 changed files with 500 additions and 157 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
ChangeLog
View File

@ -1,3 +1,26 @@
2010-02-27 Tatsuhiro Tsujikawa <t-tujikawa@users.sourceforge.net>
Supported name attribute of metalink::metaurl element and
multi-file torrent with Metalink4. Files with same metaurl are
grouped and downloaded in one RequestGroup.
* src/BtDependency.cc
* src/FileEntry.h
* src/Metalink2RequestGroup.cc
* src/Metalink2RequestGroup.h
* src/MetalinkEntry.cc
* src/MetalinkEntry.h
* src/MetalinkHelper.cc
* src/MetalinkHelper.h
* src/MetalinkMetaurl.cc
* src/MetalinkMetaurl.h
* src/MetalinkParserController.cc
* src/RequestGroup.cc
* src/RequestGroup.h
* src/bittorrent_helper.cc
* test/BittorrentHelperTest.cc
* test/BtDependencyTest.cc
* test/MetalinkHelperTest.cc
2010-02-26 Tatsuhiro Tsujikawa <t-tujikawa@users.sourceforge.net>
Store name attribute of metalink:metaurl element in MetalinkMetaurl.

62
src/BtDependency.cc
View File

@ -47,6 +47,7 @@
#include "File.h"
#include "bittorrent_helper.h"
#include "DlAbortEx.h"
#include "StringFormat.h"
namespace aria2 {
@ -58,6 +59,18 @@ BtDependency::BtDependency(const RequestGroupWeakHandle& dependant,
BtDependency::~BtDependency() {}
static void copyValues(const SharedHandle<FileEntry>& d,
const SharedHandle<FileEntry>& s)
{
d->setRequested(true);
d->setPath(s->getPath());
d->addUris(s->getRemainingUris().begin(),
s->getRemainingUris().end());
if(!s->isSingleHostMultiConnectionEnabled()) {
d->disableSingleHostMultiConnection();
}
}
bool BtDependency::resolve()
{
if(_dependee->getNumCommand() == 0 && _dependee->downloadFinished()) {
@ -80,23 +93,38 @@ bool BtDependency::resolve()
bittorrent::loadFromMemory
(content, context, File(dependee->getFirstFilePath()).getBasename());
}
if(context->getFileEntries().size() !=
_dependant->getDownloadContext()->getFileEntries().size()) {
throw DL_ABORT_EX("The number of file in torrent doesn't match to"
" the dependent.");
}
// Copy file path in _dependant's FileEntries to newly created
// context's FileEntries to endorse the path structure of
// _dependant. URIs and singleHostMultiConnection are also copied.
for(std::vector<SharedHandle<FileEntry> >::const_iterator s =
_dependant->getDownloadContext()->getFileEntries().begin(),
d = context->getFileEntries().begin();
d != context->getFileEntries().end(); ++s, ++d) {
(*d)->setPath((*s)->getPath());
(*d)->addUris((*s)->getRemainingUris().begin(),
(*s)->getRemainingUris().end());
if(!(*s)->isSingleHostMultiConnectionEnabled()) {
(*d)->disableSingleHostMultiConnection();
const std::vector<SharedHandle<FileEntry> >& fileEntries =
context->getFileEntries();
const std::vector<SharedHandle<FileEntry> >& dependantFileEntries =
_dependant->getDownloadContext()->getFileEntries();
// If dependant's FileEntry::getOriginalName() is empty, we
// assume that torrent is single file. In Metalink3, this is
// always assumed.
if(fileEntries.size() == 1 && dependantFileEntries.size() == 1 &&
dependantFileEntries[0]->getOriginalName().empty()) {
copyValues(fileEntries[0], dependantFileEntries[0]);
} else {
std::for_each(fileEntries.begin(), fileEntries.end(),
std::bind2nd(mem_fun_sh(&FileEntry::setRequested),false));
// Copy file path in _dependant's FileEntries to newly created
// context's FileEntries to endorse the path structure of
// _dependant. URIs and singleHostMultiConnection are also copied.
for(std::vector<SharedHandle<FileEntry> >::const_iterator s =
dependantFileEntries.begin(); s != dependantFileEntries.end();
++s){
std::vector<SharedHandle<FileEntry> >::const_iterator d =
context->getFileEntries().begin();
for(; d != context->getFileEntries().end(); ++d) {
if((*d)->getOriginalName() == (*s)->getOriginalName()) {
break;
}
}
if(d == context->getFileEntries().end()) {
throw DL_ABORT_EX
(StringFormat("No entry %s in torrent file",
(*s)->getOriginalName().c_str()).str());
}
copyValues(*d, *s);
}
}
} catch(RecoverableException& e) {

11
src/FileEntry.h
View File

@ -70,6 +70,7 @@ private:
// available.
std::deque<URIResult> _uriResults;
bool _singleHostMultiConnection;
std::string _originalName;
Logger* _logger;
void storePool(const SharedHandle<Request>& request);
@ -227,6 +228,16 @@ public:
void reuseUri(size_t num);
void releaseRuntimeResource();
void setOriginalName(const std::string& originalName)
{
_originalName = originalName;
}
const std::string& getOriginalName() const
{
return _originalName;
}
};
typedef SharedHandle<FileEntry> FileEntryHandle;

207
src/Metalink2RequestGroup.cc
View File

@ -136,57 +136,66 @@ void removeMetalinkContentTypes(const SharedHandle<RequestGroup>& group)
void
Metalink2RequestGroup::createRequestGroup
(std::deque<SharedHandle<RequestGroup> >& groups,
std::deque<SharedHandle<MetalinkEntry> > entries,
const std::deque<SharedHandle<MetalinkEntry> >& entries,
const SharedHandle<Option>& option)
{
if(entries.size() == 0) {
if(entries.empty()) {
_logger->notice(EX_NO_RESULT_WITH_YOUR_PREFS);
return;
}
std::deque<int32_t> selectIndexes =
util::parseIntRange(option->get(PREF_SELECT_FILE)).flush();
bool useIndex;
if(selectIndexes.size()) {
useIndex = true;
} else {
useIndex = false;
std::sort(selectIndexes.begin(), selectIndexes.end());
std::vector<SharedHandle<MetalinkEntry> > selectedEntries;
selectedEntries.reserve(entries.size());
std::deque<std::string> locations;
if(option->defined(PREF_METALINK_LOCATION)) {
util::split(option->get(PREF_METALINK_LOCATION),
std::back_inserter(locations), ",", true);
std::transform
(locations.begin(), locations.end(), locations.begin(), util::toLower);
}
int32_t count = 0;
for(std::deque<SharedHandle<MetalinkEntry> >::iterator itr = entries.begin(); itr != entries.end();
++itr, ++count) {
SharedHandle<MetalinkEntry>& entry = *itr;
if(option->defined(PREF_METALINK_LOCATION)) {
std::deque<std::string> locations;
util::split(option->get(PREF_METALINK_LOCATION),
std::back_inserter(locations), ",", true);
std::transform
(locations.begin(), locations.end(), locations.begin(), util::toLower);
entry->setLocationPriority
(locations, -MetalinkResource::getLowestPriority());
}
if(option->get(PREF_METALINK_PREFERRED_PROTOCOL) != V_NONE) {
entry->setProtocolPriority
(option->get(PREF_METALINK_PREFERRED_PROTOCOL),
-MetalinkResource::getLowestPriority());
}
if(useIndex) {
if(std::find(selectIndexes.begin(), selectIndexes.end(), count+1) ==
selectIndexes.end()) {
std::string preferredProtocol;
if(option->get(PREF_METALINK_PREFERRED_PROTOCOL) != V_NONE) {
preferredProtocol = option->get(PREF_METALINK_PREFERRED_PROTOCOL);
}
{
int32_t count = 1;
for(std::deque<SharedHandle<MetalinkEntry> >::const_iterator i =
entries.begin(); i != entries.end(); ++i, ++count) {
(*i)->dropUnsupportedResource();
if((*i)->resources.empty() && (*i)->metaurls.empty()) {
continue;
}
(*i)->setLocationPriority
(locations, -MetalinkResource::getLowestPriority());
if(!preferredProtocol.empty()) {
(*i)->setProtocolPriority
(preferredProtocol, -MetalinkResource::getLowestPriority());
}
if(selectIndexes.empty() ||
std::binary_search(selectIndexes.begin(), selectIndexes.end(), count)){
selectedEntries.push_back(*i);
}
}
entry->dropUnsupportedResource();
if(entry->resources.empty() && entry->metaurls.empty()) {
continue;
}
_logger->info(MSG_METALINK_QUEUEING, entry->getPath().c_str());
}
std::for_each(entries.begin(), entries.end(),
mem_fun_sh(&MetalinkEntry::reorderMetaurlsByPriority));
std::vector<std::pair<std::string,
std::vector<SharedHandle<MetalinkEntry> > > > entryGroups;
MetalinkHelper::groupEntryByMetaurlName(entryGroups, selectedEntries);
for(std::vector<std::pair<std::string,
std::vector<SharedHandle<MetalinkEntry> > > >::const_iterator itr =
entryGroups.begin(); itr != entryGroups.end(); ++itr) {
const std::string& metaurl = (*itr).first;
const std::vector<SharedHandle<MetalinkEntry> >& mes = (*itr).second;
_logger->info("Processing metaurl group metaurl=%s", metaurl.c_str());
#ifdef ENABLE_BITTORRENT
SharedHandle<RequestGroup> torrentRg;
if(!entry->metaurls.empty()) {
entry->reorderMetaurlsByPriority();
// there is torrent entry
if(!metaurl.empty()) {
std::deque<std::string> uris;
uris.push_back(entry->metaurls[0]->url);
uris.push_back(metaurl);
{
std::deque<SharedHandle<RequestGroup> > result;
createRequestGroupForUri(result, option, uris,
@ -213,62 +222,94 @@ Metalink2RequestGroup::createRequestGroup
}
}
#endif // ENABLE_BITTORRENT
entry->reorderResourcesByPriority();
std::deque<std::string> uris;
std::for_each(entry->resources.begin(), entry->resources.end(),
AccumulateNonP2PUrl(uris));
SharedHandle<RequestGroup> rg(new RequestGroup(option));
// If piece hash is specified in the metalink,
// make segment size equal to piece hash size.
size_t pieceLength;
SharedHandle<DownloadContext> dctx;
if(mes.size() == 1) {
SharedHandle<MetalinkEntry> entry = mes[0];
_logger->info(MSG_METALINK_QUEUEING, entry->getPath().c_str());
entry->reorderResourcesByPriority();
std::deque<std::string> uris;
std::for_each(entry->resources.begin(), entry->resources.end(),
AccumulateNonP2PUrl(uris));
// If piece hash is specified in the metalink,
// make segment size equal to piece hash size.
size_t pieceLength;
#ifdef ENABLE_MESSAGE_DIGEST
if(entry->chunkChecksum.isNull()) {
pieceLength = option->getAsInt(PREF_SEGMENT_SIZE);
} else {
pieceLength = entry->chunkChecksum->getChecksumLength();
}
#else
pieceLength = option->getAsInt(PREF_SEGMENT_SIZE);
#endif // ENABLE_MESSAGE_DIGEST
SharedHandle<DownloadContext> dctx
(new DownloadContext
(pieceLength,
entry->getLength(),
util::applyDir(option->get(PREF_DIR), entry->file->getPath())));
dctx->setDir(option->get(PREF_DIR));
dctx->getFirstFileEntry()->setUris(uris);
if(option->getAsBool(PREF_METALINK_ENABLE_UNIQUE_PROTOCOL)) {
dctx->getFirstFileEntry()->disableSingleHostMultiConnection();
}
#ifdef ENABLE_MESSAGE_DIGEST
if(entry->chunkChecksum.isNull()) {
if(!entry->checksum.isNull()) {
dctx->setChecksum(entry->checksum->getMessageDigest());
dctx->setChecksumHashAlgo(entry->checksum->getAlgo());
if(entry->chunkChecksum.isNull()) {
pieceLength = option->getAsInt(PREF_SEGMENT_SIZE);
} else {
pieceLength = entry->chunkChecksum->getChecksumLength();
}
} else {
dctx->setPieceHashes(entry->chunkChecksum->getChecksums().begin(),
entry->chunkChecksum->getChecksums().end());
dctx->setPieceHashAlgo(entry->chunkChecksum->getAlgo());
}
#else
pieceLength = option->getAsInt(PREF_SEGMENT_SIZE);
#endif // ENABLE_MESSAGE_DIGEST
dctx->setSignature(entry->getSignature());
dctx.reset(new DownloadContext
(pieceLength,
entry->getLength(),
util::applyDir(option->get(PREF_DIR),
entry->file->getPath())));
dctx->getFirstFileEntry()->setUris(uris);
if(option->getAsBool(PREF_METALINK_ENABLE_UNIQUE_PROTOCOL)) {
dctx->getFirstFileEntry()->disableSingleHostMultiConnection();
}
#ifdef ENABLE_MESSAGE_DIGEST
if(entry->chunkChecksum.isNull()) {
if(!entry->checksum.isNull()) {
dctx->setChecksum(entry->checksum->getMessageDigest());
dctx->setChecksumHashAlgo(entry->checksum->getAlgo());
}
} else {
dctx->setPieceHashes(entry->chunkChecksum->getChecksums().begin(),
entry->chunkChecksum->getChecksums().end());
dctx->setPieceHashAlgo(entry->chunkChecksum->getAlgo());
}
#endif // ENABLE_MESSAGE_DIGEST
dctx->setSignature(entry->getSignature());
rg->setNumConcurrentCommand
(entry->maxConnections < 0 ?
option->getAsInt(PREF_METALINK_SERVERS) :
std::min(option->getAsInt(PREF_METALINK_SERVERS),
static_cast<int32_t>(entry->maxConnections)));
} else {
dctx.reset(new DownloadContext());
// piece length is overridden by the one in torrent file.
dctx->setPieceLength(option->getAsInt(PREF_SEGMENT_SIZE));
std::vector<SharedHandle<FileEntry> > fileEntries;
off_t offset = 0;
for(std::deque<SharedHandle<MetalinkEntry> >::const_iterator i =
entries.begin(); i != entries.end(); ++i) {
_logger->info("Metalink: Queueing %s for download as a member.",
(*i)->getPath().c_str());
_logger->debug("originalName = %s", (*i)->metaurls[0]->name.c_str());
(*i)->reorderResourcesByPriority();
std::deque<std::string> uris;
std::for_each((*i)->resources.begin(), (*i)->resources.end(),
AccumulateNonP2PUrl(uris));
SharedHandle<FileEntry> fe
(new FileEntry
(util::applyDir(option->get(PREF_DIR), (*i)->file->getPath()),
(*i)->file->getLength(), offset, uris));
if(option->getAsBool(PREF_METALINK_ENABLE_UNIQUE_PROTOCOL)) {
fe->disableSingleHostMultiConnection();
}
fe->setOriginalName((*i)->metaurls[0]->name);
fileEntries.push_back(fe);
offset += (*i)->file->getLength();
}
dctx->setFileEntries(fileEntries.begin(), fileEntries.end());
rg->setNumConcurrentCommand(option->getAsInt(PREF_METALINK_SERVERS));
}
dctx->setDir(option->get(PREF_DIR));
rg->setDownloadContext(dctx);
rg->setNumConcurrentCommand
(entry->maxConnections < 0 ?
option->getAsInt(PREF_METALINK_SERVERS) :
std::min(option->getAsInt(PREF_METALINK_SERVERS),
static_cast<int32_t>(entry->maxConnections)));
// remove "metalink" from Accept Type list to avoid loop in tranparent
// metalink
// remove "metalink" from Accept Type list to avoid loop in
// tranparent metalink
removeMetalinkContentTypes(rg);
#ifdef ENABLE_BITTORRENT
// Inject depenency between rg and torrentRg here if torrentRg.isNull() == false
// Inject depenency between rg and torrentRg here if
// torrentRg.isNull() == false
if(!torrentRg.isNull()) {
SharedHandle<Dependency> dep(new BtDependency(rg, torrentRg));
rg->dependsOn(dep);
torrentRg->belongsTo(rg->getGID());
}
#endif // ENABLE_BITTORRENT

2
src/Metalink2RequestGroup.h
View File

@ -54,7 +54,7 @@ private:
void
createRequestGroup(std::deque<SharedHandle<RequestGroup> >& groups,
std::deque<SharedHandle<MetalinkEntry> > entries,
const std::deque<SharedHandle<MetalinkEntry> >& entries,
const SharedHandle<Option>& option);
public:
Metalink2RequestGroup();

4
src/MetalinkEntry.cc
View File

@ -51,7 +51,7 @@
namespace aria2 {
MetalinkEntry::MetalinkEntry():
file(0),
sizeKnown(false),
maxConnections(-1)
{}
@ -94,7 +94,7 @@ MetalinkEntry& MetalinkEntry::operator=(const MetalinkEntry& metalinkEntry)
return *this;
}
std::string MetalinkEntry::getPath() const
const std::string& MetalinkEntry::getPath() const
{
return file->getPath();
}

4
src/MetalinkEntry.h
View File

@ -61,6 +61,8 @@ public:
std::string version;
std::vector<std::string> languages;
std::vector<std::string> oses;
// True if size is specified in Metalink document.
bool sizeKnown;
std::deque<SharedHandle<MetalinkResource> > resources;
std::vector<SharedHandle<MetalinkMetaurl> > metaurls;
int maxConnections; // Metalink3Spec
@ -77,7 +79,7 @@ public:
MetalinkEntry& operator=(const MetalinkEntry& metalinkEntry);
std::string getPath() const;
const std::string& getPath() const;
uint64_t getLength() const;

38
src/MetalinkHelper.cc
View File

@ -41,6 +41,7 @@
#include "prefs.h"
#include "DlAbortEx.h"
#include "BinaryStream.h"
#include "MetalinkMetaurl.h"
namespace aria2 {
@ -79,4 +80,41 @@ void MetalinkHelper::query
option->get(PREF_METALINK_OS));
}
void MetalinkHelper::groupEntryByMetaurlName
(std::vector<
std::pair<std::string, std::vector<SharedHandle<MetalinkEntry> > > >& result,
const std::vector<SharedHandle<MetalinkEntry> >& entries)
{
for(std::vector<SharedHandle<MetalinkEntry> >::const_iterator eiter =
entries.begin(); eiter != entries.end(); ++eiter) {
if((*eiter)->metaurls.empty()) {
std::pair<std::string, std::vector<SharedHandle<MetalinkEntry> > > p;
p.second.push_back(*eiter);
result.push_back(p);
} else {
std::vector<
std::pair<std::string,
std::vector<SharedHandle<MetalinkEntry> > > >::iterator i =
result.begin();
if((*eiter)->metaurls[0]->name.empty() ||
!(*eiter)->sizeKnown) {
i = result.end();
}
for(; i != result.end(); ++i) {
if((*i).first == (*eiter)->metaurls[0]->url &&
!(*i).second[0]->metaurls[0]->name.empty()) {
(*i).second.push_back(*eiter);
break;
}
}
if(i == result.end()) {
std::pair<std::string, std::vector<SharedHandle<MetalinkEntry> > > p;
p.first = (*eiter)->metaurls[0]->url;
p.second.push_back(*eiter);
result.push_back(p);
}
}
}
}
} // namespace aria2

10
src/MetalinkHelper.h
View File

@ -36,9 +36,12 @@
#define _D_METALINK_HELPER_H_
#include "common.h"
#include "SharedHandle.h"
#include <string>
#include <deque>
#include <vector>
#include "SharedHandle.h"
namespace aria2 {
@ -65,6 +68,11 @@ public:
static void parseAndQuery
(std::deque<SharedHandle<MetalinkEntry> >& result,
const SharedHandle<BinaryStream>& binaryStream, const Option* option);
static void groupEntryByMetaurlName
(std::vector<
std::pair<std::string, std::vector<SharedHandle<MetalinkEntry> > > >& result,
const std::vector<SharedHandle<MetalinkEntry> >& entries);
};
} // namespace aria2

5
src/MetalinkMetaurl.cc
View File

@ -42,4 +42,9 @@ const std::string MetalinkMetaurl::MEDIATYPE_TORRENT("torrent");
MetalinkMetaurl::MetalinkMetaurl():
priority(MetalinkResource::getLowestPriority()) {}
MetalinkMetaurl::MetalinkMetaurl
(const std::string& url, const std::string& mediatype,
const std::string& name, int priority):
url(url), mediatype(mediatype), name(name), priority(priority) {}
} // namespace aria2

5
src/MetalinkMetaurl.h
View File

@ -45,11 +45,14 @@ class MetalinkMetaurl {
public:
std::string url;
std::string mediatype;
int priority;
std::string name;
int priority;
MetalinkMetaurl();
MetalinkMetaurl(const std::string& url, const std::string& mediatype,
const std::string& name, int priority);
static const std::string MEDIATYPE_TORRENT;
};

1
src/MetalinkParserController.cc
View File

@ -100,6 +100,7 @@ void MetalinkParserController::setFileLengthOfEntry(uint64_t length)
} else {
_tEntry->file->setLength(length);
}
_tEntry->sizeKnown = true;
}
void MetalinkParserController::setVersionOfEntry(const std::string& version)

129
src/RequestGroup.cc
View File

@ -205,6 +205,8 @@ void RequestGroup::createInitialCommand
SharedHandle<BtRegistry> btRegistry = e->getBtRegistry();
if(!btRegistry->getDownloadContext
(torrentAttrs[bittorrent::INFO_HASH].s()).isNull()) {
// TODO If metadataGetMode == false and each FileEntry has
// URI, then go without BT.
throw DOWNLOAD_FAILURE_EXCEPTION
(StringFormat
("InfoHash %s is already registered.",
@ -284,14 +286,7 @@ void RequestGroup::createInitialCommand
return;
}
// Remove the control file if download file doesn't exist
if(progressInfoFile->exists() && !_pieceStorage->getDiskAdaptor()->fileExists()) {
progressInfoFile->removeFile();
_logger->notice(MSG_REMOVED_DEFUNCT_CONTROL_FILE,
progressInfoFile->getFilename().c_str(),
_downloadContext->getBasePath().c_str());
}
removeDefunctControlFile(progressInfoFile);
{
uint64_t actualFileSize = _pieceStorage->getDiskAdaptor()->size();
if(actualFileSize == _downloadContext->getTotalLength()) {
@ -372,42 +367,80 @@ void RequestGroup::createInitialCommand
}
}
#endif // ENABLE_BITTORRENT
// TODO Currently, BitTorrent+WEB-Seeding is only way to download
// multiple files in one RequestGroup. In this context, we don't
// have BitTorrent, so add assertion here. This situation will be
// changed if Metalink spec is formalized to support multi-file
// torrent.
assert(_downloadContext->getFileEntries().size() == 1);
// TODO I assume here when totallength is set to DownloadContext and it is
// not 0, then filepath is also set DownloadContext correctly....
if(_option->getAsBool(PREF_DRY_RUN) ||
_downloadContext->getTotalLength() == 0) {
createNextCommand(commands, e, 1);
}else {
if(_downloadContext->getFileEntries().size() == 1) {
// TODO I assume here when totallength is set to DownloadContext and it is
// not 0, then filepath is also set DownloadContext correctly....
if(_option->getAsBool(PREF_DRY_RUN) ||
_downloadContext->getTotalLength() == 0) {
createNextCommand(commands, e, 1);
}else {
if(e->_requestGroupMan->isSameFileBeingDownloaded(this)) {
throw DOWNLOAD_FAILURE_EXCEPTION
(StringFormat(EX_DUPLICATE_FILE_DOWNLOAD,
_downloadContext->getBasePath().c_str()).str());
}
adjustFilename
(SharedHandle<BtProgressInfoFile>(new DefaultBtProgressInfoFile
(_downloadContext,
SharedHandle<PieceStorage>(),
_option.get())));
initPieceStorage();
BtProgressInfoFileHandle infoFile
(new DefaultBtProgressInfoFile(_downloadContext, _pieceStorage,
_option.get()));
if(!infoFile->exists() && downloadFinishedByFileLength()) {
_pieceStorage->markAllPiecesDone();
_logger->notice(MSG_DOWNLOAD_ALREADY_COMPLETED,
_gid, _downloadContext->getBasePath().c_str());
} else {
loadAndOpenFile(infoFile);
SharedHandle<CheckIntegrityEntry> checkIntegrityEntry
(new StreamCheckIntegrityEntry(this));
processCheckIntegrityEntry(commands, checkIntegrityEntry, e);
}
}
} else {
// In this context, multiple FileEntry objects are in
// DownloadContext.
if(e->_requestGroupMan->isSameFileBeingDownloaded(this)) {
throw DOWNLOAD_FAILURE_EXCEPTION
(StringFormat(EX_DUPLICATE_FILE_DOWNLOAD,
_downloadContext->getBasePath().c_str()).str());
}
adjustFilename
(SharedHandle<BtProgressInfoFile>(new DefaultBtProgressInfoFile
(_downloadContext,
SharedHandle<PieceStorage>(),
_option.get())));
initPieceStorage();
BtProgressInfoFileHandle infoFile
(new DefaultBtProgressInfoFile(_downloadContext, _pieceStorage,
_option.get()));
if(!infoFile->exists() && downloadFinishedByFileLength()) {
_pieceStorage->markAllPiecesDone();
_logger->notice(MSG_DOWNLOAD_ALREADY_COMPLETED,
_gid, _downloadContext->getBasePath().c_str());
} else {
loadAndOpenFile(infoFile);
SharedHandle<CheckIntegrityEntry> checkIntegrityEntry
(new StreamCheckIntegrityEntry(this));
processCheckIntegrityEntry(commands, checkIntegrityEntry, e);
if(_downloadContext->getFileEntries().size() > 1) {
_pieceStorage->setupFileFilter();
}
SharedHandle<DefaultBtProgressInfoFile> progressInfoFile
(new DefaultBtProgressInfoFile(_downloadContext,
_pieceStorage,
_option.get()));
removeDefunctControlFile(progressInfoFile);
// Call Load, Save and file allocation command here
if(progressInfoFile->exists()) {
// load .aria2 file if it exists.
progressInfoFile->load();
_pieceStorage->getDiskAdaptor()->openFile();
} else {
if(_pieceStorage->getDiskAdaptor()->fileExists()) {
if(!_option->getAsBool(PREF_CHECK_INTEGRITY) &&
!_option->getAsBool(PREF_ALLOW_OVERWRITE)) {
// TODO we need this->haltRequested = true?
throw DOWNLOAD_FAILURE_EXCEPTION
(StringFormat
(MSG_FILE_ALREADY_EXISTS,
_downloadContext->getBasePath().c_str()).str());
} else {
_pieceStorage->getDiskAdaptor()->openFile();
}
} else {
_pieceStorage->getDiskAdaptor()->openFile();
}
}
_progressInfoFile = progressInfoFile;
SharedHandle<CheckIntegrityEntry> checkIntegrityEntry
(new StreamCheckIntegrityEntry(this));
processCheckIntegrityEntry(commands, checkIntegrityEntry, e);
}
}
@ -543,6 +576,19 @@ void RequestGroup::adjustFilename
}
}
void RequestGroup::removeDefunctControlFile
(const SharedHandle<BtProgressInfoFile>& progressInfoFile)
{
// Remove the control file if download file doesn't exist
if(progressInfoFile->exists() &&
!_pieceStorage->getDiskAdaptor()->fileExists()) {
progressInfoFile->removeFile();
_logger->notice(MSG_REMOVED_DEFUNCT_CONTROL_FILE,
progressInfoFile->getFilename().c_str(),
_downloadContext->getBasePath().c_str());
}
}
void RequestGroup::loadAndOpenFile(const BtProgressInfoFileHandle& progressInfoFile)
{
try {
@ -550,14 +596,7 @@ void RequestGroup::loadAndOpenFile(const BtProgressInfoFileHandle& progressInfoF
_pieceStorage->getDiskAdaptor()->initAndOpenFile();
return;
}
// Remove the control file if download file doesn't exist
if(progressInfoFile->exists() && !_pieceStorage->getDiskAdaptor()->fileExists()) {
progressInfoFile->removeFile();
_logger->notice(MSG_REMOVED_DEFUNCT_CONTROL_FILE,
progressInfoFile->getFilename().c_str(),
_downloadContext->getBasePath().c_str());
}
removeDefunctControlFile(progressInfoFile);
if(progressInfoFile->exists()) {
progressInfoFile->load();
_pieceStorage->getDiskAdaptor()->openExistingFile();

3
src/RequestGroup.h
View File

@ -180,6 +180,9 @@ private:
// _uriResults, then last result code is returned. Otherwise
// returns downloadresultcode::UNKNOWN_ERROR.
downloadresultcode::RESULT downloadResult() const;
void removeDefunctControlFile
(const SharedHandle<BtProgressInfoFile>& progressInfoFile);
public:
// The copy of option is stored in RequestGroup object.
RequestGroup(const SharedHandle<Option>& option);

2
src/bittorrent_helper.cc
View File

@ -263,6 +263,7 @@ static void extractFileEntries
(new FileEntry(util::applyDir(ctx->getDir(), path),
fileLengthData.i(),
offset, uris));
fileEntry->setOriginalName(path);
fileEntries.push_back(fileEntry);
offset += fileEntry->getLength();
}
@ -291,6 +292,7 @@ static void extractFileEntries
SharedHandle<FileEntry> fileEntry
(new FileEntry(util::applyDir(ctx->getDir(), name),totalLength, 0,
uris));
fileEntry->setOriginalName(name);
fileEntries.push_back(fileEntry);
}
ctx->setFileEntries(fileEntries.begin(), fileEntries.end());

4
test/BittorrentHelperTest.cc
View File

@ -168,6 +168,8 @@ void BittorrentHelperTest::testGetFileEntries() {
SharedHandle<FileEntry> fileEntry1 = *itr;
CPPUNIT_ASSERT_EQUAL(std::string("./aria2-test/aria2/src/aria2c"),
fileEntry1->getPath());
CPPUNIT_ASSERT_EQUAL(std::string("aria2-test/aria2/src/aria2c"),
fileEntry1->getOriginalName());
itr++;
SharedHandle<FileEntry> fileEntry2 = *itr;
CPPUNIT_ASSERT_EQUAL(std::string("./aria2-test/aria2-0.2.2.tar.bz2"),
@ -186,6 +188,8 @@ void BittorrentHelperTest::testGetFileEntriesSingle() {
SharedHandle<FileEntry> fileEntry1 = *itr;
CPPUNIT_ASSERT_EQUAL(std::string("./aria2-0.8.2.tar.bz2"),
fileEntry1->getPath());
CPPUNIT_ASSERT_EQUAL(std::string("aria2-0.8.2.tar.bz2"),
fileEntry1->getOriginalName());
}
void BittorrentHelperTest::testGetTotalLength() {

70
test/BtDependencyTest.cc
View File

@ -23,6 +23,9 @@ class BtDependencyTest:public CppUnit::TestFixture {
CPPUNIT_TEST_SUITE(BtDependencyTest);
CPPUNIT_TEST(testResolve);
CPPUNIT_TEST(testResolve_originalNameNoMatch);
CPPUNIT_TEST(testResolve_singleFileWithoutOriginalName);
CPPUNIT_TEST(testResolve_multiFile);
CPPUNIT_TEST(testResolve_metadata);
CPPUNIT_TEST(testResolve_loadError);
CPPUNIT_TEST(testResolve_dependeeFailure);
@ -37,7 +40,9 @@ class BtDependencyTest:public CppUnit::TestFixture {
dctx->setDir("/tmp");
std::deque<std::string> uris;
uris.push_back("http://localhost/outfile.path");
dctx->getFirstFileEntry()->setUris(uris);
SharedHandle<FileEntry> fileEntry = dctx->getFirstFileEntry();
fileEntry->setUris(uris);
fileEntry->setOriginalName("aria2-0.8.2.tar.bz2");
dependant->setDownloadContext(dctx);
return dependant;
}
@ -65,6 +70,9 @@ public:
}
void testResolve();
void testResolve_originalNameNoMatch();
void testResolve_singleFileWithoutOriginalName();
void testResolve_multiFile();
void testResolve_metadata();
void testResolve_loadError();
void testResolve_dependeeFailure();
@ -93,6 +101,64 @@ void BtDependencyTest::testResolve()
CPPUNIT_ASSERT_EQUAL(std::string("/tmp/outfile.path"),
firstFileEntry->getPath());
CPPUNIT_ASSERT_EQUAL((size_t)1, firstFileEntry->getRemainingUris().size());
CPPUNIT_ASSERT(firstFileEntry->isRequested());
}
void BtDependencyTest::testResolve_originalNameNoMatch()
{
std::string filename = "single.torrent";
SharedHandle<RequestGroup> dependant = createDependant(_option);
dependant->getDownloadContext()->getFirstFileEntry()->setOriginalName
("aria2-1.1.0.tar.bz2");
SharedHandle<RequestGroup> dependee =
createDependee(_option, filename, File(filename).size());
dependee->getPieceStorage()->markAllPiecesDone();
BtDependency dep(dependant, dependee);
CPPUNIT_ASSERT(dep.resolve());
CPPUNIT_ASSERT(!dependant->getDownloadContext()->hasAttribute
(bittorrent::BITTORRENT));
}
void BtDependencyTest::testResolve_singleFileWithoutOriginalName()
{
std::string filename = "single.torrent";
SharedHandle<RequestGroup> dependant = createDependant(_option);
dependant->getDownloadContext()->getFirstFileEntry()->setOriginalName("");
SharedHandle<RequestGroup> dependee =
createDependee(_option, filename, File(filename).size());
dependee->getPieceStorage()->markAllPiecesDone();
BtDependency dep(dependant, dependee);
CPPUNIT_ASSERT(dep.resolve());
CPPUNIT_ASSERT(dependant->getDownloadContext()->hasAttribute
(bittorrent::BITTORRENT));
}
void BtDependencyTest::testResolve_multiFile()
{
std::string filename = "test.torrent";
SharedHandle<RequestGroup> dependant = createDependant(_option);
dependant->getDownloadContext()->getFirstFileEntry()->setOriginalName
("aria2-test/aria2/src/aria2c");
SharedHandle<RequestGroup> dependee =
createDependee(_option, filename, File(filename).size());
dependee->getPieceStorage()->markAllPiecesDone();
BtDependency dep(dependant, dependee);
CPPUNIT_ASSERT(dep.resolve());
CPPUNIT_ASSERT(dependant->getDownloadContext()->hasAttribute
(bittorrent::BITTORRENT));
const std::vector<SharedHandle<FileEntry> >& fileEntries =
dependant->getDownloadContext()->getFileEntries();
CPPUNIT_ASSERT_EQUAL(std::string("/tmp/outfile.path"),
fileEntries[0]->getPath());
CPPUNIT_ASSERT(fileEntries[0]->isRequested());
CPPUNIT_ASSERT_EQUAL(std::string("/tmp/aria2-test/aria2-0.2.2.tar.bz2"),
fileEntries[1]->getPath());
CPPUNIT_ASSERT(!fileEntries[1]->isRequested());
}
void BtDependencyTest::testResolve_metadata()
@ -121,6 +187,8 @@ void BtDependencyTest::testResolve_metadata()
CPPUNIT_ASSERT_EQUAL
(std::string("cd41c7fdddfd034a15a04d7ff881216e01c4ceaf"),
bittorrent::getInfoHashString(dependant->getDownloadContext()));
CPPUNIT_ASSERT
(dependant->getDownloadContext()->getFirstFileEntry()->isRequested());
}
void BtDependencyTest::testResolve_loadError()

77
test/MetalinkHelperTest.cc
View File

@ -1,8 +1,11 @@
#include "MetalinkHelper.h"
#include <cppunit/extensions/HelperMacros.h>
#include "MetalinkEntry.h"
#include "Option.h"
#include "prefs.h"
#include <cppunit/extensions/HelperMacros.h>
#include "MetalinkMetaurl.h"
namespace aria2 {
@ -11,16 +14,14 @@ class MetalinkHelperTest:public CppUnit::TestFixture {
CPPUNIT_TEST_SUITE(MetalinkHelperTest);
CPPUNIT_TEST(testParseAndQuery);
CPPUNIT_TEST(testParseAndQuery_version);
CPPUNIT_TEST(testGroupEntryByMetaurlName);
CPPUNIT_TEST_SUITE_END();
private:
public:
void setUp() {}
void tearDown() {}
void testParseAndQuery();
void testParseAndQuery_version();
void testGroupEntryByMetaurlName();
};
@ -45,4 +46,70 @@ void MetalinkHelperTest::testParseAndQuery_version()
CPPUNIT_ASSERT_EQUAL(std::string("aria2-0.5.1.tar.bz2"), entry->getPath());
}
void MetalinkHelperTest::testGroupEntryByMetaurlName()
{
std::vector<SharedHandle<MetalinkEntry> > entries;
SharedHandle<MetalinkEntry> e1(new MetalinkEntry());
e1->version = "1";
e1->sizeKnown = true;
// no name
e1->metaurls.push_back
(SharedHandle<MetalinkMetaurl>
(new MetalinkMetaurl("http://meta1", "torrent", "", 1)));
SharedHandle<MetalinkEntry> e2(new MetalinkEntry());
e2->version = "2";
e2->sizeKnown = true;
SharedHandle<MetalinkEntry> e3(new MetalinkEntry());
e3->version = "3";
e3->sizeKnown = true;
e3->metaurls.push_back
(SharedHandle<MetalinkMetaurl>
(new MetalinkMetaurl("http://meta2", "torrent", "f3", 1)));
SharedHandle<MetalinkEntry> e4(new MetalinkEntry());
e4->version = "4";
e4->sizeKnown = true;
e4->metaurls.push_back
(SharedHandle<MetalinkMetaurl>
(new MetalinkMetaurl("http://meta1", "torrent", "f4", 1)));
SharedHandle<MetalinkEntry> e5(new MetalinkEntry());
e5->version = "5";
// no size
e5->metaurls.push_back
(SharedHandle<MetalinkMetaurl>
(new MetalinkMetaurl("http://meta1", "torrent", "f5", 1)));
SharedHandle<MetalinkEntry> e6(new MetalinkEntry());
e6->version = "6";
e6->sizeKnown = true;
e6->metaurls.push_back
(SharedHandle<MetalinkMetaurl>
(new MetalinkMetaurl("http://meta1", "torrent", "f6", 1)));
entries.push_back(e1);
entries.push_back(e2);
entries.push_back(e3);
entries.push_back(e4);
entries.push_back(e5);
entries.push_back(e6);
std::vector<std::pair<std::string,
std::vector<SharedHandle<MetalinkEntry> > > > result;
MetalinkHelper::groupEntryByMetaurlName(result, entries);
CPPUNIT_ASSERT_EQUAL(std::string("http://meta1"), result[0].first);
CPPUNIT_ASSERT_EQUAL(std::string("1"), result[0].second[0]->version);
CPPUNIT_ASSERT_EQUAL(std::string(""), result[1].first);
CPPUNIT_ASSERT_EQUAL(std::string("2"), result[1].second[0]->version);
CPPUNIT_ASSERT_EQUAL(std::string("http://meta2"), result[2].first);
CPPUNIT_ASSERT_EQUAL(std::string("3"), result[2].second[0]->version);
CPPUNIT_ASSERT_EQUAL(std::string("http://meta1"), result[3].first);
CPPUNIT_ASSERT_EQUAL(std::string("4"), result[3].second[0]->version);
CPPUNIT_ASSERT_EQUAL(std::string("6"), result[3].second[1]->version);
}
} // namespace aria2