aria2/src/Metalink2RequestGroup.cc

367 lines
14 KiB
C++

/* <!-- copyright */
/*
* aria2 - The high speed download utility
*
* Copyright (C) 2006 Tatsuhiro Tsujikawa
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*
* In addition, as a special exception, the copyright holders give
* permission to link the code of portions of this program with the
* OpenSSL library under certain conditions as described in each
* individual source file, and distribute linked combinations
* including the two.
* You must obey the GNU General Public License in all respects
* for all of the code used other than OpenSSL. If you modify
* file(s) with this exception, you may extend this exception to your
* version of the file(s), but you are not obligated to do so. If you
* do not wish to do so, delete this exception statement from your
* version. If you delete this exception statement from all source
* files in the program, then also delete it here.
*/
/* copyright --> */
#include "Metalink2RequestGroup.h"
#include <algorithm>
#include "RequestGroup.h"
#include "Option.h"
#include "LogFactory.h"
#include "Logger.h"
#include "prefs.h"
#include "util.h"
#include "message.h"
#include "DownloadContext.h"
#include "metalink_helper.h"
#include "BinaryStream.h"
#include "MemoryBufferPreDownloadHandler.h"
#include "TrueRequestGroupCriteria.h"
#include "MetalinkEntry.h"
#include "MetalinkResource.h"
#include "MetalinkMetaurl.h"
#include "FileEntry.h"
#include "A2STR.h"
#include "a2functional.h"
#include "download_helper.h"
#include "fmt.h"
#include "SegList.h"
#include "DownloadFailureException.h"
#ifdef ENABLE_BITTORRENT
# include "BtDependency.h"
# include "download_helper.h"
#endif // ENABLE_BITTORRENT
#ifdef ENABLE_MESSAGE_DIGEST
# include "Checksum.h"
# include "ChunkChecksum.h"
#endif // ENABLE_MESSAGE_DIGEST
namespace aria2 {
Metalink2RequestGroup::Metalink2RequestGroup() {}
namespace {
class AccumulateNonP2PUri {
private:
std::vector<std::string>& urisPtr;
public:
AccumulateNonP2PUri(std::vector<std::string>& urisPtr)
:urisPtr(urisPtr) {}
void operator()(const SharedHandle<MetalinkResource>& resource) {
switch(resource->type) {
case MetalinkResource::TYPE_HTTP:
case MetalinkResource::TYPE_HTTPS:
case MetalinkResource::TYPE_FTP:
urisPtr.push_back(resource->url);
break;
default:
break;
}
}
};
} // namespace
namespace {
class FindBitTorrentUri {
public:
FindBitTorrentUri() {}
bool operator()(const SharedHandle<MetalinkResource>& resource) {
if(resource->type == MetalinkResource::TYPE_BITTORRENT) {
return true;
} else {
return false;
}
}
};
} // namespace
void
Metalink2RequestGroup::generate
(std::vector<SharedHandle<RequestGroup> >& groups,
const std::string& metalinkFile,
const SharedHandle<Option>& option,
const std::string& baseUri)
{
std::vector<SharedHandle<MetalinkEntry> > entries;
metalink::parseAndQuery(entries, metalinkFile, option.get(), baseUri);
std::vector<SharedHandle<RequestGroup> > tempgroups;
createRequestGroup(tempgroups, entries, option);
SharedHandle<MetadataInfo> mi;
if(metalinkFile == DEV_STDIN) {
mi.reset(new MetadataInfo());
} else {
// TODO Downloads from local metalink file does not save neither
// its gid nor MetadataInfo's gid.
mi.reset(new MetadataInfo(GroupId::create(), metalinkFile));
}
setMetadataInfo(tempgroups.begin(), tempgroups.end(), mi);
groups.insert(groups.end(), tempgroups.begin(), tempgroups.end());
}
void
Metalink2RequestGroup::generate
(std::vector<SharedHandle<RequestGroup> >& groups,
const SharedHandle<BinaryStream>& binaryStream,
const SharedHandle<Option>& option,
const std::string& baseUri)
{
std::vector<SharedHandle<MetalinkEntry> > entries;
metalink::parseAndQuery(entries, binaryStream.get(), option.get(), baseUri);
std::vector<SharedHandle<RequestGroup> > tempgroups;
createRequestGroup(tempgroups, entries, option);
SharedHandle<MetadataInfo> mi(new MetadataInfo());
setMetadataInfo(tempgroups.begin(), tempgroups.end(), mi);
groups.insert(groups.end(), tempgroups.begin(), tempgroups.end());
}
void
Metalink2RequestGroup::createRequestGroup
(std::vector<SharedHandle<RequestGroup> >& groups,
const std::vector<SharedHandle<MetalinkEntry> >& entries,
const SharedHandle<Option>& optionTemplate)
{
if(entries.empty()) {
A2_LOG_NOTICE(EX_NO_RESULT_WITH_YOUR_PREFS);
return;
}
std::vector<std::string> locations;
if(optionTemplate->defined(PREF_METALINK_LOCATION)) {
const std::string& loc = optionTemplate->get(PREF_METALINK_LOCATION);
util::split(loc.begin(), loc.end(),
std::back_inserter(locations), ',', true);
for(std::vector<std::string>::iterator i = locations.begin(),
eoi = locations.end(); i != eoi; ++i) {
util::lowercase(*i);
}
}
std::string preferredProtocol;
if(optionTemplate->get(PREF_METALINK_PREFERRED_PROTOCOL) != V_NONE) {
preferredProtocol = optionTemplate->get(PREF_METALINK_PREFERRED_PROTOCOL);
}
for(std::vector<SharedHandle<MetalinkEntry> >::const_iterator i =
entries.begin(), eoi = entries.end(); i != eoi; ++i) {
(*i)->dropUnsupportedResource();
if((*i)->resources.empty() && (*i)->metaurls.empty()) {
continue;
}
(*i)->setLocationPriority
(locations, -MetalinkResource::getLowestPriority());
if(!preferredProtocol.empty()) {
(*i)->setProtocolPriority
(preferredProtocol, -MetalinkResource::getLowestPriority());
}
}
std::vector<SharedHandle<MetalinkEntry> > selectedEntries;
SegList<int> sgl;
util::parseIntSegments(sgl, optionTemplate->get(PREF_SELECT_FILE));
sgl.normalize();
if(!sgl.hasNext()) {
selectedEntries.assign(entries.begin(), entries.end());
} else {
selectedEntries.reserve(entries.size());
for(size_t i = 0, len = entries.size(); i < len && sgl.hasNext(); ++i) {
size_t j = sgl.peek()-1;
if(i == j) {
selectedEntries.push_back(entries[i]);
sgl.next();
}
}
}
std::for_each(selectedEntries.begin(), selectedEntries.end(),
mem_fun_sh(&MetalinkEntry::reorderMetaurlsByPriority));
std::vector<std::pair<std::string,
std::vector<SharedHandle<MetalinkEntry> > > > entryGroups;
metalink::groupEntryByMetaurlName(entryGroups, selectedEntries);
for(std::vector<std::pair<std::string,
std::vector<SharedHandle<MetalinkEntry> > > >::const_iterator itr =
entryGroups.begin(), eoi = entryGroups.end(); itr != eoi; ++itr) {
const std::string& metaurl = (*itr).first;
const std::vector<SharedHandle<MetalinkEntry> >& mes = (*itr).second;
A2_LOG_INFO(fmt("Processing metaurl group metaurl=%s", metaurl.c_str()));
#ifdef ENABLE_BITTORRENT
SharedHandle<RequestGroup> torrentRg;
if(!metaurl.empty()) {
std::vector<std::string> uris;
uris.push_back(metaurl);
{
std::vector<SharedHandle<RequestGroup> > result;
createRequestGroupForUri(result, optionTemplate, uris,
/* ignoreForceSequential = */true,
/* ignoreLocalPath = */true);
if(!uris.empty()) {
torrentRg = result[0];
}
}
if(torrentRg) {
torrentRg->setNumConcurrentCommand(1);
torrentRg->clearPreDownloadHandler();
torrentRg->clearPostDownloadHandler();
// remove "metalink" from Accept Type list to avoid loop in
// tranparent metalink
torrentRg->getDownloadContext()->setAcceptMetalink(false);
// make it in-memory download
SharedHandle<PreDownloadHandler> preh
(new MemoryBufferPreDownloadHandler());
SharedHandle<RequestGroupCriteria> cri(new TrueRequestGroupCriteria());
preh->setCriteria(cri);
torrentRg->addPreDownloadHandler(preh);
groups.push_back(torrentRg);
}
}
#endif // ENABLE_BITTORRENT
SharedHandle<Option> option = util::copy(optionTemplate);
SharedHandle<RequestGroup> rg(new RequestGroup(GroupId::create(), option));
SharedHandle<DownloadContext> dctx;
int numSplit = option->getAsInt(PREF_SPLIT);
int maxConn = option->getAsInt(PREF_MAX_CONNECTION_PER_SERVER);
if(mes.size() == 1) {
SharedHandle<MetalinkEntry> entry = mes[0];
A2_LOG_INFO(fmt(MSG_METALINK_QUEUEING, entry->getPath().c_str()));
entry->reorderResourcesByPriority();
std::vector<std::string> uris;
std::for_each(entry->resources.begin(), entry->resources.end(),
AccumulateNonP2PUri(uris));
// If piece hash is specified in the metalink,
// make segment size equal to piece hash size.
int32_t pieceLength;
#ifdef ENABLE_MESSAGE_DIGEST
if(!entry->chunkChecksum) {
pieceLength = option->getAsInt(PREF_PIECE_LENGTH);
} else {
pieceLength = entry->chunkChecksum->getPieceLength();
}
#else
pieceLength = option->getAsInt(PREF_PIECE_LENGTH);
#endif // ENABLE_MESSAGE_DIGEST
dctx.reset(new DownloadContext
(pieceLength,
entry->getLength(),
util::applyDir(option->get(PREF_DIR),
entry->file->getPath())));
dctx->getFirstFileEntry()->setUris(uris);
dctx->getFirstFileEntry()->setMaxConnectionPerServer(maxConn);
if(option->getAsBool(PREF_METALINK_ENABLE_UNIQUE_PROTOCOL)) {
dctx->getFirstFileEntry()->setUniqueProtocol(true);
}
#ifdef ENABLE_MESSAGE_DIGEST
if(entry->checksum) {
dctx->setDigest(entry->checksum->getHashType(),
entry->checksum->getDigest());
}
if(entry->chunkChecksum) {
dctx->setPieceHashes(entry->chunkChecksum->getHashType(),
entry->chunkChecksum->getPieceHashes().begin(),
entry->chunkChecksum->getPieceHashes().end());
}
#endif // ENABLE_MESSAGE_DIGEST
dctx->setSignature(entry->getSignature());
rg->setNumConcurrentCommand
(entry->maxConnections < 0 ?
numSplit : std::min(numSplit, entry->maxConnections));
} else {
dctx.reset(new DownloadContext());
// piece length is overridden by the one in torrent file.
dctx->setPieceLength(option->getAsInt(PREF_PIECE_LENGTH));
std::vector<SharedHandle<FileEntry> > fileEntries;
int64_t offset = 0;
for(std::vector<SharedHandle<MetalinkEntry> >::const_iterator i =
mes.begin(), eoi = mes.end(); i != eoi; ++i) {
A2_LOG_INFO(fmt("Metalink: Queueing %s for download as a member.",
(*i)->getPath().c_str()));
A2_LOG_DEBUG(fmt("originalName = %s", (*i)->metaurls[0]->name.c_str()));
(*i)->reorderResourcesByPriority();
std::vector<std::string> uris;
std::for_each((*i)->resources.begin(), (*i)->resources.end(),
AccumulateNonP2PUri(uris));
SharedHandle<FileEntry> fe
(new FileEntry
(util::applyDir(option->get(PREF_DIR), (*i)->file->getPath()),
(*i)->file->getLength(), offset, uris));
fe->setMaxConnectionPerServer(maxConn);
if(option->getAsBool(PREF_METALINK_ENABLE_UNIQUE_PROTOCOL)) {
fe->setUniqueProtocol(true);
}
fe->setOriginalName((*i)->metaurls[0]->name);
fileEntries.push_back(fe);
if(offset >
std::numeric_limits<int64_t>::max() - (*i)->file->getLength()) {
throw DOWNLOAD_FAILURE_EXCEPTION(fmt(EX_TOO_LARGE_FILE, offset));
}
offset += (*i)->file->getLength();
}
dctx->setFileEntries(fileEntries.begin(), fileEntries.end());
rg->setNumConcurrentCommand(numSplit);
}
rg->setDownloadContext(dctx);
rg->setPauseRequested(option->getAsBool(PREF_PAUSE));
removeOneshotOption(option);
// remove "metalink" from Accept Type list to avoid loop in
// tranparent metalink
dctx->setAcceptMetalink(false);
#ifdef ENABLE_BITTORRENT
// Inject depenency between rg and torrentRg here if
// torrentRg is true
if(torrentRg) {
SharedHandle<Dependency> dep(new BtDependency(rg.get(), torrentRg));
rg->dependsOn(dep);
torrentRg->belongsTo(rg->getGID());
// metadata download may take very long time. If URIs are
// available, give up metadata download in at most 30 seconds.
const time_t btStopTimeout = 30;
time_t currentBtStopTimeout =
torrentRg->getOption()->getAsInt(PREF_BT_STOP_TIMEOUT);
if(currentBtStopTimeout == 0 || currentBtStopTimeout > btStopTimeout) {
std::vector<SharedHandle<FileEntry> >::const_iterator i;
std::vector<SharedHandle<FileEntry> >::const_iterator eoi
= dctx->getFileEntries().end();
for(i = dctx->getFileEntries().begin(); i != eoi; ++i) {
if((*i)->getRemainingUris().empty()) {
break;
}
}
if(i == dctx->getFileEntries().end()) {
torrentRg->getOption()->put
(PREF_BT_STOP_TIMEOUT, util::itos(btStopTimeout));
}
}
}
#endif // ENABLE_BITTORRENT
groups.push_back(rg);
}
}
} // namespace aria2