github
/
aria2
mirror of https://github.com/aria2/aria2
1
0
Fork 0
Code Issues Releases Wiki Activity
aria2/src/RequestGroup.cc

1269 lines
39 KiB
C++
Raw Blame History

/* <!-- copyright */
/*
* aria2 - The high speed download utility
*
* Copyright (C) 2006 Tatsuhiro Tsujikawa
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*
* In addition, as a special exception, the copyright holders give
* permission to link the code of portions of this program with the
* OpenSSL library under certain conditions as described in each
* individual source file, and distribute linked combinations
* including the two.
* You must obey the GNU General Public License in all respects
* for all of the code used other than OpenSSL. If you modify
* file(s) with this exception, you may extend this exception to your
* version of the file(s), but you are not obligated to do so. If you
* do not wish to do so, delete this exception statement from your
* version. If you delete this exception statement from all source
* files in the program, then also delete it here.
*/
/* copyright --> */
#include "RequestGroup.h"
#include <cassert>
#include <algorithm>
#include "PostDownloadHandler.h"
#include "DownloadEngine.h"
#include "SegmentMan.h"
#include "NullProgressInfoFile.h"
#include "Dependency.h"
#include "prefs.h"
#include "CreateRequestCommand.h"
#include "File.h"
#include "message.h"
#include "util.h"
#include "LogFactory.h"
#include "Logger.h"
#include "DiskAdaptor.h"
#include "DiskWriterFactory.h"
#include "RecoverableException.h"
#include "StreamCheckIntegrityEntry.h"
#include "CheckIntegrityCommand.h"
#include "UnknownLengthPieceStorage.h"
#include "DownloadContext.h"
#include "DlAbortEx.h"
#include "DownloadFailureException.h"
#include "RequestGroupMan.h"
#include "DefaultBtProgressInfoFile.h"
#include "DefaultPieceStorage.h"
#include "download_handlers.h"
#include "MemoryBufferPreDownloadHandler.h"
#include "DownloadHandlerConstants.h"
#include "Option.h"
#include "FileEntry.h"
#include "Request.h"
#include "FileAllocationIterator.h"
#include "fmt.h"
#include "A2STR.h"
#include "URISelector.h"
#include "InorderURISelector.h"
#include "PieceSelector.h"
#include "a2functional.h"
#include "SocketCore.h"
#include "SimpleRandomizer.h"
#include "Segment.h"
#include "SocketRecvBuffer.h"
#include "RequestGroupCriteria.h"
#include "CheckIntegrityCommand.h"
#include "ChecksumCheckIntegrityEntry.h"
#ifdef ENABLE_BITTORRENT
# include "bittorrent_helper.h"
# include "BtRegistry.h"
# include "BtCheckIntegrityEntry.h"
# include "DefaultPeerStorage.h"
# include "DefaultBtAnnounce.h"
# include "BtRuntime.h"
# include "BtSetup.h"
# include "BtPostDownloadHandler.h"
# include "DHTSetup.h"
# include "DHTRegistry.h"
# include "DHTNode.h"
# include "DHTRoutingTable.h"
# include "DHTTaskQueue.h"
# include "DHTTaskFactory.h"
# include "DHTTokenTracker.h"
# include "DHTMessageDispatcher.h"
# include "DHTMessageReceiver.h"
# include "DHTMessageFactory.h"
# include "DHTMessageCallback.h"
# include "BtMessageFactory.h"
# include "BtRequestFactory.h"
# include "BtMessageDispatcher.h"
# include "BtMessageReceiver.h"
# include "PeerConnection.h"
# include "ExtensionMessageFactory.h"
# include "DHTPeerAnnounceStorage.h"
# include "DHTEntryPointNameResolveCommand.h"
# include "LongestSequencePieceSelector.h"
# include "PriorityPieceSelector.h"
# include "bittorrent_helper.h"
#endif // ENABLE_BITTORRENT
#ifdef ENABLE_METALINK
# include "MetalinkPostDownloadHandler.h"
#endif // ENABLE_METALINK
namespace aria2 {
RequestGroup::RequestGroup(const std::shared_ptr<GroupId>& gid,
const std::shared_ptr<Option>& option)
: belongsToGID_(0),
gid_(gid),
option_(option),
progressInfoFile_(new NullProgressInfoFile()),
uriSelector_(make_unique<InorderURISelector>()),
requestGroupMan_(nullptr),
#ifdef ENABLE_BITTORRENT
btRuntime_(nullptr),
peerStorage_(nullptr),
#endif // ENABLE_BITTORRENT
lastModifiedTime_(Time::null()),
timeout_(option->getAsInt(PREF_TIMEOUT)),
state_(STATE_WAITING),
numConcurrentCommand_(option->getAsInt(PREF_SPLIT)),
numStreamConnection_(0),
numStreamCommand_(0),
numCommand_(0),
fileNotFoundCount_(0),
maxDownloadSpeedLimit_(option->getAsInt(PREF_MAX_DOWNLOAD_LIMIT)),
maxUploadSpeedLimit_(option->getAsInt(PREF_MAX_UPLOAD_LIMIT)),
resumeFailureCount_(0),
haltReason_(RequestGroup::NONE),
lastErrorCode_(error_code::UNDEFINED),
saveControlFile_(true),
preLocalFileCheckEnabled_(true),
haltRequested_(false),
forceHaltRequested_(false),
pauseRequested_(false),
inMemoryDownload_(false)
{
fileAllocationEnabled_ = option_->get(PREF_FILE_ALLOCATION) != V_NONE;
if(!option_->getAsBool(PREF_DRY_RUN)) {
initializePreDownloadHandler();
initializePostDownloadHandler();
}
}
RequestGroup::~RequestGroup() {}
bool RequestGroup::isCheckIntegrityReady()
{
return option_->getAsBool(PREF_CHECK_INTEGRITY) &&
((downloadContext_->isChecksumVerificationAvailable() &&
downloadFinishedByFileLength()) ||
downloadContext_->isPieceHashVerificationAvailable());
}
bool RequestGroup::downloadFinished() const
{
if(!pieceStorage_) {
return false;
}
return pieceStorage_->downloadFinished();
}
bool RequestGroup::allDownloadFinished() const
{
if(!pieceStorage_) {
return false;
}
return pieceStorage_->allDownloadFinished();
}
error_code::Value RequestGroup::downloadResult() const
{
if(downloadFinished() && !downloadContext_->isChecksumVerificationNeeded()) {
return error_code::FINISHED;
}
if(haltReason_ == RequestGroup::USER_REQUEST) {
return error_code::REMOVED;
}
if(lastErrorCode_ == error_code::UNDEFINED) {
if(haltReason_ == RequestGroup::SHUTDOWN_SIGNAL) {
return error_code::IN_PROGRESS;
}
return error_code::UNKNOWN_ERROR;
}
return lastErrorCode_;
}
void RequestGroup::closeFile()
{
if(pieceStorage_) {
pieceStorage_->flushWrDiskCacheEntry();
pieceStorage_->getDiskAdaptor()->closeFile();
}
}
// TODO The function name is not intuitive at all.. it does not convey
// that this function open file.
std::unique_ptr<CheckIntegrityEntry> RequestGroup::createCheckIntegrityEntry()
{
auto infoFile = std::make_shared<DefaultBtProgressInfoFile>
(downloadContext_, pieceStorage_, option_.get());
if(option_->getAsBool(PREF_CHECK_INTEGRITY) &&
downloadContext_->isPieceHashVerificationAvailable()) {
// When checking piece hash, we don't care file is downloaded and
// infoFile exists.
loadAndOpenFile(infoFile);
return make_unique<StreamCheckIntegrityEntry>(this);
}
if (isPreLocalFileCheckEnabled() &&
(infoFile->exists() ||
(File(getFirstFilePath()).exists() && option_->getAsBool(PREF_CONTINUE))
)) {
// If infoFile exists or -c option is given, we need to check
// download has been completed (which is determined after
// loadAndOpenFile()). If so, use ChecksumCheckIntegrityEntry when
// verification is enabled, because CreateRequestCommand does not
// issue checksum verification and download fails without it.
loadAndOpenFile(infoFile);
if(downloadFinished()) {
if(downloadContext_->isChecksumVerificationNeeded()) {
A2_LOG_INFO(MSG_HASH_CHECK_NOT_DONE);
auto tempEntry = make_unique<ChecksumCheckIntegrityEntry>(this);
tempEntry->setRedownload(true);
return std::move(tempEntry);
}
downloadContext_->setChecksumVerified(true);
A2_LOG_NOTICE(fmt(MSG_DOWNLOAD_ALREADY_COMPLETED,
gid_->toHex().c_str(),
downloadContext_->getBasePath().c_str()));
return nullptr;
}
return make_unique<StreamCheckIntegrityEntry>(this);
}
if (downloadFinishedByFileLength() &&
downloadContext_->isChecksumVerificationAvailable()) {
pieceStorage_->markAllPiecesDone();
loadAndOpenFile(infoFile);
auto tempEntry = make_unique<ChecksumCheckIntegrityEntry>(this);
tempEntry->setRedownload(true);
return std::move(tempEntry);
}
loadAndOpenFile(infoFile);
return make_unique<StreamCheckIntegrityEntry>(this);
}
void RequestGroup::createInitialCommand
(std::vector<std::unique_ptr<Command>>& commands, DownloadEngine* e)
{
// Start session timer here. When file size becomes known, it will
// be reset again in *FileAllocationEntry, because hash check and
// file allocation takes a time. For downloads in which file size
// is unknown, session timer will not be reset.
downloadContext_->resetDownloadStartTime();
#ifdef ENABLE_BITTORRENT
if (downloadContext_->hasAttribute(CTX_ATTR_BT)) {
auto torrentAttrs = bittorrent::getTorrentAttrs(downloadContext_);
bool metadataGetMode = torrentAttrs->metadata.empty();
if (option_->getAsBool(PREF_DRY_RUN)) {
throw DOWNLOAD_FAILURE_EXCEPTION
("Cancel BitTorrent download in dry-run context.");
}
auto& btRegistry = e->getBtRegistry();
if (btRegistry->getDownloadContext(torrentAttrs->infoHash)) {
// TODO If metadataGetMode == false and each FileEntry has
// URI, then go without BT.
throw DOWNLOAD_FAILURE_EXCEPTION2(fmt("InfoHash %s is already registered.",
bittorrent::getInfoHashString(downloadContext_).c_str()),
error_code::DUPLICATE_INFO_HASH);
}
if (metadataGetMode) {
// Use UnknownLengthPieceStorage.
initPieceStorage();
}
else if(e->getRequestGroupMan()->isSameFileBeingDownloaded(this)) {
throw DOWNLOAD_FAILURE_EXCEPTION2(fmt(EX_DUPLICATE_FILE_DOWNLOAD,
downloadContext_->getBasePath().c_str()),
error_code::DUPLICATE_DOWNLOAD);
}
else {
initPieceStorage();
if(downloadContext_->getFileEntries().size() > 1) {
pieceStorage_->setupFileFilter();
}
}
std::shared_ptr<DefaultBtProgressInfoFile> progressInfoFile;
if(!metadataGetMode) {
progressInfoFile = std::make_shared<DefaultBtProgressInfoFile>
(downloadContext_,
pieceStorage_,
option_.get());
}
auto btRuntime = std::make_shared<BtRuntime>();
btRuntime->setMaxPeers(option_->getAsInt(PREF_BT_MAX_PEERS));
btRuntime_ = btRuntime.get();
if(progressInfoFile) {
progressInfoFile->setBtRuntime(btRuntime);
}
auto peerStorage = std::make_shared<DefaultPeerStorage>();
peerStorage->setBtRuntime(btRuntime);
peerStorage->setPieceStorage(pieceStorage_);
peerStorage_ = peerStorage.get();
if(progressInfoFile) {
progressInfoFile->setPeerStorage(peerStorage);
}
auto btAnnounce = std::make_shared<DefaultBtAnnounce>
(downloadContext_.get(),
option_.get());
btAnnounce->setBtRuntime(btRuntime);
btAnnounce->setPieceStorage(pieceStorage_);
btAnnounce->setPeerStorage(peerStorage);
btAnnounce->setUserDefinedInterval
(option_->getAsInt(PREF_BT_TRACKER_INTERVAL));
btAnnounce->shuffleAnnounce();
assert(!btRegistry->get(gid_->getNumericId()));
btRegistry->put(gid_->getNumericId(), make_unique<BtObject>
(downloadContext_,
pieceStorage_,
peerStorage,
btAnnounce,
btRuntime,
(progressInfoFile ?
progressInfoFile : progressInfoFile_)));
if (option_->getAsBool(PREF_ENABLE_DHT) ||
(!e->getOption()->getAsBool(PREF_DISABLE_IPV6) &&
option_->getAsBool(PREF_ENABLE_DHT6))) {
if (option_->getAsBool(PREF_ENABLE_DHT)) {
e->addCommand(DHTSetup().setup(e, AF_INET));
}
if (!e->getOption()->getAsBool(PREF_DISABLE_IPV6) &&
option_->getAsBool(PREF_ENABLE_DHT6)) {
e->addCommand(DHTSetup().setup(e, AF_INET6));
}
const auto& nodes = torrentAttrs->nodes;
// TODO Are nodes in torrent IPv4 only?
if(!torrentAttrs->privateTorrent &&
!nodes.empty() && DHTRegistry::isInitialized()) {
auto command = make_unique<DHTEntryPointNameResolveCommand>
(e->newCUID(), e, nodes);
command->setTaskQueue(DHTRegistry::getData().taskQueue.get());
command->setTaskFactory(DHTRegistry::getData().taskFactory.get());
command->setRoutingTable(DHTRegistry::getData().routingTable.get());
command->setLocalNode(DHTRegistry::getData().localNode);
e->addCommand(std::move(command));
}
} else if(metadataGetMode) {
A2_LOG_NOTICE(_("For BitTorrent Magnet URI, enabling DHT is strongly"
" recommended. See --enable-dht option."));
}
if (metadataGetMode) {
BtCheckIntegrityEntry{this}.onDownloadIncomplete(commands, e);
return;
}
removeDefunctControlFile(progressInfoFile);
{
int64_t actualFileSize = pieceStorage_->getDiskAdaptor()->size();
if (actualFileSize == downloadContext_->getTotalLength()) {
// First, make DiskAdaptor read-only mode to allow the
// program to seed file in read-only media.
pieceStorage_->getDiskAdaptor()->enableReadOnly();
}
else {
// Open file in writable mode to allow the program
// truncate the file to downloadContext_->getTotalLength()
A2_LOG_DEBUG(fmt("File size not match. File is opened in writable"
" mode. Expected:%" PRId64 " Actual:%" PRId64 "",
downloadContext_->getTotalLength(),
actualFileSize));
}
}
// Call Load, Save and file allocation command here
if(progressInfoFile->exists()) {
// load .aria2 file if it exists.
progressInfoFile->load();
pieceStorage_->getDiskAdaptor()->openFile();
}
else if(pieceStorage_->getDiskAdaptor()->fileExists()) {
if(!option_->getAsBool(PREF_CHECK_INTEGRITY) &&
!option_->getAsBool(PREF_ALLOW_OVERWRITE) &&
!option_->getAsBool(PREF_BT_SEED_UNVERIFIED)) {
// TODO we need this->haltRequested = true?
throw DOWNLOAD_FAILURE_EXCEPTION2(fmt(MSG_FILE_ALREADY_EXISTS,
downloadContext_->getBasePath().c_str()),
error_code::FILE_ALREADY_EXISTS);
}
pieceStorage_->getDiskAdaptor()->openFile();
if(option_->getAsBool(PREF_BT_SEED_UNVERIFIED)) {
pieceStorage_->markAllPiecesDone();
}
}
else {
pieceStorage_->getDiskAdaptor()->openFile();
}
progressInfoFile_ = progressInfoFile;
auto entry = make_unique<BtCheckIntegrityEntry>(this);
// --bt-seed-unverified=true is given and download has completed, skip
// validation for piece hashes.
if (option_->getAsBool(PREF_BT_SEED_UNVERIFIED) &&
pieceStorage_->downloadFinished()) {
entry->onDownloadFinished(commands, e);
}
else {
processCheckIntegrityEntry(commands, std::move(entry), e);
}
return;
}
#endif // ENABLE_BITTORRENT
if (downloadContext_->getFileEntries().size() == 1) {
// TODO I assume here when totallength is set to DownloadContext and it is
// not 0, then filepath is also set DownloadContext correctly....
if (option_->getAsBool(PREF_DRY_RUN) ||
downloadContext_->getTotalLength() == 0) {
createNextCommand(commands, e, 1);
return;
}
auto progressInfoFile = std::make_shared<DefaultBtProgressInfoFile>
(downloadContext_,
nullptr,
option_.get());
adjustFilename(progressInfoFile);
initPieceStorage();
auto checkEntry = createCheckIntegrityEntry();
if(checkEntry) {
processCheckIntegrityEntry(commands, std::move(checkEntry), e);
}
return;
}
// TODO --dry-run is not supported for multifile download for now.
if (option_->getAsBool(PREF_DRY_RUN)) {
throw DOWNLOAD_FAILURE_EXCEPTION
("--dry-run in multi-file download is not supported yet.");
}
// TODO file size is known in this context?
// In this context, multiple FileEntry objects are in
// DownloadContext.
if (e->getRequestGroupMan()->isSameFileBeingDownloaded(this)) {
throw DOWNLOAD_FAILURE_EXCEPTION2(fmt(EX_DUPLICATE_FILE_DOWNLOAD,
downloadContext_->getBasePath().c_str()),
error_code::DUPLICATE_DOWNLOAD);
}
initPieceStorage();
if (downloadContext_->getFileEntries().size() > 1) {
pieceStorage_->setupFileFilter();
}
auto progressInfoFile = std::make_shared<DefaultBtProgressInfoFile>
(downloadContext_,
pieceStorage_,
option_.get());
removeDefunctControlFile(progressInfoFile);
// Call Load, Save and file allocation command here
if (progressInfoFile->exists()) {
// load .aria2 file if it exists.
progressInfoFile->load();
pieceStorage_->getDiskAdaptor()->openFile();
}
else if (pieceStorage_->getDiskAdaptor()->fileExists()) {
if(!isCheckIntegrityReady() && !option_->getAsBool(PREF_ALLOW_OVERWRITE)) {
// TODO we need this->haltRequested = true?
throw DOWNLOAD_FAILURE_EXCEPTION2(fmt(MSG_FILE_ALREADY_EXISTS,
downloadContext_->getBasePath().c_str()),
error_code::FILE_ALREADY_EXISTS);
}
pieceStorage_->getDiskAdaptor()->openFile();
} else {
pieceStorage_->getDiskAdaptor()->openFile();
}
progressInfoFile_ = progressInfoFile;
processCheckIntegrityEntry(commands,
make_unique<StreamCheckIntegrityEntry>(this),
e);
}
void RequestGroup::processCheckIntegrityEntry
(std::vector<std::unique_ptr<Command>>& commands,
std::unique_ptr<CheckIntegrityEntry> entry, DownloadEngine* e)
{
int64_t actualFileSize = pieceStorage_->getDiskAdaptor()->size();
if(actualFileSize > downloadContext_->getTotalLength()) {
entry->cutTrailingGarbage();
}
if((option_->getAsBool(PREF_CHECK_INTEGRITY) ||
downloadContext_->isChecksumVerificationNeeded()) &&
entry->isValidationReady()) {
entry->initValidator();
// Don't save control file(.aria2 file) when user presses
// control-c key while aria2 is checking hashes. If control file
// doesn't exist when aria2 launched, the completed length in
// saved control file will be 0 byte and this confuses user.
// enableSaveControlFile() will be called after hash checking is
// done. See CheckIntegrityCommand.
disableSaveControlFile();
e->getCheckIntegrityMan()->pushEntry(std::move(entry));
return;
}
entry->onDownloadIncomplete(commands, e);
}
void RequestGroup::initPieceStorage()
{
std::shared_ptr<PieceStorage> tempPieceStorage;
if(downloadContext_->knowsTotalLength() &&
// Following conditions are needed for chunked encoding with
// content-length = 0. Google's dl server used this before.
(downloadContext_->getTotalLength() > 0
#ifdef ENABLE_BITTORRENT
|| downloadContext_->hasAttribute(CTX_ATTR_BT)
#endif // ENABLE_BITTORRENT
)) {
#ifdef ENABLE_BITTORRENT
auto ps = std::make_shared<DefaultPieceStorage>
(downloadContext_, option_.get());
if (downloadContext_->hasAttribute(CTX_ATTR_BT)) {
if (isUriSuppliedForRequsetFileEntry
(downloadContext_->getFileEntries().begin(),
downloadContext_->getFileEntries().end())) {
// Use LongestSequencePieceSelector when HTTP/FTP/BitTorrent
// integrated downloads. Currently multi-file integrated
// download is not supported.
A2_LOG_DEBUG("Using LongestSequencePieceSelector");
ps->setPieceSelector(make_unique<LongestSequencePieceSelector>());
}
if (option_->defined(PREF_BT_PRIORITIZE_PIECE)) {
std::vector<size_t> result;
util::parsePrioritizePieceRange(result,
option_->get(PREF_BT_PRIORITIZE_PIECE),
downloadContext_->getFileEntries(),
downloadContext_->getPieceLength());
if (!result.empty()) {
std::random_shuffle(std::begin(result), std::end(result),
*SimpleRandomizer::getInstance());
auto priSelector = make_unique<PriorityPieceSelector>
(ps->popPieceSelector());
priSelector->setPriorityPiece(std::begin(result), std::end(result));
ps->setPieceSelector(std::move(priSelector));
}
}
}
#else // !ENABLE_BITTORRENT
auto ps = std::make_shared<DefaultPieceStorage>
(downloadContext_, option_.get());
#endif // !ENABLE_BITTORRENT
if (requestGroupMan_) {
ps->setWrDiskCache(requestGroupMan_->getWrDiskCache());
}
if (diskWriterFactory_) {
ps->setDiskWriterFactory(diskWriterFactory_);
}
tempPieceStorage = ps;
}
else {
auto ps = std::make_shared<UnknownLengthPieceStorage>(downloadContext_);
if (diskWriterFactory_) {
ps->setDiskWriterFactory(diskWriterFactory_);
}
tempPieceStorage = ps;
}
tempPieceStorage->initStorage();
if(requestGroupMan_) {
tempPieceStorage->getDiskAdaptor()->setOpenedFileCounter
(requestGroupMan_->getOpenedFileCounter());
}
segmentMan_ = std::make_shared<SegmentMan>(downloadContext_, tempPieceStorage);
pieceStorage_ = tempPieceStorage;
}
void RequestGroup::dropPieceStorage()
{
segmentMan_.reset();
pieceStorage_.reset();
}
bool RequestGroup::downloadFinishedByFileLength()
{
// assuming that a control file doesn't exist.
if(!isPreLocalFileCheckEnabled() ||
option_->getAsBool(PREF_ALLOW_OVERWRITE)) {
return false;
}
if(!downloadContext_->knowsTotalLength()) {
return false;
}
File outfile(getFirstFilePath());
if(outfile.exists() && downloadContext_->getTotalLength() == outfile.size()) {
return true;
}
return false;
}
void RequestGroup::adjustFilename
(const std::shared_ptr<BtProgressInfoFile>& infoFile)
{
if(!isPreLocalFileCheckEnabled()) {
// OK, no need to care about filename.
return;
}
// TODO need this?
if(requestGroupMan_) {
if(requestGroupMan_->isSameFileBeingDownloaded(this)) {
// The file name must be renamed
tryAutoFileRenaming();
A2_LOG_NOTICE(fmt(MSG_FILE_RENAMED, getFirstFilePath().c_str()));
return;
}
}
if (!option_->getAsBool(PREF_DRY_RUN) &&
option_->getAsBool(PREF_REMOVE_CONTROL_FILE) &&
infoFile->exists()) {
infoFile->removeFile();
A2_LOG_NOTICE(fmt(_("Removed control file for %s because it is requested by"
" user."),
infoFile->getFilename().c_str()));
}
if (infoFile->exists()) {
// Use current filename
return;
}
File outfile(getFirstFilePath());
if(outfile.exists() && option_->getAsBool(PREF_CONTINUE) &&
outfile.size() <= downloadContext_->getTotalLength()) {
// File exists but user decided to resume it.
}
else if(outfile.exists() && isCheckIntegrityReady()) {
// check-integrity existing file
}
else {
shouldCancelDownloadForSafety();
}
}
void RequestGroup::removeDefunctControlFile
(const std::shared_ptr<BtProgressInfoFile>& progressInfoFile)
{
// Remove the control file if download file doesn't exist
if(progressInfoFile->exists() &&
!pieceStorage_->getDiskAdaptor()->fileExists()) {
progressInfoFile->removeFile();
A2_LOG_NOTICE(fmt(MSG_REMOVED_DEFUNCT_CONTROL_FILE,
progressInfoFile->getFilename().c_str(),
downloadContext_->getBasePath().c_str()));
}
}
void RequestGroup::loadAndOpenFile
(const std::shared_ptr<BtProgressInfoFile>& progressInfoFile)
{
try {
if(!isPreLocalFileCheckEnabled()) {
pieceStorage_->getDiskAdaptor()->initAndOpenFile();
return;
}
removeDefunctControlFile(progressInfoFile);
if(progressInfoFile->exists()) {
progressInfoFile->load();
pieceStorage_->getDiskAdaptor()->openExistingFile();
}
else {
File outfile(getFirstFilePath());
if (outfile.exists() && option_->getAsBool(PREF_CONTINUE) &&
outfile.size() <= getTotalLength()) {
pieceStorage_->getDiskAdaptor()->openExistingFile();
pieceStorage_->markPiecesDone(outfile.size());
}
else if (outfile.exists() && isCheckIntegrityReady()) {
pieceStorage_->getDiskAdaptor()->openExistingFile();
}
else {
pieceStorage_->getDiskAdaptor()->initAndOpenFile();
}
}
setProgressInfoFile(progressInfoFile);
}
catch(RecoverableException& e) {
throw DOWNLOAD_FAILURE_EXCEPTION2(fmt(EX_DOWNLOAD_ABORTED), e);
}
}
// assuming that a control file does not exist
void RequestGroup::shouldCancelDownloadForSafety()
{
if (option_->getAsBool(PREF_ALLOW_OVERWRITE)) {
return;
}
File outfile(getFirstFilePath());
if (!outfile.exists()) {
return;
}
tryAutoFileRenaming();
A2_LOG_NOTICE(fmt(MSG_FILE_RENAMED, getFirstFilePath().c_str()));
}
void RequestGroup::tryAutoFileRenaming()
{
if (!option_->getAsBool(PREF_AUTO_FILE_RENAMING)) {
throw DOWNLOAD_FAILURE_EXCEPTION2(fmt(MSG_FILE_ALREADY_EXISTS,
getFirstFilePath().c_str()),
error_code::FILE_ALREADY_EXISTS);
}
std::string filepath = getFirstFilePath();
if(filepath.empty()) {
throw DOWNLOAD_FAILURE_EXCEPTION2(fmt("File renaming failed: %s",
getFirstFilePath().c_str()),
error_code::FILE_RENAMING_FAILED);
}
for (int i = 1; i < 10000; ++i) {
auto newfilename = fmt("%s.%d", filepath.c_str(), i);
File newfile(newfilename);
File ctrlfile(newfile.getPath() + DefaultBtProgressInfoFile::getSuffix());
if (!newfile.exists() || (newfile.exists() && ctrlfile.exists())) {
downloadContext_->getFirstFileEntry()->setPath(newfile.getPath());
return;
}
}
throw DOWNLOAD_FAILURE_EXCEPTION2(fmt("File renaming failed: %s",
getFirstFilePath().c_str()),
error_code::FILE_RENAMING_FAILED);
}
void RequestGroup::createNextCommandWithAdj
(std::vector<std::unique_ptr<Command>>& commands, DownloadEngine* e,
int numAdj)
{
int numCommand;
if (getTotalLength() == 0) {
numCommand = 1+numAdj;
}
else {
numCommand = std::min(downloadContext_->getNumPieces(),
static_cast<size_t>(numConcurrentCommand_));
numCommand += numAdj;
}
if (numCommand > 0) {
createNextCommand(commands, e, numCommand);
}
}
void RequestGroup::createNextCommand
(std::vector<std::unique_ptr<Command>>& commands, DownloadEngine* e)
{
int numCommand;
if (getTotalLength() == 0) {
if (numStreamCommand_ > 0) {
numCommand = 0;
} else {
numCommand = 1;
}
}
else if (numStreamCommand_ >= numConcurrentCommand_) {
numCommand = 0;
}
else {
numCommand = std::min(downloadContext_->getNumPieces(),
static_cast<size_t>(numConcurrentCommand_-numStreamCommand_));
}
if (numCommand > 0) {
createNextCommand(commands, e, numCommand);
}
}
void RequestGroup::createNextCommand
(std::vector<std::unique_ptr<Command>>& commands, DownloadEngine* e,
int numCommand)
{
for (; numCommand > 0; --numCommand) {
commands.push_back(make_unique<CreateRequestCommand>
(e->newCUID(), this, e));
}
if (!commands.empty()) {
e->setNoWait(true);
}
}
std::string RequestGroup::getFirstFilePath() const
{
assert(downloadContext_);
if (inMemoryDownload()) {
return "[MEMORY]" + File(downloadContext_->getFirstFileEntry()->getPath())
.getBasename();
}
return downloadContext_->getFirstFileEntry()->getPath();
}
int64_t RequestGroup::getTotalLength() const
{
if (!pieceStorage_) {
return 0;
}
if (pieceStorage_->isSelectiveDownloadingMode()) {
return pieceStorage_->getFilteredTotalLength();
}
return pieceStorage_->getTotalLength();
}
int64_t RequestGroup::getCompletedLength() const
{
if (!pieceStorage_) {
return 0;
}
if (pieceStorage_->isSelectiveDownloadingMode()) {
return pieceStorage_->getFilteredCompletedLength();
}
return pieceStorage_->getCompletedLength();
}
void RequestGroup::validateFilename(const std::string& expectedFilename,
const std::string& actualFilename) const
{
if(expectedFilename.empty()) {
return;
}
if(expectedFilename != actualFilename) {
throw DL_ABORT_EX(fmt(EX_FILENAME_MISMATCH,
expectedFilename.c_str(),
actualFilename.c_str()));
}
}
void RequestGroup::validateTotalLength(int64_t expectedTotalLength,
int64_t actualTotalLength) const
{
if(expectedTotalLength <= 0) {
return;
}
if(expectedTotalLength != actualTotalLength) {
throw DL_ABORT_EX(fmt(EX_SIZE_MISMATCH, expectedTotalLength,
actualTotalLength));
}
}
void RequestGroup::validateFilename(const std::string& actualFilename) const
{
validateFilename(downloadContext_->getFileEntries().front()->getBasename(),
actualFilename);
}
void RequestGroup::validateTotalLength(int64_t actualTotalLength) const
{
validateTotalLength(getTotalLength(), actualTotalLength);
}
void RequestGroup::increaseStreamCommand()
{
++numStreamCommand_;
}
void RequestGroup::decreaseStreamCommand()
{
--numStreamCommand_;
}
void RequestGroup::increaseStreamConnection()
{
++numStreamConnection_;
}
void RequestGroup::decreaseStreamConnection()
{
--numStreamConnection_;
}
int RequestGroup::getNumConnection() const
{
int numConnection = numStreamConnection_;
#ifdef ENABLE_BITTORRENT
if(btRuntime_) {
numConnection += btRuntime_->getConnections();
}
#endif // ENABLE_BITTORRENT
return numConnection;
}
void RequestGroup::increaseNumCommand()
{
++numCommand_;
}
void RequestGroup::decreaseNumCommand()
{
--numCommand_;
if(!numCommand_ && requestGroupMan_) {
A2_LOG_DEBUG(fmt("GID#%s - Request queue check", gid_->toHex().c_str()));
requestGroupMan_->requestQueueCheck();
}
}
TransferStat RequestGroup::calculateStat() const
{
TransferStat stat = downloadContext_->getNetStat().toTransferStat();
#ifdef ENABLE_BITTORRENT
if(btRuntime_) {
stat.allTimeUploadLength = btRuntime_->getUploadLengthAtStartup() +
stat.sessionUploadLength;
}
#endif // ENABLE_BITTORRENT
return stat;
}
void RequestGroup::setHaltRequested(bool f, HaltReason haltReason)
{
haltRequested_ = f;
if(haltRequested_) {
pauseRequested_ = false;
haltReason_ = haltReason;
}
#ifdef ENABLE_BITTORRENT
if(btRuntime_) {
btRuntime_->setHalt(f);
}
#endif // ENABLE_BITTORRENT
}
void RequestGroup::setForceHaltRequested(bool f, HaltReason haltReason)
{
setHaltRequested(f, haltReason);
forceHaltRequested_ = f;
}
void RequestGroup::setPauseRequested(bool f)
{
pauseRequested_ = f;
}
void RequestGroup::releaseRuntimeResource(DownloadEngine* e)
{
#ifdef ENABLE_BITTORRENT
e->getBtRegistry()->remove(gid_->getNumericId());
btRuntime_ = nullptr;
peerStorage_ = nullptr;
#endif // ENABLE_BITTORRENT
if(pieceStorage_) {
pieceStorage_->removeAdvertisedPiece(0);
}
// Don't reset segmentMan_ and pieceStorage_ here to provide
// progress information via RPC
progressInfoFile_ = std::make_shared<NullProgressInfoFile>();
downloadContext_->releaseRuntimeResource();
}
void RequestGroup::preDownloadProcessing()
{
A2_LOG_DEBUG(fmt("Finding PreDownloadHandler for path %s.",
getFirstFilePath().c_str()));
try {
for (const auto& pdh: preDownloadHandlers_) {
if(pdh->canHandle(this)) {
pdh->execute(this);
return;
}
}
}
catch (RecoverableException& ex) {
A2_LOG_ERROR_EX(EX_EXCEPTION_CAUGHT, ex);
return;
}
A2_LOG_DEBUG("No PreDownloadHandler found.");
return;
}
void RequestGroup::postDownloadProcessing
(std::vector<std::shared_ptr<RequestGroup> >& groups)
{
A2_LOG_DEBUG(fmt("Finding PostDownloadHandler for path %s.",
getFirstFilePath().c_str()));
try {
for (const auto& pdh: postDownloadHandlers_) {
if (pdh->canHandle(this)) {
pdh->getNextRequestGroups(groups, this);
return;
}
}
}
catch(RecoverableException& ex) {
A2_LOG_ERROR_EX(EX_EXCEPTION_CAUGHT, ex);
}
A2_LOG_DEBUG("No PostDownloadHandler found.");
}
void RequestGroup::initializePreDownloadHandler()
{
#ifdef ENABLE_BITTORRENT
if(option_->get(PREF_FOLLOW_TORRENT) == V_MEM) {
preDownloadHandlers_.push_back
(download_handlers::getBtPreDownloadHandler());
}
#endif // ENABLE_BITTORRENT
#ifdef ENABLE_METALINK
if(option_->get(PREF_FOLLOW_METALINK) == V_MEM) {
preDownloadHandlers_.push_back
(download_handlers::getMetalinkPreDownloadHandler());
}
#endif // ENABLE_METALINK
}
void RequestGroup::initializePostDownloadHandler()
{
#ifdef ENABLE_BITTORRENT
if(option_->getAsBool(PREF_FOLLOW_TORRENT) ||
option_->get(PREF_FOLLOW_TORRENT) == V_MEM) {
postDownloadHandlers_.push_back
(download_handlers::getBtPostDownloadHandler());
}
#endif // ENABLE_BITTORRENT
#ifdef ENABLE_METALINK
if(option_->getAsBool(PREF_FOLLOW_METALINK) ||
option_->get(PREF_FOLLOW_METALINK) == V_MEM) {
postDownloadHandlers_.push_back
(download_handlers::getMetalinkPostDownloadHandler());
}
#endif // ENABLE_METALINK
}
bool RequestGroup::isDependencyResolved()
{
if(!dependency_) {
return true;
}
return dependency_->resolve();
}
void RequestGroup::dependsOn(const std::shared_ptr<Dependency>& dep)
{
dependency_ = dep;
}
void RequestGroup::setDiskWriterFactory
(const std::shared_ptr<DiskWriterFactory>& diskWriterFactory)
{
diskWriterFactory_ = diskWriterFactory;
}
void RequestGroup::addPostDownloadHandler(const PostDownloadHandler* handler)
{
postDownloadHandlers_.push_back(handler);
}
void RequestGroup::addPreDownloadHandler(const PreDownloadHandler* handler)
{
preDownloadHandlers_.push_back(handler);
}
void RequestGroup::clearPostDownloadHandler()
{
postDownloadHandlers_.clear();
}
void RequestGroup::clearPreDownloadHandler()
{
preDownloadHandlers_.clear();
}
void RequestGroup::setPieceStorage
(const std::shared_ptr<PieceStorage>& pieceStorage)
{
pieceStorage_ = pieceStorage;
}
void RequestGroup::setProgressInfoFile
(const std::shared_ptr<BtProgressInfoFile>& progressInfoFile)
{
progressInfoFile_ = progressInfoFile;
}
bool RequestGroup::needsFileAllocation() const
{
return isFileAllocationEnabled() &&
option_->getAsLLInt(PREF_NO_FILE_ALLOCATION_LIMIT) <= getTotalLength() &&
!pieceStorage_->getDiskAdaptor()->fileAllocationIterator()->finished();
}
std::shared_ptr<DownloadResult> RequestGroup::createDownloadResult() const
{
A2_LOG_DEBUG(fmt("GID#%s - Creating DownloadResult.",
gid_->toHex().c_str()));
TransferStat st = calculateStat();
std::shared_ptr<DownloadResult> res(new DownloadResult());
res->gid = gid_;
res->fileEntries = downloadContext_->getFileEntries();
res->inMemoryDownload = inMemoryDownload_;
res->sessionDownloadLength = st.sessionDownloadLength;
res->sessionTime = downloadContext_->calculateSessionTime();
res->result = downloadResult();
res->followedBy = followedByGIDs_;
res->belongsTo = belongsToGID_;
res->option = option_;
res->metadataInfo = metadataInfo_;
res->totalLength = getTotalLength();
res->completedLength = getCompletedLength();
res->uploadLength = st.allTimeUploadLength;
if(pieceStorage_ && pieceStorage_->getBitfieldLength() > 0) {
res->bitfield.assign(pieceStorage_->getBitfield(),
pieceStorage_->getBitfield() +
pieceStorage_->getBitfieldLength());
}
#ifdef ENABLE_BITTORRENT
if(downloadContext_->hasAttribute(CTX_ATTR_BT)) {
const unsigned char* p = bittorrent::getInfoHash(downloadContext_);
res->infoHash.assign(p, p+INFO_HASH_LENGTH);
}
#endif // ENABLE_BITTORRENT
res->pieceLength = downloadContext_->getPieceLength();
res->numPieces = downloadContext_->getNumPieces();
res->dir = option_->get(PREF_DIR);
return res;
}
void RequestGroup::reportDownloadFinished()
{
A2_LOG_NOTICE(fmt(MSG_FILE_DOWNLOAD_COMPLETED,
downloadContext_->getBasePath().c_str()));
uriSelector_->resetCounters();
#ifdef ENABLE_BITTORRENT
if (downloadContext_->hasAttribute(CTX_ATTR_BT)) {
TransferStat stat = calculateStat();
int64_t completedLength = getCompletedLength();
double shareRatio = completedLength == 0 ?
0.0 :
1.0 * stat.allTimeUploadLength / completedLength;
auto attrs = bittorrent::getTorrentAttrs(downloadContext_);
if(!attrs->metadata.empty()) {
A2_LOG_NOTICE(fmt(MSG_SHARE_RATIO_REPORT,
shareRatio,
util::abbrevSize(stat.allTimeUploadLength).c_str(),
util::abbrevSize(completedLength).c_str()));
}
}
#endif // ENABLE_BITTORRENT
}
void RequestGroup::setURISelector(std::unique_ptr<URISelector> uriSelector)
{
uriSelector_ = std::move(uriSelector);
}
void RequestGroup::applyLastModifiedTimeToLocalFiles()
{
if (!pieceStorage_ || !lastModifiedTime_.good()) {
return;
}
A2_LOG_INFO(fmt("Applying Last-Modified time: %s",
lastModifiedTime_.toHTTPDate().c_str()));
size_t n = pieceStorage_->getDiskAdaptor()->utime(Time(), lastModifiedTime_);
A2_LOG_INFO(fmt("Last-Modified attrs of %lu files were updated.",
static_cast<unsigned long>(n)));
}
void RequestGroup::updateLastModifiedTime(const Time& time)
{
if (time.good() && lastModifiedTime_ < time) {
lastModifiedTime_ = time;
}
}
void RequestGroup::increaseAndValidateFileNotFoundCount()
{
++fileNotFoundCount_;
const int maxCount = option_->getAsInt(PREF_MAX_FILE_NOT_FOUND);
if (maxCount > 0 && fileNotFoundCount_ >= maxCount &&
downloadContext_->getNetStat().getSessionDownloadLength() == 0) {
throw DOWNLOAD_FAILURE_EXCEPTION2(fmt("Reached max-file-not-found count=%d",
maxCount),
error_code::MAX_FILE_NOT_FOUND);
}
}
void RequestGroup::markInMemoryDownload()
{
inMemoryDownload_ = true;
}
void RequestGroup::setTimeout(time_t timeout)
{
timeout_ = timeout;
}
bool RequestGroup::doesDownloadSpeedExceed()
{
int spd = downloadContext_->getNetStat().calculateDownloadSpeed();
return maxDownloadSpeedLimit_ > 0 && maxDownloadSpeedLimit_ < spd;
}
bool RequestGroup::doesUploadSpeedExceed()
{
int spd = downloadContext_->getNetStat().calculateUploadSpeed();
return maxUploadSpeedLimit_ > 0 && maxUploadSpeedLimit_ < spd;
}
void RequestGroup::saveControlFile() const
{
if(saveControlFile_) {
progressInfoFile_->save();
}
}
void RequestGroup::removeControlFile() const
{
progressInfoFile_->removeFile();
}
void RequestGroup::setDownloadContext
(const std::shared_ptr<DownloadContext>& downloadContext)
{
downloadContext_ = downloadContext;
if(downloadContext_) {
downloadContext_->setOwnerRequestGroup(this);
}
}
bool RequestGroup::p2pInvolved() const
{
#ifdef ENABLE_BITTORRENT
return downloadContext_->hasAttribute(CTX_ATTR_BT);
#else // !ENABLE_BITTORRENT
return false;
#endif // !ENABLE_BITTORRENT
}
} // namespace aria2
Reference in New Issue View Git Blame Copy Permalink