github
/
aria2
mirror of https://github.com/aria2/aria2
1
0
Fork 0
Code Issues Releases Wiki Activity

Added --deferred-input option. 

If true is given to --deferred-input option, aria2 does not read all
URIs and options from file specified by -i option at startup, but it
reads one by one when it needs later. This may reduce memory usage if
input file contains a lot of URIs to download. If false is given,
aria2 reads all URIs and options at startup. This option defaults to
false.
pull/9/head
Tatsuhiro Tsujikawa 2012-02-05 18:57:16 +09:00
parent b8121faf93
commit f2d55f0edc
12 changed files with 141 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
doc/aria2c.1.asciidoc
View File

@ -42,6 +42,7 @@ Basic Options
URI. This optional line must start with one or more white spaces and have
one option per single line.
See *<<_input_file, Input File>>* subsection for details.
See also *<<aria2_optref_deferred_input, --deferred-input>>* option.
[[aria2_optref_log]]*-l*, *--log*=LOG::
The file name of the log file. If '-' is specified, log is written to
@ -916,6 +917,16 @@ Advanced Options
and standard input, standard output and standard error will be
redirected to '/dev/null'. Default: 'false'
[[aria2_optref_deferred_input]]*--deferred-input*[='true'|'false']::
If 'true' is given, aria2 does not read all URIs and options from file
specified by *<<aria2_optref_input_file, -i>>* option at startup,
but it reads one by one when it
needs later. This may reduce memory usage if input file contains a
lot of URIs to download. If 'false' is given, aria2 reads all URIs
and options at startup.
Default: 'false'
[[aria2_optref_disable_ipv6]]*--disable-ipv6*[='true'|'false']::
Disable IPv6. This is useful if you have to use broken DNS and want

10
src/MultiUrlRequestInfo.cc
View File

@ -60,6 +60,7 @@
#include "fmt.h"
#include "SocketCore.h"
#include "OutputFile.h"
#include "UriListParser.h"
#ifdef ENABLE_SSL
# include "TLSContext.h"
#endif // ENABLE_SSL
@ -143,11 +144,13 @@ MultiUrlRequestInfo::MultiUrlRequestInfo
(const std::vector<SharedHandle<RequestGroup> >& requestGroups,
const SharedHandle<Option>& op,
const SharedHandle<StatCalc>& statCalc,
const SharedHandle<OutputFile>& summaryOut)
const SharedHandle<OutputFile>& summaryOut,
const SharedHandle<UriListParser>& uriListParser)
: requestGroups_(requestGroups),
option_(op),
statCalc_(statCalc),
summaryOut_(summaryOut)
summaryOut_(summaryOut),
uriListParser_(uriListParser)
{}
MultiUrlRequestInfo::~MultiUrlRequestInfo() {}
@ -235,6 +238,9 @@ error_code::Value MultiUrlRequestInfo::execute()
(option_->getAsInt(PREF_SERVER_STAT_TIMEOUT));
}
e->setStatCalc(statCalc_);
if(uriListParser_) {
e->getRequestGroupMan()->setUriListParser(uriListParser_);
}
#ifdef SIGHUP
util::setGlobalSignalHandler(SIGHUP, handler, 0);
#endif // SIGHUP

6
src/MultiUrlRequestInfo.h
View File

@ -48,6 +48,7 @@ class RequestGroup;
class Option;
class StatCalc;
class OutputFile;
class UriListParser;
class MultiUrlRequestInfo {
private:
@ -59,13 +60,16 @@ private:
SharedHandle<OutputFile> summaryOut_;
SharedHandle<UriListParser> uriListParser_;
void printMessageForContinue();
public:
MultiUrlRequestInfo
(const std::vector<SharedHandle<RequestGroup> >& requestGroups,
const SharedHandle<Option>& op,
const SharedHandle<StatCalc>& statCalc,
const SharedHandle<OutputFile>& summaryOut);
const SharedHandle<OutputFile>& summaryOut,
const SharedHandle<UriListParser>& uriListParser);
virtual ~MultiUrlRequestInfo();

9
src/OptionHandlerFactory.cc
View File

@ -201,6 +201,15 @@ OptionHandlerFactory::createOptionHandlers()
op->addTag(TAG_ADVANCED);
handlers.push_back(op);
}
{
SharedHandle<OptionHandler> op(new BooleanOptionHandler
(PREF_DEFERRED_INPUT,
TEXT_DEFERRED_INPUT,
A2_V_FALSE,
OptionHandler::OPT_ARG));
op->addTag(TAG_ADVANCED);
handlers.push_back(op);
}
{
SharedHandle<OptionHandler> op(new DefaultOptionHandler
(PREF_DIR,

24
src/RequestGroupMan.cc
View File

@ -74,6 +74,8 @@
#include "Triplet.h"
#include "Signature.h"
#include "OutputFile.h"
#include "download_helper.h"
#include "UriListParser.h"
namespace aria2 {
@ -485,7 +487,21 @@ void RequestGroupMan::fillRequestGroupFromReserver(DownloadEngine* e)
std::vector<SharedHandle<RequestGroup> > temp;
int count = 0;
int num = maxSimultaneousDownloads_-requestGroups_.size();
while(count < num && !reservedGroups_.empty()) {
while(count < num && (uriListParser_ || !reservedGroups_.empty())) {
if(uriListParser_ && reservedGroups_.empty()) {
std::vector<SharedHandle<RequestGroup> > groups;
bool ok = createRequestGroupFromUriListParser(groups, option_,
uriListParser_.get());
if(ok) {
reservedGroups_.insert(reservedGroups_.end(), groups.begin(),
groups.end());
} else {
uriListParser_.reset();
if(reservedGroups_.empty()) {
break;
}
}
}
SharedHandle<RequestGroup> groupToAdd = reservedGroups_.front();
reservedGroups_.pop_front();
std::vector<Command*> commands;
@ -962,4 +978,10 @@ void RequestGroupMan::getUsedHosts
std::back_inserter(usedHosts), Tuple2Pair<1, 3>());
}
void RequestGroupMan::setUriListParser
(const SharedHandle<UriListParser>& uriListParser)
{
uriListParser_ = uriListParser;
}
} // namespace aria2

6
src/RequestGroupMan.h
View File

@ -55,6 +55,7 @@ class ServerStatMan;
class ServerStat;
class Option;
class OutputFile;
class UriListParser;
class RequestGroupMan {
private:
@ -85,6 +86,9 @@ private:
int maxDownloadResult_;
// UriListParser for deferred input.
SharedHandle<UriListParser> uriListParser_;
void formatDownloadResultFull
(OutputFile& out,
const std::string& status,
@ -309,6 +313,8 @@ public:
{
maxDownloadResult_ = v;
}
void setUriListParser(const SharedHandle<UriListParser>& uriListParser);
};
typedef SharedHandle<RequestGroupMan> RequestGroupManHandle;

53
src/download_helper.cc
View File

@ -412,21 +412,23 @@ void createRequestGroupForUri
}
}
namespace {
void createRequestGroupForUriList
bool createRequestGroupFromUriListParser
(std::vector<SharedHandle<RequestGroup> >& result,
const SharedHandle<Option>& option,
const std::string& filename)
const Option* option,
UriListParser* uriListParser)
{
UriListParser p(filename);
while(p.hasNext()) {
// Since result already contains some entries, we cache the size of
// it. Later, we use this value to determine RequestGroup is
// actually created.
size_t num = result.size();
while(uriListParser->hasNext()) {
std::vector<std::string> uris;
Option tempOption;
p.parseNext(uris, tempOption);
uriListParser->parseNext(uris, tempOption);
if(uris.empty()) {
continue;
}
SharedHandle<Option> requestOption(new Option(*option.get()));
SharedHandle<Option> requestOption(new Option(*option));
requestOption->remove(PREF_OUT);
const SharedHandle<OptionParser>& oparser = OptionParser::getInstance();
for(size_t i = 1, len = option::countOption(); i < len; ++i) {
@ -436,25 +438,38 @@ void createRequestGroupForUriList
requestOption->put(pref, tempOption.get(pref));
}
}
// This does not throw exception because throwOnError = false.
createRequestGroupForUri(result, requestOption, uris);
if(num < result.size()) {
return true;
}
}
return false;
}
SharedHandle<UriListParser> openUriListParser(const std::string& filename)
{
std::string listPath;
if(filename == "-") {
listPath = DEV_STDIN;
} else {
if(!File(filename).isFile()) {
throw DL_ABORT_EX
(fmt(EX_FILE_OPEN, filename.c_str(), "No such file"));
}
listPath = filename;
}
return SharedHandle<UriListParser>(new UriListParser(listPath));
}
} // namespace
void createRequestGroupForUriList
(std::vector<SharedHandle<RequestGroup> >& result,
const SharedHandle<Option>& option)
{
if(option->get(PREF_INPUT_FILE) == "-") {
createRequestGroupForUriList(result, option, DEV_STDIN);
} else {
if(!File(option->get(PREF_INPUT_FILE)).isFile()) {
throw DL_ABORT_EX
(fmt(EX_FILE_OPEN, option->get(PREF_INPUT_FILE).c_str(),
"No such file"));
}
createRequestGroupForUriList(result, option, option->get(PREF_INPUT_FILE));
}
SharedHandle<UriListParser> uriListParser = openUriListParser
(option->get(PREF_INPUT_FILE));
while(createRequestGroupFromUriListParser(result, option.get(),
uriListParser.get()));
}
SharedHandle<MetadataInfo>

21
src/download_helper.h
View File

@ -49,6 +49,7 @@ class RequestGroup;
class Option;
class MetadataInfo;
class DownloadContext;
class UriListParser;
#ifdef ENABLE_BITTORRENT
// Create RequestGroup object using torrent file specified by
@ -75,6 +76,26 @@ void createRequestGroupForMetalink
const std::string& metalinkData = "");
#endif // ENABLE_METALINK
// Reads one entry from uriListParser and creates RequestGroups from
// it and store them in result. If the bad entry is found, this
// function just skips it and reads next entry. If at least one
// RequestGroup is created successfully, this function returns true
// and created RequestGroups are stored in result. If no RequestGroup
// is created and uriListParser reads all input, this function returns
// false. The option is used as a option template.
bool createRequestGroupFromUriListParser
(std::vector<SharedHandle<RequestGroup> >& result,
const Option* option,
UriListParser* uriListParser);
// Creates UriListParser using given filename. If filename is "-",
// then UriListParser is configured to read from standard input.
// Otherwise, this function first checks file denoted by filename
// exists. If it does not exist, this function throws exception.
// This function returns SharedHandle<UriListParser> object if it
// succeeds.
SharedHandle<UriListParser> openUriListParser(const std::string& filename);
// Create RequestGroup objects from reading file specified by input-file option.
// If the value of input-file option is "-", stdin is used as a input source.
// Each line is treated as if it is provided in command-line argument.

14
src/main.cc
View File

@ -70,6 +70,7 @@
#include "fmt.h"
#include "NullOutputFile.h"
#include "console.h"
#include "UriListParser.h"
#ifdef ENABLE_BITTORRENT
# include "bittorrent_helper.h"
#endif // ENABLE_BITTORRENT
@ -225,6 +226,7 @@ error_code::Value main(int argc, char* argv[])
util::setGlobalSignalHandler(SIGCHLD, SIG_IGN, 0);
#endif // SIGCHILD
std::vector<SharedHandle<RequestGroup> > requestGroups;
SharedHandle<UriListParser> uriListParser;
#ifdef ENABLE_BITTORRENT
if(!op->blank(PREF_TORRENT_FILE)) {
if(op->get(PREF_SHOW_FILES) == A2_V_TRUE) {
@ -248,7 +250,11 @@ error_code::Value main(int argc, char* argv[])
else
#endif // ENABLE_METALINK
if(!op->blank(PREF_INPUT_FILE)) {
createRequestGroupForUriList(requestGroups, op);
if(op->getAsBool(PREF_DEFERRED_INPUT)) {
uriListParser = openUriListParser(op->get(PREF_INPUT_FILE));
} else {
createRequestGroupForUriList(requestGroups, op);
}
#if defined ENABLE_BITTORRENT || defined ENABLE_METALINK
} else if(op->get(PREF_SHOW_FILES) == A2_V_TRUE) {
showFiles(args, op);
@ -269,11 +275,13 @@ error_code::Value main(int argc, char* argv[])
op->remove(PREF_SELECT_FILE);
op->remove(PREF_PAUSE);
op->remove(PREF_CHECKSUM);
if(!op->getAsBool(PREF_ENABLE_RPC) && requestGroups.empty()) {
if(!op->getAsBool(PREF_ENABLE_RPC) && requestGroups.empty() &&
!uriListParser) {
global::cout()->printf("%s\n", MSG_NO_FILES_TO_DOWNLOAD);
} else {
exitStatus = MultiUrlRequestInfo(requestGroups, op, getStatCalc(op),
getSummaryOut(op)).execute();
getSummaryOut(op),
uriListParser).execute();
}
return exitStatus;
}

2
src/prefs.cc
View File

@ -213,6 +213,8 @@ const Pref* PREF_NO_NETRC = makePref("no-netrc");
const Pref* PREF_MAX_DOWNLOADS = makePref("max-downloads");
// value: string that your file system recognizes as a file name.
const Pref* PREF_INPUT_FILE = makePref("input-file");
// value: true | false
const Pref* PREF_DEFERRED_INPUT = makePref("deferred-input");
// value: 1*digit
const Pref* PREF_MAX_CONCURRENT_DOWNLOADS = makePref("max-concurrent-downloads");
// value: true | false

2
src/prefs.h
View File

@ -156,6 +156,8 @@ extern const Pref* PREF_MAX_OVERALL_DOWNLOAD_LIMIT;
extern const Pref* PREF_MAX_DOWNLOADS;
// value: string that your file system recognizes as a file name.
extern const Pref* PREF_INPUT_FILE;
// value: true | false
extern const Pref* PREF_DEFERRED_INPUT;
// value: 1*digit
extern const Pref* PREF_MAX_CONCURRENT_DOWNLOADS;
// value: true | false

10
src/usage_text.h
View File

@ -226,7 +226,7 @@
" line of URI. This optional line must start with\n" \
" one or more white spaces and have one option per\n" \
" single line. See INPUT FILE section of man page\n" \
" for details.")
" for details. See also --deferred-input option.")
#define TEXT_MAX_CONCURRENT_DOWNLOADS \
_(" -j, --max-concurrent-downloads=N Set maximum number of parallel downloads for\n" \
" every static (HTTP/FTP) URL, torrent and metalink.\n" \
@ -859,3 +859,11 @@
" with its own pid and when parent process exits\n" \
" for some reason, aria2 can detect it and shutdown\n" \
" itself.")
#define TEXT_DEFERRED_INPUT \
_(" --deferred-input[=true|false] If true is given, aria2 does not read all URIs\n" \
" and options from file specified by -i option at\n" \
" startup, but it reads one by one when it needs\n" \
" later. This may reduce memory usage if input\n" \
" file contains a lot of URIs to download.\n" \
" If false is given, aria2 reads all URIs and\n" \
" options at startup.")