2008-11-11 Tatsuhiro Tsujikawa <t-tujikawa@users.sourceforge.net>

Added the ability to specify output filename and directory in
	input file.
	Additional parameters are added in the following line of URIs
	with proceeding white space(s), 1 parameter in 1 line.
	The parameter names are the same with the command-line option
	name without proceeding "--". Not all the options are available
	here: at the moment, 'dir' and 'out' options are avialable.
	Please note that out option has no effect against Metalink or
	BitTorrentdownloads.
	Example input file:
	
	http://host/foo-1.1.tar.bz2
	  out=foo.tar.bz2
	  dir=/tmp/downloads
	http://host/thundermonkey-2.0.tar.bz2

	And then invoke 'aria2c -i url.txt --dir ~/mydownloads'.
	foo-1.1.tar.bz2 is saved as /tmp/downloads/foo.tar.bz2, whereas
	thundermonkey-2.0.tar.bz2 is saved as
	~/mydownloads/thundermonkey-2.0.tar.bz2.
	* src/BtPostDownloadHandler.cc
	* src/Metalink2RequestGroup.cc
	* src/Metalink2RequestGroup.h
	* src/MetalinkPostDownloadHandler.cc
	* src/UriListParser.cc
	* src/UriListParser.h
	* src/main.cc
	* test/Metalink2RequestGroupTest.cc
	* test/UriListParserTest.cc
	* test/filelist1.txt
pull/1/head
Tatsuhiro Tsujikawa 2008-11-11 14:56:46 +00:00
parent 9a581e2cab
commit 05a9313e19
11 changed files with 220 additions and 69 deletions

View File

@ -1,3 +1,36 @@
2008-11-11 Tatsuhiro Tsujikawa <t-tujikawa@users.sourceforge.net>
Added the ability to specify output filename and directory in input
file.
Additional parameters are added in the following line of URIs
with proceeding white space(s), 1 parameter in 1 line.
The parameter names are the same with the command-line option name
without proceeding "--". Not all the options are available here:
at the moment, 'dir' and 'out' options are avialable.
Please note that out option has no effect against Metalink or BitTorrent
downloads.
Example input file:
http://host/foo-1.1.tar.bz2
out=foo.tar.bz2
dir=/tmp/downloads
http://host/thundermonkey-2.0.tar.bz2
And then invoke 'aria2c -i url.txt --dir ~/mydownloads'.
foo-1.1.tar.bz2 is saved as /tmp/downloads/foo.tar.bz2, whereas
thundermonkey-2.0.tar.bz2 is saved as
~/mydownloads/thundermonkey-2.0.tar.bz2.
* src/BtPostDownloadHandler.cc
* src/Metalink2RequestGroup.cc
* src/Metalink2RequestGroup.h
* src/MetalinkPostDownloadHandler.cc
* src/UriListParser.cc
* src/UriListParser.h
* src/main.cc
* test/Metalink2RequestGroupTest.cc
* test/UriListParserTest.cc
* test/filelist1.txt
2008-11-11 Tatsuhiro Tsujikawa <t-tujikawa@users.sourceforge.net>
Fixed error when SSL library is not found.

View File

@ -81,7 +81,7 @@ void BtPostDownloadHandler::getNextRequestGroups
if(op->defined(PREF_PEER_ID_PREFIX)) {
btContext->setPeerIdPrefix(op->get(PREF_PEER_ID_PREFIX));
}
btContext->setDir(op->get(PREF_DIR));
btContext->setDir(requestGroup->getDownloadContext()->getDir());
rg->setDownloadContext(btContext);
btContext->setOwnerRequestGroup(rg.get());

View File

@ -101,26 +101,29 @@ public:
void
Metalink2RequestGroup::generate(std::deque<SharedHandle<RequestGroup> >& groups,
const std::string& metalinkFile)
const std::string& metalinkFile,
const Option& requestOption)
{
std::deque<SharedHandle<MetalinkEntry> > entries;
MetalinkHelper::parseAndQuery(entries, metalinkFile, _option);
createRequestGroup(groups, entries);
createRequestGroup(groups, entries, requestOption);
}
void
Metalink2RequestGroup::generate(std::deque<SharedHandle<RequestGroup> >& groups,
const SharedHandle<BinaryStream>& binaryStream)
const SharedHandle<BinaryStream>& binaryStream,
const Option& requestOption)
{
std::deque<SharedHandle<MetalinkEntry> > entries;
MetalinkHelper::parseAndQuery(entries, binaryStream, _option);
createRequestGroup(groups, entries);
createRequestGroup(groups, entries, requestOption);
}
void
Metalink2RequestGroup::createRequestGroup
(std::deque<SharedHandle<RequestGroup> >& groups,
std::deque<SharedHandle<MetalinkEntry> > entries)
std::deque<SharedHandle<MetalinkEntry> > entries,
const Option& requestOption)
{
if(entries.size() == 0) {
_logger->notice(EX_NO_RESULT_WITH_YOUR_PREFS);
@ -210,7 +213,7 @@ Metalink2RequestGroup::createRequestGroup
entry->getLength(),
A2STR::NIL,
entry->file->getPath()));
dctx->setDir(_option->get(PREF_DIR));
dctx->setDir(requestOption.get(PREF_DIR));
#ifdef ENABLE_MESSAGE_DIGEST
if(entry->chunkChecksum.isNull()) {
if(!entry->checksum.isNull()) {

View File

@ -56,17 +56,20 @@ private:
void
createRequestGroup(std::deque<SharedHandle<RequestGroup> >& groups,
std::deque<SharedHandle<MetalinkEntry> > entries);
std::deque<SharedHandle<MetalinkEntry> > entries,
const Option& requestOption);
public:
Metalink2RequestGroup(const Option* option);
~Metalink2RequestGroup();
void generate(std::deque<SharedHandle<RequestGroup> >& groups,
const std::string& metalinkFile);
const std::string& metalinkFile,
const Option& requestOption);
void generate(std::deque<SharedHandle<RequestGroup> >& groups,
const SharedHandle<BinaryStream>& binaryStream);
const SharedHandle<BinaryStream>& binaryStream,
const Option& requestOption);
};
} // namespace aria2

View File

@ -41,6 +41,9 @@
#include "DownloadHandlerConstants.h"
#include "ContentTypeRequestGroupCriteria.h"
#include "Exception.h"
#include "prefs.h"
#include "Option.h"
#include "DownloadContext.h"
namespace aria2 {
@ -64,7 +67,10 @@ void MetalinkPostDownloadHandler::getNextRequestGroups
SharedHandle<DiskAdaptor> diskAdaptor = requestGroup->getPieceStorage()->getDiskAdaptor();
try {
diskAdaptor->openExistingFile();
Metalink2RequestGroup(op).generate(groups, diskAdaptor);
Option requestOption;
requestOption.put(PREF_DIR, requestGroup->getDownloadContext()->getDir());
Metalink2RequestGroup(op).generate(groups, diskAdaptor, requestOption);
diskAdaptor->closeFile();
} catch(Exception& e) {
diskAdaptor->closeFile();

View File

@ -33,26 +33,50 @@
*/
/* copyright --> */
#include "UriListParser.h"
#include "Util.h"
#include <istream>
#include "Util.h"
#include "Option.h"
namespace aria2 {
UriListParser::UriListParser() {}
UriListParser::UriListParser(std::istream& in):_in(in) {}
UriListParser::~UriListParser() {}
std::deque<std::string> UriListParser::parseNext(std::istream& in)
static void getOptions(Option& op, std::string& line, std::istream& in)
{
std::deque<std::string> uris;
std::string line;
while(getline(in, line)) {
if(!Util::trim(line).empty()) {
Util::slice(uris, line, '\t', true);
return uris;
if(Util::startsWith(line, " ")) {
std::pair<std::string, std::string> p = Util::split(line, "=");
op.put(p.first, p.second);
} else {
break;
}
}
return uris;
}
void UriListParser::parseNext(std::deque<std::string>& uris, Option& op)
{
if(_line.empty()) {
getline(_in, _line);
}
if(!_in) {
return;
}
do {
if(!Util::trim(_line).empty()) {
Util::slice(uris, _line, '\t', true);
getOptions(op, _line, _in);
return;
}
} while(getline(_in, _line));
}
bool UriListParser::hasNext() const
{
return _in;
}
} // namespace aria2

View File

@ -36,19 +36,28 @@
#define _D_URI_LIST_PARSER_H_
#include "common.h"
#include <string>
#include <deque>
#include <iosfwd>
#include "Option.h"
namespace aria2 {
class UriListParser {
private:
std::istream& _in;
std::string _line;
public:
UriListParser();
UriListParser(std::istream& in);
~UriListParser();
std::deque<std::string> parseNext(std::istream& in);
void parseNext(std::deque<std::string>& uris, Option& op);
bool hasNext() const;
};
} // namespace aria2

View File

@ -104,16 +104,17 @@ std::deque<std::string> unfoldURI(const std::deque<std::string>& args)
return nargs;
}
RequestGroupHandle createRequestGroup(const Option* op, const std::deque<std::string>& uris,
const std::string& ufilename = A2STR::NIL)
RequestGroupHandle createRequestGroup
(const Option* op, const std::deque<std::string>& uris,
const Option& requestOption)
{
RequestGroupHandle rg(new RequestGroup(op, uris));
SingleFileDownloadContextHandle dctx
(new SingleFileDownloadContext(op->getAsInt(PREF_SEGMENT_SIZE),
0,
A2STR::NIL,
ufilename));
dctx->setDir(op->get(PREF_DIR));
0,
A2STR::NIL,
requestOption.get(PREF_OUT)));
dctx->setDir(requestOption.get(PREF_DIR));
rg->setDownloadContext(dctx);
return rg;
}
@ -145,7 +146,8 @@ extern Option* option_processing(int argc, char* const argv[]);
SharedHandle<RequestGroup>
createBtRequestGroup(const std::string& torrentFilePath,
Option* op,
const std::deque<std::string>& auxUris)
const std::deque<std::string>& auxUris,
const Option& requestOption)
{
SharedHandle<RequestGroup> rg(new RequestGroup(op, auxUris));
SharedHandle<DefaultBtContext> btContext(new DefaultBtContext());
@ -153,7 +155,7 @@ createBtRequestGroup(const std::string& torrentFilePath,
if(op->defined(PREF_PEER_ID_PREFIX)) {
btContext->setPeerIdPrefix(op->get(PREF_PEER_ID_PREFIX));
}
btContext->setDir(op->get(PREF_DIR));
btContext->setDir(requestOption.get(PREF_DIR));
rg->setDownloadContext(btContext);
btContext->setOwnerRequestGroup(rg.get());
return rg;
@ -171,7 +173,7 @@ int32_t downloadBitTorrent(Option* op, const std::deque<std::string>& uris)
size_t numSplit = op->getAsInt(PREF_SPLIT);
if(nargs.size() >= numSplit) {
RequestGroupHandle rg = createBtRequestGroup(op->get(PREF_TORRENT_FILE),
op, nargs);
op, nargs, *op);
rg->setNumConcurrentCommand(numSplit);
groups.push_back(rg);
} else {
@ -181,7 +183,7 @@ int32_t downloadBitTorrent(Option* op, const std::deque<std::string>& uris)
xargs.erase(xargs.begin()+numSplit, xargs.end());
}
RequestGroupHandle rg = createBtRequestGroup(op->get(PREF_TORRENT_FILE),
op, xargs);
op, xargs, *op);
rg->setNumConcurrentCommand(numSplit);
groups.push_back(rg);
}
@ -193,7 +195,7 @@ int32_t downloadBitTorrent(Option* op, const std::deque<std::string>& uris)
int32_t downloadMetalink(Option* op)
{
RequestGroups groups;
Metalink2RequestGroup(op).generate(groups, op->get(PREF_METALINK_FILE));
Metalink2RequestGroup(op).generate(groups, op->get(PREF_METALINK_FILE), *op);
if(groups.empty()) {
throw FatalException("No files to download.");
}
@ -206,10 +208,12 @@ private:
std::deque<SharedHandle<RequestGroup> >& _requestGroups;
ProtocolDetector _detector;
Option* _op;
Option& _requestOption;
public:
AccRequestGroup(std::deque<SharedHandle<RequestGroup> >& requestGroups,
Option* op):
_requestGroups(requestGroups), _op(op) {}
Option* op,
Option& requestOption):
_requestGroups(requestGroups), _op(op), _requestOption(requestOption) {}
void
operator()(const std::string& uri)
@ -219,14 +223,15 @@ public:
for(size_t count = _op->getAsInt(PREF_SPLIT); count; --count) {
xuris.push_back(uri);
}
RequestGroupHandle rg = createRequestGroup(_op, xuris);
RequestGroupHandle rg = createRequestGroup(_op, xuris, _requestOption);
_requestGroups.push_back(rg);
}
#ifdef ENABLE_BITTORRENT
else if(_detector.guessTorrentFile(uri)) {
try {
_requestGroups.push_back(createBtRequestGroup(uri, _op,
std::deque<std::string>()));
std::deque<std::string>(),
_requestOption));
} catch(RecoverableException& e) {
// error occurred while parsing torrent file.
// We simply ignore it.
@ -237,7 +242,8 @@ public:
#ifdef ENABLE_METALINK
else if(_detector.guessMetalinkFile(uri)) {
try {
Metalink2RequestGroup(_op).generate(_requestGroups, uri);
Metalink2RequestGroup(_op).generate(_requestGroups, uri,
_requestOption);
} catch(RecoverableException& e) {
// error occurred while parsing metalink file.
// We simply ignore it.
@ -251,29 +257,43 @@ public:
}
};
void copyIfndef(Option& dest, const Option& src, const std::string& name)
{
if(!dest.defined(name)) {
dest.put(name, src.get(name));
}
}
int32_t downloadUriList(Option* op, std::istream& in)
{
UriListParser p;
UriListParser p(in);
RequestGroups groups;
while(in) {
std::deque<std::string> uris = p.parseNext(in);
while(p.hasNext()) {
std::deque<std::string> uris;
Option requestOption;
p.parseNext(uris, requestOption);
copyIfndef(requestOption, *op, PREF_DIR);
copyIfndef(requestOption, *op, PREF_OUT);
if(uris.size() == 1 && op->get(PREF_PARAMETERIZED_URI) == V_TRUE) {
std::deque<std::string> unfoldedURIs = unfoldURI(uris);
std::for_each(unfoldedURIs.begin(), unfoldedURIs.end(),
AccRequestGroup(groups, op));
AccRequestGroup(groups, op, requestOption));
} else if(uris.size() == 1) {
std::for_each(uris.begin(), uris.end(), AccRequestGroup(groups, op));
std::for_each(uris.begin(), uris.end(),
AccRequestGroup(groups, op, requestOption));
} else if(!uris.empty()) {
size_t numSplit = op->getAsInt(PREF_SPLIT);
if(uris.size() >= numSplit) {
SharedHandle<RequestGroup> rg = createRequestGroup(op, uris);
SharedHandle<RequestGroup> rg =
createRequestGroup(op, uris, requestOption);
rg->setNumConcurrentCommand(numSplit);
groups.push_back(rg);
} else {
std::deque<std::string> xuris;
ncopy(uris.begin(), uris.end(), numSplit, std::back_inserter(xuris));
xuris.erase(xuris.begin()+numSplit, xuris.end());
SharedHandle<RequestGroup> rg = createRequestGroup(op, xuris);
SharedHandle<RequestGroup> rg =
createRequestGroup(op, xuris, requestOption);
rg->setNumConcurrentCommand(numSplit);
groups.push_back(rg);
}
@ -316,7 +336,7 @@ int32_t downloadUri(Option* op, const std::deque<std::string>& uris)
}
RequestGroups groups;
if(op->get(PREF_FORCE_SEQUENTIAL) == V_TRUE) {
std::for_each(nargs.begin(), nargs.end(), AccRequestGroup(groups, op));
std::for_each(nargs.begin(), nargs.end(), AccRequestGroup(groups, op, *op));
} else {
std::deque<std::string>::iterator strmProtoEnd =
std::stable_partition(nargs.begin(), nargs.end(), StreamProtocolFilter());
@ -325,19 +345,19 @@ int32_t downloadUri(Option* op, const std::deque<std::string>& uris)
size_t numURIs = std::distance(nargs.begin(), strmProtoEnd);
if(numURIs >= numSplit) {
std::deque<std::string> xargs(nargs.begin(), strmProtoEnd);
RequestGroupHandle rg = createRequestGroup(op, xargs, op->get(PREF_OUT));
RequestGroupHandle rg = createRequestGroup(op, xargs, *op);
rg->setNumConcurrentCommand(numSplit);
groups.push_back(rg);
} else if(numURIs > 0) {
std::deque<std::string> xargs;
ncopy(nargs.begin(), strmProtoEnd, numSplit, std::back_inserter(xargs));
xargs.erase(xargs.begin()+numSplit, xargs.end());
RequestGroupHandle rg = createRequestGroup(op, xargs, op->get(PREF_OUT));
RequestGroupHandle rg = createRequestGroup(op, xargs, *op);
rg->setNumConcurrentCommand(numSplit);
groups.push_back(rg);
}
// process remaining URIs(local metalink, BitTorrent files)
std::for_each(strmProtoEnd, nargs.end(), AccRequestGroup(groups, op));
std::for_each(strmProtoEnd, nargs.end(), AccRequestGroup(groups, op, *op));
}
return MultiUrlRequestInfo(groups, op, getStatCalc(op), getSummaryOut(op)).execute();
}

View File

@ -33,7 +33,10 @@ CPPUNIT_TEST_SUITE_REGISTRATION( Metalink2RequestGroupTest );
void Metalink2RequestGroupTest::testGenerate()
{
std::deque<SharedHandle<RequestGroup> > groups;
Metalink2RequestGroup(_option.get()).generate(groups, "test.xml");
Option requestOption;
requestOption.put(PREF_DIR, "/tmp");
Metalink2RequestGroup(_option.get()).generate(groups, "test.xml",
requestOption);
// first file
{
SharedHandle<RequestGroup> rg = groups[0];
@ -41,15 +44,22 @@ void Metalink2RequestGroupTest::testGenerate()
rg->getURIs(uris);
std::sort(uris.begin(), uris.end());
CPPUNIT_ASSERT_EQUAL((size_t)2, uris.size());
CPPUNIT_ASSERT_EQUAL(std::string("ftp://ftphost/aria2-0.5.2.tar.bz2"), uris[0]);
CPPUNIT_ASSERT_EQUAL(std::string("http://httphost/aria2-0.5.2.tar.bz2"), uris[1]);
CPPUNIT_ASSERT_EQUAL
(std::string("ftp://ftphost/aria2-0.5.2.tar.bz2"), uris[0]);
CPPUNIT_ASSERT_EQUAL
(std::string("http://httphost/aria2-0.5.2.tar.bz2"), uris[1]);
SharedHandle<SingleFileDownloadContext> dctx
(dynamic_pointer_cast<SingleFileDownloadContext>(rg->getDownloadContext()));
(dynamic_pointer_cast<SingleFileDownloadContext>
(rg->getDownloadContext()));
CPPUNIT_ASSERT(!dctx.isNull());
CPPUNIT_ASSERT_EQUAL((uint64_t)0ULL, dctx->getTotalLength());
CPPUNIT_ASSERT_EQUAL(std::string("/tmp"), dctx->getDir());
#ifdef ENABLE_MESSAGE_DIGEST
CPPUNIT_ASSERT_EQUAL(std::string("sha1"), dctx->getChecksumHashAlgo());
CPPUNIT_ASSERT_EQUAL(std::string("a96cf3f0266b91d87d5124cf94326422800b627d"),
CPPUNIT_ASSERT_EQUAL
(std::string("a96cf3f0266b91d87d5124cf94326422800b627d"),
dctx->getChecksum());
#endif // ENABLE_MESSAGE_DIGEST
CPPUNIT_ASSERT(!dctx->getSignature().isNull());
@ -61,9 +71,13 @@ void Metalink2RequestGroupTest::testGenerate()
std::deque<std::string> uris;
rg->getURIs(uris);
CPPUNIT_ASSERT_EQUAL((size_t)2, uris.size());
SharedHandle<SingleFileDownloadContext> dctx
(dynamic_pointer_cast<SingleFileDownloadContext>(rg->getDownloadContext()));
(dynamic_pointer_cast<SingleFileDownloadContext>
(rg->getDownloadContext()));
CPPUNIT_ASSERT(!dctx.isNull());
CPPUNIT_ASSERT_EQUAL(std::string("/tmp"), dctx->getDir());
#ifdef ENABLE_MESSAGE_DIGEST
CPPUNIT_ASSERT_EQUAL(std::string("sha1"), dctx->getPieceHashAlgo());
CPPUNIT_ASSERT_EQUAL((size_t)2, dctx->getPieceHashes().size());
@ -81,11 +95,15 @@ void Metalink2RequestGroupTest::testGenerate()
std::deque<std::string> uris;
rg->getURIs(uris);
CPPUNIT_ASSERT_EQUAL((size_t)1, uris.size());
CPPUNIT_ASSERT_EQUAL(std::string("http://host/torrent-http.integrated.torrent"),
uris[0]);
CPPUNIT_ASSERT_EQUAL
(std::string("http://host/torrent-http.integrated.torrent"), uris[0]);
SharedHandle<SingleFileDownloadContext> dctx
(dynamic_pointer_cast<SingleFileDownloadContext>(rg->getDownloadContext()));
(dynamic_pointer_cast<SingleFileDownloadContext>
(rg->getDownloadContext()));
CPPUNIT_ASSERT(!dctx.isNull());
// PREF_DIR has no effect for internal torrent file download
CPPUNIT_ASSERT_EQUAL(std::string("."), dctx->getDir());
}
#endif // ENABLE_BITTORRENT
@ -99,10 +117,15 @@ void Metalink2RequestGroupTest::testGenerate()
std::deque<std::string> uris;
rg->getURIs(uris);
CPPUNIT_ASSERT_EQUAL((size_t)1, uris.size());
CPPUNIT_ASSERT_EQUAL(std::string("http://host/torrent-http.integrated"), uris[0]);
CPPUNIT_ASSERT_EQUAL
(std::string("http://host/torrent-http.integrated"), uris[0]);
SharedHandle<SingleFileDownloadContext> dctx
(dynamic_pointer_cast<SingleFileDownloadContext>(rg->getDownloadContext()));
(dynamic_pointer_cast<SingleFileDownloadContext>
(rg->getDownloadContext()));
CPPUNIT_ASSERT(!dctx.isNull());
CPPUNIT_ASSERT_EQUAL(std::string("/tmp"), dctx->getDir());
}
}

View File

@ -1,13 +1,17 @@
#include "UriListParser.h"
#include "Exception.h"
#include "Util.h"
#include <sstream>
#include <algorithm>
#include <iostream>
#include <fstream>
#include <iterator>
#include <cppunit/extensions/HelperMacros.h>
#include "Exception.h"
#include "Util.h"
#include "prefs.h"
namespace aria2 {
class UriListParserTest : public CppUnit::TestFixture {
@ -36,16 +40,40 @@ std::string UriListParserTest::list2String(const std::deque<std::string>& src)
void UriListParserTest::testHasNext()
{
UriListParser flp;
std::ifstream in("filelist1.txt");
CPPUNIT_ASSERT_EQUAL(std::string("http://localhost/index.html http://localhost2/index.html"), list2String(flp.parseNext(in)));
UriListParser flp(in);
std::deque<std::string> uris;
Option reqOp;
CPPUNIT_ASSERT(flp.hasNext());
flp.parseNext(uris, reqOp);
CPPUNIT_ASSERT_EQUAL
(std::string("http://localhost/index.html http://localhost2/index.html"),
list2String(uris));
uris.clear();
reqOp.clear();
CPPUNIT_ASSERT(flp.hasNext());
flp.parseNext(uris, reqOp);
CPPUNIT_ASSERT_EQUAL(std::string("ftp://localhost/aria2.tar.bz2"),
list2String(flp.parseNext(in)));
CPPUNIT_ASSERT_EQUAL(std::string(""),
list2String(flp.parseNext(in)));
CPPUNIT_ASSERT(!in);
list2String(uris));
CPPUNIT_ASSERT_EQUAL(std::string("/tmp"), reqOp.get(PREF_DIR));
CPPUNIT_ASSERT_EQUAL(std::string("chunky_chocolate"), reqOp.get(PREF_OUT));
uris.clear();
reqOp.clear();
CPPUNIT_ASSERT(!flp.hasNext());
flp.parseNext(uris, reqOp);
CPPUNIT_ASSERT_EQUAL(std::string(""), list2String(uris));
CPPUNIT_ASSERT(!flp.hasNext());
}
} // namespace aria2

View File

@ -1,3 +1,5 @@
http://localhost/index.html http://localhost2/index.html
ftp://localhost/aria2.tar.bz2
dir=/tmp
out=chunky_chocolate