github
/
aria2
mirror of https://github.com/aria2/aria2
1
0
Fork 0
Code Issues Releases Wiki Activity

Save gid option with --save-session option

pull/38/head
Tatsuhiro Tsujikawa 2012-12-17 00:29:15 +09:00
parent 983cb3683a
commit 77a4ee4de0
11 changed files with 126 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
doc/manual-src/en/aria2c.rst
View File

@ -1334,14 +1334,30 @@ Advanced Options
.. option:: --save-session=<FILE>
Save error/unfinished downloads to FILE on exit. You can pass this
output file to aria2c with :option:`--input-file <-i>` option on restart. Please note that
downloads added by :func:`aria2.addTorrent` and
:func:`aria2.addMetalink`
RPC method and whose metadata could not be saved as a file are not saved.
Downloads removed using
:func:`aria2.remove` and
:func:`aria2.forceRemove`
will not be saved.
output file to aria2c with :option:`--input-file <-i>` option on
restart. Please note that downloads added by
:func:`aria2.addTorrent` and :func:`aria2.addMetalink` RPC method
and whose metadata could not be saved as a file are not saved.
Downloads removed using :func:`aria2.remove` and
:func:`aria2.forceRemove` will not be saved. GID is also saved with
:option:`gid <--gid>`, but there are some restrictions, see below.
.. note::
Normally, GID of the download itself is saved. But some downloads
use metadata (e.g., BitTorrent and Metalink). In this case, there
are some restrictions.
1. magnet URI, and followed by torrent download
GID of BitTorrent metadata download is saved.
2. URI to torrent file, and followed by torrent download
GID of torrent file download is saved.
3. URI to metalink file, and followed by file downloads described in metalink file
GID of metalink file download is saved.
4. local torrent file
GID of torrent download is saved.
5. local metalink file
Any meaningful GID is not saved.
.. option:: --stop=<SEC>

3
src/BtPostDownloadHandler.cc
View File

@ -108,7 +108,8 @@ void BtPostDownloadHandler::getNextRequestGroups
torrent);
requestGroup->followedBy(newRgs.begin(), newRgs.end());
SharedHandle<MetadataInfo> mi =
createMetadataInfoFromFirstFileEntry(requestGroup->getDownloadContext());
createMetadataInfoFromFirstFileEntry(requestGroup->getGroupId(),
requestGroup->getDownloadContext());
if(mi) {
setMetadataInfo(newRgs.begin(), newRgs.end(), mi);
}

17
src/MetadataInfo.cc
View File

@ -36,22 +36,13 @@
namespace aria2 {
int64_t MetadataInfo::count_ = 0;
MetadataInfo::MetadataInfo(const std::string& uri)
: id_(genId()), uri_(uri), dataOnly_(false)
MetadataInfo::MetadataInfo(const SharedHandle<GroupId>& gid,
const std::string& uri)
: gid_(gid), uri_(uri)
{}
MetadataInfo::MetadataInfo():id_(genId()), dataOnly_(true) {}
MetadataInfo::MetadataInfo() {}
MetadataInfo::~MetadataInfo() {}
int64_t MetadataInfo::genId()
{
if(count_ == INT64_MAX) {
count_ = 0;
}
return ++count_;
}
} // namespace aria2

18
src/MetadataInfo.h
View File

@ -39,16 +39,17 @@
#include <string>
#include "SharedHandle.h"
#include "GroupId.h"
namespace aria2 {
class MetadataInfo {
private:
int64_t id_;
SharedHandle<GroupId> gid_;
std::string uri_;
bool dataOnly_;
static int64_t count_;
public:
MetadataInfo(const std::string& uri);
MetadataInfo(const SharedHandle<GroupId>& gid, const std::string& uri);
MetadataInfo();
@ -56,7 +57,7 @@ public:
bool dataOnly() const
{
return dataOnly_;
return !gid_;
}
const std::string& getUri() const
@ -64,12 +65,11 @@ public:
return uri_;
}
int64_t getId() const
a2_gid_t getGID() const
{
return id_;
assert(gid_);
return gid_->getNumericId();
}
static int64_t genId();
};
} // namespace aria2

4
src/Metalink2RequestGroup.cc
View File

@ -123,7 +123,9 @@ Metalink2RequestGroup::generate
if(metalinkFile == DEV_STDIN) {
mi.reset(new MetadataInfo());
} else {
mi.reset(new MetadataInfo(metalinkFile));
// TODO Downloads from local metalink file does not save neither
// its gid nor MetadataInfo's gid.
mi.reset(new MetadataInfo(GroupId::create(), metalinkFile));
}
setMetadataInfo(tempgroups.begin(), tempgroups.end(), mi);
groups.insert(groups.end(), tempgroups.begin(), tempgroups.end());

3
src/MetalinkPostDownloadHandler.cc
View File

@ -106,7 +106,8 @@ void MetalinkPostDownloadHandler::getNextRequestGroups
requestGroup->getOption(), baseUri);
requestGroup->followedBy(newRgs.begin(), newRgs.end());
SharedHandle<MetadataInfo> mi =
createMetadataInfoFromFirstFileEntry(requestGroup->getDownloadContext());
createMetadataInfoFromFirstFileEntry(requestGroup->getGroupId(),
requestGroup->getDownloadContext());
if(mi) {
setMetadataInfo(newRgs.begin(), newRgs.end(), mi);
}

5
src/RequestGroup.h
View File

@ -265,6 +265,11 @@ public:
return gid_->getNumericId();
}
const SharedHandle<GroupId>& getGroupId() const
{
return gid_;
}
TransferStat calculateStat() const;
const SharedHandle<DownloadContext>& getDownloadContext() const

39
src/SessionSerializer.cc
View File

@ -107,9 +107,23 @@ bool writeOption(BufferedFile& fp, const SharedHandle<Option>& op)
}
} // namespace
// The downloads whose followedBy() is empty is persisited with its
// GID without no problem. For other cases, there are several patterns.
//
// 1. magnet URI
// GID of metadata download is persisted.
// 2. URI to torrent file
// GID of torrent file download is persisted.
// 3. URI to metalink file
// GID of metalink file download is persisted.
// 4. local torrent file
// GID of torrent download itself is persisted.
// 5. local metalink file
// No GID is persisted. GID is saved but it is just a random GID.
namespace {
bool writeDownloadResult
(BufferedFile& fp, std::set<int64_t>& metainfoCache,
(BufferedFile& fp, std::set<a2_gid_t>& metainfoCache,
const SharedHandle<DownloadResult>& dr)
{
const SharedHandle<MetadataInfo>& mi = dr->metadataInfo;
@ -117,6 +131,15 @@ bool writeDownloadResult
return true;
}
if(!mi) {
// With --force-save option, same gid may be saved twice. (e.g.,
// Downloading .meta4 followed by its conent download. First
// .meta4 download is saved and second content download is also
// saved with the same gid.)
if(metainfoCache.count(dr->gid->getNumericId()) != 0) {
return true;
} else {
metainfoCache.insert(dr->gid->getNumericId());
}
// only save first file entry
if(dr->fileEntries.empty()) {
return true;
@ -136,14 +159,22 @@ bool writeDownloadResult
if(fp.write("\n", 1) != 1) {
return false;
}
if(fp.printf(" gid=%s\n", dr->gid->toHex().c_str()) < 0) {
return false;
}
} else {
if(metainfoCache.count(mi->getId()) != 0) {
if(metainfoCache.count(mi->getGID()) != 0) {
return true;
} else {
metainfoCache.insert(mi->getId());
metainfoCache.insert(mi->getGID());
if(fp.printf("%s\n", mi->getUri().c_str()) < 0) {
return false;
}
// For downloads generated by metadata (e.g., BitTorrent,
// Metalink), save gid of Metadata download.
if(fp.printf(" gid=%s\n", GroupId::toHex(mi->getGID()).c_str()) < 0) {
return false;
}
}
}
return writeOption(fp, dr->option);
@ -152,7 +183,7 @@ bool writeDownloadResult
bool SessionSerializer::save(BufferedFile& fp) const
{
std::set<int64_t> metainfoCache;
std::set<a2_gid_t> metainfoCache;
const std::deque<SharedHandle<DownloadResult> >& results =
rgman_->getDownloadResults();
for(std::deque<SharedHandle<DownloadResult> >::const_iterator itr =

20
src/download_helper.cc
View File

@ -165,9 +165,10 @@ SharedHandle<RequestGroup> createRequestGroup
#if defined ENABLE_BITTORRENT || ENABLE_METALINK
namespace {
SharedHandle<MetadataInfo> createMetadataInfo(const std::string& uri)
SharedHandle<MetadataInfo> createMetadataInfo(const SharedHandle<GroupId>& gid,
const std::string& uri)
{
return SharedHandle<MetadataInfo>(new MetadataInfo(uri));
return SharedHandle<MetadataInfo>(new MetadataInfo(gid, uri));
}
} // namespace
@ -190,7 +191,8 @@ createBtRequestGroup(const std::string& metaInfoUri,
bool adjustAnnounceUri = true)
{
SharedHandle<Option> option = util::copy(optionTemplate);
SharedHandle<RequestGroup> rg(new RequestGroup(getGID(option), option));
SharedHandle<GroupId> gid = getGID(option);
SharedHandle<RequestGroup> rg(new RequestGroup(gid, option));
SharedHandle<DownloadContext> dctx(new DownloadContext());
// may throw exception
bittorrent::loadFromMemory(torrent, dctx, option, auxUris,
@ -198,7 +200,7 @@ createBtRequestGroup(const std::string& metaInfoUri,
if(metaInfoUri.empty()) {
rg->setMetadataInfo(createMetadataInfoDataOnly());
} else {
rg->setMetadataInfo(createMetadataInfo(metaInfoUri));
rg->setMetadataInfo(createMetadataInfo(gid, metaInfoUri));
}
if(adjustAnnounceUri) {
bittorrent::adjustAnnounceUri(bittorrent::getTorrentAttrs(dctx), option);
@ -232,7 +234,8 @@ createBtMagnetRequestGroup
const SharedHandle<Option>& optionTemplate)
{
SharedHandle<Option> option = util::copy(optionTemplate);
SharedHandle<RequestGroup> rg(new RequestGroup(getGID(option), option));
SharedHandle<GroupId> gid = getGID(option);
SharedHandle<RequestGroup> rg(new RequestGroup(gid, option));
SharedHandle<DownloadContext> dctx
(new DownloadContext(METADATA_PIECE_SIZE, 0,
A2STR::NIL));
@ -252,7 +255,7 @@ createBtMagnetRequestGroup
rg->addPostDownloadHandler(utMetadataPostHandler);
rg->setDiskWriterFactory
(SharedHandle<DiskWriterFactory>(new ByteArrayDiskWriterFactory()));
rg->setMetadataInfo(createMetadataInfo(magnetLink));
rg->setMetadataInfo(createMetadataInfo(gid, magnetLink));
rg->markInMemoryDownload();
rg->setPauseRequested(option->getAsBool(PREF_PAUSE));
removeOneshotOption(option);
@ -535,7 +538,8 @@ void createRequestGroupForUriList
}
SharedHandle<MetadataInfo>
createMetadataInfoFromFirstFileEntry(const SharedHandle<DownloadContext>& dctx)
createMetadataInfoFromFirstFileEntry(const SharedHandle<GroupId>& gid,
const SharedHandle<DownloadContext>& dctx)
{
if(dctx->getFileEntries().empty()) {
return SharedHandle<MetadataInfo>();
@ -545,7 +549,7 @@ createMetadataInfoFromFirstFileEntry(const SharedHandle<DownloadContext>& dctx)
if(uris.empty()) {
return SharedHandle<MetadataInfo>();
}
return SharedHandle<MetadataInfo>(new MetadataInfo(uris[0]));
return SharedHandle<MetadataInfo>(new MetadataInfo(gid, uris[0]));
}
}

4
src/download_helper.h
View File

@ -51,6 +51,7 @@ class MetadataInfo;
class DownloadContext;
class UriListParser;
class ValueBase;
class GroupId;
#ifdef ENABLE_BITTORRENT
// Create RequestGroup object using torrent file specified by
@ -144,7 +145,8 @@ void setMetadataInfo
}
SharedHandle<MetadataInfo>
createMetadataInfoFromFirstFileEntry(const SharedHandle<DownloadContext>& dctx);
createMetadataInfoFromFirstFileEntry(const SharedHandle<GroupId>& gid,
const SharedHandle<DownloadContext>& dctx);
// Removes option value which is only effective at the first
// construction time.

32
test/SessionSerializerTest.cc
View File

@ -66,11 +66,14 @@ void SessionSerializerTest::testSave()
SharedHandle<RequestGroupMan> rgman
(new RequestGroupMan(result, 1, option.get()));
SessionSerializer s(rgman);
// REMOVED downloads will not be saved.
rgman->addDownloadResult
(createDownloadResult(error_code::REMOVED, "http://removed"));
rgman->addDownloadResult
(createDownloadResult(error_code::TIME_OUT, "http://error"));
SharedHandle<DownloadResult> drs[] = {
// REMOVED downloads will not be saved.
createDownloadResult(error_code::REMOVED, "http://removed"),
createDownloadResult(error_code::TIME_OUT, "http://error")
};
for(size_t i = 0; i < sizeof(drs)/sizeof(drs[0]); ++i) {
rgman->addDownloadResult(drs[i]);
}
std::string filename = A2_TEST_OUT_DIR"/aria2_SessionSerializerTest_testSave";
s.save(filename);
std::ifstream ss(filename.c_str(), std::ios::binary);
@ -78,20 +81,39 @@ void SessionSerializerTest::testSave()
std::getline(ss, line);
CPPUNIT_ASSERT_EQUAL(std::string("http://error\t"), line);
std::getline(ss, line);
CPPUNIT_ASSERT_EQUAL(fmt(" gid=%s", drs[1]->gid->toHex().c_str()), line);
std::getline(ss, line);
CPPUNIT_ASSERT_EQUAL(uris[0]+"\t"+uris[1]+"\t", line);
std::getline(ss, line);
CPPUNIT_ASSERT_EQUAL(fmt(" gid=%s",
GroupId::toHex(result[0]->getGID()).c_str()),
line);
std::getline(ss, line);
CPPUNIT_ASSERT_EQUAL(std::string(" dir=/tmp"), line);
std::getline(ss, line);
CPPUNIT_ASSERT_EQUAL(uris[2], line);
std::getline(ss, line);
CPPUNIT_ASSERT_EQUAL(fmt(" gid=%s",
GroupId::toHex(result[1]->getGID()).c_str()),
line);
std::getline(ss, line);
CPPUNIT_ASSERT_EQUAL(std::string(" dir=/tmp"), line);
std::getline(ss, line);
CPPUNIT_ASSERT_EQUAL(uris[3], line);
std::getline(ss, line);
// local metalink download does not save meaningful GID
CPPUNIT_ASSERT(fmt(" gid=%s",
GroupId::toHex(result[2]->getGID()).c_str())
!= line);
std::getline(ss, line);
CPPUNIT_ASSERT_EQUAL(std::string(" dir=/tmp"), line);
std::getline(ss, line);
CPPUNIT_ASSERT_EQUAL(uris[4], line);
std::getline(ss, line);
CPPUNIT_ASSERT_EQUAL(fmt(" gid=%s",
GroupId::toHex(result[4]->getGID()).c_str()),
line);
std::getline(ss, line);
CPPUNIT_ASSERT_EQUAL(std::string(" dir=/tmp"), line);
std::getline(ss, line);
CPPUNIT_ASSERT(!ss);