Save gid option with --save-session option

doc/manual-src/en/aria2c.rst

@@ -1334,14 +1334,30 @@ Advanced Options
 .. option:: --save-session=<FILE>
 
   Save error/unfinished downloads to FILE on exit.  You can pass this
-  output file to aria2c with :option:`--input-file <-i>` option on restart. Please note that
+  output file to aria2c with :option:`--input-file <-i>` option on
-  downloads added by :func:`aria2.addTorrent` and
+  restart. Please note that downloads added by
-  :func:`aria2.addMetalink`
+  :func:`aria2.addTorrent` and :func:`aria2.addMetalink` RPC method
-  RPC method and whose metadata could not be saved as a file are not saved.
+  and whose metadata could not be saved as a file are not saved.
-  Downloads removed using
+  Downloads removed using :func:`aria2.remove` and
-  :func:`aria2.remove` and
+  :func:`aria2.forceRemove` will not be saved. GID is also saved with
-  :func:`aria2.forceRemove`
+  :option:`gid <--gid>`, but there are some restrictions, see below.
-  will not be saved.
+
+  .. note::
+
+    Normally, GID of the download itself is saved. But some downloads
+    use metadata (e.g., BitTorrent and Metalink). In this case, there
+    are some restrictions.
+
+    1. magnet URI, and followed by torrent download
+        GID of BitTorrent metadata download is saved.
+    2. URI to torrent file, and followed by torrent download
+        GID of torrent file download is saved.
+    3. URI to metalink file, and followed by file downloads described in metalink file
+        GID of metalink file download is saved.
+    4. local torrent file
+        GID of torrent download is saved.
+    5. local metalink file
+        Any meaningful GID is not saved.
 
 .. option:: --stop=<SEC>
 

src/BtPostDownloadHandler.cc

@@ -108,7 +108,8 @@ void BtPostDownloadHandler::getNextRequestGroups
                                   torrent);
   requestGroup->followedBy(newRgs.begin(), newRgs.end());
   SharedHandle<MetadataInfo> mi =
-    createMetadataInfoFromFirstFileEntry(requestGroup->getDownloadContext());
+    createMetadataInfoFromFirstFileEntry(requestGroup->getGroupId(),
+                                         requestGroup->getDownloadContext());
   if(mi) {
     setMetadataInfo(newRgs.begin(), newRgs.end(), mi);
   }

src/MetadataInfo.cc

@@ -36,22 +36,13 @@
 
 namespace aria2 {
 
-int64_t MetadataInfo::count_ = 0;
+MetadataInfo::MetadataInfo(const SharedHandle<GroupId>& gid,
-
+                           const std::string& uri)
-MetadataInfo::MetadataInfo(const std::string& uri)
+  : gid_(gid), uri_(uri)
-  : id_(genId()), uri_(uri), dataOnly_(false)
 {}
 
-MetadataInfo::MetadataInfo():id_(genId()), dataOnly_(true) {}
+MetadataInfo::MetadataInfo() {}
 
 MetadataInfo::~MetadataInfo() {}
 
-int64_t MetadataInfo::genId()
-{
-  if(count_ == INT64_MAX) {
-    count_ = 0;
-  }
-  return ++count_;
-}
-
 } // namespace aria2

src/MetadataInfo.h

@@ -39,16 +39,17 @@
 
 #include <string>
 
+#include "SharedHandle.h"
+#include "GroupId.h"
+
 namespace aria2 {
 
 class MetadataInfo {
 private:
-  int64_t id_;
+  SharedHandle<GroupId> gid_;
   std::string uri_;
-  bool dataOnly_;
-  static int64_t count_;
 public:
-  MetadataInfo(const std::string& uri);
+  MetadataInfo(const SharedHandle<GroupId>& gid, const std::string& uri);
 
   MetadataInfo();
 
@@ -56,7 +57,7 @@ public:
 
   bool dataOnly() const
   {
-    return dataOnly_;
+    return !gid_;
   }
 
   const std::string& getUri() const
@@ -64,12 +65,11 @@ public:
     return uri_;
   }
 
-  int64_t getId() const
+  a2_gid_t getGID() const
   {
-    return id_;
+    assert(gid_);
+    return gid_->getNumericId();
   }
-
-  static int64_t genId();
 };
 
 } // namespace aria2

src/Metalink2RequestGroup.cc

@@ -123,7 +123,9 @@ Metalink2RequestGroup::generate
   if(metalinkFile == DEV_STDIN) {
     mi.reset(new MetadataInfo());
   } else {
-    mi.reset(new MetadataInfo(metalinkFile));
+    // TODO Downloads from local metalink file does not save neither
+    // its gid nor MetadataInfo's gid.
+    mi.reset(new MetadataInfo(GroupId::create(), metalinkFile));
   }
   setMetadataInfo(tempgroups.begin(), tempgroups.end(), mi);
   groups.insert(groups.end(), tempgroups.begin(), tempgroups.end());

src/MetalinkPostDownloadHandler.cc

@@ -106,7 +106,8 @@ void MetalinkPostDownloadHandler::getNextRequestGroups
                                      requestGroup->getOption(), baseUri);
     requestGroup->followedBy(newRgs.begin(), newRgs.end());
     SharedHandle<MetadataInfo> mi =
-      createMetadataInfoFromFirstFileEntry(requestGroup->getDownloadContext());
+      createMetadataInfoFromFirstFileEntry(requestGroup->getGroupId(),
+                                           requestGroup->getDownloadContext());
     if(mi) {
       setMetadataInfo(newRgs.begin(), newRgs.end(), mi);
     }

src/RequestGroup.h

@@ -265,6 +265,11 @@ public:
     return gid_->getNumericId();
   }
 
+  const SharedHandle<GroupId>& getGroupId() const
+  {
+    return gid_;
+  }
+
   TransferStat calculateStat() const;
 
   const SharedHandle<DownloadContext>& getDownloadContext() const

src/SessionSerializer.cc

@@ -107,9 +107,23 @@ bool writeOption(BufferedFile& fp, const SharedHandle<Option>& op)
 }
 } // namespace
 
+// The downloads whose followedBy() is empty is persisited with its
+// GID without no problem. For other cases, there are several patterns.
+//
+// 1. magnet URI
+//  GID of metadata download is persisted.
+// 2. URI to torrent file
+//  GID of torrent file download is persisted.
+// 3. URI to metalink file
+//  GID of metalink file download is persisted.
+// 4. local torrent file
+//  GID of torrent download itself is persisted.
+// 5. local metalink file
+//  No GID is persisted. GID is saved but it is just a random GID.
+
 namespace {
 bool writeDownloadResult
-(BufferedFile& fp, std::set<int64_t>& metainfoCache,
+(BufferedFile& fp, std::set<a2_gid_t>& metainfoCache,
  const SharedHandle<DownloadResult>& dr)
 {
   const SharedHandle<MetadataInfo>& mi = dr->metadataInfo;
@@ -117,6 +131,15 @@ bool writeDownloadResult
     return true;
   }
   if(!mi) {
+    // With --force-save option, same gid may be saved twice. (e.g.,
+    // Downloading .meta4 followed by its conent download. First
+    // .meta4 download is saved and second content download is also
+    // saved with the same gid.)
+    if(metainfoCache.count(dr->gid->getNumericId()) != 0) {
+      return true;
+    } else {
+      metainfoCache.insert(dr->gid->getNumericId());
+    }
     // only save first file entry
     if(dr->fileEntries.empty()) {
       return true;
@@ -136,14 +159,22 @@ bool writeDownloadResult
     if(fp.write("\n", 1) != 1) {
       return false;
     }
+    if(fp.printf(" gid=%s\n", dr->gid->toHex().c_str()) < 0) {
+      return false;
+    }
   } else {
-    if(metainfoCache.count(mi->getId()) != 0) {
+    if(metainfoCache.count(mi->getGID()) != 0) {
       return true;
     } else {
-      metainfoCache.insert(mi->getId());
+      metainfoCache.insert(mi->getGID());
       if(fp.printf("%s\n", mi->getUri().c_str()) < 0) {
         return false;
       }
+      // For downloads generated by metadata (e.g., BitTorrent,
+      // Metalink), save gid of Metadata download.
+      if(fp.printf(" gid=%s\n", GroupId::toHex(mi->getGID()).c_str()) < 0) {
+        return false;
+      }
     }
   }
   return writeOption(fp, dr->option);
@@ -152,7 +183,7 @@ bool writeDownloadResult
 
 bool SessionSerializer::save(BufferedFile& fp) const
 {
-  std::set<int64_t> metainfoCache;
+  std::set<a2_gid_t> metainfoCache;
   const std::deque<SharedHandle<DownloadResult> >& results =
     rgman_->getDownloadResults();
   for(std::deque<SharedHandle<DownloadResult> >::const_iterator itr =

src/download_helper.cc

@@ -165,9 +165,10 @@ SharedHandle<RequestGroup> createRequestGroup
 
 #if defined ENABLE_BITTORRENT || ENABLE_METALINK
 namespace {
-SharedHandle<MetadataInfo> createMetadataInfo(const std::string& uri)
+SharedHandle<MetadataInfo> createMetadataInfo(const SharedHandle<GroupId>& gid,
+                                              const std::string& uri)
 {
-  return SharedHandle<MetadataInfo>(new MetadataInfo(uri));
+  return SharedHandle<MetadataInfo>(new MetadataInfo(gid, uri));
 }
 } // namespace
 
@@ -190,7 +191,8 @@ createBtRequestGroup(const std::string& metaInfoUri,
                      bool adjustAnnounceUri = true)
 {
   SharedHandle<Option> option = util::copy(optionTemplate);
-  SharedHandle<RequestGroup> rg(new RequestGroup(getGID(option), option));
+  SharedHandle<GroupId> gid = getGID(option);
+  SharedHandle<RequestGroup> rg(new RequestGroup(gid, option));
   SharedHandle<DownloadContext> dctx(new DownloadContext());
   // may throw exception
   bittorrent::loadFromMemory(torrent, dctx, option, auxUris,
@@ -198,7 +200,7 @@ createBtRequestGroup(const std::string& metaInfoUri,
   if(metaInfoUri.empty()) {
     rg->setMetadataInfo(createMetadataInfoDataOnly());
   } else {
-    rg->setMetadataInfo(createMetadataInfo(metaInfoUri));
+    rg->setMetadataInfo(createMetadataInfo(gid, metaInfoUri));
   }
   if(adjustAnnounceUri) {
     bittorrent::adjustAnnounceUri(bittorrent::getTorrentAttrs(dctx), option);
@@ -232,7 +234,8 @@ createBtMagnetRequestGroup
  const SharedHandle<Option>& optionTemplate)
 {
   SharedHandle<Option> option = util::copy(optionTemplate);
-  SharedHandle<RequestGroup> rg(new RequestGroup(getGID(option), option));
+  SharedHandle<GroupId> gid = getGID(option);
+  SharedHandle<RequestGroup> rg(new RequestGroup(gid, option));
   SharedHandle<DownloadContext> dctx
     (new DownloadContext(METADATA_PIECE_SIZE, 0,
                          A2STR::NIL));
@@ -252,7 +255,7 @@ createBtMagnetRequestGroup
   rg->addPostDownloadHandler(utMetadataPostHandler);
   rg->setDiskWriterFactory
     (SharedHandle<DiskWriterFactory>(new ByteArrayDiskWriterFactory()));
-  rg->setMetadataInfo(createMetadataInfo(magnetLink));
+  rg->setMetadataInfo(createMetadataInfo(gid, magnetLink));
   rg->markInMemoryDownload();
   rg->setPauseRequested(option->getAsBool(PREF_PAUSE));
   removeOneshotOption(option);
@@ -535,7 +538,8 @@ void createRequestGroupForUriList
 }
 
 SharedHandle<MetadataInfo>
-createMetadataInfoFromFirstFileEntry(const SharedHandle<DownloadContext>& dctx)
+createMetadataInfoFromFirstFileEntry(const SharedHandle<GroupId>& gid,
+                                     const SharedHandle<DownloadContext>& dctx)
 {
   if(dctx->getFileEntries().empty()) {
     return SharedHandle<MetadataInfo>();
@@ -545,7 +549,7 @@ createMetadataInfoFromFirstFileEntry(const SharedHandle<DownloadContext>& dctx)
     if(uris.empty()) {
       return SharedHandle<MetadataInfo>();
     }
-    return SharedHandle<MetadataInfo>(new MetadataInfo(uris[0]));
+    return SharedHandle<MetadataInfo>(new MetadataInfo(gid, uris[0]));
   }
 }
 

src/download_helper.h

@@ -51,6 +51,7 @@ class MetadataInfo;
 class DownloadContext;
 class UriListParser;
 class ValueBase;
+class GroupId;
 
 #ifdef ENABLE_BITTORRENT
 // Create RequestGroup object using torrent file specified by
@@ -144,7 +145,8 @@ void setMetadataInfo
 }
 
 SharedHandle<MetadataInfo>
-createMetadataInfoFromFirstFileEntry(const SharedHandle<DownloadContext>& dctx);
+createMetadataInfoFromFirstFileEntry(const SharedHandle<GroupId>& gid,
+                                     const SharedHandle<DownloadContext>& dctx);
 
 // Removes option value which is only effective at the first
 // construction time.

test/SessionSerializerTest.cc

@@ -66,11 +66,14 @@ void SessionSerializerTest::testSave()
   SharedHandle<RequestGroupMan> rgman
     (new RequestGroupMan(result, 1, option.get()));
   SessionSerializer s(rgman);
-  // REMOVED downloads will not be saved.
+  SharedHandle<DownloadResult> drs[] = {
-  rgman->addDownloadResult
+    // REMOVED downloads will not be saved.
-    (createDownloadResult(error_code::REMOVED, "http://removed"));
+    createDownloadResult(error_code::REMOVED, "http://removed"),
-  rgman->addDownloadResult
+    createDownloadResult(error_code::TIME_OUT, "http://error")
-    (createDownloadResult(error_code::TIME_OUT, "http://error"));
+  };
+  for(size_t i = 0; i < sizeof(drs)/sizeof(drs[0]); ++i) {
+    rgman->addDownloadResult(drs[i]);
+  }
   std::string filename = A2_TEST_OUT_DIR"/aria2_SessionSerializerTest_testSave";
   s.save(filename);
   std::ifstream ss(filename.c_str(), std::ios::binary);
@@ -78,20 +81,39 @@ void SessionSerializerTest::testSave()
   std::getline(ss, line);
   CPPUNIT_ASSERT_EQUAL(std::string("http://error\t"), line);
   std::getline(ss, line);
+  CPPUNIT_ASSERT_EQUAL(fmt(" gid=%s", drs[1]->gid->toHex().c_str()), line);
+  std::getline(ss, line);
   CPPUNIT_ASSERT_EQUAL(uris[0]+"\t"+uris[1]+"\t", line);
   std::getline(ss, line);
+  CPPUNIT_ASSERT_EQUAL(fmt(" gid=%s",
+                           GroupId::toHex(result[0]->getGID()).c_str()),
+                       line);
+  std::getline(ss, line);
   CPPUNIT_ASSERT_EQUAL(std::string(" dir=/tmp"), line);
   std::getline(ss, line);
   CPPUNIT_ASSERT_EQUAL(uris[2], line);
   std::getline(ss, line);
+  CPPUNIT_ASSERT_EQUAL(fmt(" gid=%s",
+                           GroupId::toHex(result[1]->getGID()).c_str()),
+                       line);
+  std::getline(ss, line);
   CPPUNIT_ASSERT_EQUAL(std::string(" dir=/tmp"), line);
   std::getline(ss, line);
   CPPUNIT_ASSERT_EQUAL(uris[3], line);
   std::getline(ss, line);
+  // local metalink download does not save meaningful GID
+  CPPUNIT_ASSERT(fmt(" gid=%s",
+                     GroupId::toHex(result[2]->getGID()).c_str())
+                 != line);
+  std::getline(ss, line);
   CPPUNIT_ASSERT_EQUAL(std::string(" dir=/tmp"), line);
   std::getline(ss, line);
   CPPUNIT_ASSERT_EQUAL(uris[4], line);
   std::getline(ss, line);
+  CPPUNIT_ASSERT_EQUAL(fmt(" gid=%s",
+                           GroupId::toHex(result[4]->getGID()).c_str()),
+                       line);
+  std::getline(ss, line);
   CPPUNIT_ASSERT_EQUAL(std::string(" dir=/tmp"), line);
   std::getline(ss, line);
   CPPUNIT_ASSERT(!ss);