Add --keep-unfinished-download-result option

This option keeps unfinished download results even if doing so exceeds
--max-download-result. This is useful if all unfinished downloads must
be saved in session file (see --save-session option). Please keep in
mind that there is no upper bound to the number of unfinished download
result to keep.  User should use this option only when they know the
total number of downloads in advance.

doc/manual-src/en/aria2c.rst

@@ -1391,6 +1391,16 @@ Advanced Options
     system doesn't have :manpage:`getifaddrs(3)`, this option doesn't accept interface
     name.
 
+.. option:: --keep-unfinished-download-result[=true|false]
+
+  Keep unfinished download results even if doing so exceeds
+  :option:`--max-download-result`.  This is useful if all unfinished
+  downloads must be saved in session file (see
+  :option:`--save-session` option).  Please keep in mind that there is
+  no upper bound to the number of unfinished download result to keep.
+  User should use this option only when they know the total number of
+  downloads in advance.  Default: ``false``
+
 .. option:: --max-download-result=<NUM>
 
   Set maximum number of download result kept in memory. The download
@@ -3240,6 +3250,7 @@ For information on the *secret* parameter, see :ref:`rpc_auth`.
 
   * :option:`bt-max-open-files <--bt-max-open-files>`
   * :option:`download-result <--download-result>`
+  * :option:`keep-unfinished-download-result <--keep-unfinished-download-result>`
   * :option:`log <-l>`
   * :option:`log-level <--log-level>`
   * :option:`max-concurrent-downloads <-j>`

doc/xmlrpc/aria2rpc

@@ -228,6 +228,9 @@ OptionParser.new do |opt|
   opt.on("-l","--log FILE"){|val| options["log"]=val}
   opt.on("--max-download-result NUM"){|val| options["max-download-result"]=val}
   opt.on("--download-result OPT"){|val| options["download-result"]=val}
+  opt.on("--keep-unfinished-download-result [BOOL]",["true","false"]){|val|
+    options["keep-unfinished-download-result"]=val||"true"
+  }
   opt.on("--save-session FILE"){|val| options["save-session"]=val}
   opt.on("--server-stat-of FILE"){|val| options["server-stat-of"]=val}
   opt.on("--save-cookies FILE"){|val| options["save-cookies"]=val}

src/OptionHandlerFactory.cc

@@ -403,6 +403,15 @@ std::vector<OptionHandler*> OptionHandlerFactory::createOptionHandlers()
     op->addTag(TAG_ADVANCED);
     handlers.push_back(op);
   }
+  {
+    OptionHandler* op(
+        new BooleanOptionHandler(PREF_KEEP_UNFINISHED_DOWNLOAD_RESULT,
+                                 TEXT_KEEP_UNFINISHED_DOWNLOAD_RESULT,
+                                 A2_V_FALSE, OptionHandler::OPT_ARG));
+    op->addTag(TAG_ADVANCED);
+    op->setChangeGlobalOption(true);
+    handlers.push_back(op);
+  }
   {
     OptionHandler* op(new DefaultOptionHandler(
         PREF_LOG, TEXT_LOG, NO_DEFAULT_VALUE, PATH_TO_FILE_STDOUT,

src/RequestGroupMan.cc

@@ -920,13 +920,21 @@ void RequestGroupMan::addDownloadResult(
   bool rv = downloadResults_.push_back(dr->gid->getNumericId(), dr);
   assert(rv);
   while (downloadResults_.size() > maxDownloadResult_) {
-    DownloadResultList::iterator i = downloadResults_.begin();
     // Save last encountered error code so that we can report it
     // later.
-    const std::shared_ptr<DownloadResult>& dr = *i;
+    const auto& dr = downloadResults_[0];
     if (dr->belongsTo == 0 && dr->result != error_code::FINISHED) {
       removedLastErrorResult_ = dr->result;
       ++removedErrorResult_;
+
+      // Keep unfinished download result, so that we can save them by
+      // SessionSerializer.
+      if (option_->getAsBool(PREF_KEEP_UNFINISHED_DOWNLOAD_RESULT)) {
+        if (dr->result != error_code::REMOVED ||
+            dr->option->getAsBool(PREF_FORCE_SAVE)) {
+          unfinishedDownloadResults_.push_back(dr);
+        }
+      }
     }
     downloadResults_.pop_front();
   }

src/RequestGroupMan.h

@@ -71,6 +71,10 @@ private:
   RequestGroupList requestGroups_;
   RequestGroupList reservedGroups_;
   DownloadResultList downloadResults_;
+  // This includes download result which did not finish, and deleted
+  // from downloadResults_.  This is used to save them in
+  // SessionSerializer.
+  std::vector<std::shared_ptr<DownloadResult>> unfinishedDownloadResults_;
 
   int maxConcurrentDownloads_;
 
@@ -261,6 +265,12 @@ public:
 
   void addDownloadResult(const std::shared_ptr<DownloadResult>& downloadResult);
 
+  const std::vector<std::shared_ptr<DownloadResult>>&
+  getUnfinishedDownloadResult() const
+  {
+    return unfinishedDownloadResults_;
+  }
+
   std::shared_ptr<ServerStat> findServerStat(const std::string& hostname,
                                              const std::string& protocol) const;
 

src/SessionSerializer.cc

@@ -272,11 +272,14 @@ bool writeDownloadResult(IOFile& fp, std::set<a2_gid_t>& metainfoCache,
 }
 } // namespace
 
-bool SessionSerializer::save(IOFile& fp) const
+namespace {
+template <typename InputIt>
+bool saveDownloadResult(IOFile& fp, std::set<a2_gid_t>& metainfoCache,
+                        InputIt first, InputIt last, bool saveInProgress,
+                        bool saveError)
 {
-  std::set<a2_gid_t> metainfoCache;
+  for (; first != last; ++first) {
-  const DownloadResultList& results = rgman_->getDownloadResults();
+    const auto& dr = *first;
-  for (const auto& dr : results) {
     auto save = false;
     switch (dr->result) {
     case error_code::FINISHED:
@@ -284,20 +287,41 @@ bool SessionSerializer::save(IOFile& fp) const
       save = dr->option->getAsBool(PREF_FORCE_SAVE);
       break;
     case error_code::IN_PROGRESS:
-      save = saveInProgress_;
+      save = saveInProgress;
       break;
     case error_code::RESOURCE_NOT_FOUND:
     case error_code::MAX_FILE_NOT_FOUND:
-      save = saveError_ && dr->option->getAsBool(PREF_SAVE_NOT_FOUND);
+      save = saveError && dr->option->getAsBool(PREF_SAVE_NOT_FOUND);
       break;
     default:
-      save = saveError_;
+      save = saveError;
       break;
     }
     if (save && !writeDownloadResult(fp, metainfoCache, dr, false)) {
       return false;
     }
   }
+  return true;
+}
+} // namespace
+
+bool SessionSerializer::save(IOFile& fp) const
+{
+  std::set<a2_gid_t> metainfoCache;
+
+  const auto& unfinishedResults = rgman_->getUnfinishedDownloadResult();
+  if (!saveDownloadResult(fp, metainfoCache, std::begin(unfinishedResults),
+                          std::end(unfinishedResults), saveInProgress_,
+                          saveError_)) {
+    return false;
+  }
+
+  const auto& results = rgman_->getDownloadResults();
+  if (!saveDownloadResult(fp, metainfoCache, std::begin(results),
+                          std::end(results), saveInProgress_, saveError_)) {
+    return false;
+  }
+
   {
     // Save active downloads.
     const RequestGroupList& groups = rgman_->getRequestGroups();

src/prefs.cc

@@ -374,6 +374,9 @@ PrefPtr PREF_SOCKET_RECV_BUFFER_SIZE = makePref("socket-recv-buffer-size");
 PrefPtr PREF_MAX_MMAP_LIMIT = makePref("max-mmap-limit");
 // value: true | false
 PrefPtr PREF_STDERR = makePref("stderr");
+// value: true | false
+PrefPtr PREF_KEEP_UNFINISHED_DOWNLOAD_RESULT =
+    makePref("keep-unfinished-download-result");
 
 /**
  * FTP related preferences

src/prefs.h

@@ -328,6 +328,8 @@ extern PrefPtr PREF_SOCKET_RECV_BUFFER_SIZE;
 extern PrefPtr PREF_MAX_MMAP_LIMIT;
 // value: true | false
 extern PrefPtr PREF_STDERR;
+// value: true | false
+extern PrefPtr PREF_KEEP_UNFINISHED_DOWNLOAD_RESULT;
 
 /**
  * FTP related preferences

src/usage_text.h

@@ -1099,5 +1099,14 @@
 #define TEXT_STDERR \
   _(" --stderr[=true|false]        Redirect all console output that would be\n" \
     "                              otherwise printed in stdout to stderr.")
-
+#define TEXT_KEEP_UNFINISHED_DOWNLOAD_RESULT \
+  _(" --keep-unfinished-download-result[=true|false]\n" \
+    "                              Keep unfinished download results even if doing\n" \
+    "                              so exceeds --max-download-result. This is useful\n" \
+    "                              if all unfinished downloads must be saved in\n" \
+    "                              session file (see --save-session option). Please\n" \
+    "                              keep in mind that there is no upper bound to the\n" \
+    "                              number of unfinished download result to keep.\n" \
+    "                              User should use this option only when they know\n" \
+    "                              the total number of downloads in advance.")
 // clang-format on