2009-12-20 Tatsuhiro Tsujikawa <t-tujikawa@users.sourceforge.net>

Added following 2 keys, followedBy and belongsTo, to the response
	of tellStatus.

	followedBy: List of GIDs which are generated by the consequence of
	this download. For example, when aria2 downloaded Metalink file,
	it generates downloads described in it(see *--follow-metalink*
	option). This value is useful to track these auto generated
	downloads. If there is no such downloads, this key will not be
	included in the response.

	belongsTo: GID of a parent download. Some downloads are a part of
	another download.  For example, if a file in Metalink has
	BitTorrent resource, the download of .torrent is a part of that
	file.  If this download has no parent, this key will not be
	included in the response.
	* src/BtPostDownloadHandler.cc
	* src/DownloadResult.h
	* src/Metalink2RequestGroup.cc
	* src/MetalinkPostDownloadHandler.cc
	* src/RequestGroup.cc
	* src/RequestGroup.h
	* src/UTMetadataPostDownloadHandler.cc
	* src/XmlRpcMethodImpl.cc
	* src/XmlRpcMethodImpl.h

src/BtPostDownloadHandler.cc

@@ -77,9 +77,12 @@ void BtPostDownloadHandler::getNextRequestGroups
     requestGroup->getPieceStorage()->getDiskAdaptor()->closeFile();
     throw;
   }
-  createRequestGroupForBitTorrent(groups, requestGroup->getOption(),
+  std::deque<SharedHandle<RequestGroup> > newRgs;
+  createRequestGroupForBitTorrent(newRgs, requestGroup->getOption(),
 				  std::deque<std::string>(),
 				  content);
+  requestGroup->followedBy(newRgs.begin(), newRgs.end());
+  groups.insert(groups.end(), newRgs.begin(), newRgs.end());
 }
 
 } // namespace aria2

src/DownloadResult.h

@@ -65,18 +65,30 @@ public:
 
   downloadresultcode::RESULT result;
 
+  // This field contains GIDs. See comment in
+  // RequestGroup.cc::_followedByGIDs.
+  std::vector<int32_t> followedBy;
+
+  // This field contains GID. See comment in
+  // RequestGroup.cc::_belongsToGID.
+  int32_t belongsTo;
+
   DownloadResult(int32_t gid,
 		 const std::vector<SharedHandle<FileEntry> >& fileEntries,
 		 bool inMemoryDownload,
 		 uint64_t sessionDownloadLength,
 		 int64_t sessionTime,
-		 downloadresultcode::RESULT result):
+		 downloadresultcode::RESULT result,
+		 const std::vector<int32_t> followedBy,
+		 int32_t belongsTo):
     gid(gid),
     fileEntries(fileEntries),
     inMemoryDownload(inMemoryDownload),
     sessionDownloadLength(sessionDownloadLength),
     sessionTime(sessionTime),
-    result(result) {}
+    result(result),
+    followedBy(followedBy),
+    belongsTo(belongsTo) {}
 };
 
 typedef SharedHandle<DownloadResult> DownloadResultHandle;

src/Metalink2RequestGroup.cc

@@ -252,6 +252,8 @@ Metalink2RequestGroup::createRequestGroup
     if(!torrentRg.isNull()) {
       SharedHandle<Dependency> dep(new BtDependency(rg, torrentRg));
       rg->dependsOn(dep);
+
+      torrentRg->belongsTo(rg->getGID());
     }
 #endif // ENABLE_BITTORRENT
     groups.push_back(rg);

src/MetalinkPostDownloadHandler.cc

@@ -71,9 +71,11 @@ void MetalinkPostDownloadHandler::getNextRequestGroups
   try {
     diskAdaptor->openExistingFile();
     //requestOption.put(PREF_DIR, requestGroup->getDownloadContext()->getDir());
-    
-    Metalink2RequestGroup().generate(groups, diskAdaptor,
+    std::deque<SharedHandle<RequestGroup> > newRgs;
+    Metalink2RequestGroup().generate(newRgs, diskAdaptor,
 				     requestGroup->getOption());
+    requestGroup->followedBy(newRgs.begin(), newRgs.end());
+    groups.insert(groups.end(), newRgs.begin(), newRgs.end());
     diskAdaptor->closeFile();
   } catch(Exception& e) {
     diskAdaptor->closeFile();

src/RequestGroup.cc

@@ -133,6 +133,7 @@ RequestGroup::RequestGroup(const SharedHandle<Option>& option):
   _inMemoryDownload(false),
   _maxDownloadSpeedLimit(option->getAsInt(PREF_MAX_DOWNLOAD_LIMIT)),
   _maxUploadSpeedLimit(option->getAsInt(PREF_MAX_UPLOAD_LIMIT)),
+  _belongsToGID(0),
   _logger(LogFactory::getInstance())
 {
   _fileAllocationEnabled = _option->get(PREF_FILE_ALLOCATION) != V_NONE;
@@ -955,7 +956,9 @@ DownloadResultHandle RequestGroup::createDownloadResult() const
 			_inMemoryDownload,
 			sessionDownloadLength,
 			_downloadContext->calculateSessionTime(),
-			downloadResult()));
+			downloadResult(),
+			_followedByGIDs,
+			_belongsToGID));
 }
   
 void RequestGroup::reportDownloadFinished()

src/RequestGroup.h

@@ -39,6 +39,7 @@
 
 #include <string>
 #include <deque>
+#include <vector>
 
 #include "SharedHandle.h"
 #include "TransferStat.h"
@@ -151,6 +152,18 @@ private:
 
   SharedHandle<URIResult> _lastUriResult;
 
+  // If this download generates another downloads when completed(for
+  // example, downloads generated by PostDownloadHandler), this field
+  // has the GID of generated RequestGroups. empty list means there is
+  // no such RequestGroup.
+  std::vector<int32_t> _followedByGIDs;
+
+  // If this download is a part of another download(for example,
+  // downloading torrent file described in Metalink file), this field
+  // has the GID of parent RequestGroup. 0 means this is a parent
+  // RequestGroup.
+  int32_t _belongsToGID;
+
   Logger* _logger;
 
   void validateFilename(const std::string& expectedFilename,
@@ -445,6 +458,30 @@ public:
 
   void disableSaveControlFile() { _saveControlFile = false; }
 
+  template<typename InputIterator>
+  void followedBy(InputIterator groupFirst, InputIterator groupLast)
+  {
+    _followedByGIDs.clear();
+    for(; groupFirst != groupLast; ++groupFirst) {
+      _followedByGIDs.push_back((*groupFirst)->getGID());
+    }
+  }
+
+  const std::vector<int32_t>& followedBy() const
+  {
+    return _followedByGIDs;
+  }
+
+  void belongsTo(int32_t gid)
+  {
+    _belongsToGID = gid;
+  }
+
+  int32_t belongsTo() const
+  {
+    return _belongsToGID;
+  }
+
   static void resetGIDCounter() { _gidCounter = 0; }
 };
 

src/UTMetadataPostDownloadHandler.cc

@@ -76,14 +76,12 @@ void UTMetadataPostDownloadHandler::getNextRequestGroups
   std::string metadata =
     util::toString(requestGroup->getPieceStorage()->getDiskAdaptor());
   std::string torrent = bittorrent::metadata2Torrent(metadata, attrs);
-  try {
+
   std::deque<SharedHandle<RequestGroup> > newRgs;
   createRequestGroupForBitTorrent(newRgs, requestGroup->getOption(),
 				  std::deque<std::string>(), torrent);
+  requestGroup->followedBy(newRgs.begin(), newRgs.end());
   groups.insert(groups.end(), newRgs.begin(), newRgs.end());
-  } catch(RecoverableException& e) {
-    _logger->error("Failed to parse BitTorrent metadata.", e);
-  }
 }
 
 } // namespace aria2

src/XmlRpcMethodImpl.cc

@@ -262,7 +262,7 @@ BDE RemoveXmlRpcMethod::process(const XmlRpcRequest& req, DownloadEngine* e)
   return createGIDResponse(gid);
 }
 
-static void gatherProgressCommon
+void gatherProgressCommon
 (BDE& entryDict, const SharedHandle<RequestGroup>& group)
 {
   entryDict["gid"] = util::itos(group->getGID());
@@ -286,6 +286,18 @@ static void gatherProgressCommon
     util::uitos(group->getDownloadContext()->getPieceLength());
   entryDict["numPieces"] =
     util::uitos(group->getDownloadContext()->getNumPieces());
+  if(!group->followedBy().empty()) {
+    BDE list = BDE::list();
+    // The element is GID.
+    for(std::vector<int32_t>::const_iterator i = group->followedBy().begin();
+	i != group->followedBy().end(); ++i) {
+      list << util::itos(*i);
+    }
+    entryDict["followedBy"] = list;
+  }
+  if(group->belongsTo()) {
+    entryDict["belongsTo"] = util::itos(group->belongsTo());
+  }
 }
 
 #ifdef ENABLE_BITTORRENT
@@ -343,7 +355,7 @@ static void gatherProgress
 #endif // ENABLE_BITTORRENT
 }
 
-static void gatherStoppedDownload
+void gatherStoppedDownload
 (BDE& entryDict, const SharedHandle<DownloadResult>& ds)
 {
   entryDict["gid"] = util::itos(ds->gid);
@@ -355,6 +367,18 @@ static void gatherStoppedDownload
   } else {
     entryDict["status"] = BDE_ERROR;
   }
+  if(!ds->followedBy.empty()) {
+    BDE list = BDE::list();
+    // The element is GID.
+    for(std::vector<int32_t>::const_iterator i = ds->followedBy.begin();
+	i != ds->followedBy.end(); ++i) {
+      list << util::itos(*i);
+    }
+    entryDict["followedBy"] = list;
+  }
+  if(ds->belongsTo) {
+    entryDict["belongsTo"] = util::itos(ds->belongsTo);
+  }
 }
 
 static

src/XmlRpcMethodImpl.h

@@ -39,6 +39,9 @@
 
 namespace aria2 {
 
+class DownloadResult;
+class RequestGroup;
+
 namespace xmlrpc {
 
 class AddUriXmlRpcMethod:public XmlRpcMethod {
@@ -122,6 +125,16 @@ protected:
   virtual BDE process(const XmlRpcRequest& req, DownloadEngine* e);
 };
 
+// Helper function to store data to entryDict from ds. This function
+// is used by tellStatus method.
+void gatherStoppedDownload
+(BDE& entryDict, const SharedHandle<DownloadResult>& ds);
+
+// Helper function to store data to entryDict from group. This
+// function is used by tellStatus/tellActive/tellWaiting method
+void gatherProgressCommon
+(BDE& entryDict, const SharedHandle<RequestGroup>& group);
+
 } // namespace xmlrpc
 
 } // namespace aria2

test/BtPostDownloadHandlerTest.cc

@@ -77,6 +77,8 @@ void BtPostDownloadHandlerTest::testGetNextRequestGroups()
   CPPUNIT_ASSERT_EQUAL
     (std::string("248d0a1cd08284299de78d5c1ed359bb46717d8c"),
      bittorrent::getInfoHashString(groups.front()->getDownloadContext()));
+  CPPUNIT_ASSERT(std::find(rg.followedBy().begin(), rg.followedBy().end(),
+			   groups.front()->getGID()) != rg.followedBy().end());
 }
 
 } // namespace aria2

test/Metalink2RequestGroupTest.cc

@@ -97,10 +97,11 @@ void Metalink2RequestGroupTest::testGenerate()
     const SharedHandle<DownloadContext>& dctx = rg->getDownloadContext();
 
     CPPUNIT_ASSERT(!dctx.isNull());
+    CPPUNIT_ASSERT_EQUAL(groups[5]->getGID(), rg->belongsTo());
   }
 #endif // ENABLE_BITTORRENT
 
-  // sixth file <- depends on thrid file
+  // sixth file <- depends on fifth file to download .torrent file.
   {
 #ifdef ENABLE_BITTORRENT
     SharedHandle<RequestGroup> rg = groups[5];

test/UTMetadataPostDownloadHandlerTest.cc

@@ -100,6 +100,10 @@ void UTMetadataPostDownloadHandlerTest::testGetNextRequestGroups()
 		       newAttrs[bittorrent::ANNOUNCE_LIST][0][0].s());
   CPPUNIT_ASSERT_EQUAL(_option->get("Hello"),
 		       newRg->getOption()->get("Hello"));
+  CPPUNIT_ASSERT
+    (std::find(_requestGroup->followedBy().begin(),
+	       _requestGroup->followedBy().end(),
+	       newRg->getGID()) != _requestGroup->followedBy().end());
 
   results.clear();
 
@@ -109,8 +113,12 @@ void UTMetadataPostDownloadHandlerTest::testGetNextRequestGroups()
   _requestGroup->getPieceStorage()->getDiskAdaptor()->writeData
     (reinterpret_cast<const unsigned char*>(metadata.data()), metadata.size(),
      0);
+  try {
     handler.getNextRequestGroups(results, _requestGroup.get());
-  CPPUNIT_ASSERT(results.empty());  
+    CPPUNIT_FAIL("exception must be thrown.");
+  } catch(RecoverableException& e) {
+    // success
+  }
 }
 
 } // namespace aria2

test/XmlRpcMethodTest.cc

@@ -18,6 +18,7 @@
 #include "TestUtil.h"
 #include "DownloadContext.h"
 #include "FeatureConfig.h"
+#include "util.h"
 #ifdef ENABLE_BITTORRENT
 # include "BtRegistry.h"
 # include "BtRuntime.h"
@@ -64,6 +65,8 @@ class XmlRpcMethodTest:public CppUnit::TestFixture {
   CPPUNIT_TEST(testTellWaiting_fail);
   CPPUNIT_TEST(testGetVersion);
   CPPUNIT_TEST(testNoSuchMethod);
+  CPPUNIT_TEST(testGatherStoppedDownload);
+  CPPUNIT_TEST(testGatherProgressCommon);
   CPPUNIT_TEST_SUITE_END();
 private:
   SharedHandle<DownloadEngine> _e;
@@ -114,6 +117,8 @@ public:
   void testTellWaiting_fail();
   void testGetVersion();
   void testNoSuchMethod();
+  void testGatherStoppedDownload();
+  void testGatherProgressCommon();
 };
 
 
@@ -624,6 +629,53 @@ void XmlRpcMethodTest::testGetVersion()
 		       features);
 }
 
+void XmlRpcMethodTest::testGatherStoppedDownload()
+{
+  std::vector<SharedHandle<FileEntry> > fileEntries;
+  std::vector<int32_t> followedBy;
+  followedBy.push_back(3);
+  followedBy.push_back(4);
+  SharedHandle<DownloadResult> d
+    (new DownloadResult(1,
+			fileEntries,
+			false,
+			UINT64_MAX,
+			1000,
+			downloadresultcode::FINISHED,
+			followedBy,
+			2));
+  BDE entry = BDE::dict();
+  gatherStoppedDownload(entry, d);
+
+  CPPUNIT_ASSERT_EQUAL(std::string("3"), entry["followedBy"][0].s());
+  CPPUNIT_ASSERT_EQUAL(std::string("4"), entry["followedBy"][1].s());
+  CPPUNIT_ASSERT_EQUAL(std::string("2"), entry["belongsTo"].s());
+}
+
+void XmlRpcMethodTest::testGatherProgressCommon()
+{
+  SharedHandle<DownloadContext> dctx(new DownloadContext());
+
+  SharedHandle<RequestGroup> group(new RequestGroup(_option));
+  group->setDownloadContext(dctx);
+  std::vector<SharedHandle<RequestGroup> > followedBy;
+  for(int i = 0; i < 2; ++i) {
+    followedBy.push_back(SharedHandle<RequestGroup>(new RequestGroup(_option)));
+  }
+
+  group->followedBy(followedBy.begin(), followedBy.end());
+  group->belongsTo(2);
+
+  BDE entry = BDE::dict();
+  gatherProgressCommon(entry, group);
+  
+  CPPUNIT_ASSERT_EQUAL(util::itos(followedBy[0]->getGID()),
+		       entry["followedBy"][0].s());
+  CPPUNIT_ASSERT_EQUAL(util::itos(followedBy[1]->getGID()),
+		       entry["followedBy"][1].s());
+  CPPUNIT_ASSERT_EQUAL(std::string("2"), entry["belongsTo"].s());
+}
+
 } // namespace xmlrpc
 
 } // namespace aria2