2010-08-24 Tatsuhiro Tsujikawa <t-tujikawa@users.sourceforge.net>

Added --bt-tracker and --bt-exclude-tracker option.  In
	--bt-tracker option, you can specify comma separated list of
	additional BitTorrent tracker's announce URI. These URIs are not
	affected by --bt-exclude-tracker option because they are added
	after URIs in --bt-exclude-tracker option are removed.  In
	--bt-exclude-tracker option, you can specify comma separated list
	of BitTorrent tracker's announce URI to remove. You can use
	special value '*' which matches all URIs, thus removes all
	announce URIs. When specifying '*' in shell command-line, don't
	forget to escape or quote it.
	* src/BtDependency.cc
	* src/OptionHandlerFactory.cc
	* src/UTMetadataPostDownloadHandler.cc
	* src/bittorrent_helper.cc
	* src/bittorrent_helper.h
	* src/download_helper.cc
	* src/download_helper.h
	* src/prefs.cc
	* src/prefs.h
	* src/usage_text.h
	* test/BittorrentHelperTest.cc
	* test/DownloadHelperTest.cc

ChangeLog

@@ -1,3 +1,28 @@
+2010-08-24  Tatsuhiro Tsujikawa  <t-tujikawa@users.sourceforge.net>
+
+	Added --bt-tracker and --bt-exclude-tracker option.  In
+	--bt-tracker option, you can specify comma separated list of
+	additional BitTorrent tracker's announce URI. These URIs are not
+	affected by --bt-exclude-tracker option because they are added
+	after URIs in --bt-exclude-tracker option are removed.  In
+	--bt-exclude-tracker option, you can specify comma separated list
+	of BitTorrent tracker's announce URI to remove. You can use
+	special value '*' which matches all URIs, thus removes all
+	announce URIs. When specifying '*' in shell command-line, don't
+	forget to escape or quote it.
+	* src/BtDependency.cc
+	* src/OptionHandlerFactory.cc
+	* src/UTMetadataPostDownloadHandler.cc
+	* src/bittorrent_helper.cc
+	* src/bittorrent_helper.h
+	* src/download_helper.cc
+	* src/download_helper.h
+	* src/prefs.cc
+	* src/prefs.h
+	* src/usage_text.h
+	* test/BittorrentHelperTest.cc
+	* test/DownloadHelperTest.cc
+
 2010-08-24  Tatsuhiro Tsujikawa  <t-tujikawa@users.sourceforge.net>
 
 	Code cleanup

src/BtDependency.cc

@@ -88,9 +88,13 @@ bool BtDependency::resolve()
           bittorrent::getTorrentAttrs(dependee->getDownloadContext());
         bittorrent::loadFromMemory
           (bittorrent::metadata2Torrent(content, attrs), context, "default");
+        // We don't call bittorrent::adjustAnnounceUri() because it
+        // has already been called with attrs.
       } else {
         bittorrent::loadFromMemory
           (content, context, File(dependee->getFirstFilePath()).getBasename());
+        bittorrent::adjustAnnounceUri(bittorrent::getTorrentAttrs(context),
+                                      dependant_->getOption());
       }
       const std::vector<SharedHandle<FileEntry> >& fileEntries =
         context->getFileEntries();

src/OptionHandlerFactory.cc

@@ -1145,6 +1145,16 @@ OptionHandlers OptionHandlerFactory::createOptionHandlers()
     op->addTag(TAG_BITTORRENT);
     handlers.push_back(op);
   }
+  {
+    SharedHandle<DefaultOptionHandler> op(new DefaultOptionHandler
+                                          (PREF_BT_EXCLUDE_TRACKER,
+                                           TEXT_BT_EXCLUDE_TRACKER,
+                                           NO_DESCRIPTION,
+                                           "URI,... "
+                                           "or *"));
+    op->addTag(TAG_BITTORRENT);
+    handlers.push_back(op);
+  }
   {
     SharedHandle<OptionHandler> op(new DefaultOptionHandler
                                    (PREF_BT_EXTERNAL_IP,
@@ -1288,6 +1298,15 @@ OptionHandlers OptionHandlerFactory::createOptionHandlers()
     op->hide();
     handlers.push_back(op);
   }
+  {
+    SharedHandle<DefaultOptionHandler> op(new DefaultOptionHandler
+                                          (PREF_BT_TRACKER,
+                                           TEXT_BT_TRACKER,
+                                           NO_DESCRIPTION,
+                                           "URI,..."));
+    op->addTag(TAG_BITTORRENT);
+    handlers.push_back(op);
+  }
   {
     SharedHandle<NumberOptionHandler> op(new NumberOptionHandler
                                          (PREF_BT_TRACKER_CONNECT_TIMEOUT,

src/UTMetadataPostDownloadHandler.cc

@@ -93,8 +93,10 @@ void UTMetadataPostDownloadHandler::getNextRequestGroups
   }
   if(!requestGroup->getOption()->getAsBool(PREF_BT_METADATA_ONLY)) {
     std::vector<SharedHandle<RequestGroup> > newRgs;
+    // Don't adjust announce URI because it has been done when
+    // RequestGroup is created with magnet URI.
     createRequestGroupForBitTorrent(newRgs, requestGroup->getOption(),
-                                    std::vector<std::string>(), torrent);
+                                    std::vector<std::string>(), torrent, false);
     requestGroup->followedBy(newRgs.begin(), newRgs.end());
     if(!requestGroup->getMetadataInfo().isNull()) {
       setMetadataInfo(newRgs.begin(), newRgs.end(),

src/bittorrent_helper.cc

@@ -55,6 +55,8 @@
 #include "bencode2.h"
 #include "TorrentAttribute.h"
 #include "SocketCore.h"
+#include "Option.h"
+#include "prefs.h"
 
 namespace aria2 {
 
@@ -1005,6 +1007,60 @@ int getCompactLength(int family)
   }
 }
 
+void removeAnnounceUri
+(const SharedHandle<TorrentAttribute>& attrs,
+ const std::vector<std::string>& uris)
+{
+  if(uris.empty()) {
+    return;
+  }
+  if(std::find(uris.begin(), uris.end(), "*") == uris.end()) {
+    for(std::vector<std::vector<std::string> >::iterator i =
+          attrs->announceList.begin(); i != attrs->announceList.end();) {
+      for(std::vector<std::string>::iterator j =(*i).begin();j != (*i).end();) {
+        if(std::find(uris.begin(), uris.end(), *j) == uris.end()) {
+          ++j;
+        } else {
+          j = (*i).erase(j);
+        }
+      }
+      if((*i).empty()) {
+        i = attrs->announceList.erase(i);
+      } else {
+        ++i;
+      }
+    }
+  } else {
+    attrs->announceList.clear();
+  }
+}
+
+void addAnnounceUri
+(const SharedHandle<TorrentAttribute>& attrs,
+ const std::vector<std::string>& uris)
+{
+  for(std::vector<std::string>::const_iterator i = uris.begin(),
+        eoi = uris.end(); i != eoi; ++i) {
+    std::vector<std::string> tier;
+    tier.push_back(*i);
+    attrs->announceList.push_back(tier);
+  }
+}
+
+void adjustAnnounceUri
+(const SharedHandle<TorrentAttribute>& attrs,
+ const SharedHandle<Option>& option)
+{
+  std::vector<std::string> excludeUris;
+  std::vector<std::string> addUris;
+  util::split(option->get(PREF_BT_EXCLUDE_TRACKER),
+              std::back_inserter(excludeUris), A2STR::COMMA_C, true);
+  util::split(option->get(PREF_BT_TRACKER),
+              std::back_inserter(addUris), A2STR::COMMA_C, true);
+  removeAnnounceUri(attrs, excludeUris);
+  addAnnounceUri(attrs, addUris);
+}
+
 } // namespace bittorrent
 
 } // namespace aria2

src/bittorrent_helper.h

@@ -52,6 +52,7 @@ namespace aria2 {
 
 class DownloadContext;
 class Randomizer;
+class Option;
 
 namespace bittorrent {
 
@@ -222,6 +223,27 @@ std::string metadata2Torrent
 // Constructs BitTorrent Magnet URI using attrs.
 std::string torrent2Magnet(const SharedHandle<TorrentAttribute>& attrs);
 
+// Removes announce URI in uris from attrs.  If uris contains '*', all
+// announce URIs are removed.
+void removeAnnounceUri
+(const SharedHandle<TorrentAttribute>& attrs,
+ const std::vector<std::string>& uris);
+
+// Adds announce URI in uris to attrs. Each URI in uris creates its
+// own tier.
+void addAnnounceUri
+(const SharedHandle<TorrentAttribute>& attrs,
+ const std::vector<std::string>& uris);
+
+// This helper function uses 2 option values: PREF_BT_TRACKER and
+// PREF_BT_EXCLUDE_TRACKER. First, the value of
+// PREF_BT_EXCLUDE_TRACKER is converted to std::vector<std::string>
+// and call removeAnnounceUri(). Then the value of PREF_BT_TRACKER is
+// converted to std::vector<std::string> and call addAnnounceUri().
+void adjustAnnounceUri
+(const SharedHandle<TorrentAttribute>& attrs,
+ const SharedHandle<Option>& option);
+
 template<typename OutputIterator>
 void extractPeer(const ValueBase* peerData, int family, OutputIterator dest)
 {

src/download_helper.cc

@@ -240,7 +240,8 @@ SharedHandle<RequestGroup>
 createBtRequestGroup(const std::string& torrentFilePath,
                      const SharedHandle<Option>& option,
                      const std::vector<std::string>& auxUris,
-                     const std::string& torrentData = "")
+                     const std::string& torrentData = "",
+                     bool adjustAnnounceUri = true)
 {
   SharedHandle<RequestGroup> rg(new RequestGroup(option));
   SharedHandle<DownloadContext> dctx(new DownloadContext());
@@ -254,6 +255,9 @@ createBtRequestGroup(const std::string& torrentFilePath,
     // exception
     rg->setMetadataInfo(createMetadataInfoDataOnly());
   }
+  if(adjustAnnounceUri) {
+    bittorrent::adjustAnnounceUri(bittorrent::getTorrentAttrs(dctx), option);
+  }
   dctx->setFileFilter(util::parseIntRange(option->get(PREF_SELECT_FILE)));
   std::istringstream indexOutIn(option->get(PREF_INDEX_OUT));
   std::map<size_t, std::string> indexPathMap =
@@ -288,6 +292,7 @@ createBtMagnetRequestGroup(const std::string& magnetLink,
   bittorrent::loadMagnet(magnetLink, dctx);
   SharedHandle<TorrentAttribute> torrentAttrs =
     bittorrent::getTorrentAttrs(dctx);
+  bittorrent::adjustAnnounceUri(torrentAttrs, rg->getOption());
   dctx->getFirstFileEntry()->setPath(torrentAttrs->name);
   rg->setDownloadContext(dctx);
   rg->clearPostDownloadHandler();
@@ -304,7 +309,8 @@ void createRequestGroupForBitTorrent
 (std::vector<SharedHandle<RequestGroup> >& result,
  const SharedHandle<Option>& option,
  const std::vector<std::string>& uris,
- const std::string& torrentData)
+ const std::string& torrentData,
+ bool adjustAnnounceUri)
 {
   std::vector<std::string> nargs;
   if(option->get(PREF_PARAMETERIZED_URI) == V_TRUE) {
@@ -316,7 +322,7 @@ void createRequestGroupForBitTorrent
   size_t numSplit = option->getAsInt(PREF_SPLIT);
   SharedHandle<RequestGroup> rg =
     createBtRequestGroup(option->get(PREF_TORRENT_FILE), option, nargs,
-                         torrentData);
+                         torrentData, adjustAnnounceUri);
   rg->setNumConcurrentCommand(numSplit);
   result.push_back(rg);
 }

src/download_helper.h

@@ -55,13 +55,16 @@ const std::set<std::string>& listRequestOptions();
 #ifdef ENABLE_BITTORRENT
 // Create RequestGroup object using torrent file specified by
 // torrent-file option.  If non-empty torrentData is specified, then
-// it is used as a content of torrent file instead. In this function,
+// it is used as a content of torrent file instead. If
+// adjustAnnounceUri is true, announce URIs are adjusted using
+// bittorrent::adjustAnnounceUri().  In this function,
 // force-sequential is ignored.
 void createRequestGroupForBitTorrent
 (std::vector<SharedHandle<RequestGroup> >& result,
  const SharedHandle<Option>& option,
  const std::vector<std::string>& uris,
- const std::string& torrentData = "");
+ const std::string& torrentData = "",
+ bool adjustAnnounceUri = true);
 #endif // ENABLE_BITTORRENT
 
 #ifdef ENABLE_METALINK

src/prefs.cc

@@ -374,6 +374,10 @@ const std::string PREF_BT_TRACKER_CONNECT_TIMEOUT("bt-tracker-connect-timeout");
 const std::string PREF_DHT_MESSAGE_TIMEOUT("dht-message-timeout");
 // values: string
 const std::string PREF_ON_BT_DOWNLOAD_COMPLETE("on-bt-download-complete");
+// values: string
+const std::string PREF_BT_TRACKER("bt-tracker");
+// values: string
+const std::string PREF_BT_EXCLUDE_TRACKER("bt-exclude-tracker");
 
 /**
  * Metalink related preferences

src/prefs.h

@@ -378,6 +378,10 @@ extern const std::string PREF_BT_TRACKER_CONNECT_TIMEOUT;
 extern const std::string PREF_DHT_MESSAGE_TIMEOUT;
 // values: string
 extern const std::string PREF_ON_BT_DOWNLOAD_COMPLETE;
+// values: string
+extern const std::string PREF_BT_TRACKER;
+// values: string
+extern const std::string PREF_BT_EXCLUDE_TRACKER;
 
 /**
  * Metalink related preferences

src/usage_text.h

@@ -728,3 +728,16 @@
     "                              network.")
 #define TEXT_DHT_FILE_PATH6                     \
   _(" --dht-file-path6=PATH        Change the IPv6 DHT routing table file to PATH.")
+#define TEXT_BT_TRACKER                                                 \
+  _(" --bt-tracker=URI[,...]       Comma separated list of additional BitTorrent\n" \
+    "                              tracker's announce URI. These URIs are not\n" \
+    "                              affected by --bt-exclude-tracker option because\n" \
+    "                              they are added after URIs in --bt-exclude-tracker\n" \
+    "                              option are removed.")
+#define TEXT_BT_EXCLUDE_TRACKER                                         \
+  _(" --bt-exclude-tracker=URI[,...] Comma separated list of BitTorrent tracker's\n" \
+    "                              announce URI to remove. You can use special value\n" \
+    "                              '*' which matches all URIs, thus removes all\n" \
+    "                              announce URIs. When specifying '*' in shell\n" \
+    "                              command-line, don't forget to escape or quote it.\n" \
+    "                              See also --bt-tracker option.")

test/BittorrentHelperTest.cc

@@ -17,6 +17,8 @@
 #include "bencode2.h"
 #include "TestUtil.h"
 #include "base32.h"
+#include "Option.h"
+#include "prefs.h"
 
 namespace aria2 {
 
@@ -67,6 +69,9 @@ class BittorrentHelperTest:public CppUnit::TestFixture {
   CPPUNIT_TEST(testExtract2PeersFromList);
   CPPUNIT_TEST(testPackcompact);
   CPPUNIT_TEST(testUnpackcompact);
+  CPPUNIT_TEST(testRemoveAnnounceUri);
+  CPPUNIT_TEST(testAddAnnounceUri);
+  CPPUNIT_TEST(testAdjustAnnounceUri);
   CPPUNIT_TEST_SUITE_END();
 public:
   void setUp() {
@@ -114,6 +119,9 @@ public:
   void testExtract2PeersFromList();
   void testPackcompact();
   void testUnpackcompact();
+  void testRemoveAnnounceUri();
+  void testAddAnnounceUri();
+  void testAdjustAnnounceUri();
 };
 
 
@@ -852,6 +860,82 @@ void BittorrentHelperTest::testUnpackcompact()
   CPPUNIT_ASSERT_EQUAL((uint16_t)6881, p.second);
 }
 
+void BittorrentHelperTest::testRemoveAnnounceUri()
+{
+  SharedHandle<TorrentAttribute> attrs(new TorrentAttribute());
+  std::vector<std::string> tier1;
+  tier1.push_back("http://host1/announce");
+  std::vector<std::string> tier2;
+  tier2.push_back("http://host2/announce");
+  tier2.push_back("http://host3/announce");
+  attrs->announceList.push_back(tier1);
+  attrs->announceList.push_back(tier2);
+  
+  std::vector<std::string> removeUris;
+  removeUris.push_back(tier1[0]);
+  removeUris.push_back(tier2[0]);
+  removeAnnounceUri(attrs, removeUris);
+  CPPUNIT_ASSERT_EQUAL((size_t)1, attrs->announceList.size());
+  CPPUNIT_ASSERT_EQUAL(std::string("http://host3/announce"),
+                       attrs->announceList[0][0]);
+
+  removeUris.clear();
+  removeUris.push_back("*");
+
+  removeAnnounceUri(attrs, removeUris);
+  CPPUNIT_ASSERT(attrs->announceList.empty());
+}
+
+void BittorrentHelperTest::testAddAnnounceUri()
+{
+  SharedHandle<TorrentAttribute> attrs(new TorrentAttribute());
+  std::vector<std::string> addUris;
+  addUris.push_back("http://host1/announce");
+  addUris.push_back("http://host2/announce");
+  addAnnounceUri(attrs, addUris);
+  CPPUNIT_ASSERT_EQUAL((size_t)2, attrs->announceList.size());
+
+  CPPUNIT_ASSERT_EQUAL((size_t)1, attrs->announceList[0].size());
+  CPPUNIT_ASSERT_EQUAL(std::string("http://host1/announce"),
+                       attrs->announceList[0][0]);
+
+  CPPUNIT_ASSERT_EQUAL((size_t)1, attrs->announceList[1].size());
+  CPPUNIT_ASSERT_EQUAL(std::string("http://host2/announce"),
+                       attrs->announceList[1][0]);
+}
+
+void BittorrentHelperTest::testAdjustAnnounceUri()
+{
+  SharedHandle<TorrentAttribute> attrs(new TorrentAttribute());
+  std::vector<std::string> tier1;
+  tier1.push_back("http://host1/announce");
+  std::vector<std::string> tier2;
+  tier2.push_back("http://host2/announce");
+  tier2.push_back("http://host3/announce");
+  attrs->announceList.push_back(tier1);
+  attrs->announceList.push_back(tier2);
+
+  SharedHandle<Option> option(new Option());
+  option->put(PREF_BT_TRACKER, "http://host1/announce,http://host4/announce");
+  option->put(PREF_BT_EXCLUDE_TRACKER,
+              "http://host1/announce,http://host2/announce");
+  adjustAnnounceUri(attrs, option);
+
+  CPPUNIT_ASSERT_EQUAL((size_t)3, attrs->announceList.size());
+
+  CPPUNIT_ASSERT_EQUAL((size_t)1, attrs->announceList[0].size());
+  CPPUNIT_ASSERT_EQUAL(std::string("http://host3/announce"),
+                       attrs->announceList[0][0]);
+
+  CPPUNIT_ASSERT_EQUAL((size_t)1, attrs->announceList[1].size());
+  CPPUNIT_ASSERT_EQUAL(std::string("http://host1/announce"),
+                       attrs->announceList[1][0]);
+
+  CPPUNIT_ASSERT_EQUAL((size_t)1, attrs->announceList[2].size());
+  CPPUNIT_ASSERT_EQUAL(std::string("http://host4/announce"),
+                       attrs->announceList[2][0]);
+}
+
 } // namespace bittorrent
 
 } // namespace aria2

test/DownloadHelperTest.cc

@@ -13,6 +13,9 @@
 #include "prefs.h"
 #include "Exception.h"
 #include "util.h"
+#ifdef ENABLE_BITTORRENT
+# include "bittorrent_helper.h"
+#endif // ENABLE_BITTORRENT
 
 namespace aria2 {
 
@@ -326,6 +329,7 @@ void DownloadHelperTest::testCreateRequestGroupForBitTorrent()
   option_->put(PREF_TORRENT_FILE, "test.torrent");
   option_->put(PREF_DIR, "/tmp");
   option_->put(PREF_OUT, "file.out");
+  option_->put(PREF_BT_EXCLUDE_TRACKER, "http://tracker1");
   {
     std::vector<SharedHandle<RequestGroup> > result;
   
@@ -343,6 +347,10 @@ void DownloadHelperTest::testCreateRequestGroupForBitTorrent()
       CPPUNIT_ASSERT_EQUAL(array[i]+"/aria2-test/aria2/src/aria2c", uris[i]);
     }
     CPPUNIT_ASSERT_EQUAL((unsigned int)5, group->getNumConcurrentCommand());
+    SharedHandle<TorrentAttribute> attrs =
+      bittorrent::getTorrentAttrs(group->getDownloadContext());
+    // http://tracker1 was deleted.
+    CPPUNIT_ASSERT_EQUAL((size_t)2, attrs->announceList.size());
   }
   {
     // no URIs are given