2008-04-26 Tatsuhiro Tsujikawa <tujikawa at rednoah dot com>

Added build-in "Accept-Feature" support. For now only "metalink" is
	used in this header field.
	This "metalink" value is removed from the list when connecting URLs
	fed by metalink file to avoid loop in "transparent" metlaink.
	* src/HttpRequest.cc
	* src/HttpRequest.h: Renamed _userHeaders as _headers. Accept-Feature
	header is also held in this variable. Also renamed setUserHeaders as
	addHeader and it was rewritten to add header not just to clear the old
	value.
	* src/HttpRequestCommand.cc
	* src/Metalink2RequestGroup.cc: Added the code to remove "metalink"
	from "Accept-Feature" list.
	* src/RequestGroup.cc: Added "metalink" to "Accept-Feature" by default.
	* src/RequestGroup.h
	* src/TaggedItem.cc: Moved Concat class to a2functional.h.
	* src/a2functional.h: Included <string> because Concat class depends on
	it.
	* test/HttpRequestTest.cc

ChangeLog

@@ -1,3 +1,24 @@
+2008-04-26  Tatsuhiro Tsujikawa  <tujikawa at rednoah dot com>
+
+	Added build-in "Accept-Feature" support. For now only "metalink" is
+	used in this header field.
+	This "metalink" value is removed from the list when connecting URLs
+	fed by metalink file to avoid loop in "transparent" metlaink.
+	* src/HttpRequest.cc
+	* src/HttpRequest.h: Renamed _userHeaders as _headers. Accept-Feature
+	header is also held in this variable. Also renamed setUserHeaders as
+	addHeader and it was rewritten to add header not just to clear the old
+	value.
+	* src/HttpRequestCommand.cc
+	* src/Metalink2RequestGroup.cc: Added the code to remove "metalink"
+	from "Accept-Feature" list.
+	* src/RequestGroup.cc: Added "metalink" to "Accept-Feature" by default.
+	* src/RequestGroup.h
+	* src/TaggedItem.cc: Moved Concat class to a2functional.h.
+	* src/a2functional.h: Included <string> because Concat class depends on
+	it.
+	* test/HttpRequestTest.cc
+
 2008-04-26  Tatsuhiro Tsujikawa  <tujikawa at rednoah dot com>
 
 	Added comment

src/HttpRequest.cc

@@ -192,8 +192,8 @@ std::string HttpRequest::createRequest() const
     requestLine += std::string("Cookie: ")+cookiesValue+"\r\n";
   }
   // append additional headers given by user.
-  for(std::deque<std::string>::const_iterator i = _userHeaders.begin();
-      i != _userHeaders.end(); ++i) {
+  for(std::deque<std::string>::const_iterator i = _headers.begin();
+      i != _headers.end(); ++i) {
     requestLine += (*i)+"\r\n";
   }
 
@@ -225,11 +225,11 @@ std::string HttpRequest::getProxyAuthString() const {
     Base64::encode(AuthConfigFactorySingleton::instance()->createAuthConfigForHttpProxy(request)->getAuthText())+"\r\n";
 }
 
-void HttpRequest::setUserHeaders(const std::string& userHeadersString)
+void HttpRequest::addHeader(const std::string& headersString)
 {
   std::deque<std::string> headers;
-  Util::slice(headers, userHeadersString, '\n', true);
-  _userHeaders = headers;
+  Util::slice(headers, headersString, '\n', true);
+  _headers.insert(_headers.end(), headers.begin(), headers.end());
 }
 
 void HttpRequest::configure(const Option* option)

src/HttpRequest.h

@@ -66,7 +66,7 @@ private:
 
   std::string userAgent;
 
-  std::deque<std::string> _userHeaders;
+  std::deque<std::string> _headers;
 
   std::string getHostText(const std::string& host, uint16_t port) const;
 
@@ -167,7 +167,8 @@ public:
     this->userAgent = userAgent;
   }
   
-  void setUserHeaders(const std::string& userHeaders);
+  // accepts multiline headers, deliminated by LF
+  void addHeader(const std::string& headers);
 };
 
 typedef SharedHandle<HttpRequest> HttpRequestHandle;

src/HttpRequestCommand.cc

@@ -44,6 +44,9 @@
 #include "Option.h"
 #include "Socket.h"
 #include "prefs.h"
+#include "a2functional.h"
+#include "Util.h"
+#include <numeric>
 
 namespace aria2 {
 
@@ -67,14 +70,24 @@ static SharedHandle<HttpRequest>
 createHttpRequest(const SharedHandle<Request>& req,
 		  const SharedHandle<Segment>& segment,
 		  uint64_t totalLength,
-		  const Option* option)
+		  const Option* option,
+		  const std::deque<std::string>& acceptFeatures)
 {
   HttpRequestHandle httpRequest(new HttpRequest());
   httpRequest->setUserAgent(option->get(PREF_USER_AGENT));
   httpRequest->setRequest(req);
   httpRequest->setSegment(segment);
   httpRequest->setEntityLength(totalLength);
-  httpRequest->setUserHeaders(option->get(PREF_HEADER));
+  httpRequest->addHeader(option->get(PREF_HEADER));
+  if(acceptFeatures.size()) {
+    std::string acceptFeaturesHeader = "Accept-Features: "+
+      Util::trim
+      (std::accumulate(acceptFeatures.begin()+1, acceptFeatures.end(),
+		       *acceptFeatures.begin(),
+		       Concat(",")),
+       ",");
+    httpRequest->addHeader(acceptFeaturesHeader);
+  }
   httpRequest->configure(option);
 
   return httpRequest;
@@ -89,7 +102,8 @@ bool HttpRequestCommand::executeInternal() {
   if(_segments.empty()) {
     HttpRequestHandle httpRequest
       (createHttpRequest(req, SharedHandle<Segment>(),
-			 _requestGroup->getTotalLength(), e->option));    
+			 _requestGroup->getTotalLength(), e->option,
+			 _requestGroup->getAcceptFeatures()));
     _httpConnection->sendRequest(httpRequest);
   } else {
     for(Segments::iterator itr = _segments.begin(); itr != _segments.end(); ++itr) {
@@ -97,7 +111,8 @@ bool HttpRequestCommand::executeInternal() {
       if(!_httpConnection->isIssued(segment)) {
 	HttpRequestHandle httpRequest
 	  (createHttpRequest(req, segment,
-			     _requestGroup->getTotalLength(), e->option));    
+			     _requestGroup->getTotalLength(), e->option,
+			     _requestGroup->getAcceptFeatures()));
 	_httpConnection->sendRequest(httpRequest);
       }
     }

src/Metalink2RequestGroup.cc

@@ -169,6 +169,9 @@ Metalink2RequestGroup::createRequestGroup(std::deque<SharedHandle<MetalinkEntry>
       torrentRg->setDownloadContext(dctx);
       torrentRg->clearPreDowloadHandler();
       torrentRg->clearPostDowloadHandler();
+      // remove "metalink" from Accept-Feature list to avoid loop in tranparent
+      // metalink
+      torrentRg->removeAcceptFeatureHeader(RequestGroup::FEATURE_METALINK);
       // make it in-memory download
       SharedHandle<PreDownloadHandler> preh(new MemoryBufferPreDownloadHandler());
       {
@@ -219,6 +222,9 @@ Metalink2RequestGroup::createRequestGroup(std::deque<SharedHandle<MetalinkEntry>
 				std::min(_option->getAsInt(PREF_METALINK_SERVERS), entry->maxConnections));
     // In metalink, multi connection to a single host is not allowed by default.
     rg->setSingleHostMultiConnectionEnabled(!_option->getAsBool(PREF_METALINK_ENABLE_UNIQUE_PROTOCOL));
+    // remove "metalink" from Accept-Feature list to avoid loop in tranparent
+    // metalink
+    rg->removeAcceptFeatureHeader(RequestGroup::FEATURE_METALINK);
 
 #ifdef ENABLE_BITTORRENT
     // Inject depenency between rg and torrentRg here if torrentRg.isNull() == false

src/RequestGroup.cc

@@ -103,6 +103,8 @@ namespace aria2 {
 
 int32_t RequestGroup::_gidCounter = 0;
 
+const std::string RequestGroup::FEATURE_METALINK = "metalink";
+
 RequestGroup::RequestGroup(const Option* option,
 			   const std::deque<std::string>& uris):
   _gid(++_gidCounter),
@@ -124,6 +126,12 @@ RequestGroup::RequestGroup(const Option* option,
   } else {
     _fileAllocationEnabled = false;
   }
+  // For now, only supported 'Accept-Features' is "metalink" used for
+  // transparent metalink.
+  // It would be good to put this value in Option so that user can tweak
+  // and add this list.
+  _acceptFeatures.push_back(FEATURE_METALINK);
+
   initializePreDownloadHandler();
   initializePostDownloadHandler();
 }
@@ -933,4 +941,24 @@ void RequestGroup::reportDownloadFinished()
 #endif // ENABLE_BITTORRENT
 }
 
+const std::deque<std::string>& RequestGroup::getAcceptFeatures() const
+{
+  return _acceptFeatures;
+}
+
+void RequestGroup::addAcceptFeatureHeader(const std::string& feature)
+{
+  if(std::find(_acceptFeatures.begin(), _acceptFeatures.end(), feature) == _acceptFeatures.end()) {
+    _acceptFeatures.push_back(feature);
+  }
+}
+
+void RequestGroup::removeAcceptFeatureHeader(const std::string& feature)
+{
+  std::deque<std::string>::iterator i = std::find(_acceptFeatures.begin(), _acceptFeatures.end(), feature);
+  if(i != _acceptFeatures.end()) {
+    _acceptFeatures.erase(i);
+  }
+}
+
 } // namespace aria2

src/RequestGroup.h

@@ -108,6 +108,8 @@ private:
 
   std::deque<SharedHandle<PostDownloadHandler> > _postDownloadHandlers;
 
+  std::deque<std::string> _acceptFeatures;
+
   const Option* _option;
 
   const Logger* _logger;
@@ -334,6 +336,14 @@ public:
   void removeURIWhoseHostnameIs(const std::string& hostname);
 
   void reportDownloadFinished();
+
+  const std::deque<std::string>& getAcceptFeatures() const;
+
+  void addAcceptFeatureHeader(const std::string& feature);
+
+  void removeAcceptFeatureHeader(const std::string& feature);
+
+  static const std::string FEATURE_METALINK;
 };
 
 typedef SharedHandle<RequestGroup> RequestGroupHandle;

src/TaggedItem.cc

@@ -33,23 +33,12 @@
  */
 /* copyright --> */
 #include "TaggedItem.h"
+#include "a2functional.h"
 #include <numeric>
 #include <algorithm>
 
 namespace aria2 {
 
-class Concat {
-private:
-  std::string _delim;
-public:
-  Concat(const std::string& delim = ""):_delim(delim) {}
-
-  std::string operator()(const std::string& s1, const std::string& s2) const
-  {
-    return s1+_delim+s2;
-  }
-};
-
 std::string TaggedItem::toTagString() const
 {
   if(_tags.size()) {

src/a2functional.h

@@ -37,6 +37,7 @@
 
 #include <functional>
 #include "SharedHandle.h"
+#include <string>
 
 namespace aria2 {
 
@@ -138,6 +139,18 @@ public:
   }
 };
 
+class Concat {
+private:
+  std::string _delim;
+public:
+  Concat(const std::string& delim = ""):_delim(delim) {}
+
+  std::string operator()(const std::string& s1, const std::string& s2) const
+  {
+    return s1+_delim+s2;
+  }
+};
+
 } // namespace aria2
 
 #endif // _D_A2_FUNCTIONAL_H_

test/HttpRequestTest.cc

@@ -22,7 +22,7 @@ class HttpRequestTest : public CppUnit::TestFixture {
   CPPUNIT_TEST(testCreateProxyRequest);
   CPPUNIT_TEST(testIsRangeSatisfied);
   CPPUNIT_TEST(testUserAgent);
-  CPPUNIT_TEST(testUserHeaders);
+  CPPUNIT_TEST(testAddHeader);
   CPPUNIT_TEST_SUITE_END();
 private:
 
@@ -37,7 +37,7 @@ public:
   void testCreateProxyRequest();
   void testIsRangeSatisfied();
   void testUserAgent();
-  void testUserHeaders();
+  void testAddHeader();
 };
 
 
@@ -596,14 +596,14 @@ void HttpRequestTest::testUserAgent()
   CPPUNIT_ASSERT_EQUAL(expectedTextForProxy, httpRequest.createProxyRequest());
 }
 
-void HttpRequestTest::testUserHeaders()
+void HttpRequestTest::testAddHeader()
 {
   SharedHandle<Request> request(new Request());
   request->setUrl("http://localhost/archives/aria2-1.0.0.tar.bz2");
 
   HttpRequest httpRequest;
   httpRequest.setRequest(request);
-  httpRequest.setUserHeaders("X-ARIA2: v0.13\nX-ARIA2-DISTRIBUTE: enabled\n");
+  httpRequest.addHeader("X-ARIA2: v0.13\nX-ARIA2-DISTRIBUTE: enabled\n");
 
   std::string expectedText = "GET /archives/aria2-1.0.0.tar.bz2 HTTP/1.1\r\n"
     "User-Agent: aria2\r\n"