2008-05-05 Tatsuhiro Tsujikawa <tujikawa at rednoah dot com>

Reimplemented transparent metalink using Accept request header.
	* src/HttpRequest.cc
	* src/HttpRequest.h
	* src/HttpRequestCommand.cc
	* src/Metalink2RequestGroup.cc
	* src/RequestGroup.cc
	* src/RequestGroup.h
	* test/HttpRequestTest.cc

ChangeLog

@@ -1,3 +1,14 @@
+2008-05-05  Tatsuhiro Tsujikawa  <tujikawa at rednoah dot com>
+
+	Reimplemented transparent metalink using Accept request header.
+	* src/HttpRequest.cc
+	* src/HttpRequest.h
+	* src/HttpRequestCommand.cc
+	* src/Metalink2RequestGroup.cc
+	* src/RequestGroup.cc
+	* src/RequestGroup.h
+	* test/HttpRequestTest.cc
+	
 2008-04-27  Tatsuhiro Tsujikawa  <tujikawa at rednoah dot com>
 
 	Bump up version number to 0.13.2a

src/HttpRequest.cc

@@ -44,6 +44,7 @@
 #include "prefs.h"
 #include "AuthConfigFactory.h"
 #include "AuthConfig.h"
+#include "a2functional.h"
 #include <numeric>
 
 namespace aria2 {
@@ -145,11 +146,17 @@ std::string HttpRequest::createRequest() const
   }
   requestLine +=
     std::string(" HTTP/1.1\r\n")+
-    "User-Agent: "+userAgent+"\r\n"+
-    "Accept: */*\r\n"+        /* */
+    "User-Agent: "+userAgent+"\r\n";
+  
+  requestLine +=
+    std::accumulate(_acceptTypes.begin(), _acceptTypes.end(),
+		    std::string("Accept: */*"), Concat(","))+"\r\n"; /* */
+
+  requestLine +=
     "Host: "+getHostText(getHost(), getPort())+"\r\n"+
     "Pragma: no-cache\r\n"+
     "Cache-Control: no-cache\r\n";
+
   if(!request->isKeepAliveEnabled() && !request->isPipeliningEnabled()) {
     requestLine += "Connection: close\r\n";
   }
@@ -232,6 +239,11 @@ void HttpRequest::addHeader(const std::string& headersString)
   _headers.insert(_headers.end(), headers.begin(), headers.end());
 }
 
+void HttpRequest::addAcceptType(const std::string& type)
+{
+  _acceptTypes.push_back(type);
+}
+
 void HttpRequest::configure(const Option* option)
 {
   authEnabled = option->get(PREF_HTTP_AUTH_ENABLED) == V_TRUE;

src/HttpRequest.h

@@ -68,6 +68,8 @@ private:
 
   std::deque<std::string> _headers;
 
+  std::deque<std::string> _acceptTypes;
+
   std::string getHostText(const std::string& host, uint16_t port) const;
 
   std::string getProxyAuthString() const;
@@ -171,6 +173,14 @@ public:
   
   // accepts multiline headers, deliminated by LF
   void addHeader(const std::string& headers);
+
+  void addAcceptType(const std::string& type);
+
+  template<typename InputIterator>
+  void addAcceptType(InputIterator first, InputIterator last)
+  {
+    _acceptTypes.insert(_acceptTypes.end(), first, last);
+  }
 };
 
 typedef SharedHandle<HttpRequest> HttpRequestHandle;

src/HttpRequestCommand.cc

@@ -71,7 +71,7 @@ createHttpRequest(const SharedHandle<Request>& req,
 		  const SharedHandle<Segment>& segment,
 		  uint64_t totalLength,
 		  const Option* option,
-		  const std::deque<std::string>& acceptFeatures)
+		  const RequestGroup* rg)
 {
   HttpRequestHandle httpRequest(new HttpRequest());
   httpRequest->setUserAgent(option->get(PREF_USER_AGENT));
@@ -79,7 +79,8 @@ createHttpRequest(const SharedHandle<Request>& req,
   httpRequest->setSegment(segment);
   httpRequest->setEntityLength(totalLength);
   httpRequest->addHeader(option->get(PREF_HEADER));
-  if(acceptFeatures.size()) {
+  if(rg->getAcceptFeatures().size()) {
+    const std::deque<std::string>& acceptFeatures = rg->getAcceptFeatures();
     std::string acceptFeaturesHeader = "Accept-Features: "+
       Util::trim
       (std::accumulate(acceptFeatures.begin()+1, acceptFeatures.end(),
@@ -88,6 +89,8 @@ createHttpRequest(const SharedHandle<Request>& req,
        ",");
     httpRequest->addHeader(acceptFeaturesHeader);
   }
+  httpRequest->addAcceptType(rg->getAcceptTypes().begin(),
+			     rg->getAcceptTypes().end());
   httpRequest->configure(option);
 
   return httpRequest;
@@ -103,7 +106,7 @@ bool HttpRequestCommand::executeInternal() {
     HttpRequestHandle httpRequest
       (createHttpRequest(req, SharedHandle<Segment>(),
 			 _requestGroup->getTotalLength(), e->option,
-			 _requestGroup->getAcceptFeatures()));
+			 _requestGroup));
     _httpConnection->sendRequest(httpRequest);
   } else {
     for(Segments::iterator itr = _segments.begin(); itr != _segments.end(); ++itr) {
@@ -112,7 +115,7 @@ bool HttpRequestCommand::executeInternal() {
 	HttpRequestHandle httpRequest
 	  (createHttpRequest(req, segment,
 			     _requestGroup->getTotalLength(), e->option,
-			     _requestGroup->getAcceptFeatures()));
+			     _requestGroup));
 	_httpConnection->sendRequest(httpRequest);
       }
     }

src/Metalink2RequestGroup.cc

@@ -169,9 +169,9 @@ Metalink2RequestGroup::createRequestGroup(std::deque<SharedHandle<MetalinkEntry>
       torrentRg->setDownloadContext(dctx);
       torrentRg->clearPreDowloadHandler();
       torrentRg->clearPostDowloadHandler();
-      // remove "metalink" from Accept-Feature list to avoid loop in tranparent
+      // remove "metalink" from Accept Type list to avoid loop in tranparent
       // metalink
-      torrentRg->removeAcceptFeatureHeader(RequestGroup::FEATURE_METALINK);
+      torrentRg->removeAcceptType(RequestGroup::ACCEPT_METALINK);
       // make it in-memory download
       SharedHandle<PreDownloadHandler> preh(new MemoryBufferPreDownloadHandler());
       {
@@ -222,9 +222,9 @@ Metalink2RequestGroup::createRequestGroup(std::deque<SharedHandle<MetalinkEntry>
 				std::min(_option->getAsInt(PREF_METALINK_SERVERS), entry->maxConnections));
     // In metalink, multi connection to a single host is not allowed by default.
     rg->setSingleHostMultiConnectionEnabled(!_option->getAsBool(PREF_METALINK_ENABLE_UNIQUE_PROTOCOL));
-    // remove "metalink" from Accept-Feature list to avoid loop in tranparent
+    // remove "metalink" from Accept Type list to avoid loop in tranparent
     // metalink
-    rg->removeAcceptFeatureHeader(RequestGroup::FEATURE_METALINK);
+    rg->removeAcceptType(RequestGroup::ACCEPT_METALINK);
 
 #ifdef ENABLE_BITTORRENT
     // Inject depenency between rg and torrentRg here if torrentRg.isNull() == false

src/RequestGroup.cc

@@ -104,7 +104,7 @@ namespace aria2 {
 
 int32_t RequestGroup::_gidCounter = 0;
 
-const std::string RequestGroup::FEATURE_METALINK = "metalink";
+const std::string RequestGroup::ACCEPT_METALINK = "application/metalink+xml";
 
 RequestGroup::RequestGroup(const Option* option,
 			   const std::deque<std::string>& uris):
@@ -127,11 +127,11 @@ RequestGroup::RequestGroup(const Option* option,
   } else {
     _fileAllocationEnabled = false;
   }
-  // For now, only supported 'Accept-Features' is "metalink" used for
-  // transparent metalink.
+  // Add types to be sent as a Accept header value here.
   // It would be good to put this value in Option so that user can tweak
   // and add this list.
-  _acceptFeatures.push_back(FEATURE_METALINK);
+  // ACCEPT_METALINK is used for `transparent metalink'.
+  addAcceptType(ACCEPT_METALINK);
 
   initializePreDownloadHandler();
   initializePostDownloadHandler();
@@ -969,4 +969,22 @@ void RequestGroup::removeAcceptFeatureHeader(const std::string& feature)
   }
 }
 
+const std::deque<std::string>& RequestGroup::getAcceptTypes() const
+{
+  return _acceptTypes;
+}
+
+void RequestGroup::addAcceptType(const std::string& type)
+{
+  if(std::find(_acceptTypes.begin(), _acceptTypes.end(), type) == _acceptTypes.end()) {
+    _acceptTypes.push_back(type);
+  }
+}
+
+void RequestGroup::removeAcceptType(const std::string& type)
+{
+  _acceptTypes.erase(std::remove(_acceptTypes.begin(), _acceptTypes.end(), type),
+		     _acceptTypes.end());
+}
+
 } // namespace aria2

src/RequestGroup.h

@@ -110,6 +110,8 @@ private:
 
   std::deque<std::string> _acceptFeatures;
 
+  std::deque<std::string> _acceptTypes;
+
   const Option* _option;
 
   Logger* _logger;
@@ -343,7 +345,13 @@ public:
 
   void removeAcceptFeatureHeader(const std::string& feature);
 
-  static const std::string FEATURE_METALINK;
+  const std::deque<std::string>& getAcceptTypes() const;
+
+  void addAcceptType(const std::string& type);
+
+  void removeAcceptType(const std::string& type);
+
+  static const std::string ACCEPT_METALINK;
 };
 
 typedef SharedHandle<RequestGroup> RequestGroupHandle;

test/HttpRequestTest.cc

@@ -7,6 +7,7 @@
 #include "Request.h"
 #include "CookieBox.h"
 #include "Option.h"
+#include "array_fun.h"
 #include <cppunit/extensions/HelperMacros.h>
 
 namespace aria2 {
@@ -24,6 +25,7 @@ class HttpRequestTest : public CppUnit::TestFixture {
   CPPUNIT_TEST(testIsRangeSatisfied);
   CPPUNIT_TEST(testUserAgent);
   CPPUNIT_TEST(testAddHeader);
+  CPPUNIT_TEST(testAddAcceptType);
   CPPUNIT_TEST_SUITE_END();
 private:
 
@@ -40,6 +42,7 @@ public:
   void testIsRangeSatisfied();
   void testUserAgent();
   void testAddHeader();
+  void testAddAcceptType();
 };
 
 
@@ -641,5 +644,29 @@ void HttpRequestTest::testAddHeader()
   CPPUNIT_ASSERT_EQUAL(expectedText, httpRequest.createRequest());
 }
 
+void HttpRequestTest::testAddAcceptType()
+{
+  std::string acceptTypes[] = { "cream/custard",
+				"muffin/chocolate" };
+
+  SharedHandle<Request> request(new Request());
+  request->setUrl("http://localhost/archives/aria2-1.0.0.tar.bz2");
+
+  HttpRequest httpRequest;
+  httpRequest.setRequest(request);
+  httpRequest.addAcceptType(&acceptTypes[0], &acceptTypes[arrayLength(acceptTypes)]);
+
+  std::string expectedText =
+    "GET /archives/aria2-1.0.0.tar.bz2 HTTP/1.1\r\n"
+    "User-Agent: aria2\r\n"
+    "Accept: */*,cream/custard,muffin/chocolate\r\n"
+    "Host: localhost\r\n"
+    "Pragma: no-cache\r\n"
+    "Cache-Control: no-cache\r\n"
+    "Connection: close\r\n"
+    "\r\n";
+
+  CPPUNIT_ASSERT_EQUAL(expectedText, httpRequest.createRequest());
+}
 
 } // namespace aria2