2008-11-23 Tatsuhiro Tsujikawa <t-tujikawa@users.sourceforge.net>

Use HEAD method to get file size from HTTP server for segmented
	downloading.
	The request to the BitTorrent Tracker always uses GET method
	because the response of the tracker is small and it doesn't need
	segmented download.
	* src/FtpNegotiationCommand.cc
	* src/HttpRequest.cc
	* src/HttpResponseCommand.cc
	* src/HttpSkipResponseCommand.cc
	* src/RequestGroup.cc
	* src/RequestGroup.h
	* src/TrackerWatcherCommand.cc
	* test/HttpRequestTest.cc

ChangeLog

@@ -1,3 +1,19 @@
+2008-11-23  Tatsuhiro Tsujikawa  <t-tujikawa@users.sourceforge.net>
+
+	Use HEAD method to get file size from HTTP server for segmented
+	downloading.
+	The request to the BitTorrent Tracker always uses GET method because
+	the response of the tracker is small and it doesn't need segmented
+	download.
+	* src/FtpNegotiationCommand.cc
+	* src/HttpRequest.cc
+	* src/HttpResponseCommand.cc
+	* src/HttpSkipResponseCommand.cc
+	* src/RequestGroup.cc
+	* src/RequestGroup.h
+	* src/TrackerWatcherCommand.cc
+	* test/HttpRequestTest.cc
+
 2008-11-23  Tatsuhiro Tsujikawa  <t-tujikawa@users.sourceforge.net>
 
 	Fixed the bug that causes floating exception when -T option is used and

src/FtpNegotiationCommand.cc

@@ -341,12 +341,6 @@ bool FtpNegotiationCommand::onFileSizeDetermined(uint64_t totalLength)
   } else {
     _requestGroup->initPieceStorage();
 
-    // TODO Is this really necessary?
-    if(req->getMethod() == Request::METHOD_HEAD) {
-      sequence = SEQ_HEAD_OK;
-      return false;
-    }
-    
     BtProgressInfoFileHandle infoFile(new DefaultBtProgressInfoFile(_requestGroup->getDownloadContext(), _requestGroup->getPieceStorage(), e->option));
     if(!infoFile->exists() && _requestGroup->downloadFinishedByFileLength()) {
       sequence = SEQ_DOWNLOAD_ALREADY_COMPLETED;

src/HttpRequest.cc

@@ -136,7 +136,7 @@ std::string HttpRequest::createRequest() const
 {
   SharedHandle<AuthConfig> authConfig =
     _authConfigFactory->createAuthConfig(request);
-  std::string requestLine = "GET ";
+  std::string requestLine = request->getMethod()+" ";
   if(!_proxyRequest.isNull()) {
     if(getProtocol() == Request::PROTO_FTP &&
        request->getUsername().empty() && !authConfig->getUser().empty()) {

src/HttpResponseCommand.cc

@@ -188,12 +188,6 @@ bool HttpResponseCommand::handleDefaultEncoding(const HttpResponseHandle& httpRe
   HttpRequestHandle httpRequest = httpResponse->getHttpRequest();
   _requestGroup->initPieceStorage();
 
-  // quick hack for method 'head',, is it necessary?
-  if(httpRequest->getMethod() == Request::METHOD_HEAD) {
-    // TODO because we don't want segment file to be saved.
-    return true;
-  }
-
   BtProgressInfoFileHandle infoFile(new DefaultBtProgressInfoFile(_requestGroup->getDownloadContext(), _requestGroup->getPieceStorage(), e->option));
   if(!infoFile->exists() && _requestGroup->downloadFinishedByFileLength()) {
     return true;
@@ -210,13 +204,20 @@ bool HttpResponseCommand::handleDefaultEncoding(const HttpResponseHandle& httpRe
     // we can't continue to use this socket because server sends all entity
     // body instead of a segment.
     // Therefore, we shutdown the socket here if pipelining is enabled.
-    if(!segment.isNull() && segment->getPositionToWrite() == 0 &&
+    if(req->getMethod() == Request::METHOD_GET &&
+       !segment.isNull() && segment->getPositionToWrite() == 0 &&
        !req->isPipeliningEnabled()) {
       command = createHttpDownloadCommand(httpResponse);
     } else {
       _requestGroup->getSegmentMan()->cancelSegment(cuid);
     }
     prepareForNextAction(command);
+    if(req->getMethod() == Request::METHOD_HEAD) {
+      if(req->supportsPersistentConnection()) {
+	e->poolSocket(req, isProxyDefined(), socket);
+      }
+      req->setMethod(Request::METHOD_GET);
+    }
   } catch(Exception& e) {
     delete command;
     throw;
@@ -261,9 +262,12 @@ static SharedHandle<Decoder> getContentEncodingDecoder
 
 bool HttpResponseCommand::handleOtherEncoding(const HttpResponseHandle& httpResponse) {
   HttpRequestHandle httpRequest = httpResponse->getHttpRequest();
-  // quick hack for method 'head',, is it necessary?
-  if(httpRequest->getMethod() == Request::METHOD_HEAD) {
-    return true;
+  if(req->getMethod() == Request::METHOD_HEAD) {
+    if(req->supportsPersistentConnection()) {
+      e->poolSocket(req, isProxyDefined(), socket);
+    }
+    req->setMethod(Request::METHOD_GET);
+    return prepareForRetry(0);
   }
   _requestGroup->initPieceStorage();
   _requestGroup->shouldCancelDownloadForSafety();
@@ -286,10 +290,11 @@ bool HttpResponseCommand::skipResponseBody
     (cuid, req, _requestGroup, httpConnection, httpResponse, e, socket);
   command->setTransferEncodingDecoder(decoder);
 
-  // If the response body is zero-length, set command's status to real time
-  // so that avoid read check blocking
-  if(httpResponse->getEntityLength() == 0 &&
-     !httpResponse->isTransferEncodingSpecified()) {
+  // If request method is HEAD or the response body is zero-length,
+  // set command's status to real time so that avoid read check blocking
+  if(req->getMethod() == Request::METHOD_HEAD ||
+     (httpResponse->getEntityLength() == 0 &&
+      !httpResponse->isTransferEncodingSpecified())) {
     command->setStatusRealtime();
     // If entity length == 0, then socket read/write check must be disabled.
     command->disableSocketCheck();

src/HttpSkipResponseCommand.cc

@@ -80,12 +80,14 @@ void HttpSkipResponseCommand::setTransferEncodingDecoder
 
 bool HttpSkipResponseCommand::executeInternal()
 {
-  if(_totalLength == 0 && _transferEncodingDecoder.isNull()) {
-    // If content-length header is present and it's value is 0, then
-    // pool socket for reuse.
+  if(req->getMethod() == Request::METHOD_HEAD ||
+     (_totalLength == 0 && _transferEncodingDecoder.isNull())) {
+    // If request method is HEAD or content-length header is present and
+    // it's value is 0, then pool socket for reuse.
     // If content-length header is not present, then EOF is expected in the end.
     // In this case, the content is thrown away and socket cannot be pooled. 
-    if(_httpResponse->getHttpHeader()->defined(HttpHeader::CONTENT_LENGTH)) {
+    if(req->getMethod() == Request::METHOD_HEAD ||
+       _httpResponse->getHttpHeader()->defined(HttpHeader::CONTENT_LENGTH)) {
       poolConnection();
     }
     return processResponse();

src/RequestGroup.cc

@@ -179,7 +179,8 @@ void RequestGroup::closeFile()
 }
 
 void RequestGroup::createInitialCommand(std::deque<Command*>& commands,
-					DownloadEngine* e)
+					DownloadEngine* e,
+					const std::string& method)
 {
 #ifdef ENABLE_BITTORRENT
   {
@@ -306,7 +307,7 @@ void RequestGroup::createInitialCommand(std::deque<Command*>& commands,
   // TODO I assume here when totallength is set to DownloadContext and it is
   // not 0, then filepath is also set DownloadContext correctly....
   if(_downloadContext->getTotalLength() == 0) {
-    createNextCommand(commands, e, 1);
+    createNextCommand(commands, e, 1, method);
   }else {
     if(e->_requestGroupMan->isSameFileBeingDownloaded(this)) {
       throw DownloadFailureException

src/RequestGroup.h

@@ -43,6 +43,7 @@
 #include "SharedHandle.h"
 #include "TransferStat.h"
 #include "TimeA2.h"
+#include "Request.h"
 
 namespace aria2 {
 
@@ -164,15 +165,23 @@ public:
 
   SharedHandle<SegmentMan> getSegmentMan() const;
 
+  // Returns first bootstrap commands to initiate a download.
+  // If this is HTTP/FTP download and file size is unknown, only 1 command
+  // (usually, HttpInitiateConnection or FtpInitiateConnection) will be created
+  // with its Request object having Requet::METHOD_HEAD in its method.
+  // This behavior can be changed by providing 3rd argument.
+  // The method has effect only for using HTTP request including FTP via HTTP
+  // proxy.
   void createInitialCommand(std::deque<Command*>& commands,
-			    DownloadEngine* e);
+			    DownloadEngine* e,
+			    const std::string& method = Request::METHOD_HEAD);
 
   void createNextCommandWithAdj(std::deque<Command*>& commands,
 				DownloadEngine* e, int numAdj);
 
   void createNextCommand(std::deque<Command*>& commands,
 			 DownloadEngine* e, unsigned int numCommand,
-			 const std::string& method = "GET");
+			 const std::string& method = Request::METHOD_GET);
   
   void addURI(const std::string& uri)
   {

src/TrackerWatcherCommand.cc

@@ -57,6 +57,7 @@
 #include "Logger.h"
 #include "A2STR.h"
 #include "SocketCore.h"
+#include "Request.h"
 
 namespace aria2 {
 
@@ -92,7 +93,8 @@ bool TrackerWatcherCommand::execute() {
     _trackerRequestGroup = createAnnounce();
     if(!_trackerRequestGroup.isNull()) {
       std::deque<Command*> commands;
-      _trackerRequestGroup->createInitialCommand(commands, e);
+      _trackerRequestGroup->createInitialCommand(commands, e,
+						 Request::METHOD_GET);
       e->addCommand(commands);
       logger->debug("added tracker request command");
     }

test/HttpRequestTest.cc

@@ -1,5 +1,7 @@
 #include "HttpRequest.h"
 
+#include <sstream>
+
 #include <cppunit/extensions/HelperMacros.h>
 
 #include "prefs.h"
@@ -11,6 +13,7 @@
 #include "Option.h"
 #include "array_fun.h"
 #include "CookieStorage.h"
+#include "Util.h"
 
 namespace aria2 {
 
@@ -23,6 +26,7 @@ class HttpRequestTest : public CppUnit::TestFixture {
   CPPUNIT_TEST(testCreateRequest_ftp);
   CPPUNIT_TEST(testCreateRequest_with_cookie);
   CPPUNIT_TEST(testCreateRequest_query);
+  CPPUNIT_TEST(testCreateRequest_head);
   CPPUNIT_TEST(testCreateProxyRequest);
   CPPUNIT_TEST(testIsRangeSatisfied);
   CPPUNIT_TEST(testUserAgent);
@@ -46,6 +50,7 @@ public:
   void testCreateRequest_ftp();
   void testCreateRequest_with_cookie();
   void testCreateRequest_query();
+  void testCreateRequest_head();
   void testCreateProxyRequest();
   void testIsRangeSatisfied();
   void testUserAgent();
@@ -450,6 +455,23 @@ void HttpRequestTest::testCreateRequest_query()
   CPPUNIT_ASSERT_EQUAL(expectedText, httpRequest.createRequest());
 }
 
+void HttpRequestTest::testCreateRequest_head()
+{
+  SharedHandle<Request> request(new Request());
+  request->setMethod(Request::METHOD_HEAD);
+  request->setUrl("http://localhost/aria2-1.0.0.tar.bz2");
+
+  HttpRequest httpRequest;
+  httpRequest.setRequest(request);
+  httpRequest.setAuthConfigFactory(_authConfigFactory);
+  
+  std::stringstream result(httpRequest.createRequest());
+  std::string line;
+  CPPUNIT_ASSERT(getline(result, line));
+  Util::trimSelf(line);
+  CPPUNIT_ASSERT_EQUAL(std::string("HEAD /aria2-1.0.0.tar.bz2 HTTP/1.1"), line);
+}
+
 void HttpRequestTest::testCreateProxyRequest()
 {
   SharedHandle<Request> request(new Request());