diff --git a/ChangeLog b/ChangeLog index f84071d4..af6b6a51 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,19 @@ +2008-11-23 Tatsuhiro Tsujikawa + + Use HEAD method to get file size from HTTP server for segmented + downloading. + The request to the BitTorrent Tracker always uses GET method because + the response of the tracker is small and it doesn't need segmented + download. + * src/FtpNegotiationCommand.cc + * src/HttpRequest.cc + * src/HttpResponseCommand.cc + * src/HttpSkipResponseCommand.cc + * src/RequestGroup.cc + * src/RequestGroup.h + * src/TrackerWatcherCommand.cc + * test/HttpRequestTest.cc + 2008-11-23 Tatsuhiro Tsujikawa Fixed the bug that causes floating exception when -T option is used and diff --git a/src/FtpNegotiationCommand.cc b/src/FtpNegotiationCommand.cc index 4f25804f..63df6e2f 100644 --- a/src/FtpNegotiationCommand.cc +++ b/src/FtpNegotiationCommand.cc @@ -341,12 +341,6 @@ bool FtpNegotiationCommand::onFileSizeDetermined(uint64_t totalLength) } else { _requestGroup->initPieceStorage(); - // TODO Is this really necessary? - if(req->getMethod() == Request::METHOD_HEAD) { - sequence = SEQ_HEAD_OK; - return false; - } - BtProgressInfoFileHandle infoFile(new DefaultBtProgressInfoFile(_requestGroup->getDownloadContext(), _requestGroup->getPieceStorage(), e->option)); if(!infoFile->exists() && _requestGroup->downloadFinishedByFileLength()) { sequence = SEQ_DOWNLOAD_ALREADY_COMPLETED; diff --git a/src/HttpRequest.cc b/src/HttpRequest.cc index 0a0656f5..4c582116 100644 --- a/src/HttpRequest.cc +++ b/src/HttpRequest.cc @@ -136,7 +136,7 @@ std::string HttpRequest::createRequest() const { SharedHandle authConfig = _authConfigFactory->createAuthConfig(request); - std::string requestLine = "GET "; + std::string requestLine = request->getMethod()+" "; if(!_proxyRequest.isNull()) { if(getProtocol() == Request::PROTO_FTP && request->getUsername().empty() && !authConfig->getUser().empty()) { diff --git a/src/HttpResponseCommand.cc b/src/HttpResponseCommand.cc index 3fe1c22a..a79d3714 100644 --- a/src/HttpResponseCommand.cc +++ b/src/HttpResponseCommand.cc @@ -188,12 +188,6 @@ bool HttpResponseCommand::handleDefaultEncoding(const HttpResponseHandle& httpRe HttpRequestHandle httpRequest = httpResponse->getHttpRequest(); _requestGroup->initPieceStorage(); - // quick hack for method 'head',, is it necessary? - if(httpRequest->getMethod() == Request::METHOD_HEAD) { - // TODO because we don't want segment file to be saved. - return true; - } - BtProgressInfoFileHandle infoFile(new DefaultBtProgressInfoFile(_requestGroup->getDownloadContext(), _requestGroup->getPieceStorage(), e->option)); if(!infoFile->exists() && _requestGroup->downloadFinishedByFileLength()) { return true; @@ -210,13 +204,20 @@ bool HttpResponseCommand::handleDefaultEncoding(const HttpResponseHandle& httpRe // we can't continue to use this socket because server sends all entity // body instead of a segment. // Therefore, we shutdown the socket here if pipelining is enabled. - if(!segment.isNull() && segment->getPositionToWrite() == 0 && + if(req->getMethod() == Request::METHOD_GET && + !segment.isNull() && segment->getPositionToWrite() == 0 && !req->isPipeliningEnabled()) { command = createHttpDownloadCommand(httpResponse); } else { _requestGroup->getSegmentMan()->cancelSegment(cuid); } prepareForNextAction(command); + if(req->getMethod() == Request::METHOD_HEAD) { + if(req->supportsPersistentConnection()) { + e->poolSocket(req, isProxyDefined(), socket); + } + req->setMethod(Request::METHOD_GET); + } } catch(Exception& e) { delete command; throw; @@ -261,9 +262,12 @@ static SharedHandle getContentEncodingDecoder bool HttpResponseCommand::handleOtherEncoding(const HttpResponseHandle& httpResponse) { HttpRequestHandle httpRequest = httpResponse->getHttpRequest(); - // quick hack for method 'head',, is it necessary? - if(httpRequest->getMethod() == Request::METHOD_HEAD) { - return true; + if(req->getMethod() == Request::METHOD_HEAD) { + if(req->supportsPersistentConnection()) { + e->poolSocket(req, isProxyDefined(), socket); + } + req->setMethod(Request::METHOD_GET); + return prepareForRetry(0); } _requestGroup->initPieceStorage(); _requestGroup->shouldCancelDownloadForSafety(); @@ -286,10 +290,11 @@ bool HttpResponseCommand::skipResponseBody (cuid, req, _requestGroup, httpConnection, httpResponse, e, socket); command->setTransferEncodingDecoder(decoder); - // If the response body is zero-length, set command's status to real time - // so that avoid read check blocking - if(httpResponse->getEntityLength() == 0 && - !httpResponse->isTransferEncodingSpecified()) { + // If request method is HEAD or the response body is zero-length, + // set command's status to real time so that avoid read check blocking + if(req->getMethod() == Request::METHOD_HEAD || + (httpResponse->getEntityLength() == 0 && + !httpResponse->isTransferEncodingSpecified())) { command->setStatusRealtime(); // If entity length == 0, then socket read/write check must be disabled. command->disableSocketCheck(); diff --git a/src/HttpSkipResponseCommand.cc b/src/HttpSkipResponseCommand.cc index 90511203..f1ae1f03 100644 --- a/src/HttpSkipResponseCommand.cc +++ b/src/HttpSkipResponseCommand.cc @@ -80,12 +80,14 @@ void HttpSkipResponseCommand::setTransferEncodingDecoder bool HttpSkipResponseCommand::executeInternal() { - if(_totalLength == 0 && _transferEncodingDecoder.isNull()) { - // If content-length header is present and it's value is 0, then - // pool socket for reuse. + if(req->getMethod() == Request::METHOD_HEAD || + (_totalLength == 0 && _transferEncodingDecoder.isNull())) { + // If request method is HEAD or content-length header is present and + // it's value is 0, then pool socket for reuse. // If content-length header is not present, then EOF is expected in the end. // In this case, the content is thrown away and socket cannot be pooled. - if(_httpResponse->getHttpHeader()->defined(HttpHeader::CONTENT_LENGTH)) { + if(req->getMethod() == Request::METHOD_HEAD || + _httpResponse->getHttpHeader()->defined(HttpHeader::CONTENT_LENGTH)) { poolConnection(); } return processResponse(); diff --git a/src/RequestGroup.cc b/src/RequestGroup.cc index 49307046..7ec1a6a6 100644 --- a/src/RequestGroup.cc +++ b/src/RequestGroup.cc @@ -179,7 +179,8 @@ void RequestGroup::closeFile() } void RequestGroup::createInitialCommand(std::deque& commands, - DownloadEngine* e) + DownloadEngine* e, + const std::string& method) { #ifdef ENABLE_BITTORRENT { @@ -306,7 +307,7 @@ void RequestGroup::createInitialCommand(std::deque& commands, // TODO I assume here when totallength is set to DownloadContext and it is // not 0, then filepath is also set DownloadContext correctly.... if(_downloadContext->getTotalLength() == 0) { - createNextCommand(commands, e, 1); + createNextCommand(commands, e, 1, method); }else { if(e->_requestGroupMan->isSameFileBeingDownloaded(this)) { throw DownloadFailureException diff --git a/src/RequestGroup.h b/src/RequestGroup.h index db0ea5fb..d54c40b4 100644 --- a/src/RequestGroup.h +++ b/src/RequestGroup.h @@ -43,6 +43,7 @@ #include "SharedHandle.h" #include "TransferStat.h" #include "TimeA2.h" +#include "Request.h" namespace aria2 { @@ -164,15 +165,23 @@ public: SharedHandle getSegmentMan() const; + // Returns first bootstrap commands to initiate a download. + // If this is HTTP/FTP download and file size is unknown, only 1 command + // (usually, HttpInitiateConnection or FtpInitiateConnection) will be created + // with its Request object having Requet::METHOD_HEAD in its method. + // This behavior can be changed by providing 3rd argument. + // The method has effect only for using HTTP request including FTP via HTTP + // proxy. void createInitialCommand(std::deque& commands, - DownloadEngine* e); + DownloadEngine* e, + const std::string& method = Request::METHOD_HEAD); void createNextCommandWithAdj(std::deque& commands, DownloadEngine* e, int numAdj); void createNextCommand(std::deque& commands, DownloadEngine* e, unsigned int numCommand, - const std::string& method = "GET"); + const std::string& method = Request::METHOD_GET); void addURI(const std::string& uri) { diff --git a/src/TrackerWatcherCommand.cc b/src/TrackerWatcherCommand.cc index 11007a7c..2eb5a1ea 100644 --- a/src/TrackerWatcherCommand.cc +++ b/src/TrackerWatcherCommand.cc @@ -57,6 +57,7 @@ #include "Logger.h" #include "A2STR.h" #include "SocketCore.h" +#include "Request.h" namespace aria2 { @@ -92,7 +93,8 @@ bool TrackerWatcherCommand::execute() { _trackerRequestGroup = createAnnounce(); if(!_trackerRequestGroup.isNull()) { std::deque commands; - _trackerRequestGroup->createInitialCommand(commands, e); + _trackerRequestGroup->createInitialCommand(commands, e, + Request::METHOD_GET); e->addCommand(commands); logger->debug("added tracker request command"); } diff --git a/test/HttpRequestTest.cc b/test/HttpRequestTest.cc index da6f3a37..7b3e11a2 100644 --- a/test/HttpRequestTest.cc +++ b/test/HttpRequestTest.cc @@ -1,5 +1,7 @@ #include "HttpRequest.h" +#include + #include #include "prefs.h" @@ -11,6 +13,7 @@ #include "Option.h" #include "array_fun.h" #include "CookieStorage.h" +#include "Util.h" namespace aria2 { @@ -23,6 +26,7 @@ class HttpRequestTest : public CppUnit::TestFixture { CPPUNIT_TEST(testCreateRequest_ftp); CPPUNIT_TEST(testCreateRequest_with_cookie); CPPUNIT_TEST(testCreateRequest_query); + CPPUNIT_TEST(testCreateRequest_head); CPPUNIT_TEST(testCreateProxyRequest); CPPUNIT_TEST(testIsRangeSatisfied); CPPUNIT_TEST(testUserAgent); @@ -46,6 +50,7 @@ public: void testCreateRequest_ftp(); void testCreateRequest_with_cookie(); void testCreateRequest_query(); + void testCreateRequest_head(); void testCreateProxyRequest(); void testIsRangeSatisfied(); void testUserAgent(); @@ -450,6 +455,23 @@ void HttpRequestTest::testCreateRequest_query() CPPUNIT_ASSERT_EQUAL(expectedText, httpRequest.createRequest()); } +void HttpRequestTest::testCreateRequest_head() +{ + SharedHandle request(new Request()); + request->setMethod(Request::METHOD_HEAD); + request->setUrl("http://localhost/aria2-1.0.0.tar.bz2"); + + HttpRequest httpRequest; + httpRequest.setRequest(request); + httpRequest.setAuthConfigFactory(_authConfigFactory); + + std::stringstream result(httpRequest.createRequest()); + std::string line; + CPPUNIT_ASSERT(getline(result, line)); + Util::trimSelf(line); + CPPUNIT_ASSERT_EQUAL(std::string("HEAD /aria2-1.0.0.tar.bz2 HTTP/1.1"), line); +} + void HttpRequestTest::testCreateProxyRequest() { SharedHandle request(new Request());