From b1edad42ffd144f449d9d4d81d414d6fb65cb46b Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 26 Nov 2008 15:22:54 +0000 Subject: [PATCH] 2008-11-27 Tatsuhiro Tsujikawa Fixed the bug that prevents aria2 from downloading 0-length files via HTTP/FTP. * src/DefaultBtContext.cc * src/DefaultBtContext.h * src/DownloadContext.h * src/FtpNegotiationCommand.cc * src/HttpResponseCommand.cc * src/HttpResponseCommand.h * src/RequestGroup.cc * src/SingleFileDownloadContext.cc * src/SingleFileDownloadContext.h * test/BtPostDownloadHandlerTest.cc * test/MetalinkPostDownloadHandlerTest.cc * test/MockBtContext.h --- ChangeLog | 17 ++++++++++++ src/DefaultBtContext.cc | 5 ++++ src/DefaultBtContext.h | 2 ++ src/DownloadContext.h | 2 ++ src/FtpNegotiationCommand.cc | 29 +++++++++++++++----- src/HttpResponseCommand.cc | 35 ++++++++++++++++++++----- src/HttpResponseCommand.h | 2 ++ src/RequestGroup.cc | 8 +++--- src/SingleFileDownloadContext.cc | 13 ++++++++- src/SingleFileDownloadContext.h | 7 +++++ test/BtPostDownloadHandlerTest.cc | 2 +- test/MetalinkPostDownloadHandlerTest.cc | 2 +- test/MockBtContext.h | 5 ++++ 13 files changed, 111 insertions(+), 18 deletions(-) diff --git a/ChangeLog b/ChangeLog index 7de4d785..08940500 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,20 @@ +2008-11-27 Tatsuhiro Tsujikawa + + Fixed the bug that prevents aria2 from downloading 0-length files + via HTTP/FTP. + * src/DefaultBtContext.cc + * src/DefaultBtContext.h + * src/DownloadContext.h + * src/FtpNegotiationCommand.cc + * src/HttpResponseCommand.cc + * src/HttpResponseCommand.h + * src/RequestGroup.cc + * src/SingleFileDownloadContext.cc + * src/SingleFileDownloadContext.h + * test/BtPostDownloadHandlerTest.cc + * test/MetalinkPostDownloadHandlerTest.cc + * test/MockBtContext.h + 2008-11-26 Tatsuhiro Tsujikawa Fixed the bug that -pfalse and -ptrue are not recognized properly. diff --git a/src/DefaultBtContext.cc b/src/DefaultBtContext.cc index 7e5b292f..0f48d899 100644 --- a/src/DefaultBtContext.cc +++ b/src/DefaultBtContext.cc @@ -391,6 +391,11 @@ uint64_t DefaultBtContext::getTotalLength() const { return totalLength; } +bool DefaultBtContext::knowsTotalLength() const +{ + return true; +} + BtContext::FILE_MODE DefaultBtContext::getFileMode() const { return fileMode; } diff --git a/src/DefaultBtContext.h b/src/DefaultBtContext.h index b83f1f8f..abab597c 100644 --- a/src/DefaultBtContext.h +++ b/src/DefaultBtContext.h @@ -110,6 +110,8 @@ private: virtual uint64_t getTotalLength() const; + virtual bool knowsTotalLength() const; + virtual FILE_MODE getFileMode() const; virtual std::deque > getFileEntries() const; diff --git a/src/DownloadContext.h b/src/DownloadContext.h index 179f159c..9440f905 100644 --- a/src/DownloadContext.h +++ b/src/DownloadContext.h @@ -75,6 +75,8 @@ public: virtual uint64_t getTotalLength() const = 0; + virtual bool knowsTotalLength() const = 0; + virtual FILE_MODE getFileMode() const = 0; virtual std::deque > getFileEntries() const = 0; diff --git a/src/FtpNegotiationCommand.cc b/src/FtpNegotiationCommand.cc index 63df6e2f..71ebc50c 100644 --- a/src/FtpNegotiationCommand.cc +++ b/src/FtpNegotiationCommand.cc @@ -333,10 +333,29 @@ bool FtpNegotiationCommand::onFileSizeDetermined(uint64_t totalLength) } if(totalLength == 0) { + if(e->option->getAsBool(PREF_FTP_PASV)) { + sequence = SEQ_SEND_PASV; + } else { + sequence = SEQ_PREPARE_SERVER_SOCKET; + } _requestGroup->initPieceStorage(); + if(dctx->knowsTotalLength() && + _requestGroup->downloadFinishedByFileLength()) { + sequence = SEQ_DOWNLOAD_ALREADY_COMPLETED; + + poolConnection(); + + return false; + } _requestGroup->shouldCancelDownloadForSafety(); _requestGroup->getPieceStorage()->getDiskAdaptor()->initAndOpenFile(); + if(dctx->knowsTotalLength()) { + sequence = SEQ_DOWNLOAD_ALREADY_COMPLETED; + poolConnection(); + return false; + } + return true; } else { _requestGroup->initPieceStorage(); @@ -387,12 +406,10 @@ bool FtpNegotiationCommand::recvSize() { // command, resuming and segmented downloading are disabled when the first // contacted FTP server doesn't support it. if(_requestGroup->getPieceStorage().isNull()) { - - if(e->option->getAsBool(PREF_FTP_PASV)) { - sequence = SEQ_SEND_PASV; - } else { - sequence = SEQ_PREPARE_SERVER_SOCKET; - } + SingleFileDownloadContextHandle dctx = + dynamic_pointer_cast + (_requestGroup->getDownloadContext()); + dctx->markTotalLengthIsUnknown(); return onFileSizeDetermined(0); } diff --git a/src/HttpResponseCommand.cc b/src/HttpResponseCommand.cc index a79d3714..6313ff14 100644 --- a/src/HttpResponseCommand.cc +++ b/src/HttpResponseCommand.cc @@ -140,6 +140,11 @@ bool HttpResponseCommand::executeInternal() shouldInflateContentEncoding(httpResponse)) { // we ignore content-length when transfer-encoding is set dctx->setTotalLength(0); + if(req->getMethod() == Request::METHOD_GET && + (totalLength != 0 || + !httpResponse->getHttpHeader()->defined(HttpHeader::CONTENT_LENGTH))){ + dctx->markTotalLengthIsUnknown(); + } return handleOtherEncoding(httpResponse); } else { return handleDefaultEncoding(httpResponse); @@ -213,9 +218,7 @@ bool HttpResponseCommand::handleDefaultEncoding(const HttpResponseHandle& httpRe } prepareForNextAction(command); if(req->getMethod() == Request::METHOD_HEAD) { - if(req->supportsPersistentConnection()) { - e->poolSocket(req, isProxyDefined(), socket); - } + poolConnection(); req->setMethod(Request::METHOD_GET); } } catch(Exception& e) { @@ -261,17 +264,30 @@ static SharedHandle getContentEncodingDecoder } bool HttpResponseCommand::handleOtherEncoding(const HttpResponseHandle& httpResponse) { + // We assume that RequestGroup::getTotalLength() == 0 here HttpRequestHandle httpRequest = httpResponse->getHttpRequest(); if(req->getMethod() == Request::METHOD_HEAD) { - if(req->supportsPersistentConnection()) { - e->poolSocket(req, isProxyDefined(), socket); - } + poolConnection(); req->setMethod(Request::METHOD_GET); return prepareForRetry(0); } _requestGroup->initPieceStorage(); + + // For zero-length file, check existing file comparing its size + if(_requestGroup->getDownloadContext()->knowsTotalLength() && + _requestGroup->downloadFinishedByFileLength()) { + poolConnection(); + return true; + } + _requestGroup->shouldCancelDownloadForSafety(); _requestGroup->getPieceStorage()->getDiskAdaptor()->initAndOpenFile(); + + if(_requestGroup->getDownloadContext()->knowsTotalLength()) { + poolConnection(); + return true; + } + e->commands.push_back (createHttpDownloadCommand(httpResponse, getTransferEncodingDecoder(httpResponse), @@ -332,4 +348,11 @@ HttpDownloadCommand* HttpResponseCommand::createHttpDownloadCommand return command; } +void HttpResponseCommand::poolConnection() +{ + if(req->supportsPersistentConnection()) { + e->poolSocket(req, isProxyDefined(), socket); + } +} + } // namespace aria2 diff --git a/src/HttpResponseCommand.h b/src/HttpResponseCommand.h index 0a622c16..f439b19f 100644 --- a/src/HttpResponseCommand.h +++ b/src/HttpResponseCommand.h @@ -62,6 +62,8 @@ private: = SharedHandle()); void updateLastModifiedTime(const Time& lastModified); + + void poolConnection(); protected: bool executeInternal(); diff --git a/src/RequestGroup.cc b/src/RequestGroup.cc index 7ec1a6a6..1792ad9a 100644 --- a/src/RequestGroup.cc +++ b/src/RequestGroup.cc @@ -347,14 +347,16 @@ void RequestGroup::processCheckIntegrityEntry(std::deque& commands, void RequestGroup::initPieceStorage() { - if(_downloadContext->getTotalLength() == 0) { - UnknownLengthPieceStorageHandle ps(new UnknownLengthPieceStorage(_downloadContext, _option)); + if(_downloadContext->knowsTotalLength()) { + DefaultPieceStorageHandle ps + (new DefaultPieceStorage(_downloadContext, _option)); if(!_diskWriterFactory.isNull()) { ps->setDiskWriterFactory(_diskWriterFactory); } _pieceStorage = ps; } else { - DefaultPieceStorageHandle ps(new DefaultPieceStorage(_downloadContext, _option)); + UnknownLengthPieceStorageHandle ps + (new UnknownLengthPieceStorage(_downloadContext, _option)); if(!_diskWriterFactory.isNull()) { ps->setDiskWriterFactory(_diskWriterFactory); } diff --git a/src/SingleFileDownloadContext.cc b/src/SingleFileDownloadContext.cc index c32ad208..ca0d54b3 100644 --- a/src/SingleFileDownloadContext.cc +++ b/src/SingleFileDownloadContext.cc @@ -46,7 +46,8 @@ SingleFileDownloadContext::SingleFileDownloadContext(size_t pieceLength, _pieceLength(pieceLength), _fileEntry(new FileEntry(filename, totalLength, 0)), _filename(filename), - _ufilename(ufilename) + _ufilename(ufilename), + _knowsTotalLength(true) { updateFileEntry(); } @@ -73,6 +74,11 @@ uint64_t SingleFileDownloadContext::getTotalLength() const return _fileEntry->getLength(); } +bool SingleFileDownloadContext::knowsTotalLength() const +{ + return _knowsTotalLength; +} + FileEntries SingleFileDownloadContext::getFileEntries() const { @@ -96,6 +102,11 @@ void SingleFileDownloadContext::setTotalLength(uint64_t totalLength) _fileEntry->setLength(totalLength); } +void SingleFileDownloadContext::markTotalLengthIsUnknown() +{ + _knowsTotalLength = false; +} + const std::string& SingleFileDownloadContext::getName() const { return _fileEntry->getPath(); diff --git a/src/SingleFileDownloadContext.h b/src/SingleFileDownloadContext.h index f1944c94..ed5cf30f 100644 --- a/src/SingleFileDownloadContext.h +++ b/src/SingleFileDownloadContext.h @@ -66,6 +66,8 @@ private: std::string _checksum; std::string _checksumHashAlgo; + bool _knowsTotalLength; + void updateFileEntry(); public: SingleFileDownloadContext(size_t pieceLength, @@ -88,6 +90,9 @@ public: virtual uint64_t getTotalLength() const; + virtual bool knowsTotalLength() const; + + virtual FILE_MODE getFileMode() const { return SINGLE; @@ -150,6 +155,8 @@ public: void setTotalLength(uint64_t totalLength); + void markTotalLengthIsUnknown(); + void setPieceHashAlgo(const std::string& algo) { _pieceHashAlgo = algo; diff --git a/test/BtPostDownloadHandlerTest.cc b/test/BtPostDownloadHandlerTest.cc index cc202dc6..f3b25dfb 100644 --- a/test/BtPostDownloadHandlerTest.cc +++ b/test/BtPostDownloadHandlerTest.cc @@ -65,7 +65,7 @@ void BtPostDownloadHandlerTest::testGetNextRequestGroups() { Option op; SharedHandle dctx - (new SingleFileDownloadContext(0, 0, "test.torrent")); + (new SingleFileDownloadContext(1024, 0, "test.torrent")); RequestGroup rg(&op, std::deque()); rg.setDownloadContext(dctx); rg.initPieceStorage(); diff --git a/test/MetalinkPostDownloadHandlerTest.cc b/test/MetalinkPostDownloadHandlerTest.cc index 87d13974..00409673 100644 --- a/test/MetalinkPostDownloadHandlerTest.cc +++ b/test/MetalinkPostDownloadHandlerTest.cc @@ -64,7 +64,7 @@ void MetalinkPostDownloadHandlerTest::testGetNextRequestGroups() { Option op; SharedHandle dctx - (new SingleFileDownloadContext(0, 0, "test.xml")); + (new SingleFileDownloadContext(1024, 0, "test.xml")); RequestGroup rg(&op, std::deque()); rg.setDownloadContext(dctx); rg.initPieceStorage(); diff --git a/test/MockBtContext.h b/test/MockBtContext.h index 740a07fe..48f69347 100644 --- a/test/MockBtContext.h +++ b/test/MockBtContext.h @@ -64,6 +64,11 @@ public: return totalLength; } + virtual bool knowsTotalLength() const + { + return true; + } + void setTotalLength(uint64_t length) { this->totalLength = length; }