From 8274432f14606d22b8c11aa55efa18c50ba20767 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Mon, 12 Jul 2010 11:55:23 +0000 Subject: [PATCH] 2010-07-12 Tatsuhiro Tsujikawa Set end byte in Range header if start byte > 0 to get more chance to pool socket. * src/DefaultPieceStorage.cc * src/DefaultPieceStorage.h * src/DownloadCommand.cc * src/HttpDownloadCommand.cc * src/HttpRequest.cc * src/HttpRequest.h * src/HttpRequestCommand.cc * src/PieceStorage.h * src/UnknownLengthPieceStorage.h * test/DefaultPieceStorageTest.cc * test/HttpRequestTest.cc * test/MockPieceStorage.h --- ChangeLog | 17 ++++++++++++ src/DefaultPieceStorage.cc | 10 +++++++ src/DefaultPieceStorage.h | 2 ++ src/DownloadCommand.cc | 5 ++++ src/HttpDownloadCommand.cc | 7 ++--- src/HttpRequest.cc | 6 ++++- src/HttpRequest.h | 7 +++++ src/HttpRequestCommand.cc | 17 ++++++++++-- src/PieceStorage.h | 4 +++ src/UnknownLengthPieceStorage.h | 1 + test/DefaultPieceStorageTest.cc | 14 ++++++++++ test/HttpRequestTest.cc | 46 +++++++++++++++++++++++++++++++++ test/MockPieceStorage.h | 5 ++++ 13 files changed, 135 insertions(+), 6 deletions(-) diff --git a/ChangeLog b/ChangeLog index 741b9dec..f2b2dcb7 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,20 @@ +2010-07-12 Tatsuhiro Tsujikawa + + Set end byte in Range header if start byte > 0 to get more chance + to pool socket. + * src/DefaultPieceStorage.cc + * src/DefaultPieceStorage.h + * src/DownloadCommand.cc + * src/HttpDownloadCommand.cc + * src/HttpRequest.cc + * src/HttpRequest.h + * src/HttpRequestCommand.cc + * src/PieceStorage.h + * src/UnknownLengthPieceStorage.h + * test/DefaultPieceStorageTest.cc + * test/HttpRequestTest.cc + * test/MockPieceStorage.h + 2010-07-11 Tatsuhiro Tsujikawa Moved segment empty check before socket check. diff --git a/src/DefaultPieceStorage.cc b/src/DefaultPieceStorage.cc index 7ae7f799..901e5a42 100644 --- a/src/DefaultPieceStorage.cc +++ b/src/DefaultPieceStorage.cc @@ -684,4 +684,14 @@ void DefaultPieceStorage::addPieceStats(size_t index) pieceStatMan_->addPieceStats(index); } +size_t DefaultPieceStorage::getNextUsedIndex(size_t index) +{ + for(size_t i = index+1; i < bitfieldMan_->countBlock(); ++i) { + if(bitfieldMan_->isUseBitSet(i) || bitfieldMan_->isBitSet(i)) { + return i; + } + } + return bitfieldMan_->countBlock(); +} + } // namespace aria2 diff --git a/src/DefaultPieceStorage.h b/src/DefaultPieceStorage.h index f282deb4..06fc64ae 100644 --- a/src/DefaultPieceStorage.h +++ b/src/DefaultPieceStorage.h @@ -227,6 +227,8 @@ public: size_t newBitfieldLength, const unsigned char* oldBitfield); + virtual size_t getNextUsedIndex(size_t index); + /** * This method is made private for test purpose only. */ diff --git a/src/DownloadCommand.cc b/src/DownloadCommand.cc index fe82521c..4b08183c 100644 --- a/src/DownloadCommand.cc +++ b/src/DownloadCommand.cc @@ -336,6 +336,11 @@ bool DownloadCommand::prepareForNextSegment() { if(!tempSegment->complete()) { return prepareForRetry(0); } + if(getRequestEndOffset() == + getFileEntry()->gtoloff + (tempSegment->getPosition()+tempSegment->getLength())) { + return prepareForRetry(0); + } SharedHandle nextSegment = getSegmentMan()->getSegment (getCuid(), tempSegment->getIndex()+1); if(nextSegment.isNull()) { diff --git a/src/HttpDownloadCommand.cc b/src/HttpDownloadCommand.cc index deed69d7..9e4798e4 100644 --- a/src/HttpDownloadCommand.cc +++ b/src/HttpDownloadCommand.cc @@ -52,7 +52,7 @@ #include "FileAllocationEntry.h" #include "CheckIntegrityEntry.h" #include "ServerStatMan.h" - +#include "Logger.h" namespace aria2 { HttpDownloadCommand::HttpDownloadCommand @@ -88,12 +88,13 @@ bool HttpDownloadCommand::prepareForNextSegment() { if(getRequest()->isPipeliningEnabled() || (getRequest()->isKeepAliveEnabled() && ( + // TODO make sure that all decoder is finished to pool socket ((!getTransferEncodingDecoder().isNull() && getTransferEncodingDecoder()->finished()) || (!getContentEncodingDecoder().isNull() && getContentEncodingDecoder()->finished())) || - getFileEntry()->getLastOffset() == - getSegments().front()->getPositionToWrite() + getRequestEndOffset() == + getFileEntry()->gtoloff(getSegments().front()->getPositionToWrite()) ) ) ) { diff --git a/src/HttpRequest.cc b/src/HttpRequest.cc index 4fa090aa..476295fd 100644 --- a/src/HttpRequest.cc +++ b/src/HttpRequest.cc @@ -58,7 +58,8 @@ const std::string HttpRequest::USER_AGENT("aria2"); HttpRequest::HttpRequest():contentEncodingEnabled_(true), userAgent_(USER_AGENT), noCache_(true), - acceptGzip_(false) + acceptGzip_(false), + endOffsetOverride_(false) {} void HttpRequest::setSegment(const SharedHandle& segment) @@ -196,6 +197,9 @@ std::string HttpRequest::createRequest() rangeHeader += "-"; if(request_->isPipeliningEnabled()) { rangeHeader += util::itos(getEndByte()); + } else if(getProtocol() != Request::PROTO_FTP && endOffsetOverride_ > 0) { + // FTP via http proxy does not support endbytes + rangeHeader += util::itos(endOffsetOverride_-1); } builtinHds.push_back(std::make_pair("Range:", rangeHeader)); } diff --git a/src/HttpRequest.h b/src/HttpRequest.h index 1bc2089e..091b5b35 100644 --- a/src/HttpRequest.h +++ b/src/HttpRequest.h @@ -87,6 +87,8 @@ private: bool acceptGzip_; + off_t endOffsetOverride_; + std::pair getProxyAuthString() const; public: HttpRequest(); @@ -272,6 +274,11 @@ public: { return acceptGzip_; } + + void setEndOffsetOverride(off_t offset) + { + endOffsetOverride_ = offset; + } }; } // namespace aria2 diff --git a/src/HttpRequestCommand.cc b/src/HttpRequestCommand.cc index 9432b6a8..e0e39a07 100644 --- a/src/HttpRequestCommand.cc +++ b/src/HttpRequestCommand.cc @@ -57,6 +57,7 @@ #include "FileAllocationEntry.h" #include "CheckIntegrityEntry.h" #include "ServerStatMan.h" +#include "PieceStorage.h" namespace aria2 { @@ -87,7 +88,8 @@ createHttpRequest(const SharedHandle& req, const RequestGroup* rg, const SharedHandle& cookieStorage, const SharedHandle& authConfigFactory, - const SharedHandle& proxyRequest) + const SharedHandle& proxyRequest, + off_t endOffset = 0) { SharedHandle httpRequest(new HttpRequest()); httpRequest->setUserAgent(option->get(PREF_USER_AGENT)); @@ -110,6 +112,7 @@ createHttpRequest(const SharedHandle& req, } else { httpRequest->disableNoCache(); } + httpRequest->setEndOffsetOverride(endOffset); return httpRequest; } @@ -148,6 +151,15 @@ bool HttpRequestCommand::executeInternal() { itr != eoi; ++itr) { const SharedHandle& segment = *itr; if(!httpConnection_->isIssued(segment)) { + off_t endOffset = 0; + if(getRequestGroup()->getTotalLength() > 0 && + !getPieceStorage().isNull()) { + size_t nextIndex = + getPieceStorage()->getNextUsedIndex(segment->getIndex()); + endOffset = std::min + (static_cast(getFileEntry()->getLength()), + getFileEntry()->gtoloff(segment->getSegmentLength()*nextIndex)); + } SharedHandle httpRequest (createHttpRequest(getRequest(), getFileEntry(), @@ -157,7 +169,8 @@ bool HttpRequestCommand::executeInternal() { getRequestGroup(), getDownloadEngine()->getCookieStorage(), getDownloadEngine()->getAuthConfigFactory(), - proxyRequest_)); + proxyRequest_, + endOffset)); httpConnection_->sendRequest(httpRequest); } } diff --git a/src/PieceStorage.h b/src/PieceStorage.h index 62734dd0..cc14bed7 100644 --- a/src/PieceStorage.h +++ b/src/PieceStorage.h @@ -244,6 +244,10 @@ public: size_t newBitfieldLength, const unsigned char* oldBitfield) = 0; + // Returns index x where all pieces in [index+1, x-1], inclusive, + // are not used and not completed. If all pieces after index+1 are + // used or completed, returns the number of pieces. + virtual size_t getNextUsedIndex(size_t index) = 0; }; typedef SharedHandle PieceStorageHandle; diff --git a/src/UnknownLengthPieceStorage.h b/src/UnknownLengthPieceStorage.h index 1ecfa09f..d90f5644 100644 --- a/src/UnknownLengthPieceStorage.h +++ b/src/UnknownLengthPieceStorage.h @@ -273,6 +273,7 @@ public: size_t newBitfieldLength, const unsigned char* oldBitfield) {} + virtual size_t getNextUsedIndex(size_t index) { return 0; } void setDiskWriterFactory(const SharedHandle& diskWriterFactory); }; diff --git a/test/DefaultPieceStorageTest.cc b/test/DefaultPieceStorageTest.cc index 2ae6a83f..f97847d8 100644 --- a/test/DefaultPieceStorageTest.cc +++ b/test/DefaultPieceStorageTest.cc @@ -31,6 +31,7 @@ class DefaultPieceStorageTest:public CppUnit::TestFixture { CPPUNIT_TEST(testCancelPiece); CPPUNIT_TEST(testMarkPiecesDone); CPPUNIT_TEST(testGetCompletedLength); + CPPUNIT_TEST(testGetNextUsedIndex); CPPUNIT_TEST_SUITE_END(); private: SharedHandle dctx_; @@ -67,6 +68,7 @@ public: void testCancelPiece(); void testMarkPiecesDone(); void testGetCompletedLength(); + void testGetNextUsedIndex(); }; @@ -302,4 +304,16 @@ void DefaultPieceStorageTest::testGetCompletedLength() CPPUNIT_ASSERT_EQUAL((uint64_t)256*1024*1024, ps.getCompletedLength()); } +void DefaultPieceStorageTest::testGetNextUsedIndex() +{ + DefaultPieceStorage pss(dctx_, option); + CPPUNIT_ASSERT_EQUAL((size_t)3, pss.getNextUsedIndex(0)); + SharedHandle piece = pss.getMissingPiece(2); + CPPUNIT_ASSERT_EQUAL((size_t)2, pss.getNextUsedIndex(0)); + pss.completePiece(piece); + CPPUNIT_ASSERT_EQUAL((size_t)2, pss.getNextUsedIndex(0)); + piece = pss.getMissingPiece(0); + CPPUNIT_ASSERT_EQUAL((size_t)2, pss.getNextUsedIndex(0)); +} + } // namespace aria2 diff --git a/test/HttpRequestTest.cc b/test/HttpRequestTest.cc index 16096391..8f350450 100644 --- a/test/HttpRequestTest.cc +++ b/test/HttpRequestTest.cc @@ -29,6 +29,7 @@ class HttpRequestTest : public CppUnit::TestFixture { CPPUNIT_TEST(testCreateRequest_query); CPPUNIT_TEST(testCreateRequest_head); CPPUNIT_TEST(testCreateRequest_ipv6LiteralAddr); + CPPUNIT_TEST(testCreateRequest_endOffsetOverride); CPPUNIT_TEST(testCreateProxyRequest); CPPUNIT_TEST(testIsRangeSatisfied); CPPUNIT_TEST(testUserAgent); @@ -55,6 +56,7 @@ public: void testCreateRequest_query(); void testCreateRequest_head(); void testCreateRequest_ipv6LiteralAddr(); + void testCreateRequest_endOffsetOverride(); void testCreateProxyRequest(); void testIsRangeSatisfied(); void testUserAgent(); @@ -494,6 +496,50 @@ void HttpRequestTest::testCreateRequest_head() CPPUNIT_ASSERT_EQUAL(std::string("HEAD /aria2-1.0.0.tar.bz2 HTTP/1.1"), line); } +void HttpRequestTest::testCreateRequest_endOffsetOverride() +{ + SharedHandle request(new Request()); + request->setUri("http://localhost/myfile"); + HttpRequest httpRequest; + httpRequest.disableContentEncoding(); + httpRequest.setRequest(request); + httpRequest.setAuthConfigFactory(authConfigFactory_, option_.get()); + SharedHandle p(new Piece(0, 1024*1024)); + SharedHandle segment(new PiecedSegment(1024*1024, p)); + httpRequest.setSegment(segment); + httpRequest.setEndOffsetOverride(1024*1024*1024); + SharedHandle fileEntry + (new FileEntry("file", 1024*1024*1024*10LL, 0)); + httpRequest.setFileEntry(fileEntry); + // End byte is not passed if start byte is 0. + std::string expectedText = + "GET /myfile HTTP/1.1\r\n" + "User-Agent: aria2\r\n" + "Accept: */*\r\n" + "Host: localhost\r\n" + "Pragma: no-cache\r\n" + "Cache-Control: no-cache\r\n" + "Connection: close\r\n" + "\r\n"; + + CPPUNIT_ASSERT_EQUAL(expectedText, httpRequest.createRequest()); + + segment->updateWrittenLength(1); + + expectedText = + "GET /myfile HTTP/1.1\r\n" + "User-Agent: aria2\r\n" + "Accept: */*\r\n" + "Host: localhost\r\n" + "Pragma: no-cache\r\n" + "Cache-Control: no-cache\r\n" + "Connection: close\r\n" + "Range: bytes=1-1073741823\r\n" + "\r\n"; + + CPPUNIT_ASSERT_EQUAL(expectedText, httpRequest.createRequest()); +} + void HttpRequestTest::testCreateProxyRequest() { SharedHandle request(new Request()); diff --git a/test/MockPieceStorage.h b/test/MockPieceStorage.h index de87c924..16a30fa7 100644 --- a/test/MockPieceStorage.h +++ b/test/MockPieceStorage.h @@ -247,6 +247,11 @@ public: virtual void updatePieceStats(const unsigned char* newBitfield, size_t newBitfieldLength, const unsigned char* oldBitfield) {} + + virtual size_t getNextUsedIndex(size_t index) + { + return 0; + } }; } // namespace aria2