github
/
aria2
mirror of https://github.com/aria2/aria2
1
0
Fork 0
Code Issues Releases Wiki Activity

2010-07-12 Tatsuhiro Tsujikawa <t-tujikawa@users.sourceforge.net> 

Set end byte in Range header if start byte > 0 to get more chance
	to pool socket.
	* src/DefaultPieceStorage.cc
	* src/DefaultPieceStorage.h
	* src/DownloadCommand.cc
	* src/HttpDownloadCommand.cc
	* src/HttpRequest.cc
	* src/HttpRequest.h
	* src/HttpRequestCommand.cc
	* src/PieceStorage.h
	* src/UnknownLengthPieceStorage.h
	* test/DefaultPieceStorageTest.cc
	* test/HttpRequestTest.cc
	* test/MockPieceStorage.h
pull/1/head
Tatsuhiro Tsujikawa 2010-07-12 11:55:23 +00:00
parent bd73c20c4d
commit 8274432f14
13 changed files with 135 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
ChangeLog
View File

@ -1,3 +1,20 @@
2010-07-12 Tatsuhiro Tsujikawa <t-tujikawa@users.sourceforge.net>
Set end byte in Range header if start byte > 0 to get more chance
to pool socket.
* src/DefaultPieceStorage.cc
* src/DefaultPieceStorage.h
* src/DownloadCommand.cc
* src/HttpDownloadCommand.cc
* src/HttpRequest.cc
* src/HttpRequest.h
* src/HttpRequestCommand.cc
* src/PieceStorage.h
* src/UnknownLengthPieceStorage.h
* test/DefaultPieceStorageTest.cc
* test/HttpRequestTest.cc
* test/MockPieceStorage.h
2010-07-11 Tatsuhiro Tsujikawa <t-tujikawa@users.sourceforge.net>
Moved segment empty check before socket check.

10
src/DefaultPieceStorage.cc
View File

@ -684,4 +684,14 @@ void DefaultPieceStorage::addPieceStats(size_t index)
pieceStatMan_->addPieceStats(index);
}
size_t DefaultPieceStorage::getNextUsedIndex(size_t index)
{
for(size_t i = index+1; i < bitfieldMan_->countBlock(); ++i) {
if(bitfieldMan_->isUseBitSet(i) || bitfieldMan_->isBitSet(i)) {
return i;
}
}
return bitfieldMan_->countBlock();
}
} // namespace aria2

2
src/DefaultPieceStorage.h
View File

@ -227,6 +227,8 @@ public:
size_t newBitfieldLength,
const unsigned char* oldBitfield);
virtual size_t getNextUsedIndex(size_t index);
/**
* This method is made private for test purpose only.
*/

5
src/DownloadCommand.cc
View File

@ -336,6 +336,11 @@ bool DownloadCommand::prepareForNextSegment() {
if(!tempSegment->complete()) {
return prepareForRetry(0);
}
if(getRequestEndOffset() ==
getFileEntry()->gtoloff
(tempSegment->getPosition()+tempSegment->getLength())) {
return prepareForRetry(0);
}
SharedHandle<Segment> nextSegment = getSegmentMan()->getSegment
(getCuid(), tempSegment->getIndex()+1);
if(nextSegment.isNull()) {

7
src/HttpDownloadCommand.cc
View File

@ -52,7 +52,7 @@
#include "FileAllocationEntry.h"
#include "CheckIntegrityEntry.h"
#include "ServerStatMan.h"
#include "Logger.h"
namespace aria2 {
HttpDownloadCommand::HttpDownloadCommand
@ -88,12 +88,13 @@ bool HttpDownloadCommand::prepareForNextSegment() {
if(getRequest()->isPipeliningEnabled() ||
(getRequest()->isKeepAliveEnabled() &&
(
// TODO make sure that all decoder is finished to pool socket
((!getTransferEncodingDecoder().isNull() &&
getTransferEncodingDecoder()->finished()) ||
(!getContentEncodingDecoder().isNull() &&
getContentEncodingDecoder()->finished())) ||
getFileEntry()->getLastOffset() ==
getSegments().front()->getPositionToWrite()
getRequestEndOffset() ==
getFileEntry()->gtoloff(getSegments().front()->getPositionToWrite())
)
)
) {

6
src/HttpRequest.cc
View File

@ -58,7 +58,8 @@ const std::string HttpRequest::USER_AGENT("aria2");
HttpRequest::HttpRequest():contentEncodingEnabled_(true),
userAgent_(USER_AGENT),
noCache_(true),
acceptGzip_(false)
acceptGzip_(false),
endOffsetOverride_(false)
{}
void HttpRequest::setSegment(const SharedHandle<Segment>& segment)
@ -196,6 +197,9 @@ std::string HttpRequest::createRequest()
rangeHeader += "-";
if(request_->isPipeliningEnabled()) {
rangeHeader += util::itos(getEndByte());
} else if(getProtocol() != Request::PROTO_FTP && endOffsetOverride_ > 0) {
// FTP via http proxy does not support endbytes
rangeHeader += util::itos(endOffsetOverride_-1);
}
builtinHds.push_back(std::make_pair("Range:", rangeHeader));
}

7
src/HttpRequest.h
View File

@ -87,6 +87,8 @@ private:
bool acceptGzip_;
off_t endOffsetOverride_;
std::pair<std::string, std::string> getProxyAuthString() const;
public:
HttpRequest();
@ -272,6 +274,11 @@ public:
{
return acceptGzip_;
}
void setEndOffsetOverride(off_t offset)
{
endOffsetOverride_ = offset;
}
};
} // namespace aria2

17
src/HttpRequestCommand.cc
View File

@ -57,6 +57,7 @@
#include "FileAllocationEntry.h"
#include "CheckIntegrityEntry.h"
#include "ServerStatMan.h"
#include "PieceStorage.h"
namespace aria2 {
@ -87,7 +88,8 @@ createHttpRequest(const SharedHandle<Request>& req,
const RequestGroup* rg,
const SharedHandle<CookieStorage>& cookieStorage,
const SharedHandle<AuthConfigFactory>& authConfigFactory,
const SharedHandle<Request>& proxyRequest)
const SharedHandle<Request>& proxyRequest,
off_t endOffset = 0)
{
SharedHandle<HttpRequest> httpRequest(new HttpRequest());
httpRequest->setUserAgent(option->get(PREF_USER_AGENT));
@ -110,6 +112,7 @@ createHttpRequest(const SharedHandle<Request>& req,
} else {
httpRequest->disableNoCache();
}
httpRequest->setEndOffsetOverride(endOffset);
return httpRequest;
}
@ -148,6 +151,15 @@ bool HttpRequestCommand::executeInternal() {
itr != eoi; ++itr) {
const SharedHandle<Segment>& segment = *itr;
if(!httpConnection_->isIssued(segment)) {
off_t endOffset = 0;
if(getRequestGroup()->getTotalLength() > 0 &&
!getPieceStorage().isNull()) {
size_t nextIndex =
getPieceStorage()->getNextUsedIndex(segment->getIndex());
endOffset = std::min
(static_cast<off_t>(getFileEntry()->getLength()),
getFileEntry()->gtoloff(segment->getSegmentLength()*nextIndex));
}
SharedHandle<HttpRequest> httpRequest
(createHttpRequest(getRequest(),
getFileEntry(),
@ -157,7 +169,8 @@ bool HttpRequestCommand::executeInternal() {
getRequestGroup(),
getDownloadEngine()->getCookieStorage(),
getDownloadEngine()->getAuthConfigFactory(),
proxyRequest_));
proxyRequest_,
endOffset));
httpConnection_->sendRequest(httpRequest);
}
}

4
src/PieceStorage.h
View File

@ -244,6 +244,10 @@ public:
size_t newBitfieldLength,
const unsigned char* oldBitfield) = 0;
// Returns index x where all pieces in [index+1, x-1], inclusive,
// are not used and not completed. If all pieces after index+1 are
// used or completed, returns the number of pieces.
virtual size_t getNextUsedIndex(size_t index) = 0;
};
typedef SharedHandle<PieceStorage> PieceStorageHandle;

1
src/UnknownLengthPieceStorage.h
View File

@ -273,6 +273,7 @@ public:
size_t newBitfieldLength,
const unsigned char* oldBitfield) {}
virtual size_t getNextUsedIndex(size_t index) { return 0; }
void setDiskWriterFactory(const SharedHandle<DiskWriterFactory>& diskWriterFactory);
};

14
test/DefaultPieceStorageTest.cc
View File

@ -31,6 +31,7 @@ class DefaultPieceStorageTest:public CppUnit::TestFixture {
CPPUNIT_TEST(testCancelPiece);
CPPUNIT_TEST(testMarkPiecesDone);
CPPUNIT_TEST(testGetCompletedLength);
CPPUNIT_TEST(testGetNextUsedIndex);
CPPUNIT_TEST_SUITE_END();
private:
SharedHandle<DownloadContext> dctx_;
@ -67,6 +68,7 @@ public:
void testCancelPiece();
void testMarkPiecesDone();
void testGetCompletedLength();
void testGetNextUsedIndex();
};
@ -302,4 +304,16 @@ void DefaultPieceStorageTest::testGetCompletedLength()
CPPUNIT_ASSERT_EQUAL((uint64_t)256*1024*1024, ps.getCompletedLength());
}
void DefaultPieceStorageTest::testGetNextUsedIndex()
{
DefaultPieceStorage pss(dctx_, option);
CPPUNIT_ASSERT_EQUAL((size_t)3, pss.getNextUsedIndex(0));
SharedHandle<Piece> piece = pss.getMissingPiece(2);
CPPUNIT_ASSERT_EQUAL((size_t)2, pss.getNextUsedIndex(0));
pss.completePiece(piece);
CPPUNIT_ASSERT_EQUAL((size_t)2, pss.getNextUsedIndex(0));
piece = pss.getMissingPiece(0);
CPPUNIT_ASSERT_EQUAL((size_t)2, pss.getNextUsedIndex(0));
}
} // namespace aria2

46
test/HttpRequestTest.cc
View File

@ -29,6 +29,7 @@ class HttpRequestTest : public CppUnit::TestFixture {
CPPUNIT_TEST(testCreateRequest_query);
CPPUNIT_TEST(testCreateRequest_head);
CPPUNIT_TEST(testCreateRequest_ipv6LiteralAddr);
CPPUNIT_TEST(testCreateRequest_endOffsetOverride);
CPPUNIT_TEST(testCreateProxyRequest);
CPPUNIT_TEST(testIsRangeSatisfied);
CPPUNIT_TEST(testUserAgent);
@ -55,6 +56,7 @@ public:
void testCreateRequest_query();
void testCreateRequest_head();
void testCreateRequest_ipv6LiteralAddr();
void testCreateRequest_endOffsetOverride();
void testCreateProxyRequest();
void testIsRangeSatisfied();
void testUserAgent();
@ -494,6 +496,50 @@ void HttpRequestTest::testCreateRequest_head()
CPPUNIT_ASSERT_EQUAL(std::string("HEAD /aria2-1.0.0.tar.bz2 HTTP/1.1"), line);
}
void HttpRequestTest::testCreateRequest_endOffsetOverride()
{
SharedHandle<Request> request(new Request());
request->setUri("http://localhost/myfile");
HttpRequest httpRequest;
httpRequest.disableContentEncoding();
httpRequest.setRequest(request);
httpRequest.setAuthConfigFactory(authConfigFactory_, option_.get());
SharedHandle<Piece> p(new Piece(0, 1024*1024));
SharedHandle<Segment> segment(new PiecedSegment(1024*1024, p));
httpRequest.setSegment(segment);
httpRequest.setEndOffsetOverride(1024*1024*1024);
SharedHandle<FileEntry> fileEntry
(new FileEntry("file", 1024*1024*1024*10LL, 0));
httpRequest.setFileEntry(fileEntry);
// End byte is not passed if start byte is 0.
std::string expectedText =
"GET /myfile HTTP/1.1\r\n"
"User-Agent: aria2\r\n"
"Accept: */*\r\n"
"Host: localhost\r\n"
"Pragma: no-cache\r\n"
"Cache-Control: no-cache\r\n"
"Connection: close\r\n"
"\r\n";
CPPUNIT_ASSERT_EQUAL(expectedText, httpRequest.createRequest());
segment->updateWrittenLength(1);
expectedText =
"GET /myfile HTTP/1.1\r\n"
"User-Agent: aria2\r\n"
"Accept: */*\r\n"
"Host: localhost\r\n"
"Pragma: no-cache\r\n"
"Cache-Control: no-cache\r\n"
"Connection: close\r\n"
"Range: bytes=1-1073741823\r\n"
"\r\n";
CPPUNIT_ASSERT_EQUAL(expectedText, httpRequest.createRequest());
}
void HttpRequestTest::testCreateProxyRequest()
{
SharedHandle<Request> request(new Request());

5
test/MockPieceStorage.h
View File

@ -247,6 +247,11 @@ public:
virtual void updatePieceStats(const unsigned char* newBitfield,
size_t newBitfieldLength,
const unsigned char* oldBitfield) {}
virtual size_t getNextUsedIndex(size_t index)
{
return 0;
}
};
} // namespace aria2