github
/
aria2
mirror of https://github.com/aria2/aria2
1
0
Fork 0
Code Issues Releases Wiki Activity

2010-07-15 Tatsuhiro Tsujikawa <t-tujikawa@users.sourceforge.net> 

Added --min-split-size=SIZE option.  aria2 does not split less
	than 2*SIZE byte range.  For example, let's consider downloading
	20MiB file. If SIZE is 10M, aria2 can split file into 2 range
	[0-10MiB) and [10MiB-20MiB) and download it using 2 sources(if
	--split >= 2, of course).  If SIZE is 15M, since 2*15M > 20MiB,
	aria2 does not split file and download it using 1 source.
	* src/BitfieldMan.cc
	* src/BitfieldMan.h
	* src/DefaultPieceStorage.cc
	* src/DefaultPieceStorage.h
	* src/OptionHandlerFactory.cc
	* src/RequestGroup.cc
	* src/prefs.cc
	* src/prefs.h
	* src/usage_text.h
	* test/BitfieldManTest.cc
	* test/SegmentManTest.cc
pull/1/head
Tatsuhiro Tsujikawa 2010-07-15 11:39:21 +00:00
parent 55748de726
commit 1ddaaf7614
12 changed files with 172 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
ChangeLog
View File

@ -1,3 +1,23 @@
2010-07-15 Tatsuhiro Tsujikawa <t-tujikawa@users.sourceforge.net>
Added --min-split-size=SIZE option. aria2 does not split less
than 2*SIZE byte range. For example, let's consider downloading
20MiB file. If SIZE is 10M, aria2 can split file into 2 range
[0-10MiB) and [10MiB-20MiB) and download it using 2 sources(if
--split >= 2, of course). If SIZE is 15M, since 2*15M > 20MiB,
aria2 does not split file and download it using 1 source.
* src/BitfieldMan.cc
* src/BitfieldMan.h
* src/DefaultPieceStorage.cc
* src/DefaultPieceStorage.h
* src/OptionHandlerFactory.cc
* src/RequestGroup.cc
* src/prefs.cc
* src/prefs.h
* src/usage_text.h
* test/BitfieldManTest.cc
* test/SegmentManTest.cc
2010-07-14 Tatsuhiro Tsujikawa <t-tujikawa@users.sourceforge.net>
FeedbackURISelector now tries to chooses URI which is not used in

70
src/BitfieldMan.cc
View File

@ -221,37 +221,59 @@ static size_t getEndIndex(size_t index, const Array& bitfield, size_t blocks) {
template<typename Array>
static bool getSparseMissingUnusedIndex
(size_t& index,
size_t minSplitSize,
const Array& bitfield,
const unsigned char* useBitfield,
size_t blockLength_,
size_t blocks)
{
BitfieldMan::Range maxRange;
BitfieldMan::Range currentRange;
{
size_t nextIndex = 0;
while(nextIndex < blocks) {
currentRange.startIndex =
getStartIndex(nextIndex, bitfield, blocks);
if(currentRange.startIndex == blocks) {
break;
}
currentRange.endIndex =
getEndIndex(currentRange.startIndex, bitfield, blocks);
if(maxRange < currentRange) {
maxRange = currentRange;
}
nextIndex = currentRange.endIndex;
size_t nextIndex = 0;
while(nextIndex < blocks) {
currentRange.startIndex =
getStartIndex(nextIndex, bitfield, blocks);
if(currentRange.startIndex == blocks) {
break;
}
currentRange.endIndex =
getEndIndex(currentRange.startIndex, bitfield, blocks);
if(currentRange.startIndex > 0) {
if(bitfield::test(useBitfield, blocks, currentRange.startIndex-1)) {
currentRange.startIndex = currentRange.getMidIndex();
}
}
// If range is equal, choose a range where its startIndex-1 is
// set.
if(maxRange < currentRange ||
(maxRange == currentRange &&
maxRange.startIndex > 0 && currentRange.startIndex > 0 &&
(!bitfield::test(bitfield, blocks, maxRange.startIndex-1) ||
bitfield::test(useBitfield, blocks, maxRange.startIndex-1))
&&
bitfield::test(bitfield, blocks, currentRange.startIndex-1) &&
!bitfield::test(useBitfield, blocks, currentRange.startIndex-1))) {
maxRange = currentRange;
}
nextIndex = currentRange.endIndex;
}
if(maxRange.getSize()) {
if(maxRange.startIndex == 0) {
index = 0;
} else if(bitfield::test(useBitfield, blocks, maxRange.startIndex-1)) {
index = maxRange.getMidIndex();
return true;
} else {
index = maxRange.startIndex;
if((!bitfield::test(useBitfield, blocks, maxRange.startIndex-1) &&
bitfield::test(bitfield, blocks, maxRange.startIndex-1)) ||
((uint64_t)(maxRange.endIndex-maxRange.startIndex)*blockLength_
>= minSplitSize)) {
index = maxRange.startIndex;
return true;
} else {
return false;
}
}
return true;
} else {
return false;
}
@ -259,17 +281,21 @@ static bool getSparseMissingUnusedIndex
bool BitfieldMan::getSparseMissingUnusedIndex
(size_t& index,
size_t minSplitSize,
const unsigned char* ignoreBitfield,
size_t ignoreBitfieldLength) const
{
if(filterEnabled_) {
return aria2::getSparseMissingUnusedIndex
(index, array(ignoreBitfield)|~array(filterBitfield_)|array(bitfield_)|array(useBitfield_),
useBitfield_, blocks_);
(index, minSplitSize,
array(ignoreBitfield)|~array(filterBitfield_)|
array(bitfield_)|array(useBitfield_),
useBitfield_, blockLength_, blocks_);
} else {
return aria2::getSparseMissingUnusedIndex
(index, array(ignoreBitfield)|array(bitfield_)|array(useBitfield_),
useBitfield_, blocks_);
(index, minSplitSize,
array(ignoreBitfield)|array(bitfield_)|array(useBitfield_),
useBitfield_, blockLength_, blocks_);
}
}

6
src/BitfieldMan.h
View File

@ -92,6 +92,11 @@ public:
bool operator<(const Range& range) const {
return getSize() < range.getSize();
}
bool operator==(const Range& range) const
{
return getSize() == range.getSize();
}
};
public:
BitfieldMan(size_t blockLength, uint64_t totalLength);
@ -144,6 +149,7 @@ public:
// affected by filter
bool getSparseMissingUnusedIndex
(size_t& index,
size_t minSplitSize,
const unsigned char* ignoreBitfield,
size_t ignoreBitfieldLength) const;

6
src/DefaultPieceStorage.cc
View File

@ -72,7 +72,8 @@ DefaultPieceStorage::DefaultPieceStorage
logger_(LogFactory::getInstance()),
option_(option),
pieceStatMan_(new PieceStatMan(downloadContext->getNumPieces(), true)),
pieceSelector_(new RarestPieceSelector(pieceStatMan_))
pieceSelector_(new RarestPieceSelector(pieceStatMan_)),
minSplitSize_(1024*1024)
{}
DefaultPieceStorage::~DefaultPieceStorage() {
@ -272,7 +273,8 @@ SharedHandle<Piece> DefaultPieceStorage::getSparseMissingUnusedPiece
(const unsigned char* ignoreBitfield, size_t length)
{
size_t index;
if(bitfieldMan_->getSparseMissingUnusedIndex(index, ignoreBitfield, length)) {
if(bitfieldMan_->getSparseMissingUnusedIndex
(index, minSplitSize_, ignoreBitfield, length)) {
return checkOutPiece(index);
} else {
return SharedHandle<Piece>();

8
src/DefaultPieceStorage.h
View File

@ -87,6 +87,8 @@ private:
SharedHandle<PieceSelector> pieceSelector_;
size_t minSplitSize_;
bool getMissingPieceIndex(size_t& index,
const unsigned char* bitfield, size_t length);
@ -251,7 +253,11 @@ public:
{
return pieceSelector_;
}
void setMinSplitSize(size_t minSplitSize)
{
minSplitSize_ = minSplitSize;
}
};
typedef SharedHandle<DefaultPieceStorage> DefaultPieceStorageHandle;

11
src/OptionHandlerFactory.cc
View File

@ -366,6 +366,17 @@ OptionHandlers OptionHandlerFactory::createOptionHandlers()
op->addTag(TAG_HTTP);
handlers.push_back(op);
}
{
SharedHandle<OptionHandler> op(new UnitNumberOptionHandler
(PREF_MIN_SPLIT_SIZE,
TEXT_MIN_SPLIT_SIZE,
"20M",
1024*1024, 1024*1024*1024));
op->addTag(TAG_ADVANCED);
op->addTag(TAG_FTP);
op->addTag(TAG_HTTP);
handlers.push_back(op);
}
{
SharedHandle<OptionHandler> op(new BooleanOptionHandler
(PREF_NO_CONF,

1
src/RequestGroup.cc
View File

@ -534,6 +534,7 @@ void RequestGroup::initPieceStorage()
if(!diskWriterFactory_.isNull()) {
ps->setDiskWriterFactory(diskWriterFactory_);
}
ps->setMinSplitSize(option_->getAsInt(PREF_MIN_SPLIT_SIZE));
tempPieceStorage = ps;
} else {
UnknownLengthPieceStorageHandle ps

2
src/prefs.cc
View File

@ -192,6 +192,8 @@ const std::string PREF_MAX_RESUME_FAILURE_TRIES("max-resume-failure-tries");
const std::string PREF_SAVE_SESSION("save-session");
// value: 1*digit
const std::string PREF_MAX_CONNECTION_PER_SERVER("max-connection-per-server");
// value: 1*digit
const std::string PREF_MIN_SPLIT_SIZE("min-split-size");
/**
* FTP related preferences

2
src/prefs.h
View File

@ -196,6 +196,8 @@ extern const std::string PREF_MAX_RESUME_FAILURE_TRIES;
extern const std::string PREF_SAVE_SESSION;
// value: 1*digit
extern const std::string PREF_MAX_CONNECTION_PER_SERVER;
// value: 1*digit
extern const std::string PREF_MIN_SPLIT_SIZE;
/**
* FTP related preferences

8
src/usage_text.h
View File

@ -686,3 +686,11 @@
#define TEXT_MAX_CONNECTION_PER_SERVER \
_(" --max-connection-per-server=NUM The maximum number of connections to one server\n"\
" for each download.")
#define TEXT_MIN_SPLIT_SIZE \
_(" --min-split-size=SIZE aria2 does not split less than 2*SIZE byte range.\n" \
" For example, let's consider downloading 20MiB\n" \
" file. If SIZE is 10M, aria2 can split file into 2\n" \
" range [0-10MiB) and [10MiB-20MiB) and download it\n" \
" using 2 sources(if --split >= 2, of course).\n" \
" If SIZE is 15M, since 2*15M > 20MiB, aria2 does\n" \
" not split file and download it using 1 source.")

87
test/BitfieldManTest.cc
View File

@ -24,6 +24,7 @@ class BitfieldManTest:public CppUnit::TestFixture {
CPPUNIT_TEST(testAddNotFilter_overflow);
CPPUNIT_TEST(testGetSparceMissingUnusedIndex);
CPPUNIT_TEST(testGetSparceMissingUnusedIndex_setBit);
CPPUNIT_TEST(testGetSparceMissingUnusedIndex_withMinSplitSize);
CPPUNIT_TEST(testIsBitSetOffsetRange);
CPPUNIT_TEST(testGetMissingUnusedLength);
CPPUNIT_TEST(testSetBitRange);
@ -53,6 +54,7 @@ public:
void testAddNotFilter_overflow();
void testGetSparceMissingUnusedIndex();
void testGetSparceMissingUnusedIndex_setBit();
void testGetSparceMissingUnusedIndex_withMinSplitSize();
void testIsBitSetOffsetRange();
void testGetMissingUnusedLength();
void testSetBitRange();
@ -280,48 +282,49 @@ void BitfieldManTest::testGetSparceMissingUnusedIndex() {
const size_t length = 2;
unsigned char ignoreBitfield[length];
memset(ignoreBitfield, 0, sizeof(ignoreBitfield));
size_t minSplitSize = 1024*1024;
size_t index;
CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index,
CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
ignoreBitfield, length));
CPPUNIT_ASSERT_EQUAL((size_t)0, index);
bitfield.setUseBit(0);
CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index,
CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
ignoreBitfield, length));
CPPUNIT_ASSERT_EQUAL((size_t)5, index);
bitfield.setUseBit(5);
CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index,
CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
ignoreBitfield, length));
CPPUNIT_ASSERT_EQUAL((size_t)3, index);
bitfield.setUseBit(3);
CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index,
CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
ignoreBitfield, length));
CPPUNIT_ASSERT_EQUAL((size_t)8, index);
bitfield.setUseBit(8);
CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index,
CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
ignoreBitfield, length));
CPPUNIT_ASSERT_EQUAL((size_t)2, index);
bitfield.setUseBit(2);
CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index,
ignoreBitfield, length));
CPPUNIT_ASSERT_EQUAL((size_t)7, index);
bitfield.setUseBit(7);
CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index,
CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
ignoreBitfield, length));
CPPUNIT_ASSERT_EQUAL((size_t)1, index);
bitfield.setUseBit(1);
CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index,
CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
ignoreBitfield, length));
CPPUNIT_ASSERT_EQUAL((size_t)4, index);
bitfield.setUseBit(4);
CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index,
CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
ignoreBitfield, length));
CPPUNIT_ASSERT_EQUAL((size_t)7, index);
bitfield.setUseBit(7);
CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
ignoreBitfield, length));
CPPUNIT_ASSERT_EQUAL((size_t)6, index);
bitfield.setUseBit(6);
CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index,
CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
ignoreBitfield, length));
CPPUNIT_ASSERT_EQUAL((size_t)9, index);
bitfield.setUseBit(9);
CPPUNIT_ASSERT(!bitfield.getSparseMissingUnusedIndex(index,
CPPUNIT_ASSERT(!bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
ignoreBitfield, length));
}
@ -330,48 +333,82 @@ void BitfieldManTest::testGetSparceMissingUnusedIndex_setBit() {
const size_t length = 2;
unsigned char ignoreBitfield[length];
memset(ignoreBitfield, 0, sizeof(ignoreBitfield));
size_t minSplitSize = 1024*1024;
size_t index;
CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index,
CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
ignoreBitfield, length));
CPPUNIT_ASSERT_EQUAL((size_t)0, index);
bitfield.setBit(0);
CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index,
CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
ignoreBitfield, length));
CPPUNIT_ASSERT_EQUAL((size_t)1, index);
bitfield.setBit(1);
CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index,
CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
ignoreBitfield, length));
CPPUNIT_ASSERT_EQUAL((size_t)2, index);
bitfield.setBit(2);
CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index,
CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
ignoreBitfield, length));
CPPUNIT_ASSERT_EQUAL((size_t)3, index);
bitfield.setBit(3);
CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index,
CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
ignoreBitfield, length));
CPPUNIT_ASSERT_EQUAL((size_t)4, index);
bitfield.setBit(4);
CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index,
CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
ignoreBitfield, length));
CPPUNIT_ASSERT_EQUAL((size_t)5, index);
bitfield.setBit(5);
CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index,
CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
ignoreBitfield, length));
CPPUNIT_ASSERT_EQUAL((size_t)6, index);
bitfield.setBit(6);
CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index,
CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
ignoreBitfield, length));
CPPUNIT_ASSERT_EQUAL((size_t)7, index);
bitfield.setBit(7);
CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index,
CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
ignoreBitfield, length));
CPPUNIT_ASSERT_EQUAL((size_t)8, index);
bitfield.setBit(8);
CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index,
CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
ignoreBitfield, length));
CPPUNIT_ASSERT_EQUAL((size_t)9, index);
bitfield.setBit(9);
CPPUNIT_ASSERT(!bitfield.getSparseMissingUnusedIndex(index,
CPPUNIT_ASSERT(!bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
ignoreBitfield, length));
}
void BitfieldManTest::testGetSparceMissingUnusedIndex_withMinSplitSize()
{
BitfieldMan bitfield(1024*1024, 10*1024*1024);
const size_t length = 2;
unsigned char ignoreBitfield[length];
memset(ignoreBitfield, 0, sizeof(ignoreBitfield));
size_t minSplitSize = 2*1024*1024;
size_t index;
bitfield.setUseBit(1);
CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
ignoreBitfield, length));
CPPUNIT_ASSERT_EQUAL((size_t)6, index);
bitfield.setBit(6);
CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
ignoreBitfield, length));
CPPUNIT_ASSERT_EQUAL((size_t)7, index);
bitfield.setUseBit(7);
CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
ignoreBitfield, length));
CPPUNIT_ASSERT_EQUAL((size_t)4, index);
bitfield.setBit(4);
CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
ignoreBitfield, length));
CPPUNIT_ASSERT_EQUAL((size_t)0, index);
bitfield.setBit(0);
CPPUNIT_ASSERT(bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
ignoreBitfield, length));
CPPUNIT_ASSERT_EQUAL((size_t)5, index);
bitfield.setBit(5);
CPPUNIT_ASSERT(!bitfield.getSparseMissingUnusedIndex(index, minSplitSize,
ignoreBitfield, length));
}

1
test/SegmentManTest.cc
View File

@ -112,6 +112,7 @@ void SegmentManTest::testGetSegment_sameFileEntry()
};
dctx->setFileEntries(&fileEntries[0], &fileEntries[3]);
SharedHandle<DefaultPieceStorage> ps(new DefaultPieceStorage(dctx, &op));
ps->setMinSplitSize(dctx->getPieceLength());
SegmentMan segman(&op, dctx, ps);
std::vector<SharedHandle<Segment> > segments;