github
/
aria2
mirror of https://github.com/aria2/aria2
1
0
Fork 0
Code Issues Releases Wiki Activity

2008-06-16 Tatsuhiro Tsujikawa <tujikawa at rednoah dot com> 

Create only requested files and files which shares a piece with
	requested file in multi-torrent downloads. Directory structures 
are
	also created in the same rule.
	MultiDiskAdaptor::fileExists() now uses FileEntry instead of
	DiskWriterEntry as in the previous implementation.
	* src/MultiDiskWriter.cc
	* src/MultiFileAllocationIterator.cc
	* src/MultiFileAllocationIterator.h
	* test/MultiFileAllocationIteratorTest.cc
pull/1/head
Tatsuhiro Tsujikawa 2008-06-15 16:19:06 +00:00
parent f9988a4ef6
commit 2f8e184fe5
5 changed files with 115 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
ChangeLog
View File

@ -1,3 +1,15 @@
2008-06-16 Tatsuhiro Tsujikawa <tujikawa at rednoah dot com>
Create only requested files and files which shares a piece with
requested file in multi-torrent downloads. Directory structures are
also created in the same rule.
MultiDiskAdaptor::fileExists() now uses FileEntry instead of
DiskWriterEntry as in the previous implementation.
* src/MultiDiskWriter.cc
* src/MultiFileAllocationIterator.cc
* src/MultiFileAllocationIterator.h
* test/MultiFileAllocationIteratorTest.cc
2008-06-14 Tatsuhiro Tsujikawa <tujikawa at rednoah dot com>
Commented out -lprofiler

99
src/MultiDiskAdaptor.cc
View File

@ -149,20 +149,77 @@ MultiDiskAdaptor::MultiDiskAdaptor():
MultiDiskAdaptor::~MultiDiskAdaptor() {}
static SharedHandle<DiskWriterEntry> createDiskWriterEntry
(const SharedHandle<FileEntry>& fileEntry,
DiskWriterFactory& dwFactory,
bool directIOAllowed)
{
SharedHandle<DiskWriterEntry> entry(new DiskWriterEntry(fileEntry));
entry->setDiskWriter(dwFactory.newDiskWriter());
entry->getDiskWriter()->setDirectIOAllowed(directIOAllowed);
return entry;
}
void MultiDiskAdaptor::resetDiskWriterEntries()
{
diskWriterEntries.clear();
for(FileEntries::const_iterator itr = fileEntries.begin();
itr != fileEntries.end(); itr++) {
DiskWriterEntryHandle entry(new DiskWriterEntry(*itr));
if((*itr)->isRequested()) {
entry->setDiskWriter(DefaultDiskWriterFactory().newDiskWriter());
} else {
SharedHandle<DiskWriter> dw(new DefaultDiskWriter());
entry->setDiskWriter(dw);
if(fileEntries.empty()) {
return;
}
DefaultDiskWriterFactory dwFactory;
if(pieceLength == 0) {
for(std::deque<SharedHandle<FileEntry> >::const_iterator itr =
fileEntries.begin(); itr != fileEntries.end(); ++itr) {
if((*itr)->isRequested()) {
diskWriterEntries.push_back
(createDiskWriterEntry(*itr, dwFactory, _directIOAllowed));
}
}
} else {
std::deque<SharedHandle<FileEntry> >::const_iterator done = fileEntries.begin();
for(std::deque<SharedHandle<FileEntry> >::const_iterator itr =
fileEntries.begin(); itr != fileEntries.end();) {
if(!(*itr)->isRequested()) {
++itr;
continue;
}
off_t pieceStartOffset = ((*itr)->getOffset()/pieceLength)*pieceLength;
std::deque<SharedHandle<DiskWriterEntry> >::iterator insertionPoint =
diskWriterEntries.end();
if(itr != fileEntries.begin()) {
for(std::deque<SharedHandle<FileEntry> >::const_iterator i = itr-1;
i != done; --i) {
if((uint64_t)pieceStartOffset < (*i)->getOffset()+(*i)->getLength()) {
insertionPoint = diskWriterEntries.insert
(insertionPoint,
createDiskWriterEntry(*i, dwFactory, _directIOAllowed));
} else {
break;
}
}
}
diskWriterEntries.push_back
(createDiskWriterEntry(*itr, dwFactory, _directIOAllowed));
++itr;
for(; itr != fileEntries.end(); ++itr) {
if((*itr)->getOffset() < pieceStartOffset+pieceLength) {
diskWriterEntries.push_back
(createDiskWriterEntry(*itr, dwFactory, _directIOAllowed));
} else {
break;
}
}
done = itr-1;
}
entry->getDiskWriter()->setDirectIOAllowed(_directIOAllowed);
diskWriterEntries.push_back(entry);
}
}
@ -173,9 +230,9 @@ std::string MultiDiskAdaptor::getTopDirPath() const
void MultiDiskAdaptor::mkdir(const std::string& topDirPath) const
{
for(FileEntries::const_iterator itr = fileEntries.begin();
itr != fileEntries.end(); itr++) {
(*itr)->setupDir(topDirPath);
for(std::deque<SharedHandle<DiskWriterEntry> >::const_iterator i =
diskWriterEntries.begin(); i != diskWriterEntries.end(); ++i) {
(*i)->getFileEntry()->setupDir(topDirPath);
}
}
@ -211,16 +268,16 @@ void MultiDiskAdaptor::openIfNot
void MultiDiskAdaptor::openFile()
{
_cachedTopDirPath = getTopDirPath();
mkdir(_cachedTopDirPath);
resetDiskWriterEntries();
mkdir(_cachedTopDirPath);
// TODO we should call openIfNot here?
}
void MultiDiskAdaptor::initAndOpenFile()
{
_cachedTopDirPath = getTopDirPath();
mkdir(_cachedTopDirPath);
resetDiskWriterEntries();
mkdir(_cachedTopDirPath);
// Call DiskWriterEntry::initAndOpenFile to make files truncated.
for(DiskWriterEntries::iterator itr = diskWriterEntries.begin();
itr != diskWriterEntries.end(); ++itr) {
@ -339,17 +396,15 @@ ssize_t MultiDiskAdaptor::readData(unsigned char* data, size_t len, off_t offset
bool MultiDiskAdaptor::fileExists()
{
if(diskWriterEntries.empty()) {
resetDiskWriterEntries();
}
// Don't use _cachedTopDirPath because they are initialized after opening files.
// This method could be called before opening files.
std::string topDirPath = getTopDirPath();
for(DiskWriterEntries::iterator itr = diskWriterEntries.begin();
itr != diskWriterEntries.end(); ++itr) {
if((*itr)->fileExists(topDirPath)) {
for(std::deque<SharedHandle<FileEntry> >::iterator i =
fileEntries.begin(); i != fileEntries.end(); ++i) {
if(File(topDirPath+"/"+(*i)->getPath()).isFile()) {
return true;
}
}
}
return false;
}

43
src/MultiFileAllocationIterator.cc
View File

@ -42,11 +42,9 @@ namespace aria2 {
MultiFileAllocationIterator::MultiFileAllocationIterator(MultiDiskAdaptor* diskAdaptor):
_diskAdaptor(diskAdaptor),
_entries(_diskAdaptor->diskWriterEntries),
_offset(0)
{
makeDiskWriterEntries(_entries, diskAdaptor->diskWriterEntries,
diskAdaptor->getPieceLength());
}
{}
MultiFileAllocationIterator::~MultiFileAllocationIterator() {}
@ -106,41 +104,4 @@ const DiskWriterEntries& MultiFileAllocationIterator::getDiskWriterEntries() con
return _entries;
}
void MultiFileAllocationIterator::makeDiskWriterEntries
(std::deque<SharedHandle<DiskWriterEntry> >& dwEntries,
const DiskWriterEntries& srcEntries, size_t pieceLength) const
{
if(pieceLength == 0) {
for(DiskWriterEntries::const_iterator itr = srcEntries.begin(); itr != srcEntries.end(); ++itr) {
if((*itr)->getFileEntry()->isRequested()) {
dwEntries.push_back(*itr);
}
}
} else {
DiskWriterEntries temp(srcEntries);
{
SharedHandle<FileEntry> f(new FileEntry());
SharedHandle<DiskWriterEntry> e(new DiskWriterEntry(f));
temp.push_front(e);
}
DiskWriterEntries::const_iterator done = temp.begin();
for(DiskWriterEntries::const_iterator itr = temp.begin()+1; itr != temp.end(); ++itr) {
const FileEntryHandle& fileEntry = (*itr)->getFileEntry();
if(!fileEntry->isRequested()) {
continue;
}
off_t pieceStartOffset = (fileEntry->getOffset()/pieceLength)*pieceLength;
for(DiskWriterEntries::const_iterator i = itr-1; i != done; --i) {
if((uint64_t)pieceStartOffset < (*i)->getFileEntry()->getOffset()+(*i)->getFileEntry()->getLength()) {
dwEntries.push_back(*i);
} else {
break;
}
}
dwEntries.push_back(*itr);
done = itr;
}
}
}
} // namespace aria2

4
src/MultiFileAllocationIterator.h
View File

@ -52,10 +52,6 @@ private:
SharedHandle<SingleFileAllocationIterator> _fileAllocationIterator;
off_t _offset;
void makeDiskWriterEntries
(std::deque<SharedHandle<DiskWriterEntry> >& dwEntries,
const std::deque<SharedHandle<DiskWriterEntry> >& srcEntries,
size_t pieceLength) const;
public:
MultiFileAllocationIterator(MultiDiskAdaptor* diskAdaptor);

43
test/MultiFileAllocationIteratorTest.cc
View File

@ -3,6 +3,7 @@
#include "MultiDiskAdaptor.h"
#include "FileEntry.h"
#include "Exception.h"
#include "array_fun.h"
#include <algorithm>
#include <iostream>
#include <cppunit/extensions/HelperMacros.h>
@ -31,27 +32,28 @@ void MultiFileAllocationIteratorTest::testMakeDiskWriterEntries()
{
SharedHandle<FileEntry> fs[] = {
SharedHandle<FileEntry>(new FileEntry("file1", 1536, 0)),
SharedHandle<FileEntry>(new FileEntry("file2", 2048, 1536)),
SharedHandle<FileEntry>(new FileEntry("file2", 2048, 1536)),// req no
SharedHandle<FileEntry>(new FileEntry("file3", 1024, 3584)),
SharedHandle<FileEntry>(new FileEntry("file4", 1024, 4608)),
SharedHandle<FileEntry>(new FileEntry("file5", 1024, 5632)),
SharedHandle<FileEntry>(new FileEntry("file6", 1024, 6656)),
SharedHandle<FileEntry>(new FileEntry("file7", 256, 7680)),
SharedHandle<FileEntry>(new FileEntry("file8", 768, 7936)),
SharedHandle<FileEntry>(new FileEntry("file9", 256, 8704)),
SharedHandle<FileEntry>(new FileEntry("fileA", 256, 8960)),
SharedHandle<FileEntry>(new FileEntry("file4", 1024, 4608)),// req no
SharedHandle<FileEntry>(new FileEntry("file5", 1024, 5632)),// req no
SharedHandle<FileEntry>(new FileEntry("file6", 1024, 6656)),// req no
SharedHandle<FileEntry>(new FileEntry("file7", 256, 7680)),// req no
SharedHandle<FileEntry>(new FileEntry("file8", 255, 7936)),
SharedHandle<FileEntry>(new FileEntry("file9", 1025, 8191)),// req no
SharedHandle<FileEntry>(new FileEntry("fileA", 1024, 9216)),// req no
SharedHandle<FileEntry>(new FileEntry("fileB", 1024, 10240)),
};
fs[1]->setRequested(false);
fs[3]->setRequested(false);
fs[4]->setRequested(false);
fs[5]->setRequested(false);
fs[6]->setRequested(false);
fs[8]->setRequested(false);
fs[9]->setRequested(false);
fs[1]->setRequested(false); // file2
fs[3]->setRequested(false); // file4
fs[4]->setRequested(false); // file5
fs[5]->setRequested(false); // file6
fs[6]->setRequested(false); // file7
fs[8]->setRequested(false); // file9
fs[9]->setRequested(false); // fileA
std::string storeDir = "/tmp/aria2_MultiFileAllocationIteratorTest_testMakeDiskWriterEntries";
SharedHandle<MultiDiskAdaptor> diskAdaptor(new MultiDiskAdaptor());
diskAdaptor->setFileEntries(std::deque<SharedHandle<FileEntry> >(&fs[0], &fs[10]));
diskAdaptor->setFileEntries(std::deque<SharedHandle<FileEntry> >(&fs[0], &fs[arrayLength(fs)]));
diskAdaptor->setPieceLength(1024);
diskAdaptor->setStoreDir(storeDir);
diskAdaptor->openFile();
@ -63,7 +65,7 @@ void MultiFileAllocationIteratorTest::testMakeDiskWriterEntries()
std::sort(entries.begin(), entries.end());
CPPUNIT_ASSERT_EQUAL((size_t)6, entries.size());
CPPUNIT_ASSERT_EQUAL((size_t)8, entries.size());
CPPUNIT_ASSERT_EQUAL(storeDir+std::string("/file1"), entries[0]->getFilePath(storeDir));
CPPUNIT_ASSERT_EQUAL(storeDir+std::string("/file2"), entries[1]->getFilePath(storeDir));
@ -71,6 +73,8 @@ void MultiFileAllocationIteratorTest::testMakeDiskWriterEntries()
CPPUNIT_ASSERT_EQUAL(storeDir+std::string("/file6"), entries[3]->getFilePath(storeDir));
CPPUNIT_ASSERT_EQUAL(storeDir+std::string("/file7"), entries[4]->getFilePath(storeDir));
CPPUNIT_ASSERT_EQUAL(storeDir+std::string("/file8"), entries[5]->getFilePath(storeDir));
CPPUNIT_ASSERT_EQUAL(storeDir+std::string("/file9"), entries[6]->getFilePath(storeDir));
CPPUNIT_ASSERT_EQUAL(storeDir+std::string("/fileB"), entries[7]->getFilePath(storeDir));
}
void MultiFileAllocationIteratorTest::testAllocate()
@ -145,9 +149,10 @@ void MultiFileAllocationIteratorTest::testAllocate()
CPPUNIT_ASSERT_EQUAL((uint64_t)length1, File(dir+"/"+topDir+"/"+fname1).size());
CPPUNIT_ASSERT_EQUAL((uint64_t)length2, File(dir+"/"+topDir+"/"+fname2).size());
CPPUNIT_ASSERT_EQUAL((uint64_t)length3, File(dir+"/"+topDir+"/"+fname3).size());
CPPUNIT_ASSERT_EQUAL(0ULL, File(dir+"/"+topDir+"/"+fname4).size());
CPPUNIT_ASSERT(!File(dir+"/"+topDir+"/"+fname4).isFile());
CPPUNIT_ASSERT_EQUAL((uint64_t)length5, File(dir+"/"+topDir+"/"+fname5).size());
CPPUNIT_ASSERT_EQUAL(0ULL, File(dir+"/"+topDir+"/"+fname6).size());
CPPUNIT_ASSERT(!File(dir+"/"+topDir+"/"+fname6).isFile());
} catch(Exception& e) {
CPPUNIT_FAIL(e.stackTrace());