diff --git a/ChangeLog b/ChangeLog index 68d718a1..adc4dd14 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,15 @@ +2008-06-16 Tatsuhiro Tsujikawa + + Create only requested files and files which shares a piece with + requested file in multi-torrent downloads. Directory structures are + also created in the same rule. + MultiDiskAdaptor::fileExists() now uses FileEntry instead of + DiskWriterEntry as in the previous implementation. + * src/MultiDiskWriter.cc + * src/MultiFileAllocationIterator.cc + * src/MultiFileAllocationIterator.h + * test/MultiFileAllocationIteratorTest.cc + 2008-06-14 Tatsuhiro Tsujikawa Commented out -lprofiler diff --git a/src/MultiDiskAdaptor.cc b/src/MultiDiskAdaptor.cc index dc59830a..a9ad762f 100644 --- a/src/MultiDiskAdaptor.cc +++ b/src/MultiDiskAdaptor.cc @@ -149,20 +149,77 @@ MultiDiskAdaptor::MultiDiskAdaptor(): MultiDiskAdaptor::~MultiDiskAdaptor() {} +static SharedHandle createDiskWriterEntry +(const SharedHandle& fileEntry, + DiskWriterFactory& dwFactory, + bool directIOAllowed) +{ + SharedHandle entry(new DiskWriterEntry(fileEntry)); + entry->setDiskWriter(dwFactory.newDiskWriter()); + entry->getDiskWriter()->setDirectIOAllowed(directIOAllowed); + + return entry; +} + + void MultiDiskAdaptor::resetDiskWriterEntries() { diskWriterEntries.clear(); - for(FileEntries::const_iterator itr = fileEntries.begin(); - itr != fileEntries.end(); itr++) { - DiskWriterEntryHandle entry(new DiskWriterEntry(*itr)); - if((*itr)->isRequested()) { - entry->setDiskWriter(DefaultDiskWriterFactory().newDiskWriter()); - } else { - SharedHandle dw(new DefaultDiskWriter()); - entry->setDiskWriter(dw); + + if(fileEntries.empty()) { + return; + } + + DefaultDiskWriterFactory dwFactory; + if(pieceLength == 0) { + for(std::deque >::const_iterator itr = + fileEntries.begin(); itr != fileEntries.end(); ++itr) { + if((*itr)->isRequested()) { + diskWriterEntries.push_back + (createDiskWriterEntry(*itr, dwFactory, _directIOAllowed)); + } + } + } else { + std::deque >::const_iterator done = fileEntries.begin(); + for(std::deque >::const_iterator itr = + fileEntries.begin(); itr != fileEntries.end();) { + if(!(*itr)->isRequested()) { + ++itr; + continue; + } + off_t pieceStartOffset = ((*itr)->getOffset()/pieceLength)*pieceLength; + std::deque >::iterator insertionPoint = + diskWriterEntries.end(); + + if(itr != fileEntries.begin()) { + for(std::deque >::const_iterator i = itr-1; + i != done; --i) { + if((uint64_t)pieceStartOffset < (*i)->getOffset()+(*i)->getLength()) { + insertionPoint = diskWriterEntries.insert + (insertionPoint, + createDiskWriterEntry(*i, dwFactory, _directIOAllowed)); + } else { + break; + } + } + } + + diskWriterEntries.push_back + (createDiskWriterEntry(*itr, dwFactory, _directIOAllowed)); + + ++itr; + + for(; itr != fileEntries.end(); ++itr) { + if((*itr)->getOffset() < pieceStartOffset+pieceLength) { + diskWriterEntries.push_back + (createDiskWriterEntry(*itr, dwFactory, _directIOAllowed)); + } else { + break; + } + } + + done = itr-1; } - entry->getDiskWriter()->setDirectIOAllowed(_directIOAllowed); - diskWriterEntries.push_back(entry); } } @@ -173,9 +230,9 @@ std::string MultiDiskAdaptor::getTopDirPath() const void MultiDiskAdaptor::mkdir(const std::string& topDirPath) const { - for(FileEntries::const_iterator itr = fileEntries.begin(); - itr != fileEntries.end(); itr++) { - (*itr)->setupDir(topDirPath); + for(std::deque >::const_iterator i = + diskWriterEntries.begin(); i != diskWriterEntries.end(); ++i) { + (*i)->getFileEntry()->setupDir(topDirPath); } } @@ -211,16 +268,16 @@ void MultiDiskAdaptor::openIfNot void MultiDiskAdaptor::openFile() { _cachedTopDirPath = getTopDirPath(); - mkdir(_cachedTopDirPath); resetDiskWriterEntries(); + mkdir(_cachedTopDirPath); // TODO we should call openIfNot here? } void MultiDiskAdaptor::initAndOpenFile() { _cachedTopDirPath = getTopDirPath(); - mkdir(_cachedTopDirPath); resetDiskWriterEntries(); + mkdir(_cachedTopDirPath); // Call DiskWriterEntry::initAndOpenFile to make files truncated. for(DiskWriterEntries::iterator itr = diskWriterEntries.begin(); itr != diskWriterEntries.end(); ++itr) { @@ -339,17 +396,15 @@ ssize_t MultiDiskAdaptor::readData(unsigned char* data, size_t len, off_t offset bool MultiDiskAdaptor::fileExists() { - if(diskWriterEntries.empty()) { - resetDiskWriterEntries(); - } // Don't use _cachedTopDirPath because they are initialized after opening files. // This method could be called before opening files. std::string topDirPath = getTopDirPath(); - for(DiskWriterEntries::iterator itr = diskWriterEntries.begin(); - itr != diskWriterEntries.end(); ++itr) { - if((*itr)->fileExists(topDirPath)) { + for(std::deque >::iterator i = + fileEntries.begin(); i != fileEntries.end(); ++i) { + + if(File(topDirPath+"/"+(*i)->getPath()).isFile()) { return true; - } + } } return false; } diff --git a/src/MultiFileAllocationIterator.cc b/src/MultiFileAllocationIterator.cc index 349d174f..1f541361 100644 --- a/src/MultiFileAllocationIterator.cc +++ b/src/MultiFileAllocationIterator.cc @@ -42,11 +42,9 @@ namespace aria2 { MultiFileAllocationIterator::MultiFileAllocationIterator(MultiDiskAdaptor* diskAdaptor): _diskAdaptor(diskAdaptor), + _entries(_diskAdaptor->diskWriterEntries), _offset(0) -{ - makeDiskWriterEntries(_entries, diskAdaptor->diskWriterEntries, - diskAdaptor->getPieceLength()); -} +{} MultiFileAllocationIterator::~MultiFileAllocationIterator() {} @@ -106,41 +104,4 @@ const DiskWriterEntries& MultiFileAllocationIterator::getDiskWriterEntries() con return _entries; } -void MultiFileAllocationIterator::makeDiskWriterEntries -(std::deque >& dwEntries, - const DiskWriterEntries& srcEntries, size_t pieceLength) const -{ - if(pieceLength == 0) { - for(DiskWriterEntries::const_iterator itr = srcEntries.begin(); itr != srcEntries.end(); ++itr) { - if((*itr)->getFileEntry()->isRequested()) { - dwEntries.push_back(*itr); - } - } - } else { - DiskWriterEntries temp(srcEntries); - { - SharedHandle f(new FileEntry()); - SharedHandle e(new DiskWriterEntry(f)); - temp.push_front(e); - } - DiskWriterEntries::const_iterator done = temp.begin(); - for(DiskWriterEntries::const_iterator itr = temp.begin()+1; itr != temp.end(); ++itr) { - const FileEntryHandle& fileEntry = (*itr)->getFileEntry(); - if(!fileEntry->isRequested()) { - continue; - } - off_t pieceStartOffset = (fileEntry->getOffset()/pieceLength)*pieceLength; - for(DiskWriterEntries::const_iterator i = itr-1; i != done; --i) { - if((uint64_t)pieceStartOffset < (*i)->getFileEntry()->getOffset()+(*i)->getFileEntry()->getLength()) { - dwEntries.push_back(*i); - } else { - break; - } - } - dwEntries.push_back(*itr); - done = itr; - } - } -} - } // namespace aria2 diff --git a/src/MultiFileAllocationIterator.h b/src/MultiFileAllocationIterator.h index 366eff16..1f32ff04 100644 --- a/src/MultiFileAllocationIterator.h +++ b/src/MultiFileAllocationIterator.h @@ -52,10 +52,6 @@ private: SharedHandle _fileAllocationIterator; off_t _offset; - void makeDiskWriterEntries - (std::deque >& dwEntries, - const std::deque >& srcEntries, - size_t pieceLength) const; public: MultiFileAllocationIterator(MultiDiskAdaptor* diskAdaptor); diff --git a/test/MultiFileAllocationIteratorTest.cc b/test/MultiFileAllocationIteratorTest.cc index b8c808ae..40c12646 100644 --- a/test/MultiFileAllocationIteratorTest.cc +++ b/test/MultiFileAllocationIteratorTest.cc @@ -3,6 +3,7 @@ #include "MultiDiskAdaptor.h" #include "FileEntry.h" #include "Exception.h" +#include "array_fun.h" #include #include #include @@ -31,27 +32,28 @@ void MultiFileAllocationIteratorTest::testMakeDiskWriterEntries() { SharedHandle fs[] = { SharedHandle(new FileEntry("file1", 1536, 0)), - SharedHandle(new FileEntry("file2", 2048, 1536)), + SharedHandle(new FileEntry("file2", 2048, 1536)),// req no SharedHandle(new FileEntry("file3", 1024, 3584)), - SharedHandle(new FileEntry("file4", 1024, 4608)), - SharedHandle(new FileEntry("file5", 1024, 5632)), - SharedHandle(new FileEntry("file6", 1024, 6656)), - SharedHandle(new FileEntry("file7", 256, 7680)), - SharedHandle(new FileEntry("file8", 768, 7936)), - SharedHandle(new FileEntry("file9", 256, 8704)), - SharedHandle(new FileEntry("fileA", 256, 8960)), + SharedHandle(new FileEntry("file4", 1024, 4608)),// req no + SharedHandle(new FileEntry("file5", 1024, 5632)),// req no + SharedHandle(new FileEntry("file6", 1024, 6656)),// req no + SharedHandle(new FileEntry("file7", 256, 7680)),// req no + SharedHandle(new FileEntry("file8", 255, 7936)), + SharedHandle(new FileEntry("file9", 1025, 8191)),// req no + SharedHandle(new FileEntry("fileA", 1024, 9216)),// req no + SharedHandle(new FileEntry("fileB", 1024, 10240)), }; - fs[1]->setRequested(false); - fs[3]->setRequested(false); - fs[4]->setRequested(false); - fs[5]->setRequested(false); - fs[6]->setRequested(false); - fs[8]->setRequested(false); - fs[9]->setRequested(false); + fs[1]->setRequested(false); // file2 + fs[3]->setRequested(false); // file4 + fs[4]->setRequested(false); // file5 + fs[5]->setRequested(false); // file6 + fs[6]->setRequested(false); // file7 + fs[8]->setRequested(false); // file9 + fs[9]->setRequested(false); // fileA std::string storeDir = "/tmp/aria2_MultiFileAllocationIteratorTest_testMakeDiskWriterEntries"; SharedHandle diskAdaptor(new MultiDiskAdaptor()); - diskAdaptor->setFileEntries(std::deque >(&fs[0], &fs[10])); + diskAdaptor->setFileEntries(std::deque >(&fs[0], &fs[arrayLength(fs)])); diskAdaptor->setPieceLength(1024); diskAdaptor->setStoreDir(storeDir); diskAdaptor->openFile(); @@ -63,7 +65,7 @@ void MultiFileAllocationIteratorTest::testMakeDiskWriterEntries() std::sort(entries.begin(), entries.end()); - CPPUNIT_ASSERT_EQUAL((size_t)6, entries.size()); + CPPUNIT_ASSERT_EQUAL((size_t)8, entries.size()); CPPUNIT_ASSERT_EQUAL(storeDir+std::string("/file1"), entries[0]->getFilePath(storeDir)); CPPUNIT_ASSERT_EQUAL(storeDir+std::string("/file2"), entries[1]->getFilePath(storeDir)); @@ -71,6 +73,8 @@ void MultiFileAllocationIteratorTest::testMakeDiskWriterEntries() CPPUNIT_ASSERT_EQUAL(storeDir+std::string("/file6"), entries[3]->getFilePath(storeDir)); CPPUNIT_ASSERT_EQUAL(storeDir+std::string("/file7"), entries[4]->getFilePath(storeDir)); CPPUNIT_ASSERT_EQUAL(storeDir+std::string("/file8"), entries[5]->getFilePath(storeDir)); + CPPUNIT_ASSERT_EQUAL(storeDir+std::string("/file9"), entries[6]->getFilePath(storeDir)); + CPPUNIT_ASSERT_EQUAL(storeDir+std::string("/fileB"), entries[7]->getFilePath(storeDir)); } void MultiFileAllocationIteratorTest::testAllocate() @@ -145,9 +149,10 @@ void MultiFileAllocationIteratorTest::testAllocate() CPPUNIT_ASSERT_EQUAL((uint64_t)length1, File(dir+"/"+topDir+"/"+fname1).size()); CPPUNIT_ASSERT_EQUAL((uint64_t)length2, File(dir+"/"+topDir+"/"+fname2).size()); CPPUNIT_ASSERT_EQUAL((uint64_t)length3, File(dir+"/"+topDir+"/"+fname3).size()); - CPPUNIT_ASSERT_EQUAL(0ULL, File(dir+"/"+topDir+"/"+fname4).size()); + CPPUNIT_ASSERT(!File(dir+"/"+topDir+"/"+fname4).isFile()); + CPPUNIT_ASSERT_EQUAL((uint64_t)length5, File(dir+"/"+topDir+"/"+fname5).size()); - CPPUNIT_ASSERT_EQUAL(0ULL, File(dir+"/"+topDir+"/"+fname6).size()); + CPPUNIT_ASSERT(!File(dir+"/"+topDir+"/"+fname6).isFile()); } catch(Exception& e) { CPPUNIT_FAIL(e.stackTrace());