diff --git a/ChangeLog b/ChangeLog index 2dbf74bd..dc1caa30 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,11 @@ +2008-06-08 Tatsuhiro Tsujikawa + + Limited the number files opened in each BitTorrent download. The default + value is 100. The option to change this value will be added. + * src/MultiDiskAdaptor.cc + * src/MultiDiskAdaptor.h + * src/MultiFileAllocationIterator.cc + 2008-06-08 Tatsuhiro Tsujikawa Fixed the bug that HTTP downloads don't start with old libraries. diff --git a/src/MultiDiskAdaptor.cc b/src/MultiDiskAdaptor.cc index b22545e0..53b4a97d 100644 --- a/src/MultiDiskAdaptor.cc +++ b/src/MultiDiskAdaptor.cc @@ -42,12 +42,14 @@ #include "DlAbortEx.h" #include "File.h" #include "StringFormat.h" +#include "Logger.h" +#include "SimpleRandomizer.h" #include namespace aria2 { DiskWriterEntry::DiskWriterEntry(const SharedHandle& fileEntry): - fileEntry(fileEntry) {} + fileEntry(fileEntry), _open(false), _directIO(false) {} DiskWriterEntry::~DiskWriterEntry() {} @@ -59,21 +61,41 @@ std::string DiskWriterEntry::getFilePath(const std::string& topDir) const void DiskWriterEntry::initAndOpenFile(const std::string& topDir) { diskWriter->initAndOpenFile(getFilePath(topDir), fileEntry->getLength()); + if(_directIO) { + diskWriter->enableDirectIO(); + } + _open = true; } void DiskWriterEntry::openFile(const std::string& topDir) { diskWriter->openFile(getFilePath(topDir), fileEntry->getLength()); + if(_directIO) { + diskWriter->enableDirectIO(); + } + _open = true; } void DiskWriterEntry::openExistingFile(const std::string& topDir) { diskWriter->openExistingFile(getFilePath(topDir), fileEntry->getLength()); + if(_directIO) { + diskWriter->enableDirectIO(); + } + _open = true; +} + +bool DiskWriterEntry::isOpen() const +{ + return _open; } void DiskWriterEntry::closeFile() { - diskWriter->closeFile(); + if(_open) { + diskWriter->closeFile(); + _open = false; + } } bool DiskWriterEntry::fileExists(const std::string& topDir) @@ -106,6 +128,22 @@ bool DiskWriterEntry::operator<(const DiskWriterEntry& entry) const return fileEntry < entry.fileEntry; } +void DiskWriterEntry::enableDirectIO() +{ + if(_open) { + diskWriter->enableDirectIO(); + } + _directIO = true; +} + +void DiskWriterEntry::disableDirectIO() +{ + if(_open) { + diskWriter->disableDirectIO(); + } + _directIO = false; +} + MultiDiskAdaptor::MultiDiskAdaptor(): pieceLength(0) {} @@ -141,36 +179,60 @@ void MultiDiskAdaptor::mkdir(const std::string& topDirPath) const } } +void MultiDiskAdaptor::openIfNot +(const SharedHandle& entry, + void (DiskWriterEntry::*open)(const std::string&), + const std::string& topDirPath) +{ + if(!entry->isOpen()) { +// logger->debug("DiskWriterEntry: Cache MISS. offset=%s", +// Util::itos(entry->getFileEntry()->getOffset()).c_str()); + + size_t numOpened = _openedDiskWriterEntries.size(); + (entry.get()->*open)(topDirPath); + if(numOpened >= OPEN_FILE_MAX) { + // Cache is full. + // Choose one DiskWriterEntry randomly and close it. + size_t index = SimpleRandomizer::getInstance()->getRandomNumber(numOpened); + std::deque >::iterator i = + _openedDiskWriterEntries.begin(); + std::advance(i, index); + (*i)->closeFile(); + (*i) = entry; + } else { + _openedDiskWriterEntries.push_back(entry); + } + } else { +// logger->debug("DiskWriterEntry: Cache HIT. offset=%s", +// Util::itos(entry->getFileEntry()->getOffset()).c_str()); + } +} + void MultiDiskAdaptor::openFile() { - const std::string topDirPath = getTopDirPath(); - mkdir(topDirPath); + _cachedTopDirPath = getTopDirPath(); + mkdir(_cachedTopDirPath); resetDiskWriterEntries(); - for(DiskWriterEntries::iterator itr = diskWriterEntries.begin(); - itr != diskWriterEntries.end(); ++itr) { - (*itr)->openFile(topDirPath); - } + // TODO we should call openIfNot here? } void MultiDiskAdaptor::initAndOpenFile() { - const std::string topDirPath = getTopDirPath(); - mkdir(topDirPath); + _cachedTopDirPath = getTopDirPath(); + mkdir(_cachedTopDirPath); resetDiskWriterEntries(); + // Call DiskWriterEntry::initAndOpenFile to make files truncated. for(DiskWriterEntries::iterator itr = diskWriterEntries.begin(); itr != diskWriterEntries.end(); ++itr) { - (*itr)->initAndOpenFile(topDirPath); + openIfNot(*itr, &DiskWriterEntry::initAndOpenFile, _cachedTopDirPath); } } void MultiDiskAdaptor::openExistingFile() { - const std::string topDirPath = getTopDirPath(); + _cachedTopDirPath = getTopDirPath(); resetDiskWriterEntries(); - for(DiskWriterEntries::iterator itr = diskWriterEntries.begin(); - itr != diskWriterEntries.end(); ++itr) { - (*itr)->openExistingFile(topDirPath); - } + // Not need to call openIfNot here. } void MultiDiskAdaptor::closeFile() @@ -240,6 +302,9 @@ void MultiDiskAdaptor::writeData(const unsigned char* data, size_t len, off_t fileOffset = offset-(*first)->getFileEntry()->getOffset(); for(DiskWriterEntries::const_iterator i = first; i != diskWriterEntries.end(); ++i) { size_t writeLength = calculateLength(*i, fileOffset, rem); + + openIfNot(*i, &DiskWriterEntry::openFile, _cachedTopDirPath); + (*i)->getDiskWriter()->writeData(data+(len-rem), writeLength, fileOffset); rem -= writeLength; fileOffset = 0; @@ -258,6 +323,9 @@ ssize_t MultiDiskAdaptor::readData(unsigned char* data, size_t len, off_t offset off_t fileOffset = offset-(*first)->getFileEntry()->getOffset(); for(DiskWriterEntries::const_iterator i = first; i != diskWriterEntries.end(); ++i) { size_t readLength = calculateLength(*i, fileOffset, rem); + + openIfNot(*i, &DiskWriterEntry::openFile, _cachedTopDirPath); + totalReadLength += (*i)->getDiskWriter()->readData(data+(len-rem), readLength, fileOffset); rem -= readLength; @@ -274,9 +342,12 @@ bool MultiDiskAdaptor::fileExists() if(diskWriterEntries.empty()) { resetDiskWriterEntries(); } + // Don't use _cachedTopDirPath because they are initialized after opening files. + // This method could be called before opening files. + std::string topDirPath = getTopDirPath(); for(DiskWriterEntries::iterator itr = diskWriterEntries.begin(); - itr != diskWriterEntries.end(); itr++) { - if((*itr)->fileExists(getTopDirPath())) { + itr != diskWriterEntries.end(); ++itr) { + if((*itr)->fileExists(topDirPath)) { return true; } } @@ -288,7 +359,7 @@ uint64_t MultiDiskAdaptor::size() const { uint64_t size = 0; for(DiskWriterEntries::const_iterator itr = diskWriterEntries.begin(); - itr != diskWriterEntries.end(); itr++) { + itr != diskWriterEntries.end(); ++itr) { size += (*itr)->size(); } return size; @@ -303,7 +374,7 @@ void MultiDiskAdaptor::enableDirectIO() { for(DiskWriterEntries::const_iterator itr = diskWriterEntries.begin(); itr != diskWriterEntries.end(); ++itr) { - (*itr)->getDiskWriter()->enableDirectIO(); + (*itr)->enableDirectIO(); } } @@ -311,7 +382,7 @@ void MultiDiskAdaptor::disableDirectIO() { for(DiskWriterEntries::const_iterator itr = diskWriterEntries.begin(); itr != diskWriterEntries.end(); ++itr) { - (*itr)->getDiskWriter()->disableDirectIO(); + (*itr)->disableDirectIO(); } } diff --git a/src/MultiDiskAdaptor.h b/src/MultiDiskAdaptor.h index 69780d77..281fbaf3 100644 --- a/src/MultiDiskAdaptor.h +++ b/src/MultiDiskAdaptor.h @@ -47,6 +47,8 @@ class DiskWriterEntry { private: SharedHandle fileEntry; SharedHandle diskWriter; + bool _open; + bool _directIO; public: DiskWriterEntry(const SharedHandle& fileEntry); @@ -62,6 +64,8 @@ public: void closeFile(); + bool isOpen() const; + bool fileExists(const std::string& topDir); uint64_t size() const; @@ -73,6 +77,16 @@ public: SharedHandle getDiskWriter() const; bool operator<(const DiskWriterEntry& entry) const; + + // Set _directIO to true. + // Additionally, if diskWriter is opened, diskWriter->enableDirectIO() is + // called. + void enableDirectIO(); + + // Set _directIO to false. + // Additionally, if diskWriter is opened, diskWriter->disableDirectIO() is + // called. + void disableDirectIO(); }; typedef SharedHandle DiskWriterEntryHandle; @@ -86,6 +100,10 @@ private: size_t pieceLength; DiskWriterEntries diskWriterEntries; + std::string _cachedTopDirPath; + + std::deque > _openedDiskWriterEntries; + bool _directIOAllowed; void resetDiskWriterEntries(); @@ -93,6 +111,13 @@ private: void mkdir(const std::string& topDirPath) const; std::string getTopDirPath() const; + + void openIfNot(const SharedHandle& entry, + void (DiskWriterEntry::*f)(const std::string&), + const std::string& topDirPath); + + static const size_t OPEN_FILE_MAX = 100; + public: MultiDiskAdaptor(); diff --git a/src/MultiFileAllocationIterator.cc b/src/MultiFileAllocationIterator.cc index 680a7bc0..349d174f 100644 --- a/src/MultiFileAllocationIterator.cc +++ b/src/MultiFileAllocationIterator.cc @@ -59,8 +59,12 @@ void MultiFileAllocationIterator::allocateChunk() DiskWriterEntryHandle entry = _entries.front(); _entries.pop_front(); FileEntryHandle fileEntry = entry->getFileEntry(); + // Open file before calling DiskWriterEntry::size() + _diskAdaptor->openIfNot(entry, &DiskWriterEntry::openFile, + _diskAdaptor->getTopDirPath()); + entry->enableDirectIO(); if(entry->size() < fileEntry->getLength()) { - entry->getDiskWriter()->enableDirectIO(); + // Calling private function of MultiDiskAdaptor. _fileAllocationIterator.reset (new SingleFileAllocationIterator(entry->getDiskWriter().get(), entry->size(),