diff --git a/doc/manual-src/en/aria2c.rst b/doc/manual-src/en/aria2c.rst index 726d3506..545089cb 100644 --- a/doc/manual-src/en/aria2c.rst +++ b/doc/manual-src/en/aria2c.rst @@ -620,7 +620,8 @@ BitTorrent Specific Options .. option:: --bt-max-open-files= - Specify maximum number of files to open in each BitTorrent download. + Specify maximum number of files to open in multi-file + BitTorrent/Metalink download globally. Default: ``100`` .. option:: --bt-max-peers= @@ -1806,7 +1807,6 @@ of URIs. These optional lines must start with white space(s). * :option:`bt-exclude-tracker <--bt-exclude-tracker>` * :option:`bt-external-ip <--bt-external-ip>` * :option:`bt-hash-check-seed <--bt-hash-check-seed>` - * :option:`bt-max-open-files <--bt-max-open-files>` * :option:`bt-max-peers <--bt-max-peers>` * :option:`bt-metadata-only <--bt-metadata-only>` * :option:`bt-min-crypto-level <--bt-min-crypto-level>` @@ -2942,6 +2942,7 @@ All code examples come from Python2.7 interpreter. struct. The following options are available: + * :option:`bt-max-open-files <--bt-max-open-files>` * :option:`download-result <--download-result>` * :option:`log <-l>` * :option:`log-level <--log-level>` diff --git a/src/DefaultPieceStorage.cc b/src/DefaultPieceStorage.cc index ff1495ff..68dc63b7 100644 --- a/src/DefaultPieceStorage.cc +++ b/src/DefaultPieceStorage.cc @@ -646,8 +646,6 @@ void DefaultPieceStorage::initStorage() multiDiskAdaptor->setFileEntries(downloadContext_->getFileEntries().begin(), downloadContext_->getFileEntries().end()); multiDiskAdaptor->setPieceLength(downloadContext_->getPieceLength()); - multiDiskAdaptor->setMaxOpenFiles - (option_->getAsInt(PREF_BT_MAX_OPEN_FILES)); diskAdaptor_ = std::move(multiDiskAdaptor); } if(option_->get(PREF_FILE_ALLOCATION) == V_FALLOC) { diff --git a/src/DiskAdaptor.cc b/src/DiskAdaptor.cc index abc70c6e..641c94de 100644 --- a/src/DiskAdaptor.cc +++ b/src/DiskAdaptor.cc @@ -38,7 +38,8 @@ namespace aria2 { DiskAdaptor::DiskAdaptor() - : fileAllocationMethod_(FILE_ALLOC_ADAPTIVE) + : fileAllocationMethod_(FILE_ALLOC_ADAPTIVE), + requestGroupMan_(nullptr) {} DiskAdaptor::~DiskAdaptor() {} diff --git a/src/DiskAdaptor.h b/src/DiskAdaptor.h index 2e792763..114f904c 100644 --- a/src/DiskAdaptor.h +++ b/src/DiskAdaptor.h @@ -48,6 +48,7 @@ namespace aria2 { class FileEntry; class FileAllocationIterator; class WrDiskCacheEntry; +class RequestGroupMan; class DiskAdaptor:public BinaryStream { public: @@ -119,10 +120,26 @@ public: return fileAllocationMethod_; } + // Closes at most |numClose| files if possible. This method is used to + // ensure that global number of open file stays under certain limit. + // Returns the number of closed files. + virtual size_t tryCloseFile(size_t numClose) { return 0; } + + void setRequestGroupMan(RequestGroupMan* rgman) + { + requestGroupMan_ = rgman; + } + + RequestGroupMan* getRequestGroupMan() const + { + return requestGroupMan_; + } private: std::vector > fileEntries_; FileAllocationMethod fileAllocationMethod_; + + RequestGroupMan* requestGroupMan_; }; } // namespace aria2 diff --git a/src/MultiDiskAdaptor.cc b/src/MultiDiskAdaptor.cc index 048e523b..069ff2a3 100644 --- a/src/MultiDiskAdaptor.cc +++ b/src/MultiDiskAdaptor.cc @@ -51,6 +51,7 @@ #include "LogFactory.h" #include "SimpleRandomizer.h" #include "WrDiskCacheEntry.h" +#include "RequestGroupMan.h" namespace aria2 { @@ -120,7 +121,6 @@ bool DiskWriterEntry::operator<(const DiskWriterEntry& entry) const MultiDiskAdaptor::MultiDiskAdaptor() : pieceLength_{0}, - maxOpenFiles_{DEFAULT_MAX_OPEN_FILES}, readOnly_{false} {} @@ -200,29 +200,37 @@ void MultiDiskAdaptor::resetDiskWriterEntries() } } +size_t MultiDiskAdaptor::tryCloseFile(size_t numClose) +{ + size_t left = numClose; + for(; !openedDiskWriterEntries_.empty() && left > 0; --left) { + // Choose one DiskWriterEntry randomly and close it. + size_t index = + SimpleRandomizer::getInstance()->getRandomNumber + (openedDiskWriterEntries_.size()); + auto i = std::begin(openedDiskWriterEntries_); + std::advance(i, index); + (*i)->closeFile(); + (*i) = openedDiskWriterEntries_.back(); + openedDiskWriterEntries_.pop_back(); + } + return numClose - left; +} + void MultiDiskAdaptor::openIfNot (DiskWriterEntry* entry, void (DiskWriterEntry::*open)()) { if(!entry->isOpen()) { - // A2_LOG_DEBUG(fmt("DiskWriterEntry: Cache MISS. offset=%s", - // util::itos(entry->getFileEntry()->getOffset()).c_str())); - int numOpened = openedDiskWriterEntries_.size(); - (entry->*open)(); - if(numOpened >= maxOpenFiles_) { - // Cache is full. - // Choose one DiskWriterEntry randomly and close it. - size_t index = - SimpleRandomizer::getInstance()->getRandomNumber(numOpened); - auto i = std::begin(openedDiskWriterEntries_); - std::advance(i, index); - (*i)->closeFile(); - *i = entry; - } else { - openedDiskWriterEntries_.push_back(entry); + // A2_LOG_NOTICE(fmt("DiskWriterEntry: Cache MISS. offset=%s", + // util::itos(entry->getFileEntry()->getOffset()).c_str())); + if(getRequestGroupMan()) { + getRequestGroupMan()->ensureMaxOpenFileLimit(1); } + (entry->*open)(); + openedDiskWriterEntries_.push_back(entry); } else { - // A2_LOG_DEBUG(fmt("DiskWriterEntry: Cache HIT. offset=%s", - // util::itos(entry->getFileEntry()->getOffset()).c_str())); + // A2_LOG_NOTICE(fmt("DiskWriterEntry: Cache HIT. offset=%s", + // util::itos(entry->getFileEntry()->getOffset()).c_str())); } } @@ -519,11 +527,6 @@ void MultiDiskAdaptor::cutTrailingGarbage() } } -void MultiDiskAdaptor::setMaxOpenFiles(int maxOpenFiles) -{ - maxOpenFiles_ = maxOpenFiles; -} - size_t MultiDiskAdaptor::utime(const Time& actime, const Time& modtime) { size_t numOK = 0; diff --git a/src/MultiDiskAdaptor.h b/src/MultiDiskAdaptor.h index 800fe788..0b540803 100644 --- a/src/MultiDiskAdaptor.h +++ b/src/MultiDiskAdaptor.h @@ -117,8 +117,6 @@ private: std::vector openedDiskWriterEntries_; - int maxOpenFiles_; - bool readOnly_; void resetDiskWriterEntries(); @@ -175,8 +173,6 @@ public: virtual void cutTrailingGarbage() CXX11_OVERRIDE; - void setMaxOpenFiles(int maxOpenFiles); - virtual size_t utime(const Time& actime, const Time& modtime) CXX11_OVERRIDE; const DiskWriterEntries& getDiskWriterEntries() const @@ -184,6 +180,7 @@ public: return diskWriterEntries_; } + virtual size_t tryCloseFile(size_t numClose) CXX11_OVERRIDE; }; } // namespace aria2 diff --git a/src/OptionHandlerFactory.cc b/src/OptionHandlerFactory.cc index 304de9e0..aa0c9e32 100644 --- a/src/OptionHandlerFactory.cc +++ b/src/OptionHandlerFactory.cc @@ -1720,9 +1720,7 @@ std::vector OptionHandlerFactory::createOptionHandlers() "100", 1)); op->addTag(TAG_BITTORRENT); - op->setInitialOption(true); op->setChangeGlobalOption(true); - op->setChangeOptionForReserved(true); handlers.push_back(op); } { diff --git a/src/RequestGroup.cc b/src/RequestGroup.cc index f8db9c91..55ea2783 100644 --- a/src/RequestGroup.cc +++ b/src/RequestGroup.cc @@ -614,6 +614,7 @@ void RequestGroup::initPieceStorage() tempPieceStorage = ps; } tempPieceStorage->initStorage(); + tempPieceStorage->getDiskAdaptor()->setRequestGroupMan(requestGroupMan_); segmentMan_ = std::make_shared(downloadContext_, tempPieceStorage); pieceStorage_ = tempPieceStorage; } diff --git a/src/RequestGroupMan.cc b/src/RequestGroupMan.cc index c25d0088..4c04f21a 100644 --- a/src/RequestGroupMan.cc +++ b/src/RequestGroupMan.cc @@ -78,6 +78,9 @@ #include "Notifier.h" #include "PeerStat.h" #include "WrDiskCache.h" +#include "PieceStorage.h" +#include "DiskAdaptor.h" +#include "SimpleRandomizer.h" #ifdef ENABLE_BITTORRENT # include "bittorrent_helper.h" #endif // ENABLE_BITTORRENT @@ -111,7 +114,8 @@ RequestGroupMan::RequestGroupMan removedErrorResult_(0), removedLastErrorResult_(error_code::FINISHED), maxDownloadResult_(option->getAsInt(PREF_MAX_DOWNLOAD_RESULT)), - wrDiskCache_(nullptr) + wrDiskCache_(nullptr), + numOpenFile_(0) { appendReservedGroup(reservedGroups_, requestGroups.begin(), requestGroups.end()); @@ -480,6 +484,8 @@ void RequestGroupMan::fillRequestGroupFromReserver(DownloadEngine* e) groupToAdd->dropPieceStorage(); configureRequestGroup(groupToAdd); groupToAdd->setRequestGroupMan(this); + groupToAdd->setState(RequestGroup::STATE_ACTIVE); + requestGroups_.push_back(groupToAdd->getGID(), groupToAdd); try { auto res = createInitialCommand(groupToAdd, e); ++count; @@ -496,8 +502,6 @@ void RequestGroupMan::fillRequestGroupFromReserver(DownloadEngine* e) // removeStoppedGroup(). requestQueueCheck(); } - groupToAdd->setState(RequestGroup::STATE_ACTIVE); - requestGroups_.push_back(groupToAdd->getGID(), groupToAdd); util::executeHookByOptName(groupToAdd, e->getOption(), PREF_ON_DOWNLOAD_START); @@ -957,4 +961,28 @@ void RequestGroupMan::initWrDiskCache() } } +void RequestGroupMan::ensureMaxOpenFileLimit(size_t numNewFile) +{ + size_t max = option_->getAsInt(PREF_BT_MAX_OPEN_FILES); + if(numOpenFile_ + numNewFile <= max) { + numOpenFile_ += numNewFile; + return; + } + assert(numNewFile <= max); + size_t numClose = numOpenFile_ + numNewFile - max; + size_t left = numClose; + auto mark = std::begin(requestGroups_); + std::advance(mark, + SimpleRandomizer::getInstance()-> + getRandomNumber(requestGroups_.size())); + for(auto i = mark; i != std::end(requestGroups_) && left > 0; ++i) { + left -= (*i)->getPieceStorage()->getDiskAdaptor()->tryCloseFile(left); + } + for(auto i = std::begin(requestGroups_); i != mark && left > 0; ++i) { + left -= (*i)->getPieceStorage()->getDiskAdaptor()->tryCloseFile(left); + } + assert(left == 0); + numOpenFile_ += numNewFile - numClose; +} + } // namespace aria2 diff --git a/src/RequestGroupMan.h b/src/RequestGroupMan.h index 0ffdb476..0cfc527c 100644 --- a/src/RequestGroupMan.h +++ b/src/RequestGroupMan.h @@ -104,6 +104,8 @@ private: WrDiskCache* wrDiskCache_; + size_t numOpenFile_; + void formatDownloadResultFull (OutputFile& out, const char* status, @@ -346,6 +348,15 @@ public: { return keepRunning_; } + + // Keeps the number of open files under the global limit specified + // in the option. The caller requests that |numNewFile| files are + // going to be opened. This function requires that |numNewFile| is + // less than or equal to the limit. + // + // Currently the only download using MultiDiskAdaptor is affected by + // the global limit. + void ensureMaxOpenFileLimit(size_t numNewFile); }; } // namespace aria2 diff --git a/src/usage_text.h b/src/usage_text.h index 72afcf92..4a93ed89 100644 --- a/src/usage_text.h +++ b/src/usage_text.h @@ -344,8 +344,9 @@ " download speed in some cases.\n" \ " You can append K or M(1K = 1024, 1M = 1024K).") #define TEXT_BT_MAX_OPEN_FILES \ - _(" --bt-max-open-files=NUM Specify maximum number of files to open in each\n" \ - " BitTorrent download.") + _(" --bt-max-open-files=NUM Specify maximum number of files to open in\n" \ + " multi-file BitTorrent/Metalink downloads\n" \ + " globally.") #define TEXT_BT_SEED_UNVERIFIED \ _(" --bt-seed-unverified[=true|false] Seed previously downloaded files without\n" \ " verifying piece hashes.") diff --git a/test/MultiDiskAdaptorTest.cc b/test/MultiDiskAdaptorTest.cc index 75e748fe..406a28f7 100644 --- a/test/MultiDiskAdaptorTest.cc +++ b/test/MultiDiskAdaptorTest.cc @@ -369,7 +369,6 @@ void MultiDiskAdaptorTest::testCutTrailingGarbage() MultiDiskAdaptor adaptor; adaptor.setFileEntries(std::begin(fileEntries), std::end(fileEntries)); - adaptor.setMaxOpenFiles(1); adaptor.setPieceLength(1); adaptor.openFile(); @@ -396,7 +395,6 @@ void MultiDiskAdaptorTest::testSize() MultiDiskAdaptor adaptor; adaptor.setFileEntries(std::begin(fileEntries), std::end(fileEntries)); - adaptor.setMaxOpenFiles(1); adaptor.setPieceLength(1); adaptor.openFile();