From 29c5ef9215be004b31cd53ef8841680c3f015d57 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 25 Mar 2009 05:43:07 +0000 Subject: [PATCH] 2009-03-25 Tatsuhiro Tsujikawa Added 'falloc' parameter for --file-allocation option. 'falloc' allocation mode uses posix_fallocate() system call to allocate file on disk. If you are using newer file systems such as ext4 (with extents support), btrfs or xfs, 'falloc' is your best choice. It allocates large(few GiB) files almost instantly. Don't use 'falloc' with legacy file systems such as ext3 because it takes almost same time as 'prealloc' and it blocks aria2 entirely until allocation finishes. 'falloc' may not be available if your system doesn't have posix_fallocate() system call. * configure.ac * src/AbstractDiskWriter.cc * src/AbstractDiskWriter.h * src/AbstractSingleDiskAdaptor.cc * src/BinaryStream.h * src/BtCheckIntegrityEntry.cc * src/ByteArrayDiskWriter.h * src/CheckIntegrityEntry.cc * src/CheckIntegrityEntry.h * src/DefaultPieceStorage.cc * src/DiskAdaptor.cc * src/DiskAdaptor.h * src/DiskWriter.h * src/FallocFileAllocationIterator.cc * src/FallocFileAllocationIterator.h * src/FileAllocationEntry.cc * src/FileAllocationEntry.h * src/Makefile.am * src/MultiFileAllocationIterator.cc * src/MultiFileAllocationIterator.h * src/OptionHandlerFactory.cc * src/RequestGroup.cc * src/StreamCheckIntegrityEntry.cc * src/prefs.cc * src/prefs.h * src/usage_text.h * test/FallocFileAllocationIteratorTest.cc * test/Makefile.am --- ChangeLog | 40 ++++++++ config.h.in | 3 + configure | 116 ++++++++++++++++++++++- configure.ac | 3 + src/AbstractDiskWriter.cc | 14 +++ src/AbstractDiskWriter.h | 5 + src/AbstractSingleDiskAdaptor.cc | 16 +++- src/BinaryStream.h | 12 ++- src/BtCheckIntegrityEntry.cc | 20 ++-- src/ByteArrayDiskWriter.h | 3 - src/CheckIntegrityEntry.cc | 21 ++++ src/CheckIntegrityEntry.h | 5 + src/DefaultPieceStorage.cc | 3 + src/DiskAdaptor.cc | 3 +- src/DiskAdaptor.h | 19 +++- src/DiskWriter.h | 2 - src/FallocFileAllocationIterator.cc | 69 ++++++++++++++ src/FallocFileAllocationIterator.h | 65 +++++++++++++ src/FileAllocationEntry.cc | 5 + src/FileAllocationEntry.h | 2 + src/Makefile.am | 4 + src/Makefile.in | 98 ++++++++++--------- src/MultiFileAllocationIterator.cc | 19 +++- src/MultiFileAllocationIterator.h | 3 +- src/OptionHandlerFactory.cc | 3 + src/RequestGroup.cc | 6 +- src/StreamCheckIntegrityEntry.cc | 11 +-- src/prefs.cc | 5 +- src/prefs.h | 3 +- src/usage_text.h | 11 ++- test/FallocFileAllocationIteratorTest.cc | 53 +++++++++++ test/Makefile.am | 4 + test/Makefile.in | 32 ++++--- 33 files changed, 563 insertions(+), 115 deletions(-) create mode 100644 src/FallocFileAllocationIterator.cc create mode 100644 src/FallocFileAllocationIterator.h create mode 100644 test/FallocFileAllocationIteratorTest.cc diff --git a/ChangeLog b/ChangeLog index 8d3d1fb7..61a8b788 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,43 @@ +2009-03-25 Tatsuhiro Tsujikawa + + Added 'falloc' parameter for --file-allocation option. 'falloc' + allocation mode uses posix_fallocate() system call to allocate + file on disk. If you are using newer file systems such as ext4 + (with extents support), btrfs or xfs, 'falloc' is your best + choice. It allocates large(few GiB) files almost instantly. Don't + use 'falloc' with legacy file systems such as ext3 because it + takes almost same time as 'prealloc' and it blocks aria2 entirely + until allocation finishes. 'falloc' may not be available if your + system doesn't have posix_fallocate() system call. + * configure.ac + * src/AbstractDiskWriter.cc + * src/AbstractDiskWriter.h + * src/AbstractSingleDiskAdaptor.cc + * src/BinaryStream.h + * src/BtCheckIntegrityEntry.cc + * src/ByteArrayDiskWriter.h + * src/CheckIntegrityEntry.cc + * src/CheckIntegrityEntry.h + * src/DefaultPieceStorage.cc + * src/DiskAdaptor.cc + * src/DiskAdaptor.h + * src/DiskWriter.h + * src/FallocFileAllocationIterator.cc + * src/FallocFileAllocationIterator.h + * src/FileAllocationEntry.cc + * src/FileAllocationEntry.h + * src/Makefile.am + * src/MultiFileAllocationIterator.cc + * src/MultiFileAllocationIterator.h + * src/OptionHandlerFactory.cc + * src/RequestGroup.cc + * src/StreamCheckIntegrityEntry.cc + * src/prefs.cc + * src/prefs.h + * src/usage_text.h + * test/FallocFileAllocationIteratorTest.cc + * test/Makefile.am + 2009-03-25 Tatsuhiro Tsujikawa Removed duplicate enableDirectIO() call. diff --git a/config.h.in b/config.h.in index 1bc86166..86248c59 100644 --- a/config.h.in +++ b/config.h.in @@ -282,6 +282,9 @@ /* Define to 1 if you have old openssl. */ #undef HAVE_OLD_LIBSSL +/* Define to 1 if you have the `posix_fallocate' function. */ +#undef HAVE_POSIX_FALLOCATE + /* Define to 1 if you have the `posix_memalign' function. */ #undef HAVE_POSIX_MEMALIGN diff --git a/configure b/configure index 8f9cb5c3..a42db444 100755 --- a/configure +++ b/configure @@ -813,6 +813,8 @@ POSUB LIBOBJS HAVE_EPOLL_TRUE HAVE_EPOLL_FALSE +HAVE_POSIX_FALLOCATE_TRUE +HAVE_POSIX_FALLOCATE_FALSE HAVE_ASCTIME_R_TRUE HAVE_ASCTIME_R_FALSE HAVE_BASENAME_TRUE @@ -20992,6 +20994,109 @@ fi +for ac_func in posix_fallocate +do +as_ac_var=`echo "ac_cv_func_$ac_func" | $as_tr_sh` +{ echo "$as_me:$LINENO: checking for $ac_func" >&5 +echo $ECHO_N "checking for $ac_func... $ECHO_C" >&6; } +if { as_var=$as_ac_var; eval "test \"\${$as_var+set}\" = set"; }; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ +/* Define $ac_func to an innocuous variant, in case declares $ac_func. + For example, HP-UX 11i declares gettimeofday. */ +#define $ac_func innocuous_$ac_func + +/* System header to define __stub macros and hopefully few prototypes, + which can conflict with char $ac_func (); below. + Prefer to if __STDC__ is defined, since + exists even on freestanding compilers. */ + +#ifdef __STDC__ +# include +#else +# include +#endif + +#undef $ac_func + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char $ac_func (); +/* The GNU C library defines this for functions which it implements + to always fail with ENOSYS. Some functions are actually named + something starting with __ and the normal name is an alias. */ +#if defined __stub_$ac_func || defined __stub___$ac_func +choke me +#endif + +int +main () +{ +return $ac_func (); + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext conftest$ac_exeext +if { (ac_try="$ac_link" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval "echo \"\$as_me:$LINENO: $ac_try_echo\"") >&5 + (eval "$ac_link") 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && { + test -z "$ac_cxx_werror_flag" || + test ! -s conftest.err + } && test -s conftest$ac_exeext && + $as_test_x conftest$ac_exeext; then + eval "$as_ac_var=yes" +else + echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + eval "$as_ac_var=no" +fi + +rm -f core conftest.err conftest.$ac_objext conftest_ipa8_conftest.oo \ + conftest$ac_exeext conftest.$ac_ext +fi +ac_res=`eval echo '${'$as_ac_var'}'` + { echo "$as_me:$LINENO: result: $ac_res" >&5 +echo "${ECHO_T}$ac_res" >&6; } +if test `eval echo '${'$as_ac_var'}'` = yes; then + cat >>confdefs.h <<_ACEOF +#define `echo "HAVE_$ac_func" | $as_tr_cpp` 1 +_ACEOF + have_posix_fallocate=yes +fi +done + + if test "x$have_posix_fallocate" = "xyes"; then + HAVE_POSIX_FALLOCATE_TRUE= + HAVE_POSIX_FALLOCATE_FALSE='#' +else + HAVE_POSIX_FALLOCATE_TRUE='#' + HAVE_POSIX_FALLOCATE_FALSE= +fi + + + for ac_func in asctime_r do as_ac_var=`echo "ac_cv_func_$ac_func" | $as_tr_sh` @@ -22345,6 +22450,13 @@ echo "$as_me: error: conditional \"HAVE_EPOLL\" was never defined. Usually this means the macro was only invoked conditionally." >&2;} { (exit 1); exit 1; }; } fi +if test -z "${HAVE_POSIX_FALLOCATE_TRUE}" && test -z "${HAVE_POSIX_FALLOCATE_FALSE}"; then + { { echo "$as_me:$LINENO: error: conditional \"HAVE_POSIX_FALLOCATE\" was never defined. +Usually this means the macro was only invoked conditionally." >&5 +echo "$as_me: error: conditional \"HAVE_POSIX_FALLOCATE\" was never defined. +Usually this means the macro was only invoked conditionally." >&2;} + { (exit 1); exit 1; }; } +fi if test -z "${HAVE_ASCTIME_R_TRUE}" && test -z "${HAVE_ASCTIME_R_FALSE}"; then { { echo "$as_me:$LINENO: error: conditional \"HAVE_ASCTIME_R\" was never defined. Usually this means the macro was only invoked conditionally." >&5 @@ -23309,6 +23421,8 @@ ac_delim='%!_!# ' for ac_last_try in false false false false false :; do cat >conf$$subs.sed <<_ACEOF HAVE_EPOLL_FALSE!$HAVE_EPOLL_FALSE$ac_delim +HAVE_POSIX_FALLOCATE_TRUE!$HAVE_POSIX_FALLOCATE_TRUE$ac_delim +HAVE_POSIX_FALLOCATE_FALSE!$HAVE_POSIX_FALLOCATE_FALSE$ac_delim HAVE_ASCTIME_R_TRUE!$HAVE_ASCTIME_R_TRUE$ac_delim HAVE_ASCTIME_R_FALSE!$HAVE_ASCTIME_R_FALSE$ac_delim HAVE_BASENAME_TRUE!$HAVE_BASENAME_TRUE$ac_delim @@ -23330,7 +23444,7 @@ HAVE_TIMEGM_FALSE!$HAVE_TIMEGM_FALSE$ac_delim LTLIBOBJS!$LTLIBOBJS$ac_delim _ACEOF - if test `sed -n "s/.*$ac_delim\$/X/p" conf$$subs.sed | grep -c X` = 20; then + if test `sed -n "s/.*$ac_delim\$/X/p" conf$$subs.sed | grep -c X` = 22; then break elif $ac_last_try; then { { echo "$as_me:$LINENO: error: could not make $CONFIG_STATUS" >&5 diff --git a/configure.ac b/configure.ac index 633468d0..ce10ae07 100644 --- a/configure.ac +++ b/configure.ac @@ -292,6 +292,9 @@ if test "x$enable_epoll" = "xyes"; then fi AM_CONDITIONAL([HAVE_EPOLL], [test "x$have_epoll" = "xyes"]) +AC_CHECK_FUNCS([posix_fallocate],[have_posix_fallocate=yes]) +AM_CONDITIONAL([HAVE_POSIX_FALLOCATE], [test "x$have_posix_fallocate" = "xyes"]) + AC_CHECK_FUNCS([asctime_r], [AM_CONDITIONAL([HAVE_ASCTIME_R], true)], [AM_CONDITIONAL([HAVE_ASCTIME_R], false)]) diff --git a/src/AbstractDiskWriter.cc b/src/AbstractDiskWriter.cc index cec3bdaf..1c8d070f 100644 --- a/src/AbstractDiskWriter.cc +++ b/src/AbstractDiskWriter.cc @@ -192,6 +192,20 @@ void AbstractDiskWriter::truncate(uint64_t length) #endif } +#ifdef HAVE_POSIX_FALLOCATE +void AbstractDiskWriter::allocate(off_t offset, uint64_t length) +{ + if(fd == -1) { + throw DlAbortEx("File not yet opened."); + } + int r = posix_fallocate(fd, offset, length); + if(r != 0) { + throw DlAbortEx(StringFormat("posix_fallocate failed. cause: %s", + strerror(r)).str()); + } +} +#endif // HAVE_POSIX_FALLOCATE + // TODO the file descriptor fd must be opened before calling this function. uint64_t AbstractDiskWriter::size() { diff --git a/src/AbstractDiskWriter.h b/src/AbstractDiskWriter.h index ef967687..ff9f3ca1 100644 --- a/src/AbstractDiskWriter.h +++ b/src/AbstractDiskWriter.h @@ -74,6 +74,11 @@ public: virtual void truncate(uint64_t length); +#ifdef HAVE_POSIX_FALLOCATE + // File must be opened before calling this function. + virtual void allocate(off_t offset, uint64_t length); +#endif // HAVE_POSIX_FALLOCATE + virtual uint64_t size(); virtual void enableDirectIO(); diff --git a/src/AbstractSingleDiskAdaptor.cc b/src/AbstractSingleDiskAdaptor.cc index d1c4d8a5..647cd33a 100644 --- a/src/AbstractSingleDiskAdaptor.cc +++ b/src/AbstractSingleDiskAdaptor.cc @@ -35,6 +35,7 @@ #include "AbstractSingleDiskAdaptor.h" #include "File.h" #include "SingleFileAllocationIterator.h" +#include "FallocFileAllocationIterator.h" #include "DiskWriter.h" namespace aria2 { @@ -90,10 +91,17 @@ void AbstractSingleDiskAdaptor::truncate(uint64_t length) FileAllocationIteratorHandle AbstractSingleDiskAdaptor::fileAllocationIterator() { - SingleFileAllocationIteratorHandle h - (new SingleFileAllocationIterator(this, size(), totalLength)); - h->init(); - return h; + + if(_fallocate) { + SharedHandle h + (new FallocFileAllocationIterator(this, size(), totalLength)); + return h; + } else { + SingleFileAllocationIteratorHandle h + (new SingleFileAllocationIterator(this, size(), totalLength)); + h->init(); + return h; + } } void AbstractSingleDiskAdaptor::enableDirectIO() diff --git a/src/BinaryStream.h b/src/BinaryStream.h index 9af8b3bf..38688e44 100644 --- a/src/BinaryStream.h +++ b/src/BinaryStream.h @@ -36,9 +36,11 @@ #define _D_BINARY_STREAM_H_ #include "common.h" -#include "SharedHandle.h" + #include +#include "SharedHandle.h" + namespace aria2 { class BinaryStream { @@ -49,7 +51,13 @@ public: virtual ssize_t readData(unsigned char* data, size_t len, off_t offset) = 0; - virtual void truncate(uint64_t length) = 0; + // Truncates a file to given length. The default implementation does + // nothing. + virtual void truncate(uint64_t length) {} + + // Allocates given length bytes of disk space from given offset. The + // default implementation does nothing. + virtual void allocate(off_t offset, uint64_t length) {} virtual void enableDirectIO() = 0; diff --git a/src/BtCheckIntegrityEntry.cc b/src/BtCheckIntegrityEntry.cc index bed582c5..1be5e251 100644 --- a/src/BtCheckIntegrityEntry.cc +++ b/src/BtCheckIntegrityEntry.cc @@ -37,7 +37,6 @@ #include "RequestGroup.h" #include "PieceStorage.h" #include "DownloadEngine.h" -#include "FileAllocationMan.h" #include "DiskAdaptor.h" #include "prefs.h" #include "Option.h" @@ -49,17 +48,6 @@ BtCheckIntegrityEntry::BtCheckIntegrityEntry(RequestGroup* requestGroup): BtCheckIntegrityEntry::~BtCheckIntegrityEntry() {} -static void proceedFileAllocation -(std::deque& commands, DownloadEngine* e, RequestGroup* requestGroup) -{ - FileAllocationEntryHandle entry(new BtFileAllocationEntry(requestGroup)); - if(requestGroup->needsFileAllocation()) { - e->_fileAllocationMan->pushEntry(entry); - } else { - entry->prepareForNextAction(commands, e); - } -} - void BtCheckIntegrityEntry::onDownloadIncomplete(std::deque& commands, DownloadEngine* e) { @@ -68,7 +56,9 @@ void BtCheckIntegrityEntry::onDownloadIncomplete(std::deque& commands, _requestGroup->getPieceStorage()->getDiskAdaptor()->disableReadOnly(); _requestGroup->getPieceStorage()->getDiskAdaptor()->openFile(); - proceedFileAllocation(commands, e, _requestGroup); + SharedHandle entry + (new BtFileAllocationEntry(_requestGroup)); + proceedFileAllocation(commands, entry, e); } void BtCheckIntegrityEntry::onDownloadFinished(std::deque& commands, @@ -80,7 +70,9 @@ void BtCheckIntegrityEntry::onDownloadFinished(std::deque& commands, // to exit rather than doing seeding. So, it would be good to toggle this // behavior. if(e->option->getAsBool(PREF_BT_HASH_CHECK_SEED)) { - proceedFileAllocation(commands, e, _requestGroup); + SharedHandle entry + (new BtFileAllocationEntry(_requestGroup)); + proceedFileAllocation(commands, entry, e); } } diff --git a/src/ByteArrayDiskWriter.h b/src/ByteArrayDiskWriter.h index 2f8007c8..2ac323d8 100644 --- a/src/ByteArrayDiskWriter.h +++ b/src/ByteArrayDiskWriter.h @@ -60,9 +60,6 @@ public: virtual void writeData(const unsigned char* data, size_t len, off_t position); virtual ssize_t readData(unsigned char* data, size_t len, off_t position); - // Not implemented yet - virtual void truncate(uint64_t length) {} - virtual uint64_t size(); virtual void enableDirectIO() {} diff --git a/src/CheckIntegrityEntry.cc b/src/CheckIntegrityEntry.cc index 5bfec8c5..7d71b7f8 100644 --- a/src/CheckIntegrityEntry.cc +++ b/src/CheckIntegrityEntry.cc @@ -37,6 +37,10 @@ #include "RequestGroup.h" #include "PieceStorage.h" #include "DiskAdaptor.h" +#include "prefs.h" +#include "FileAllocationEntry.h" +#include "DownloadEngine.h" +#include "Option.h" namespace aria2 { @@ -80,4 +84,21 @@ void CheckIntegrityEntry::cutTrailingGarbage() _requestGroup->getPieceStorage()->getDiskAdaptor()->cutTrailingGarbage(); } +void CheckIntegrityEntry::proceedFileAllocation +(std::deque& commands, + const SharedHandle& entry, + DownloadEngine* e) +{ + if(_requestGroup->needsFileAllocation()) { + e->_fileAllocationMan->pushEntry(entry); + } else { + entry->prepareForNextAction(commands, e); + } + // Disable directIO when fallocation() is going to be used. + if(e->option->get(PREF_FILE_ALLOCATION) == V_FALLOC) { + entry->disableDirectIO(); + } +} + + } // namespace aria2 diff --git a/src/CheckIntegrityEntry.h b/src/CheckIntegrityEntry.h index 300c7536..a2991eb6 100644 --- a/src/CheckIntegrityEntry.h +++ b/src/CheckIntegrityEntry.h @@ -43,11 +43,16 @@ namespace aria2 { class IteratableValidator; class DownloadEngine; +class FileAllocationEntry; class CheckIntegrityEntry : public RequestGroupEntry, public ProgressAwareEntry { protected: SharedHandle _validator; + + void proceedFileAllocation(std::deque& commands, + const SharedHandle& entry, + DownloadEngine* e); public: CheckIntegrityEntry(RequestGroup* requestGroup, Command* nextCommand = 0); diff --git a/src/DefaultPieceStorage.cc b/src/DefaultPieceStorage.cc index c8f31568..ce4b37b5 100644 --- a/src/DefaultPieceStorage.cc +++ b/src/DefaultPieceStorage.cc @@ -498,6 +498,9 @@ void DefaultPieceStorage::initStorage() } diskAdaptor->setStoreDir(downloadContext->getDir()); diskAdaptor->setFileEntries(downloadContext->getFileEntries()); + if(option->get(PREF_FILE_ALLOCATION) == V_FALLOC) { + diskAdaptor->enableFallocate(); + } } void DefaultPieceStorage::setBitfield(const unsigned char* bitfield, diff --git a/src/DiskAdaptor.cc b/src/DiskAdaptor.cc index fef04d50..d90b1b9e 100644 --- a/src/DiskAdaptor.cc +++ b/src/DiskAdaptor.cc @@ -42,7 +42,8 @@ namespace aria2 { -DiskAdaptor::DiskAdaptor():logger(LogFactory::getInstance()) {} +DiskAdaptor::DiskAdaptor(): + _fallocate(false), logger(LogFactory::getInstance()) {} DiskAdaptor::~DiskAdaptor() {} diff --git a/src/DiskAdaptor.h b/src/DiskAdaptor.h index 307ce155..dd102274 100644 --- a/src/DiskAdaptor.h +++ b/src/DiskAdaptor.h @@ -50,6 +50,7 @@ class DiskAdaptor:public BinaryStream { protected: std::string storeDir; std::deque > fileEntries; + bool _fallocate; Logger* logger; public: DiskAdaptor(); @@ -69,9 +70,6 @@ public: virtual uint64_t size() = 0; - // optional behavior - virtual void truncate(uint64_t length) {} - void setFileEntries(const std::deque >& fileEntries); SharedHandle getFileEntryFromPath(const std::string& fileEntryPath) const; @@ -110,6 +108,21 @@ public: // Returns the number of files, the actime and modtime of which are // successfully changed. virtual size_t utime(const Time& actime, const Time& modtime) = 0; + + void enableFallocate() + { + _fallocate = true; + } + + void disableFallocate() + { + _fallocate = false; + } + + bool doesFallocate() const + { + return _fallocate; + } }; typedef SharedHandle DiskAdaptorHandle; diff --git a/src/DiskWriter.h b/src/DiskWriter.h index dfd9fd97..ede90fcf 100644 --- a/src/DiskWriter.h +++ b/src/DiskWriter.h @@ -74,8 +74,6 @@ public: */ virtual void openExistingFile(const std::string& filename, uint64_t totalLength = 0) = 0; - virtual void truncate(uint64_t length) = 0; - // Returns file length virtual uint64_t size() = 0; diff --git a/src/FallocFileAllocationIterator.cc b/src/FallocFileAllocationIterator.cc new file mode 100644 index 00000000..ca0a0161 --- /dev/null +++ b/src/FallocFileAllocationIterator.cc @@ -0,0 +1,69 @@ +/* */ +#include "FallocFileAllocationIterator.h" +#include "DlAbortEx.h" + +namespace aria2 { + +FallocFileAllocationIterator::FallocFileAllocationIterator +(BinaryStream* stream, off_t offset, uint64_t totalLength): + _stream(stream), _offset(offset), _totalLength(totalLength) {} + +void FallocFileAllocationIterator::allocateChunk() +{ + if(static_cast(_offset) > _totalLength) { + throw DlAbortEx("FallocFileAllocationIterator: offset is larger than" + " totalLength"); + } + _stream->allocate(_offset, _totalLength-_offset); + _offset = _totalLength; +} + +bool FallocFileAllocationIterator::finished() +{ + return static_cast(_offset) == _totalLength; +} + +off_t FallocFileAllocationIterator::getCurrentLength() +{ + return _offset; +} + +uint64_t FallocFileAllocationIterator::getTotalLength() +{ + return _totalLength; +} + +} // namespace aria2 diff --git a/src/FallocFileAllocationIterator.h b/src/FallocFileAllocationIterator.h new file mode 100644 index 00000000..6f6cfa65 --- /dev/null +++ b/src/FallocFileAllocationIterator.h @@ -0,0 +1,65 @@ +/* */ +#ifndef _D_FALLOC_FILE_ALLOCATION_ITERATOR_H_ +#define _D_FALLOC_FILE_ALLOCATION_ITERATOR_H_ + +#include "FileAllocationIterator.h" +#include "BinaryStream.h" + +namespace aria2 { + +// Allocate disk space using posix_fallocate() system call. +class FallocFileAllocationIterator:public FileAllocationIterator { +private: + BinaryStream* _stream; + off_t _offset; + uint64_t _totalLength; +public: + FallocFileAllocationIterator(BinaryStream* stream, off_t offset, + uint64_t totalLength); + + virtual void allocateChunk(); + + virtual bool finished(); + + virtual off_t getCurrentLength(); + + virtual uint64_t getTotalLength(); +}; + +} // namespace aria2 + +#endif // _D_FALLOC_FILE_ALLOCATION_ITERATOR_H_ + diff --git a/src/FileAllocationEntry.cc b/src/FileAllocationEntry.cc index a0e1df05..6a6cd732 100644 --- a/src/FileAllocationEntry.cc +++ b/src/FileAllocationEntry.cc @@ -73,4 +73,9 @@ void FileAllocationEntry::allocateChunk() _fileAllocationIterator->allocateChunk(); } +void FileAllocationEntry::disableDirectIO() +{ + _requestGroup->getPieceStorage()->getDiskAdaptor()->disableDirectIO(); +} + } // namespace aria2 diff --git a/src/FileAllocationEntry.h b/src/FileAllocationEntry.h index 97e1d594..56a42617 100644 --- a/src/FileAllocationEntry.h +++ b/src/FileAllocationEntry.h @@ -63,6 +63,8 @@ public: virtual void prepareForNextAction(std::deque& commands, DownloadEngine* e) = 0; + + void disableDirectIO(); }; typedef SharedHandle FileAllocationEntryHandle; diff --git a/src/Makefile.am b/src/Makefile.am index 0ae20f20..c8e94f3f 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -207,6 +207,10 @@ SRCS = Socket.h\ PieceSelector.h\ LongestSequencePieceSelector.cc LongestSequencePieceSelector.h +if HAVE_POSIX_FALLOCATE +SRCS += FallocFileAllocationIterator.cc FallocFileAllocationIterator.h +endif # HAVE_POSIX_FALLOCATE + if HAVE_EPOLL SRCS += EpollEventPoll.cc EpollEventPoll.h endif # HAVE_EPOLL diff --git a/src/Makefile.in b/src/Makefile.in index f14f8051..33ecc216 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -35,14 +35,15 @@ build_triplet = @build@ host_triplet = @host@ target_triplet = @target@ bin_PROGRAMS = aria2c$(EXEEXT) -@HAVE_EPOLL_TRUE@am__append_1 = EpollEventPoll.cc EpollEventPoll.h -@ENABLE_SSL_TRUE@am__append_2 = TLSContext.h -@HAVE_LIBGNUTLS_TRUE@am__append_3 = LibgnutlsTLSContext.cc LibgnutlsTLSContext.h -@HAVE_LIBSSL_TRUE@am__append_4 = LibsslTLSContext.cc LibsslTLSContext.h -@HAVE_LIBZ_TRUE@am__append_5 = GZipDecoder.cc GZipDecoder.h -@HAVE_SQLITE3_TRUE@am__append_6 = Sqlite3MozCookieParser.cc Sqlite3MozCookieParser.h -@ENABLE_ASYNC_DNS_TRUE@am__append_7 = AsyncNameResolver.cc AsyncNameResolver.h -@ENABLE_MESSAGE_DIGEST_TRUE@am__append_8 = IteratableChunkChecksumValidator.cc IteratableChunkChecksumValidator.h\ +@HAVE_POSIX_FALLOCATE_TRUE@am__append_1 = FallocFileAllocationIterator.cc FallocFileAllocationIterator.h +@HAVE_EPOLL_TRUE@am__append_2 = EpollEventPoll.cc EpollEventPoll.h +@ENABLE_SSL_TRUE@am__append_3 = TLSContext.h +@HAVE_LIBGNUTLS_TRUE@am__append_4 = LibgnutlsTLSContext.cc LibgnutlsTLSContext.h +@HAVE_LIBSSL_TRUE@am__append_5 = LibsslTLSContext.cc LibsslTLSContext.h +@HAVE_LIBZ_TRUE@am__append_6 = GZipDecoder.cc GZipDecoder.h +@HAVE_SQLITE3_TRUE@am__append_7 = Sqlite3MozCookieParser.cc Sqlite3MozCookieParser.h +@ENABLE_ASYNC_DNS_TRUE@am__append_8 = AsyncNameResolver.cc AsyncNameResolver.h +@ENABLE_MESSAGE_DIGEST_TRUE@am__append_9 = IteratableChunkChecksumValidator.cc IteratableChunkChecksumValidator.h\ @ENABLE_MESSAGE_DIGEST_TRUE@ IteratableChecksumValidator.cc IteratableChecksumValidator.h\ @ENABLE_MESSAGE_DIGEST_TRUE@ CheckIntegrityDispatcherCommand.cc CheckIntegrityDispatcherCommand.h\ @ENABLE_MESSAGE_DIGEST_TRUE@ CheckIntegrityCommand.cc CheckIntegrityCommand.h\ @@ -52,7 +53,7 @@ bin_PROGRAMS = aria2c$(EXEEXT) @ENABLE_MESSAGE_DIGEST_TRUE@ Checksum.h\ @ENABLE_MESSAGE_DIGEST_TRUE@ ChunkChecksum.h -@ENABLE_BITTORRENT_TRUE@am__append_9 = PeerMessageUtil.cc PeerMessageUtil.h\ +@ENABLE_BITTORRENT_TRUE@am__append_10 = PeerMessageUtil.cc PeerMessageUtil.h\ @ENABLE_BITTORRENT_TRUE@ PeerAbstractCommand.cc PeerAbstractCommand.h\ @ENABLE_BITTORRENT_TRUE@ PeerInitiateConnectionCommand.cc PeerInitiateConnectionCommand.h\ @ENABLE_BITTORRENT_TRUE@ PeerInteractionCommand.cc PeerInteractionCommand.h\ @@ -219,7 +220,7 @@ bin_PROGRAMS = aria2c$(EXEEXT) @ENABLE_BITTORRENT_TRUE@ RangeBtMessageValidator.h\ @ENABLE_BITTORRENT_TRUE@ IndexBtMessageValidator.h -@ENABLE_METALINK_TRUE@am__append_10 = Metalinker.cc Metalinker.h\ +@ENABLE_METALINK_TRUE@am__append_11 = Metalinker.cc Metalinker.h\ @ENABLE_METALINK_TRUE@ MetalinkEntry.cc MetalinkEntry.h\ @ENABLE_METALINK_TRUE@ MetalinkResource.cc MetalinkResource.h\ @ENABLE_METALINK_TRUE@ MetalinkProcessor.h\ @@ -248,17 +249,17 @@ bin_PROGRAMS = aria2c$(EXEEXT) @ENABLE_METALINK_TRUE@ MetalinkPostDownloadHandler.cc MetalinkPostDownloadHandler.h\ @ENABLE_METALINK_TRUE@ MetalinkHelper.cc MetalinkHelper.h -@ENABLE_LIBXML2_TRUE@am__append_11 = XML2SAXMetalinkProcessor.cc XML2SAXMetalinkProcessor.h -@ENABLE_LIBEXPAT_TRUE@am__append_12 = ExpatMetalinkProcessor.cc ExpatMetalinkProcessor.h -@HAVE_ASCTIME_R_FALSE@am__append_13 = asctime_r.c asctime_r.h -@HAVE_BASENAME_FALSE@am__append_14 = libgen.c libgen.h -@HAVE_GETADDRINFO_FALSE@am__append_15 = getaddrinfo.c getaddrinfo.h -@HAVE_GAI_STRERROR_FALSE@am__append_16 = gai_strerror.c gai_strerror.h -@HAVE_GETTIMEOFDAY_FALSE@am__append_17 = gettimeofday.c gettimeofday.h -@HAVE_INET_ATON_FALSE@am__append_18 = inet_aton.c inet_aton.h -@HAVE_LOCALTIME_R_FALSE@am__append_19 = localtime_r.c localtime_r.h -@HAVE_STRPTIME_FALSE@am__append_20 = strptime.c strptime.h -@HAVE_TIMEGM_FALSE@am__append_21 = timegm.c timegm.h +@ENABLE_LIBXML2_TRUE@am__append_12 = XML2SAXMetalinkProcessor.cc XML2SAXMetalinkProcessor.h +@ENABLE_LIBEXPAT_TRUE@am__append_13 = ExpatMetalinkProcessor.cc ExpatMetalinkProcessor.h +@HAVE_ASCTIME_R_FALSE@am__append_14 = asctime_r.c asctime_r.h +@HAVE_BASENAME_FALSE@am__append_15 = libgen.c libgen.h +@HAVE_GETADDRINFO_FALSE@am__append_16 = getaddrinfo.c getaddrinfo.h +@HAVE_GAI_STRERROR_FALSE@am__append_17 = gai_strerror.c gai_strerror.h +@HAVE_GETTIMEOFDAY_FALSE@am__append_18 = gettimeofday.c gettimeofday.h +@HAVE_INET_ATON_FALSE@am__append_19 = inet_aton.c inet_aton.h +@HAVE_LOCALTIME_R_FALSE@am__append_20 = localtime_r.c localtime_r.h +@HAVE_STRPTIME_FALSE@am__append_21 = strptime.c strptime.h +@HAVE_TIMEGM_FALSE@am__append_22 = timegm.c timegm.h subdir = src DIST_COMMON = $(srcdir)/Makefile.am $(srcdir)/Makefile.in alloca.c ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 @@ -416,7 +417,8 @@ am__libaria2c_a_SOURCES_DIST = Socket.h SocketCore.cc SocketCore.h \ HttpServerCommand.h HttpServerResponseCommand.cc \ HttpServerResponseCommand.h HttpServer.cc HttpServer.h \ PieceSelector.h LongestSequencePieceSelector.cc \ - LongestSequencePieceSelector.h EpollEventPoll.cc \ + LongestSequencePieceSelector.h FallocFileAllocationIterator.cc \ + FallocFileAllocationIterator.h EpollEventPoll.cc \ EpollEventPoll.h TLSContext.h LibgnutlsTLSContext.cc \ LibgnutlsTLSContext.h LibsslTLSContext.cc LibsslTLSContext.h \ GZipDecoder.cc GZipDecoder.h Sqlite3MozCookieParser.cc \ @@ -576,21 +578,22 @@ am__libaria2c_a_SOURCES_DIST = Socket.h SocketCore.cc SocketCore.h \ gai_strerror.h gettimeofday.c gettimeofday.h inet_aton.c \ inet_aton.h localtime_r.c localtime_r.h strptime.c strptime.h \ timegm.c timegm.h -@HAVE_EPOLL_TRUE@am__objects_1 = EpollEventPoll.$(OBJEXT) -am__objects_2 = -@HAVE_LIBGNUTLS_TRUE@am__objects_3 = LibgnutlsTLSContext.$(OBJEXT) -@HAVE_LIBSSL_TRUE@am__objects_4 = LibsslTLSContext.$(OBJEXT) -@HAVE_LIBZ_TRUE@am__objects_5 = GZipDecoder.$(OBJEXT) -@HAVE_SQLITE3_TRUE@am__objects_6 = Sqlite3MozCookieParser.$(OBJEXT) -@ENABLE_ASYNC_DNS_TRUE@am__objects_7 = AsyncNameResolver.$(OBJEXT) -@ENABLE_MESSAGE_DIGEST_TRUE@am__objects_8 = IteratableChunkChecksumValidator.$(OBJEXT) \ +@HAVE_POSIX_FALLOCATE_TRUE@am__objects_1 = FallocFileAllocationIterator.$(OBJEXT) +@HAVE_EPOLL_TRUE@am__objects_2 = EpollEventPoll.$(OBJEXT) +am__objects_3 = +@HAVE_LIBGNUTLS_TRUE@am__objects_4 = LibgnutlsTLSContext.$(OBJEXT) +@HAVE_LIBSSL_TRUE@am__objects_5 = LibsslTLSContext.$(OBJEXT) +@HAVE_LIBZ_TRUE@am__objects_6 = GZipDecoder.$(OBJEXT) +@HAVE_SQLITE3_TRUE@am__objects_7 = Sqlite3MozCookieParser.$(OBJEXT) +@ENABLE_ASYNC_DNS_TRUE@am__objects_8 = AsyncNameResolver.$(OBJEXT) +@ENABLE_MESSAGE_DIGEST_TRUE@am__objects_9 = IteratableChunkChecksumValidator.$(OBJEXT) \ @ENABLE_MESSAGE_DIGEST_TRUE@ IteratableChecksumValidator.$(OBJEXT) \ @ENABLE_MESSAGE_DIGEST_TRUE@ CheckIntegrityDispatcherCommand.$(OBJEXT) \ @ENABLE_MESSAGE_DIGEST_TRUE@ CheckIntegrityCommand.$(OBJEXT) \ @ENABLE_MESSAGE_DIGEST_TRUE@ ChecksumCheckIntegrityEntry.$(OBJEXT) \ @ENABLE_MESSAGE_DIGEST_TRUE@ messageDigest.$(OBJEXT) \ @ENABLE_MESSAGE_DIGEST_TRUE@ MessageDigestHelper.$(OBJEXT) -@ENABLE_BITTORRENT_TRUE@am__objects_9 = PeerMessageUtil.$(OBJEXT) \ +@ENABLE_BITTORRENT_TRUE@am__objects_10 = PeerMessageUtil.$(OBJEXT) \ @ENABLE_BITTORRENT_TRUE@ PeerAbstractCommand.$(OBJEXT) \ @ENABLE_BITTORRENT_TRUE@ PeerInitiateConnectionCommand.$(OBJEXT) \ @ENABLE_BITTORRENT_TRUE@ PeerInteractionCommand.$(OBJEXT) \ @@ -701,7 +704,7 @@ am__objects_2 = @ENABLE_BITTORRENT_TRUE@ RangeBtMessage.$(OBJEXT) \ @ENABLE_BITTORRENT_TRUE@ IndexBtMessage.$(OBJEXT) \ @ENABLE_BITTORRENT_TRUE@ ZeroBtMessage.$(OBJEXT) -@ENABLE_METALINK_TRUE@am__objects_10 = Metalinker.$(OBJEXT) \ +@ENABLE_METALINK_TRUE@am__objects_11 = Metalinker.$(OBJEXT) \ @ENABLE_METALINK_TRUE@ MetalinkEntry.$(OBJEXT) \ @ENABLE_METALINK_TRUE@ MetalinkResource.$(OBJEXT) \ @ENABLE_METALINK_TRUE@ MetalinkProcessorFactory.$(OBJEXT) \ @@ -727,20 +730,20 @@ am__objects_2 = @ENABLE_METALINK_TRUE@ Metalink2RequestGroup.$(OBJEXT) \ @ENABLE_METALINK_TRUE@ MetalinkPostDownloadHandler.$(OBJEXT) \ @ENABLE_METALINK_TRUE@ MetalinkHelper.$(OBJEXT) -@ENABLE_LIBXML2_TRUE@am__objects_11 = \ +@ENABLE_LIBXML2_TRUE@am__objects_12 = \ @ENABLE_LIBXML2_TRUE@ XML2SAXMetalinkProcessor.$(OBJEXT) -@ENABLE_LIBEXPAT_TRUE@am__objects_12 = \ +@ENABLE_LIBEXPAT_TRUE@am__objects_13 = \ @ENABLE_LIBEXPAT_TRUE@ ExpatMetalinkProcessor.$(OBJEXT) -@HAVE_ASCTIME_R_FALSE@am__objects_13 = asctime_r.$(OBJEXT) -@HAVE_BASENAME_FALSE@am__objects_14 = libgen.$(OBJEXT) -@HAVE_GETADDRINFO_FALSE@am__objects_15 = getaddrinfo.$(OBJEXT) -@HAVE_GAI_STRERROR_FALSE@am__objects_16 = gai_strerror.$(OBJEXT) -@HAVE_GETTIMEOFDAY_FALSE@am__objects_17 = gettimeofday.$(OBJEXT) -@HAVE_INET_ATON_FALSE@am__objects_18 = inet_aton.$(OBJEXT) -@HAVE_LOCALTIME_R_FALSE@am__objects_19 = localtime_r.$(OBJEXT) -@HAVE_STRPTIME_FALSE@am__objects_20 = strptime.$(OBJEXT) -@HAVE_TIMEGM_FALSE@am__objects_21 = timegm.$(OBJEXT) -am__objects_22 = SocketCore.$(OBJEXT) Command.$(OBJEXT) \ +@HAVE_ASCTIME_R_FALSE@am__objects_14 = asctime_r.$(OBJEXT) +@HAVE_BASENAME_FALSE@am__objects_15 = libgen.$(OBJEXT) +@HAVE_GETADDRINFO_FALSE@am__objects_16 = getaddrinfo.$(OBJEXT) +@HAVE_GAI_STRERROR_FALSE@am__objects_17 = gai_strerror.$(OBJEXT) +@HAVE_GETTIMEOFDAY_FALSE@am__objects_18 = gettimeofday.$(OBJEXT) +@HAVE_INET_ATON_FALSE@am__objects_19 = inet_aton.$(OBJEXT) +@HAVE_LOCALTIME_R_FALSE@am__objects_20 = localtime_r.$(OBJEXT) +@HAVE_STRPTIME_FALSE@am__objects_21 = strptime.$(OBJEXT) +@HAVE_TIMEGM_FALSE@am__objects_22 = timegm.$(OBJEXT) +am__objects_23 = SocketCore.$(OBJEXT) Command.$(OBJEXT) \ AbstractCommand.$(OBJEXT) \ InitiateConnectionCommandFactory.$(OBJEXT) \ DownloadCommand.$(OBJEXT) \ @@ -825,8 +828,8 @@ am__objects_22 = SocketCore.$(OBJEXT) Command.$(OBJEXT) \ $(am__objects_11) $(am__objects_12) $(am__objects_13) \ $(am__objects_14) $(am__objects_15) $(am__objects_16) \ $(am__objects_17) $(am__objects_18) $(am__objects_19) \ - $(am__objects_20) $(am__objects_21) -am_libaria2c_a_OBJECTS = $(am__objects_22) + $(am__objects_20) $(am__objects_21) $(am__objects_22) +am_libaria2c_a_OBJECTS = $(am__objects_23) libaria2c_a_OBJECTS = $(am_libaria2c_a_OBJECTS) am__installdirs = "$(DESTDIR)$(bindir)" binPROGRAMS_INSTALL = $(INSTALL_PROGRAM) @@ -1161,7 +1164,7 @@ SRCS = Socket.h SocketCore.cc SocketCore.h BinaryStream.h Command.cc \ $(am__append_12) $(am__append_13) $(am__append_14) \ $(am__append_15) $(am__append_16) $(am__append_17) \ $(am__append_18) $(am__append_19) $(am__append_20) \ - $(am__append_21) + $(am__append_21) $(am__append_22) noinst_LIBRARIES = libaria2c.a libaria2c_a_SOURCES = $(SRCS) aria2c_LDADD = libaria2c.a @LIBINTL@ @ALLOCA@ @LIBGNUTLS_LIBS@\ @@ -1390,6 +1393,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/EpollEventPoll.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Exception.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ExpatMetalinkProcessor.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FallocFileAllocationIterator.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FeatureConfig.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FeedbackURISelector.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/File.Po@am__quote@ diff --git a/src/MultiFileAllocationIterator.cc b/src/MultiFileAllocationIterator.cc index 63654a0e..647d4ebf 100644 --- a/src/MultiFileAllocationIterator.cc +++ b/src/MultiFileAllocationIterator.cc @@ -36,6 +36,7 @@ #include "MultiDiskAdaptor.h" #include "FileEntry.h" #include "SingleFileAllocationIterator.h" +#include "FallocFileAllocationIterator.h" #include "DiskWriter.h" namespace aria2 { @@ -61,11 +62,19 @@ void MultiFileAllocationIterator::allocateChunk() _diskAdaptor->openIfNot(entry, &DiskWriterEntry::openFile); if(entry->needsFileAllocation() && entry->size() < fileEntry->getLength()) { // Calling private function of MultiDiskAdaptor. - _fileAllocationIterator.reset - (new SingleFileAllocationIterator(entry->getDiskWriter().get(), - entry->size(), - fileEntry->getLength())); - _fileAllocationIterator->init(); + if(_diskAdaptor->doesFallocate()) { + _fileAllocationIterator.reset + (new FallocFileAllocationIterator(entry->getDiskWriter().get(), + entry->size(), + fileEntry->getLength())); + } else { + SharedHandle fa + (new SingleFileAllocationIterator(entry->getDiskWriter().get(), + entry->size(), + fileEntry->getLength())); + fa->init(); + _fileAllocationIterator = fa; + } } } if(finished()) { diff --git a/src/MultiFileAllocationIterator.h b/src/MultiFileAllocationIterator.h index 1f32ff04..682e6686 100644 --- a/src/MultiFileAllocationIterator.h +++ b/src/MultiFileAllocationIterator.h @@ -42,14 +42,13 @@ namespace aria2 { class MultiDiskAdaptor; class DiskWriterEntry; -class SingleFileAllocationIterator; class MultiFileAllocationIterator:public FileAllocationIterator { private: MultiDiskAdaptor* _diskAdaptor; std::deque > _entries; - SharedHandle _fileAllocationIterator; + SharedHandle _fileAllocationIterator; off_t _offset; public: diff --git a/src/OptionHandlerFactory.cc b/src/OptionHandlerFactory.cc index 84625d61..b4a92f97 100644 --- a/src/OptionHandlerFactory.cc +++ b/src/OptionHandlerFactory.cc @@ -203,6 +203,9 @@ OptionHandlers OptionHandlerFactory::createOptionHandlers() TEXT_FILE_ALLOCATION, V_PREALLOC, V_NONE, V_PREALLOC, +#ifdef HAVE_POSIX_FALLOCATE + V_FALLOC, +#endif // HAVE_POSIX_FALLOCATE 'a')); op->addTag(TAG_BASIC); handlers.push_back(op); diff --git a/src/RequestGroup.cc b/src/RequestGroup.cc index 09299505..871c0d41 100644 --- a/src/RequestGroup.cc +++ b/src/RequestGroup.cc @@ -134,11 +134,7 @@ RequestGroup::RequestGroup(const Option* option, _option(option), _logger(LogFactory::getInstance()) { - if(_option->get(PREF_FILE_ALLOCATION) == V_PREALLOC) { - _fileAllocationEnabled = true; - } else { - _fileAllocationEnabled = false; - } + _fileAllocationEnabled = _option->get(PREF_FILE_ALLOCATION) != V_NONE; // Add types to be sent as a Accept header value here. // It would be good to put this value in Option so that user can tweak // and add this list. diff --git a/src/StreamCheckIntegrityEntry.cc b/src/StreamCheckIntegrityEntry.cc index e8685422..5a4014c3 100644 --- a/src/StreamCheckIntegrityEntry.cc +++ b/src/StreamCheckIntegrityEntry.cc @@ -36,7 +36,6 @@ #include "RequestGroup.h" #include "DownloadEngine.h" #include "StreamFileAllocationEntry.h" -#include "FileAllocationMan.h" #include "Request.h" namespace aria2 { @@ -50,17 +49,13 @@ StreamCheckIntegrityEntry::StreamCheckIntegrityEntry(const RequestHandle& curren StreamCheckIntegrityEntry::~StreamCheckIntegrityEntry() {} -void StreamCheckIntegrityEntry::onDownloadIncomplete(std::deque& commands, - DownloadEngine* e) +void StreamCheckIntegrityEntry::onDownloadIncomplete +(std::deque& commands, DownloadEngine* e) { FileAllocationEntryHandle entry (new StreamFileAllocationEntry(_currentRequest, _requestGroup, popNextCommand())); - if(_requestGroup->needsFileAllocation()) { - e->_fileAllocationMan->pushEntry(entry); - } else { - entry->prepareForNextAction(commands, e); - } + proceedFileAllocation(commands, entry, e); } void StreamCheckIntegrityEntry::onDownloadFinished(std::deque& commands, diff --git a/src/prefs.cc b/src/prefs.cc index 9ef9ed07..0cca4040 100644 --- a/src/prefs.cc +++ b/src/prefs.cc @@ -81,10 +81,11 @@ const std::string PREF_MAX_OVERALL_DOWNLOAD_LIMIT("max-overall-download-limit"); const std::string PREF_MAX_DOWNLOAD_LIMIT("max-download-limit"); // value: 1*digit const std::string PREF_STARTUP_IDLE_TIME("startup-idle-time"); -// value: prealloc | none +// value: prealloc | fallc | none const std::string PREF_FILE_ALLOCATION("file-allocation"); const std::string V_PREALLOC("prealloc"); -#// value: 1*digit +const std::string V_FALLOC("falloc"); +// value: 1*digit const std::string PREF_NO_FILE_ALLOCATION_LIMIT("no-file-allocation-limit"); // value: true | false const std::string PREF_ALLOW_OVERWRITE("allow-overwrite"); diff --git a/src/prefs.h b/src/prefs.h index 3c45e39f..c7974c3e 100644 --- a/src/prefs.h +++ b/src/prefs.h @@ -83,9 +83,10 @@ extern const std::string PREF_SEGMENT_SIZE; extern const std::string PREF_MAX_DOWNLOAD_LIMIT; // value: 1*digit extern const std::string PREF_STARTUP_IDLE_TIME; -// value: prealloc | none +// value: prealloc | falloc | none extern const std::string PREF_FILE_ALLOCATION; extern const std::string V_PREALLOC; +extern const std::string V_FALLOC; // value: 1*digit extern const std::string PREF_NO_FILE_ALLOCATION_LIMIT; // value: true | false diff --git a/src/usage_text.h b/src/usage_text.h index 43a5d66e..e78441af 100644 --- a/src/usage_text.h +++ b/src/usage_text.h @@ -120,7 +120,16 @@ _(" --file-allocation=METHOD Specify file allocation method.\n"\ " 'none' doesn't pre-allocate file space. 'prealloc'\n"\ " pre-allocates file space before download begins.\n"\ " This may take some time depending on the size of\n"\ - " the file.") + " the file.\n"\ + " If you are using newer file systems such as ext4\n"\ + " (with extents support), btrfs or xfs, 'falloc' is\n"\ + " your best choice. It allocates large(few GiB)\n"\ + " files almost instantly. Don't use 'falloc' with\n"\ + " legacy file systems such as ext3 because it takes\n"\ + " almost same time as 'prealloc' and it blocks aria2\n"\ + " entirely until allocation finishes. 'falloc' may\n"\ + " not be available if your system doesn't have\n"\ + " posix_fallocate() system call.") #define TEXT_NO_FILE_ALLOCATION_LIMIT \ _(" --no-file-allocation-limit=SIZE No file allocation is made for files whose\n"\ " size is smaller than SIZE.\n"\ diff --git a/test/FallocFileAllocationIteratorTest.cc b/test/FallocFileAllocationIteratorTest.cc new file mode 100644 index 00000000..28945c64 --- /dev/null +++ b/test/FallocFileAllocationIteratorTest.cc @@ -0,0 +1,53 @@ +#include "FallocFileAllocationIterator.h" + +#include +#include + +#include "File.h" +#include "DefaultDiskWriter.h" + +namespace aria2 { + +class FallocFileAllocationIteratorTest:public CppUnit::TestFixture { + + CPPUNIT_TEST_SUITE(FallocFileAllocationIteratorTest); + CPPUNIT_TEST(testAllocate); + CPPUNIT_TEST_SUITE_END(); +private: + +public: + void setUp() {} + + void testAllocate(); +}; + + +CPPUNIT_TEST_SUITE_REGISTRATION( FallocFileAllocationIteratorTest ); + +void FallocFileAllocationIteratorTest::testAllocate() +{ + std::string dir = "/tmp"; + std::string fname = "aria2_FallocFileAllocationIteratorTest_testAllocate"; + std::string fn = dir+"/"+fname; + std::ofstream of(fn.c_str(), std::ios::binary); + of << "0123456789"; + of.close(); + + File f(fn); + CPPUNIT_ASSERT_EQUAL((uint64_t)10, f.size()); + + DefaultDiskWriter writer; + int64_t offset = 10; + int64_t totalLength = 40960; + + // we have to open file first. + writer.openExistingFile(fn); + FallocFileAllocationIterator itr(&writer, offset, totalLength); + + itr.allocateChunk(); + CPPUNIT_ASSERT(itr.finished()); + + CPPUNIT_ASSERT_EQUAL((uint64_t)40960, f.size()); +} + +} // namespace aria2 diff --git a/test/Makefile.am b/test/Makefile.am index d1cd23fa..7bda5b09 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -69,6 +69,10 @@ aria2c_SOURCES = AllTest.cc\ LongestSequencePieceSelectorTest.cc\ a2algoTest.cc +if HAVE_POSIX_FALLOCATE +aria2c_SOURCES += FallocFileAllocationIteratorTest.cc +endif # HAVE_POSIX_FALLOCATE + if HAVE_LIBZ aria2c_SOURCES += GZipDecoderTest.cc endif # HAVE_LIBZ diff --git a/test/Makefile.in b/test/Makefile.in index c1167d28..bbd12d90 100644 --- a/test/Makefile.in +++ b/test/Makefile.in @@ -34,13 +34,14 @@ host_triplet = @host@ target_triplet = @target@ TESTS = aria2c$(EXEEXT) check_PROGRAMS = $(am__EXEEXT_1) -@HAVE_LIBZ_TRUE@am__append_1 = GZipDecoderTest.cc -@HAVE_SQLITE3_TRUE@am__append_2 = Sqlite3MozCookieParserTest.cc -@ENABLE_MESSAGE_DIGEST_TRUE@am__append_3 = MessageDigestHelperTest.cc\ +@HAVE_POSIX_FALLOCATE_TRUE@am__append_1 = FallocFileAllocationIteratorTest.cc +@HAVE_LIBZ_TRUE@am__append_2 = GZipDecoderTest.cc +@HAVE_SQLITE3_TRUE@am__append_3 = Sqlite3MozCookieParserTest.cc +@ENABLE_MESSAGE_DIGEST_TRUE@am__append_4 = MessageDigestHelperTest.cc\ @ENABLE_MESSAGE_DIGEST_TRUE@ IteratableChunkChecksumValidatorTest.cc\ @ENABLE_MESSAGE_DIGEST_TRUE@ IteratableChecksumValidatorTest.cc -@ENABLE_BITTORRENT_TRUE@am__append_4 = BtAllowedFastMessageTest.cc\ +@ENABLE_BITTORRENT_TRUE@am__append_5 = BtAllowedFastMessageTest.cc\ @ENABLE_BITTORRENT_TRUE@ BtBitfieldMessageTest.cc\ @ENABLE_BITTORRENT_TRUE@ BtCancelMessageTest.cc\ @ENABLE_BITTORRENT_TRUE@ BtChokeMessageTest.cc\ @@ -126,7 +127,7 @@ check_PROGRAMS = $(am__EXEEXT_1) @ENABLE_BITTORRENT_TRUE@ MockExtensionMessageFactory.h\ @ENABLE_BITTORRENT_TRUE@ MockPieceStorage.h -@ENABLE_METALINK_TRUE@am__append_5 = MetalinkerTest.cc\ +@ENABLE_METALINK_TRUE@am__append_6 = MetalinkerTest.cc\ @ENABLE_METALINK_TRUE@ MetalinkEntryTest.cc\ @ENABLE_METALINK_TRUE@ Metalink2RequestGroupTest.cc\ @ENABLE_METALINK_TRUE@ MetalinkPostDownloadHandlerTest.cc\ @@ -193,8 +194,8 @@ am__aria2c_SOURCES_DIST = AllTest.cc TestUtil.cc TestUtil.h \ SimpleDNSCacheTest.cc DownloadHelperTest.cc BencodeTest.cc \ SequentialPickerTest.cc RarestPieceSelectorTest.cc \ LongestSequencePieceSelectorTest.cc a2algoTest.cc \ - GZipDecoderTest.cc Sqlite3MozCookieParserTest.cc \ - MessageDigestHelperTest.cc \ + FallocFileAllocationIteratorTest.cc GZipDecoderTest.cc \ + Sqlite3MozCookieParserTest.cc MessageDigestHelperTest.cc \ IteratableChunkChecksumValidatorTest.cc \ IteratableChecksumValidatorTest.cc BtAllowedFastMessageTest.cc \ BtBitfieldMessageTest.cc BtCancelMessageTest.cc \ @@ -242,14 +243,15 @@ am__aria2c_SOURCES_DIST = AllTest.cc TestUtil.cc TestUtil.h \ Metalink2RequestGroupTest.cc \ MetalinkPostDownloadHandlerTest.cc MetalinkHelperTest.cc \ MetalinkParserControllerTest.cc MetalinkProcessorTest.cc -@HAVE_LIBZ_TRUE@am__objects_1 = GZipDecoderTest.$(OBJEXT) -@HAVE_SQLITE3_TRUE@am__objects_2 = \ +@HAVE_POSIX_FALLOCATE_TRUE@am__objects_1 = FallocFileAllocationIteratorTest.$(OBJEXT) +@HAVE_LIBZ_TRUE@am__objects_2 = GZipDecoderTest.$(OBJEXT) +@HAVE_SQLITE3_TRUE@am__objects_3 = \ @HAVE_SQLITE3_TRUE@ Sqlite3MozCookieParserTest.$(OBJEXT) -@ENABLE_MESSAGE_DIGEST_TRUE@am__objects_3 = \ +@ENABLE_MESSAGE_DIGEST_TRUE@am__objects_4 = \ @ENABLE_MESSAGE_DIGEST_TRUE@ MessageDigestHelperTest.$(OBJEXT) \ @ENABLE_MESSAGE_DIGEST_TRUE@ IteratableChunkChecksumValidatorTest.$(OBJEXT) \ @ENABLE_MESSAGE_DIGEST_TRUE@ IteratableChecksumValidatorTest.$(OBJEXT) -@ENABLE_BITTORRENT_TRUE@am__objects_4 = \ +@ENABLE_BITTORRENT_TRUE@am__objects_5 = \ @ENABLE_BITTORRENT_TRUE@ BtAllowedFastMessageTest.$(OBJEXT) \ @ENABLE_BITTORRENT_TRUE@ BtBitfieldMessageTest.$(OBJEXT) \ @ENABLE_BITTORRENT_TRUE@ BtCancelMessageTest.$(OBJEXT) \ @@ -317,7 +319,7 @@ am__aria2c_SOURCES_DIST = AllTest.cc TestUtil.cc TestUtil.h \ @ENABLE_BITTORRENT_TRUE@ ARC4Test.$(OBJEXT) \ @ENABLE_BITTORRENT_TRUE@ MSEHandshakeTest.$(OBJEXT) \ @ENABLE_BITTORRENT_TRUE@ DHTUtilTest.$(OBJEXT) -@ENABLE_METALINK_TRUE@am__objects_5 = MetalinkerTest.$(OBJEXT) \ +@ENABLE_METALINK_TRUE@am__objects_6 = MetalinkerTest.$(OBJEXT) \ @ENABLE_METALINK_TRUE@ MetalinkEntryTest.$(OBJEXT) \ @ENABLE_METALINK_TRUE@ Metalink2RequestGroupTest.$(OBJEXT) \ @ENABLE_METALINK_TRUE@ MetalinkPostDownloadHandlerTest.$(OBJEXT) \ @@ -363,7 +365,8 @@ am_aria2c_OBJECTS = AllTest.$(OBJEXT) TestUtil.$(OBJEXT) \ RarestPieceSelectorTest.$(OBJEXT) \ LongestSequencePieceSelectorTest.$(OBJEXT) \ a2algoTest.$(OBJEXT) $(am__objects_1) $(am__objects_2) \ - $(am__objects_3) $(am__objects_4) $(am__objects_5) + $(am__objects_3) $(am__objects_4) $(am__objects_5) \ + $(am__objects_6) aria2c_OBJECTS = $(am_aria2c_OBJECTS) am__DEPENDENCIES_1 = aria2c_DEPENDENCIES = ../src/libaria2c.a ../src/download_helper.o \ @@ -589,7 +592,7 @@ aria2c_SOURCES = AllTest.cc TestUtil.cc TestUtil.h SocketCoreTest.cc \ SequentialPickerTest.cc RarestPieceSelectorTest.cc \ LongestSequencePieceSelectorTest.cc a2algoTest.cc \ $(am__append_1) $(am__append_2) $(am__append_3) \ - $(am__append_4) $(am__append_5) + $(am__append_4) $(am__append_5) $(am__append_6) #aria2c_CXXFLAGS = ${CPPUNIT_CFLAGS} -I../src -I../lib -Wall -D_FILE_OFFSET_BITS=64 #aria2c_LDFLAGS = ${CPPUNIT_LIBS} @@ -753,6 +756,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DownloadHandlerFactoryTest.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DownloadHelperTest.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ExceptionTest.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FallocFileAllocationIteratorTest.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FeatureConfigTest.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FeedbackURISelectorTest.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FileEntryTest.Po@am__quote@