Merge pull request #1644 from aliemjay/sync-caches

prevent corrupt downloads after app and/or system crash
pull/1657/head
Tatsuhiro Tsujikawa 2020-06-25 11:42:25 +09:00 committed by GitHub
commit 15cad965eb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 56 additions and 8 deletions

View File

@ -590,4 +590,16 @@ void AbstractDiskWriter::dropCache(int64_t len, int64_t offset)
#endif // HAVE_POSIX_FADVISE #endif // HAVE_POSIX_FADVISE
} }
void AbstractDiskWriter::flushOSBuffers()
{
if (fd_ == A2_BAD_FD) {
return;
}
#ifdef __MINGW32__
FlushFileBuffers(fd_);
#else // !__MINGW32__
fsync(fd_);
#endif // __MINGW32__
}
} // namespace aria2 } // namespace aria2

View File

@ -100,6 +100,8 @@ public:
virtual void enableMmap() CXX11_OVERRIDE; virtual void enableMmap() CXX11_OVERRIDE;
virtual void dropCache(int64_t len, int64_t offset) CXX11_OVERRIDE; virtual void dropCache(int64_t len, int64_t offset) CXX11_OVERRIDE;
virtual void flushOSBuffers() CXX11_OVERRIDE;
}; };
} // namespace aria2 } // namespace aria2

View File

@ -103,6 +103,11 @@ void AbstractSingleDiskAdaptor::writeCache(const WrDiskCacheEntry* entry)
} }
} }
void AbstractSingleDiskAdaptor::flushOSBuffers()
{
diskWriter_->flushOSBuffers();
}
bool AbstractSingleDiskAdaptor::fileExists() bool AbstractSingleDiskAdaptor::fileExists()
{ {
return File(getFilePath()).exists(); return File(getFilePath()).exists();

View File

@ -72,6 +72,8 @@ public:
virtual void writeCache(const WrDiskCacheEntry* entry) CXX11_OVERRIDE; virtual void writeCache(const WrDiskCacheEntry* entry) CXX11_OVERRIDE;
virtual void flushOSBuffers() CXX11_OVERRIDE;
virtual bool fileExists() CXX11_OVERRIDE; virtual bool fileExists() CXX11_OVERRIDE;
virtual int64_t size() CXX11_OVERRIDE; virtual int64_t size() CXX11_OVERRIDE;

View File

@ -685,7 +685,7 @@ std::shared_ptr<DiskAdaptor> DefaultPieceStorage::getDiskAdaptor()
WrDiskCache* DefaultPieceStorage::getWrDiskCache() { return wrDiskCache_; } WrDiskCache* DefaultPieceStorage::getWrDiskCache() { return wrDiskCache_; }
void DefaultPieceStorage::flushWrDiskCacheEntry() void DefaultPieceStorage::flushWrDiskCacheEntry(bool releaseEntries)
{ {
if (!wrDiskCache_) { if (!wrDiskCache_) {
return; return;
@ -697,10 +697,12 @@ void DefaultPieceStorage::flushWrDiskCacheEntry()
auto ce = piece->getWrDiskCacheEntry(); auto ce = piece->getWrDiskCacheEntry();
if (ce) { if (ce) {
piece->flushWrCache(wrDiskCache_); piece->flushWrCache(wrDiskCache_);
if (releaseEntries) {
piece->releaseWrCache(wrDiskCache_); piece->releaseWrCache(wrDiskCache_);
} }
} }
} }
}
int32_t DefaultPieceStorage::getPieceLength(size_t index) int32_t DefaultPieceStorage::getPieceLength(size_t index)
{ {

View File

@ -234,7 +234,7 @@ public:
virtual WrDiskCache* getWrDiskCache() CXX11_OVERRIDE; virtual WrDiskCache* getWrDiskCache() CXX11_OVERRIDE;
virtual void flushWrDiskCacheEntry() CXX11_OVERRIDE; virtual void flushWrDiskCacheEntry(bool releaseEntries) CXX11_OVERRIDE;
virtual int32_t getPieceLength(size_t index) CXX11_OVERRIDE; virtual int32_t getPieceLength(size_t index) CXX11_OVERRIDE;

View File

@ -114,6 +114,9 @@ public:
// Writes cached data to the underlying disk. // Writes cached data to the underlying disk.
virtual void writeCache(const WrDiskCacheEntry* entry) = 0; virtual void writeCache(const WrDiskCacheEntry* entry) = 0;
// Force physical write of data from OS buffer cache.
virtual void flushOSBuffers() {};
void setFileAllocationMethod(FileAllocationMethod method) void setFileAllocationMethod(FileAllocationMethod method)
{ {
fileAllocationMethod_ = method; fileAllocationMethod_ = method;

View File

@ -85,6 +85,9 @@ public:
// Drops cache in range [offset, offset + len) // Drops cache in range [offset, offset + len)
virtual void dropCache(int64_t len, int64_t offset) {} virtual void dropCache(int64_t len, int64_t offset) {}
// Force physical write of data from OS buffer cache.
virtual void flushOSBuffers() {}
}; };
} // namespace aria2 } // namespace aria2

View File

@ -419,6 +419,17 @@ void MultiDiskAdaptor::writeCache(const WrDiskCacheEntry* entry)
} }
} }
void MultiDiskAdaptor::flushOSBuffers()
{
for (auto& dwent : openedDiskWriterEntries_) {
auto& dw = dwent->getDiskWriter();
if (!dw) {
continue;
}
dw->flushOSBuffers();
}
}
bool MultiDiskAdaptor::fileExists() bool MultiDiskAdaptor::fileExists()
{ {
return std::find_if(std::begin(getFileEntries()), std::end(getFileEntries()), return std::find_if(std::begin(getFileEntries()), std::end(getFileEntries()),

View File

@ -135,6 +135,8 @@ public:
virtual void writeCache(const WrDiskCacheEntry* entry) CXX11_OVERRIDE; virtual void writeCache(const WrDiskCacheEntry* entry) CXX11_OVERRIDE;
virtual void flushOSBuffers() CXX11_OVERRIDE;
virtual bool fileExists() CXX11_OVERRIDE; virtual bool fileExists() CXX11_OVERRIDE;
virtual int64_t size() CXX11_OVERRIDE; virtual int64_t size() CXX11_OVERRIDE;

View File

@ -226,8 +226,9 @@ public:
virtual WrDiskCache* getWrDiskCache() = 0; virtual WrDiskCache* getWrDiskCache() = 0;
// Flushes write disk cache for in-flight piece and evicts them. // Flushes write disk cache for in-flight piece
virtual void flushWrDiskCacheEntry() = 0; // and optionally releases the associated cache entries.
virtual void flushWrDiskCacheEntry(bool releaseEntries) = 0;
virtual int32_t getPieceLength(size_t index) = 0; virtual int32_t getPieceLength(size_t index) = 0;

View File

@ -210,7 +210,8 @@ std::pair<error_code::Value, std::string> RequestGroup::downloadResult() const
void RequestGroup::closeFile() void RequestGroup::closeFile()
{ {
if (pieceStorage_) { if (pieceStorage_) {
pieceStorage_->flushWrDiskCacheEntry(); pieceStorage_->flushWrDiskCacheEntry(true);
pieceStorage_->getDiskAdaptor()->flushOSBuffers();
pieceStorage_->getDiskAdaptor()->closeFile(); pieceStorage_->getDiskAdaptor()->closeFile();
} }
} }
@ -1290,6 +1291,10 @@ bool RequestGroup::doesUploadSpeedExceed()
void RequestGroup::saveControlFile() const void RequestGroup::saveControlFile() const
{ {
if (saveControlFile_) { if (saveControlFile_) {
if (pieceStorage_) {
pieceStorage_->flushWrDiskCacheEntry(false);
pieceStorage_->getDiskAdaptor()->flushOSBuffers();
}
progressInfoFile_->save(); progressInfoFile_->save();
} }
} }

View File

@ -216,7 +216,7 @@ public:
virtual WrDiskCache* getWrDiskCache() CXX11_OVERRIDE { return nullptr; } virtual WrDiskCache* getWrDiskCache() CXX11_OVERRIDE { return nullptr; }
virtual void flushWrDiskCacheEntry() CXX11_OVERRIDE {} virtual void flushWrDiskCacheEntry(bool releaseEntries) CXX11_OVERRIDE {}
virtual int32_t getPieceLength(size_t index) CXX11_OVERRIDE; virtual int32_t getPieceLength(size_t index) CXX11_OVERRIDE;

View File

@ -228,7 +228,7 @@ public:
virtual WrDiskCache* getWrDiskCache() CXX11_OVERRIDE { return 0; } virtual WrDiskCache* getWrDiskCache() CXX11_OVERRIDE { return 0; }
virtual void flushWrDiskCacheEntry() CXX11_OVERRIDE {} virtual void flushWrDiskCacheEntry(bool releaseEntries) CXX11_OVERRIDE {}
void setDiskAdaptor(const std::shared_ptr<DiskAdaptor>& adaptor) void setDiskAdaptor(const std::shared_ptr<DiskAdaptor>& adaptor)
{ {