github
/
aria2
mirror of https://github.com/aria2/aria2
1
0
Fork 0
Code Issues Releases Wiki Activity

Don't write control file if content is not changed from previous one 

This is desirable so that we don't have to wake up disk if control
file is not changed, and it is not have to be written again.  We use
the same method (SHA1 hash) to check the content is the same.  The
limitation is the hash is stored in memory, so we have to write the
first time in each session.

See GH-382
pull/481/head
Tatsuhiro Tsujikawa 2015-11-10 23:02:21 +09:00
parent 5ccd5b6953
commit 269520ee69
6 changed files with 288 additions and 155 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

180
src/DefaultBtProgressInfoFile.cc
View File

@ -55,6 +55,7 @@
#include "array_fun.h"
#include "DownloadContext.h"
#include "BufferedFile.h"
#include "SHA1IOFile.h"
#ifdef ENABLE_BITTORRENT
# include "PeerStorage.h"
# include "BtRuntime.h"
@ -105,95 +106,116 @@ bool DefaultBtProgressInfoFile::isTorrentDownload()
}
// Since version 0001, Integers are saved in binary form, network byte order.
void DefaultBtProgressInfoFile::save(IOFile& fp)
{
#ifdef ENABLE_BITTORRENT
bool torrentDownload = isTorrentDownload();
#else // !ENABLE_BITTORRENT
bool torrentDownload = false;
#endif // !ENABLE_BITTORRENT
// file version: 16 bits
// values: '1'
char version[] = {0x00u, 0x01u};
WRITE_CHECK(fp, version, sizeof(version));
// extension: 32 bits
// If this is BitTorrent download, then 0x00000001
// Otherwise, 0x00000000
char extension[4];
memset(extension, 0, sizeof(extension));
if (torrentDownload) {
extension[3] = 1;
}
WRITE_CHECK(fp, extension, sizeof(extension));
if (torrentDownload) {
#ifdef ENABLE_BITTORRENT
// infoHashLength:
// length: 32 bits
const unsigned char* infoHash = bittorrent::getInfoHash(dctx_);
uint32_t infoHashLengthNL = htonl(INFO_HASH_LENGTH);
WRITE_CHECK(fp, &infoHashLengthNL, sizeof(infoHashLengthNL));
// infoHash:
WRITE_CHECK(fp, infoHash, INFO_HASH_LENGTH);
#endif // ENABLE_BITTORRENT
} else {
// infoHashLength:
// length: 32 bits
uint32_t infoHashLength = 0;
WRITE_CHECK(fp, &infoHashLength, sizeof(infoHashLength));
}
// pieceLength: 32 bits
uint32_t pieceLengthNL = htonl(dctx_->getPieceLength());
WRITE_CHECK(fp, &pieceLengthNL, sizeof(pieceLengthNL));
// totalLength: 64 bits
uint64_t totalLengthNL = hton64(dctx_->getTotalLength());
WRITE_CHECK(fp, &totalLengthNL, sizeof(totalLengthNL));
// uploadLength: 64 bits
uint64_t uploadLengthNL = 0;
#ifdef ENABLE_BITTORRENT
if (torrentDownload) {
uploadLengthNL = hton64(btRuntime_->getUploadLengthAtStartup() +
dctx_->getNetStat().getSessionUploadLength());
}
#endif // ENABLE_BITTORRENT
WRITE_CHECK(fp, &uploadLengthNL, sizeof(uploadLengthNL));
// bitfieldLength: 32 bits
uint32_t bitfieldLengthNL = htonl(pieceStorage_->getBitfieldLength());
WRITE_CHECK(fp, &bitfieldLengthNL, sizeof(bitfieldLengthNL));
// bitfield
WRITE_CHECK(fp, pieceStorage_->getBitfield(),
pieceStorage_->getBitfieldLength());
// the number of in-flight piece: 32 bits
// TODO implement this
uint32_t numInFlightPieceNL = htonl(pieceStorage_->countInFlightPiece());
WRITE_CHECK(fp, &numInFlightPieceNL, sizeof(numInFlightPieceNL));
std::vector<std::shared_ptr<Piece>> inFlightPieces;
inFlightPieces.reserve(pieceStorage_->countInFlightPiece());
pieceStorage_->getInFlightPieces(inFlightPieces);
for (std::vector<std::shared_ptr<Piece>>::const_iterator
itr = inFlightPieces.begin(),
eoi = inFlightPieces.end();
itr != eoi; ++itr) {
uint32_t indexNL = htonl((*itr)->getIndex());
WRITE_CHECK(fp, &indexNL, sizeof(indexNL));
uint32_t lengthNL = htonl((*itr)->getLength());
WRITE_CHECK(fp, &lengthNL, sizeof(lengthNL));
uint32_t bitfieldLengthNL = htonl((*itr)->getBitfieldLength());
WRITE_CHECK(fp, &bitfieldLengthNL, sizeof(bitfieldLengthNL));
WRITE_CHECK(fp, (*itr)->getBitfield(), (*itr)->getBitfieldLength());
}
if (fp.close() == EOF) {
throw DL_ABORT_EX(fmt(EX_SEGMENT_FILE_WRITE, filename_.c_str()));
}
}
void DefaultBtProgressInfoFile::save()
{
SHA1IOFile sha1io;
save(sha1io);
auto digest = sha1io.digest();
if (digest == lastDigest_) {
// We don't write control file if the content is not changed.
return;
}
lastDigest_ = std::move(digest);
A2_LOG_INFO(fmt(MSG_SAVING_SEGMENT_FILE, filename_.c_str()));
std::string filenameTemp = filename_;
filenameTemp += "__temp";
{
BufferedFile fp(filenameTemp.c_str(), BufferedFile::WRITE);
if(!fp) {
if (!fp) {
throw DL_ABORT_EX(fmt(EX_SEGMENT_FILE_WRITE, filename_.c_str()));
}
#ifdef ENABLE_BITTORRENT
bool torrentDownload = isTorrentDownload();
#else // !ENABLE_BITTORRENT
bool torrentDownload = false;
#endif // !ENABLE_BITTORRENT
// file version: 16 bits
// values: '1'
char version[] = { 0x00u, 0x01u };
WRITE_CHECK(fp, version, sizeof(version));
// extension: 32 bits
// If this is BitTorrent download, then 0x00000001
// Otherwise, 0x00000000
char extension[4];
memset(extension, 0, sizeof(extension));
if(torrentDownload) {
extension[3] = 1;
}
WRITE_CHECK(fp, extension, sizeof(extension));
if(torrentDownload) {
#ifdef ENABLE_BITTORRENT
// infoHashLength:
// length: 32 bits
const unsigned char* infoHash = bittorrent::getInfoHash(dctx_);
uint32_t infoHashLengthNL = htonl(INFO_HASH_LENGTH);
WRITE_CHECK(fp, &infoHashLengthNL, sizeof(infoHashLengthNL));
// infoHash:
WRITE_CHECK(fp, infoHash, INFO_HASH_LENGTH);
#endif // ENABLE_BITTORRENT
} else {
// infoHashLength:
// length: 32 bits
uint32_t infoHashLength = 0;
WRITE_CHECK(fp, &infoHashLength, sizeof(infoHashLength));
}
// pieceLength: 32 bits
uint32_t pieceLengthNL = htonl(dctx_->getPieceLength());
WRITE_CHECK(fp, &pieceLengthNL, sizeof(pieceLengthNL));
// totalLength: 64 bits
uint64_t totalLengthNL = hton64(dctx_->getTotalLength());
WRITE_CHECK(fp, &totalLengthNL, sizeof(totalLengthNL));
// uploadLength: 64 bits
uint64_t uploadLengthNL = 0;
#ifdef ENABLE_BITTORRENT
if(torrentDownload) {
uploadLengthNL = hton64(btRuntime_->getUploadLengthAtStartup()+
dctx_->getNetStat().getSessionUploadLength());
}
#endif // ENABLE_BITTORRENT
WRITE_CHECK(fp, &uploadLengthNL, sizeof(uploadLengthNL));
// bitfieldLength: 32 bits
uint32_t bitfieldLengthNL = htonl(pieceStorage_->getBitfieldLength());
WRITE_CHECK(fp, &bitfieldLengthNL, sizeof(bitfieldLengthNL));
// bitfield
WRITE_CHECK(fp, pieceStorage_->getBitfield(),
pieceStorage_->getBitfieldLength());
// the number of in-flight piece: 32 bits
// TODO implement this
uint32_t numInFlightPieceNL = htonl(pieceStorage_->countInFlightPiece());
WRITE_CHECK(fp, &numInFlightPieceNL, sizeof(numInFlightPieceNL));
std::vector<std::shared_ptr<Piece> > inFlightPieces;
inFlightPieces.reserve(pieceStorage_->countInFlightPiece());
pieceStorage_->getInFlightPieces(inFlightPieces);
for(std::vector<std::shared_ptr<Piece> >::const_iterator itr =
inFlightPieces.begin(), eoi = inFlightPieces.end();
itr != eoi; ++itr) {
uint32_t indexNL = htonl((*itr)->getIndex());
WRITE_CHECK(fp, &indexNL, sizeof(indexNL));
uint32_t lengthNL = htonl((*itr)->getLength());
WRITE_CHECK(fp, &lengthNL, sizeof(lengthNL));
uint32_t bitfieldLengthNL = htonl((*itr)->getBitfieldLength());
WRITE_CHECK(fp, &bitfieldLengthNL, sizeof(bitfieldLengthNL));
WRITE_CHECK(fp, (*itr)->getBitfield(), (*itr)->getBitfieldLength());
}
if(fp.close() == EOF) {
throw DL_ABORT_EX(fmt(EX_SEGMENT_FILE_WRITE, filename_.c_str()));
}
A2_LOG_INFO(MSG_SAVED_SEGMENT_FILE);
save(fp);
}
if(!File(filenameTemp).renameTo(filename_)) {
A2_LOG_INFO(MSG_SAVED_SEGMENT_FILE);
if (!File(filenameTemp).renameTo(filename_)) {
throw DL_ABORT_EX(fmt(EX_SEGMENT_FILE_WRITE, filename_.c_str()));
}
}

7
src/DefaultBtProgressInfoFile.h
View File

@ -46,6 +46,7 @@ class PieceStorage;
class PeerStorage;
class BtRuntime;
class Option;
class IOFile;
class DefaultBtProgressInfoFile : public BtProgressInfoFile {
private:
@ -57,8 +58,14 @@ private:
#endif // ENABLE_BITTORRENT
const Option* option_;
std::string filename_;
// Last SHA1 digest value of the content written. Initially, this
// is empty string. This is used to avoid to write same content
// repeatedly, which could wake up disk that may be sleeping.
std::string lastDigest_;
bool isTorrentDownload();
void save(IOFile& fp);
public:
DefaultBtProgressInfoFile(const std::shared_ptr<DownloadContext>& btContext,
const std::shared_ptr<PieceStorage>& pieceStorage,

3
src/Makefile.am
View File

@ -267,7 +267,8 @@ SRCS = \
WrDiskCache.cc WrDiskCache.h\
WrDiskCacheEntry.cc WrDiskCacheEntry.h\
XmlRpcRequestParserController.cc XmlRpcRequestParserController.h\
OpenedFileCounter.cc OpenedFileCounter.h
OpenedFileCounter.cc OpenedFileCounter.h \
SHA1IOFile.cc SHA1IOFile.h
if ANDROID
SRCS += android/android.c

105
src/SHA1IOFile.cc Normal file
View File

@ -0,0 +1,105 @@
/* <!-- copyright */
/*
* aria2 - The high speed download utility
*
* Copyright (C) 2015 Tatsuhiro Tsujikawa
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*
* In addition, as a special exception, the copyright holders give
* permission to link the code of portions of this program with the
* OpenSSL library under certain conditions as described in each
* individual source file, and distribute linked combinations
* including the two.
* You must obey the GNU General Public License in all respects
* for all of the code used other than OpenSSL. If you modify
* file(s) with this exception, you may extend this exception to your
* version of the file(s), but you are not obligated to do so. If you
* do not wish to do so, delete this exception statement from your
* version. If you delete this exception statement from all source
* files in the program, then also delete it here.
*/
/* copyright --> */
#include "SHA1IOFile.h"
#include <cassert>
namespace aria2 {
SHA1IOFile::SHA1IOFile() : sha1_(MessageDigest::sha1())
{
}
std::string SHA1IOFile::digest()
{
return sha1_->digest();
}
size_t SHA1IOFile::onRead(void* ptr, size_t count)
{
assert(0);
return 0;
}
size_t SHA1IOFile::onWrite(const void* ptr, size_t count)
{
sha1_->update(ptr, count);
return count;
}
char* SHA1IOFile::onGets(char* s, int size)
{
assert(0);
return nullptr;
}
int SHA1IOFile::onVprintf(const char* format, va_list va)
{
assert(0);
return -1;
}
int SHA1IOFile::onFlush()
{
return 0;
}
int SHA1IOFile::onClose()
{
return 0;
}
bool SHA1IOFile::onSupportsColor()
{
return false;
}
bool SHA1IOFile::isError() const
{
return false;
}
bool SHA1IOFile::isEOF() const
{
return false;
}
bool SHA1IOFile::isOpen() const
{
return true;
}
} // namespace aria2

72
src/SHA1IOFile.h Normal file
View File

@ -0,0 +1,72 @@
/* <!-- copyright */
/*
* aria2 - The high speed download utility
*
* Copyright (C) 2015 Tatsuhiro Tsujikawa
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*
* In addition, as a special exception, the copyright holders give
* permission to link the code of portions of this program with the
* OpenSSL library under certain conditions as described in each
* individual source file, and distribute linked combinations
* including the two.
* You must obey the GNU General Public License in all respects
* for all of the code used other than OpenSSL. If you modify
* file(s) with this exception, you may extend this exception to your
* version of the file(s), but you are not obligated to do so. If you
* do not wish to do so, delete this exception statement from your
* version. If you delete this exception statement from all source
* files in the program, then also delete it here.
*/
/* copyright --> */
#ifndef D_SHA1_IO_FILE_H
#define D_SHA1_IO_FILE_H
#include "IOFile.h"
#include "MessageDigest.h"
#include <memory>
namespace aria2 {
// Class to calculate SHA1 hash value for data written into this
// object. No file I/O is done in this class.
class SHA1IOFile : public IOFile {
public:
SHA1IOFile();
std::string digest();
protected:
// Not implemented
virtual size_t onRead(void* ptr, size_t count) CXX11_OVERRIDE;
virtual size_t onWrite(const void* ptr, size_t count) CXX11_OVERRIDE;
// Not implemented
virtual char* onGets(char* s, int size) CXX11_OVERRIDE;
virtual int onVprintf(const char* format, va_list va) CXX11_OVERRIDE;
virtual int onFlush() CXX11_OVERRIDE;
virtual int onClose() CXX11_OVERRIDE;
virtual bool onSupportsColor() CXX11_OVERRIDE;
virtual bool isError() const CXX11_OVERRIDE;
virtual bool isEOF() const CXX11_OVERRIDE;
virtual bool isOpen() const CXX11_OVERRIDE;
private:
std::unique_ptr<MessageDigest> sha1_;
};
} // namespace aria2
#endif // D_SHA1_IO_FILE_H

76
src/SessionSerializer.cc
View File

@ -53,7 +53,7 @@
#include "BufferedFile.h"
#include "OptionParser.h"
#include "OptionHandler.h"
#include "MessageDigest.h"
#include "SHA1IOFile.h"
#if HAVE_ZLIB
#include "GZipFile.h"
@ -327,80 +327,6 @@ bool SessionSerializer::save(IOFile& fp) const
return true;
}
namespace {
// Class to calculate SHA1 hash value for data written into this
// object. No file I/O is done in this class.
class SHA1IOFile : public IOFile {
public:
SHA1IOFile()
: sha1_(MessageDigest::sha1())
{}
std::string digest()
{
return sha1_->digest();
}
protected:
virtual size_t onRead(void* ptr, size_t count) CXX11_OVERRIDE
{
assert(0);
return 0;
}
virtual size_t onWrite(const void* ptr, size_t count) CXX11_OVERRIDE
{
sha1_->update(ptr, count);
return count;
}
virtual char* onGets(char* s, int size) CXX11_OVERRIDE
{
assert(0);
return nullptr;
}
virtual int onVprintf(const char* format, va_list va) CXX11_OVERRIDE
{
assert(0);
return -1;
}
virtual int onFlush() CXX11_OVERRIDE
{
return 0;
}
virtual int onClose() CXX11_OVERRIDE
{
return 0;
}
virtual bool onSupportsColor() CXX11_OVERRIDE
{
return false;
}
virtual bool isError() const CXX11_OVERRIDE
{
return false;
}
virtual bool isEOF() const CXX11_OVERRIDE
{
return false;
}
virtual bool isOpen() const CXX11_OVERRIDE
{
return true;
}
private:
std::unique_ptr<MessageDigest> sha1_;
};
} // namespace
std::string SessionSerializer::calculateHash() const
{
SHA1IOFile sha1io;