From a82f08765e9be6f65ae5338dc27611fd9d7c715f Mon Sep 17 00:00:00 2001 From: Nils Maier Date: Sat, 24 May 2014 22:42:23 +0200 Subject: [PATCH] Fix (unknown length) downloads larger than 2GiB Closes #215 --- src/AbstractDiskWriter.cc | 1 + src/GrowSegment.cc | 2 +- src/GrowSegment.h | 12 ++++++------ src/PiecedSegment.cc | 10 +++++----- src/PiecedSegment.h | 12 ++++++------ src/Segment.h | 10 +++++----- src/SegmentMan.cc | 10 +++++----- src/SegmentMan.h | 2 +- test/GZipDecodingStreamFilterTest.cc | 2 +- test/GrowSegmentTest.cc | 4 ++-- test/MockSegment.h | 10 +++++----- test/SegmentManTest.cc | 6 +++--- test/SegmentTest.cc | 6 +++--- test/SinkStreamFilterTest.cc | 10 +++++----- 14 files changed, 49 insertions(+), 48 deletions(-) diff --git a/src/AbstractDiskWriter.cc b/src/AbstractDiskWriter.cc index 25a2d620..fe547e62 100644 --- a/src/AbstractDiskWriter.cc +++ b/src/AbstractDiskWriter.cc @@ -310,6 +310,7 @@ ssize_t AbstractDiskWriter::readDataInternal(unsigned char* data, size_t len, void AbstractDiskWriter::seek(int64_t offset) { + assert(offset >= 0); #ifdef __MINGW32__ LARGE_INTEGER fileLength; fileLength.QuadPart = offset; diff --git a/src/GrowSegment.cc b/src/GrowSegment.cc index 433813df..45e8aac1 100644 --- a/src/GrowSegment.cc +++ b/src/GrowSegment.cc @@ -43,7 +43,7 @@ GrowSegment::GrowSegment(const std::shared_ptr& piece): GrowSegment::~GrowSegment() {} -void GrowSegment::updateWrittenLength(int32_t bytes) +void GrowSegment::updateWrittenLength(int64_t bytes) { writtenLength_ += bytes; piece_->reconfigure(writtenLength_); diff --git a/src/GrowSegment.h b/src/GrowSegment.h index 70aed4c0..27ee5c9e 100644 --- a/src/GrowSegment.h +++ b/src/GrowSegment.h @@ -42,7 +42,7 @@ namespace aria2 { class GrowSegment:public Segment { private: std::shared_ptr piece_; - int32_t writtenLength_; + int64_t writtenLength_; public: GrowSegment(const std::shared_ptr& piece); @@ -68,25 +68,25 @@ public: return writtenLength_; } - virtual int32_t getLength() const CXX11_OVERRIDE + virtual int64_t getLength() const CXX11_OVERRIDE { return 0; } - virtual int32_t getSegmentLength() const CXX11_OVERRIDE + virtual int64_t getSegmentLength() const CXX11_OVERRIDE { return 0; } - virtual int32_t getWrittenLength() const CXX11_OVERRIDE + virtual int64_t getWrittenLength() const CXX11_OVERRIDE { return writtenLength_; } - virtual void updateWrittenLength(int32_t bytes) CXX11_OVERRIDE; + virtual void updateWrittenLength(int64_t bytes) CXX11_OVERRIDE; virtual bool updateHash - (int32_t begin, + (int64_t begin, const unsigned char* data, size_t dataLength) CXX11_OVERRIDE { diff --git a/src/PiecedSegment.cc b/src/PiecedSegment.cc index 819e8d70..a6146b66 100644 --- a/src/PiecedSegment.cc +++ b/src/PiecedSegment.cc @@ -71,16 +71,16 @@ int64_t PiecedSegment::getPositionToWrite() const return getPosition()+writtenLength_; } -int32_t PiecedSegment::getLength() const +int64_t PiecedSegment::getLength() const { return piece_->getLength(); } -void PiecedSegment::updateWrittenLength(int32_t bytes) +void PiecedSegment::updateWrittenLength(int64_t bytes) { - int32_t newWrittenLength = writtenLength_+bytes; + auto newWrittenLength = writtenLength_+bytes; assert(newWrittenLength <= piece_->getLength()); - for(size_t i = writtenLength_/piece_->getBlockLength(), + for(auto i = writtenLength_/piece_->getBlockLength(), end = newWrittenLength/piece_->getBlockLength(); i < end; ++i) { piece_->completeBlock(i); } @@ -91,7 +91,7 @@ void PiecedSegment::updateWrittenLength(int32_t bytes) } bool PiecedSegment::updateHash -(int32_t begin, +(int64_t begin, const unsigned char* data, size_t dataLength) { diff --git a/src/PiecedSegment.h b/src/PiecedSegment.h index 30270f7b..da969a0a 100644 --- a/src/PiecedSegment.h +++ b/src/PiecedSegment.h @@ -47,7 +47,7 @@ private: * The last piece likely have shorter length than the other length. */ int32_t pieceLength_; - int32_t writtenLength_; + int64_t writtenLength_; public: PiecedSegment(int32_t pieceLength, const std::shared_ptr& piece); @@ -62,23 +62,23 @@ public: virtual int64_t getPositionToWrite() const CXX11_OVERRIDE; - virtual int32_t getLength() const CXX11_OVERRIDE; + virtual int64_t getLength() const CXX11_OVERRIDE; - virtual int32_t getSegmentLength() const CXX11_OVERRIDE + virtual int64_t getSegmentLength() const CXX11_OVERRIDE { return pieceLength_; } - virtual int32_t getWrittenLength() const CXX11_OVERRIDE + virtual int64_t getWrittenLength() const CXX11_OVERRIDE { return writtenLength_; } - virtual void updateWrittenLength(int32_t bytes) CXX11_OVERRIDE; + virtual void updateWrittenLength(int64_t bytes) CXX11_OVERRIDE; // `begin' is a offset inside this segment. virtual bool updateHash - (int32_t begin, + (int64_t begin, const unsigned char* data, size_t dataLength) CXX11_OVERRIDE; diff --git a/src/Segment.h b/src/Segment.h index ad9355ea..a11459cb 100644 --- a/src/Segment.h +++ b/src/Segment.h @@ -58,17 +58,17 @@ public: virtual int64_t getPositionToWrite() const = 0; - virtual int32_t getLength() const = 0; + virtual int64_t getLength() const = 0; - virtual int32_t getSegmentLength() const = 0; + virtual int64_t getSegmentLength() const = 0; - virtual int32_t getWrittenLength() const = 0; + virtual int64_t getWrittenLength() const = 0; - virtual void updateWrittenLength(int32_t bytes) = 0; + virtual void updateWrittenLength(int64_t bytes) = 0; // `begin' is a offset inside this segment. virtual bool updateHash - (int32_t begin, + (int64_t begin, const unsigned char* data, size_t dataLength) = 0; diff --git a/src/SegmentMan.cc b/src/SegmentMan.cc index 5c216fce..dc94d851 100644 --- a/src/SegmentMan.cc +++ b/src/SegmentMan.cc @@ -156,8 +156,8 @@ std::shared_ptr SegmentMan::checkoutSegment } std::shared_ptr entry(new SegmentEntry(cuid, segment)); usedSegmentEntries_.push_back(entry); - A2_LOG_DEBUG(fmt("index=%lu, length=%d, segmentLength=%d," - " writtenLength=%d", + A2_LOG_DEBUG(fmt("index=%lu, length=%" PRId64 ", segmentLength=%" PRId64 "," + " writtenLength=%" PRId64, static_cast(segment->getIndex()), segment->getLength(), segment->getSegmentLength(), @@ -166,8 +166,8 @@ std::shared_ptr SegmentMan::checkoutSegment if(piece->getLength() > 0) { auto positr = segmentWrittenLengthMemo_.find(segment->getIndex()); if(positr != segmentWrittenLengthMemo_.end()) { - const int32_t writtenLength = (*positr).second; - A2_LOG_DEBUG(fmt("writtenLength(in memo)=%d, writtenLength=%d", + const auto writtenLength = (*positr).second; + A2_LOG_DEBUG(fmt("writtenLength(in memo)=%" PRId64 ", writtenLength=%" PRId64, writtenLength, segment->getWrittenLength())); // If the difference between cached writtenLength and segment's // writtenLength is less than one block, we assume that these @@ -296,7 +296,7 @@ void SegmentMan::cancelSegmentInternal piece->setUsedBySegment(false); pieceStorage_->cancelPiece(piece, cuid); segmentWrittenLengthMemo_[segment->getIndex()] = segment->getWrittenLength(); - A2_LOG_DEBUG(fmt("Memorized segment index=%lu, writtenLength=%d", + A2_LOG_DEBUG(fmt("Memorized segment index=%lu, writtenLength=%" PRId64, static_cast(segment->getIndex()), segment->getWrittenLength())); } diff --git a/src/SegmentMan.h b/src/SegmentMan.h index 60dbd625..c5cd98d7 100644 --- a/src/SegmentMan.h +++ b/src/SegmentMan.h @@ -83,7 +83,7 @@ private: // Remember writtenLength for each segment. The key is an index of a // segment. The value is writtenLength for that segment. - std::map segmentWrittenLengthMemo_; + std::map segmentWrittenLengthMemo_; // Used for calculating download speed. std::vector > peerStats_; diff --git a/test/GZipDecodingStreamFilterTest.cc b/test/GZipDecodingStreamFilterTest.cc index f9579d78..95713399 100644 --- a/test/GZipDecodingStreamFilterTest.cc +++ b/test/GZipDecodingStreamFilterTest.cc @@ -28,7 +28,7 @@ class GZipDecodingStreamFilterTest:public CppUnit::TestFixture { public: MockSegment2():positionToWrite_(0) {} - virtual void updateWrittenLength(int32_t bytes) CXX11_OVERRIDE + virtual void updateWrittenLength(int64_t bytes) CXX11_OVERRIDE { positionToWrite_ += bytes; } diff --git a/test/GrowSegmentTest.cc b/test/GrowSegmentTest.cc index c973a39e..b71689e5 100644 --- a/test/GrowSegmentTest.cc +++ b/test/GrowSegmentTest.cc @@ -36,9 +36,9 @@ void GrowSegmentTest::testClear() { GrowSegment segment(std::shared_ptr(new Piece())); segment.updateWrittenLength(32*1024); - CPPUNIT_ASSERT_EQUAL(32*1024, segment.getWrittenLength()); + CPPUNIT_ASSERT_EQUAL((int64_t)32*1024, segment.getWrittenLength()); segment.clear(nullptr); - CPPUNIT_ASSERT_EQUAL(0, segment.getWrittenLength()); + CPPUNIT_ASSERT_EQUAL((int64_t)0, segment.getWrittenLength()); } } // namespace aria2 diff --git a/test/MockSegment.h b/test/MockSegment.h index 68dc4e0c..37886fa4 100644 --- a/test/MockSegment.h +++ b/test/MockSegment.h @@ -29,26 +29,26 @@ public: return 0; } - virtual int32_t getLength() const CXX11_OVERRIDE + virtual int64_t getLength() const CXX11_OVERRIDE { return 0; } - virtual int32_t getSegmentLength() const CXX11_OVERRIDE + virtual int64_t getSegmentLength() const CXX11_OVERRIDE { return 0; } - virtual int32_t getWrittenLength() const CXX11_OVERRIDE + virtual int64_t getWrittenLength() const CXX11_OVERRIDE { return 0; } - virtual void updateWrittenLength(int32_t bytes) CXX11_OVERRIDE {} + virtual void updateWrittenLength(int64_t bytes) CXX11_OVERRIDE {} // `begin' is a offset inside this segment. virtual bool updateHash - (int32_t begin, const unsigned char* data, size_t dataLength) CXX11_OVERRIDE + (int64_t begin, const unsigned char* data, size_t dataLength) CXX11_OVERRIDE { return false; } diff --git a/test/SegmentManTest.cc b/test/SegmentManTest.cc index 0e1a2fc5..30e38ad9 100644 --- a/test/SegmentManTest.cc +++ b/test/SegmentManTest.cc @@ -66,9 +66,9 @@ void SegmentManTest::testNullBitfield() std::shared_ptr segment = segmentMan.getSegment(1, minSplitSize); CPPUNIT_ASSERT(segment); CPPUNIT_ASSERT_EQUAL((size_t)0, segment->getIndex()); - CPPUNIT_ASSERT_EQUAL(0, segment->getLength()); - CPPUNIT_ASSERT_EQUAL(0, segment->getSegmentLength()); - CPPUNIT_ASSERT_EQUAL(0, segment->getWrittenLength()); + CPPUNIT_ASSERT_EQUAL((int64_t)0, segment->getLength()); + CPPUNIT_ASSERT_EQUAL((int64_t)0, segment->getSegmentLength()); + CPPUNIT_ASSERT_EQUAL((int64_t)0, segment->getWrittenLength()); std::shared_ptr segment2 = segmentMan.getSegment(2, minSplitSize); CPPUNIT_ASSERT(!segment2); diff --git a/test/SegmentTest.cc b/test/SegmentTest.cc index 0f32e310..a2a1cfc6 100644 --- a/test/SegmentTest.cc +++ b/test/SegmentTest.cc @@ -32,7 +32,7 @@ void SegmentTest::testUpdateWrittenLength() { std::shared_ptr p(new Piece(0, 16*1024*10)); PiecedSegment s(16*1024*10, p); - CPPUNIT_ASSERT_EQUAL(0, s.getWrittenLength()); + CPPUNIT_ASSERT_EQUAL((int64_t)0, s.getWrittenLength()); s.updateWrittenLength(16*1024); CPPUNIT_ASSERT(p->hasBlock(0)); @@ -67,9 +67,9 @@ void SegmentTest::testClear() std::shared_ptr p(new Piece(0, 16*1024*10)); PiecedSegment s(16*1024*10, p); s.updateWrittenLength(16*1024*10); - CPPUNIT_ASSERT_EQUAL(16*1024*10, s.getWrittenLength()); + CPPUNIT_ASSERT_EQUAL((int64_t)16*1024*10, s.getWrittenLength()); s.clear(nullptr); - CPPUNIT_ASSERT_EQUAL(0, s.getWrittenLength()); + CPPUNIT_ASSERT_EQUAL((int64_t)0, s.getWrittenLength()); } } // namespace aria2 diff --git a/test/SinkStreamFilterTest.cc b/test/SinkStreamFilterTest.cc index e711a534..3f5ecb88 100644 --- a/test/SinkStreamFilterTest.cc +++ b/test/SinkStreamFilterTest.cc @@ -22,23 +22,23 @@ class SinkStreamFilterTest:public CppUnit::TestFixture { public: MockSegment2(int32_t length):length(length), writtenLength(0) {} - virtual int32_t getLength() const CXX11_OVERRIDE + virtual int64_t getLength() const CXX11_OVERRIDE { return length; } - virtual int32_t getWrittenLength() const CXX11_OVERRIDE + virtual int64_t getWrittenLength() const CXX11_OVERRIDE { return writtenLength; } - virtual void updateWrittenLength(int32_t bytes) CXX11_OVERRIDE + virtual void updateWrittenLength(int64_t bytes) CXX11_OVERRIDE { writtenLength += bytes; } - int32_t length; - int32_t writtenLength; + int64_t length; + int64_t writtenLength; }; std::shared_ptr filter_;