2007-10-12 Tatsuhiro Tsujikawa <tujikawa at rednoah dot com>

Throw exception when chunck checksum verification fails.
	* src/DownloadCommand.cc (validatePieceHash): New function.
	* src/PiecedSegment.{h, cc} (clear): New function.
	* src/GrowSegment.{h, cc} (clear): New function.
	* src/Segment.h (clear): New function.
	* src/SegmentMan.{h, cc} (validatePieceHash): Removed.
	* test/SegmentTest.cc
	* test/GrowSegmentTest.cc

ChangeLog

@@ -1,3 +1,14 @@
+2007-10-12  Tatsuhiro Tsujikawa  <tujikawa at rednoah dot com>
+
+	Throw exception when chunck checksum verification fails.
+	* src/DownloadCommand.cc (validatePieceHash): New function.
+	* src/PiecedSegment.{h, cc} (clear): New function.
+	* src/GrowSegment.{h, cc} (clear): New function.
+	* src/Segment.h (clear): New function.
+	* src/SegmentMan.{h, cc} (validatePieceHash): Removed.
+	* test/SegmentTest.cc
+	* test/GrowSegmentTest.cc
+
 2007-10-12  Tatsuhiro Tsujikawa  <tujikawa at rednoah dot com>
 
 	Do not send referer when redirected.

src/DownloadCommand.cc

@@ -47,6 +47,9 @@
 #include "Segment.h"
 #include "PieceStorage.h"
 #include "Option.h"
+#ifdef ENABLE_MESSAGE_DIGEST
+#include "MessageDigestHelper.h"
+#endif // ENABLE_MESSAGE_DIGEST
 #include <stdlib.h>
 
 DownloadCommand::DownloadCommand(int cuid,
@@ -121,30 +124,13 @@ bool DownloadCommand::executeInternal() {
      || bufSize == 0) {
     if(!transferDecoder.isNull()) transferDecoder->end();
     logger->info(MSG_SEGMENT_DOWNLOAD_COMPLETED, cuid);
-    _requestGroup->getSegmentMan()->completeSegment(cuid, segment);
+    validatePieceHash(segment);
-    // TODO According to the current plan, checksum is held by DownloadContext.
-#ifdef ENABLE_MESSAGE_DIGEST
-    if(e->option->get(PREF_REALTIME_CHUNK_CHECKSUM) == V_TRUE) {
-      string pieceHash = _requestGroup->getDownloadContext()->getPieceHash(segment->getIndex());
-      if(!pieceHash.empty()) {
-	_requestGroup->getSegmentMan()->validatePieceHash(segment, pieceHash);
-      }
-    }
-#endif // ENABLE_MESSAGE_DIGEST
-    /*
-#ifdef ENABLE_MESSAGE_DIGEST
-    if(e->option->get(PREF_REALTIME_CHUNK_CHECKSUM) == V_TRUE) {
-      _requestGroup->getSegmentMan()->tryChunkChecksumValidation(segment, _requestGroup->getChunkChecksum());
-    }
-#endif // ENABLE_MESSAGE_DIGEST
-    */
     // this unit is going to download another segment.
     return prepareForNextSegment();
   } else {
     e->commands.push_back(this);
     return false;
   }
-  
 }
 
 bool DownloadCommand::prepareForNextSegment() {
@@ -175,9 +161,8 @@ bool DownloadCommand::prepareForNextSegment() {
 	  return prepareForRetry(0);
 	}
 	nextSegment->updateWrittenLength(tempSegment->getOverflowLength());
-	//tempSegment->writtenLength-tempSegment->length;
 	if(nextSegment->complete()) {
-	  _requestGroup->getSegmentMan()->completeSegment(cuid, nextSegment);
+	  validatePieceHash(nextSegment);
 	  tempSegment = nextSegment;
 	} else {
 	  segment = nextSegment;
@@ -190,6 +175,38 @@ bool DownloadCommand::prepareForNextSegment() {
   }
 }
 
+void DownloadCommand::validatePieceHash(const SegmentHandle& segment)
+{
+#ifdef ENABLE_MESSAGE_DIGEST
+  string expectedPieceHash =
+    _requestGroup->getDownloadContext()->getPieceHash(segment->getIndex());
+  if(e->option->get(PREF_REALTIME_CHUNK_CHECKSUM) == V_TRUE &&
+     !expectedPieceHash.empty()) {
+    string actualPieceHash =
+      MessageDigestHelper::digest("sha1",
+				  _requestGroup->getPieceStorage()->getDiskAdaptor(),
+				  segment->getPosition(),
+				  segment->getLength());
+    if(actualPieceHash == expectedPieceHash) {
+      logger->info(MSG_GOOD_CHUNK_CHECKSUM, actualPieceHash.c_str());
+      _requestGroup->getSegmentMan()->completeSegment(cuid, segment);
+    } else {
+      logger->info(EX_INVALID_CHUNK_CHECKSUM,
+		   segment->getIndex(),
+		   Util::llitos(segment->getPosition(), true).c_str(),
+		   expectedPieceHash.c_str(),
+		   actualPieceHash.c_str());
+      segment->clear();
+      _requestGroup->getSegmentMan()->cancelSegment(cuid);
+      throw new DlRetryEx("Invalid checksum index=%d", segment->getIndex());
+    }
+  } else
+#endif // ENABLE_MESSAGE_DIGEST
+    {
+      _requestGroup->getSegmentMan()->completeSegment(cuid, segment);
+    }
+}
+
 void DownloadCommand::setTransferDecoder(const TransferEncodingHandle& transferDecoder)
 {
   this->transferDecoder = transferDecoder;

src/DownloadCommand.h

@@ -48,12 +48,16 @@ private:
   int32_t startupIdleTime;
   int32_t lowestDownloadSpeedLimit;
   PeerStatHandle peerStat;
+
+  void validatePieceHash(const SegmentHandle& segment);
+
 protected:
   TransferEncodingHandle transferDecoder;
 
   virtual bool executeInternal();
 
   virtual bool prepareForNextSegment();
+
 public:
   DownloadCommand(int cuid,
 		  const RequestHandle req,

src/GrowSegment.cc

@@ -47,6 +47,12 @@ void GrowSegment::updateWrittenLength(int32_t bytes)
   _piece->setAllBlock();
 }
 
+void GrowSegment::clear()
+{
+  _writtenLength = 0;
+  _piece->clearAllBlock();
+}
+
 PieceHandle GrowSegment::getPiece() const
 {
   return _piece;

src/GrowSegment.h

@@ -88,6 +88,8 @@ public:
 
   virtual void updateWrittenLength(int32_t bytes);
 
+  virtual void clear();
+
   virtual PieceHandle getPiece() const;
 };
 

src/PiecedSegment.cc

@@ -84,6 +84,13 @@ void PiecedSegment::updateWrittenLength(int32_t bytes)
   _writtenLength = newWrittenLength;
 }
 
+void PiecedSegment::clear()
+{
+  _writtenLength = 0;
+  _overflowLength = 0;
+  _piece->clearAllBlock();
+}
+
 PieceHandle PiecedSegment::getPiece() const
 {
   return _piece;

src/PiecedSegment.h

@@ -80,6 +80,8 @@ public:
 
   virtual void updateWrittenLength(int32_t bytes);
 
+  virtual void clear();
+
   virtual PieceHandle getPiece() const;
 
   bool operator==(const PiecedSegment& segment) const;

src/Segment.h

@@ -62,6 +62,8 @@ public:
 
   virtual void updateWrittenLength(int32_t bytes) = 0;
 
+  virtual void clear() = 0;
+
   virtual PieceHandle getPiece() const = 0;
 };
 

src/SegmentMan.cc

@@ -47,9 +47,6 @@
 #include "Option.h"
 #include "DownloadContext.h"
 #include "Piece.h"
-#ifdef ENABLE_MESSAGE_DIGEST
-#include "MessageDigestHelper.h"
-#endif // ENABLE_MESSAGE_DIGEST
 #include "a2io.h"
 #include <errno.h>
 
@@ -282,96 +279,6 @@ void SegmentMan::markPieceDone(int64_t length)
   */
 }
 
-#ifdef ENABLE_MESSAGE_DIGEST
-void SegmentMan::validatePieceHash(const SegmentHandle& segment,
-				   const string& expectedPieceHash)
-{
-  string actualPieceHash =
-    MessageDigestHelper::digest("sha1",
-				_pieceStorage->getDiskAdaptor(),
-				segment->getPosition(),
-				segment->getLength());
-  if(actualPieceHash == expectedPieceHash) {
-    logger->info(MSG_GOOD_CHUNK_CHECKSUM, actualPieceHash.c_str());
-  } else {
-    _pieceStorage->markPieceMissing(segment->getIndex());
-    logger->info(EX_INVALID_CHUNK_CHECKSUM,
-		 segment->getIndex(),
-		 Util::llitos(segment->getPosition(), true).c_str(),
-		 expectedPieceHash.c_str(),
-		 actualPieceHash.c_str());
-  }
-}
-
-/*
-bool SegmentMan::isChunkChecksumValidationReady(const ChunkChecksumHandle& chunkChecksum) const {
-  return false;
-  // TODO fix this
-  return !chunkChecksum.isNull() && !_downloadContext.isNull() && totalSize > 0 &&
-    chunkChecksum->getEstimatedDataLength() >= totalSize;
-}
-*/
-#endif // ENABLE_MESSAGE_DIGEST
-
-#ifdef ENABLE_MESSAGE_DIGEST
-  /*
-void SegmentMan::tryChunkChecksumValidation(const SegmentHandle& segment, const ChunkChecksumHandle& chunkChecksum)
-{
-  // TODO implement this function later
-  if(!isChunkChecksumValidationReady(chunkChecksum)) {
-    return;
-  }
-  int32_t hashStartIndex;
-  int32_t hashEndIndex;
-  Util::indexRange(hashStartIndex, hashEndIndex,
-		   segment->getPosition(),
-		   segment->writtenLength,
-		   chunkChecksum->getChecksumLength());
-  if(!bitfield->isBitSetOffsetRange((int64_t)hashStartIndex*chunkChecksum->getChecksumLength(),
-				    chunkChecksum->getChecksumLength())) {
-    ++hashStartIndex;
-  }
-  if(!bitfield->isBitSetOffsetRange((int64_t)hashEndIndex*chunkChecksum->getChecksumLength(),
-				    chunkChecksum->getChecksumLength())) {
-    --hashEndIndex;
-  }
-  logger->debug("hashStartIndex=%d, hashEndIndex=%d",
-		hashStartIndex, hashEndIndex);
-  if(hashStartIndex > hashEndIndex) {
-    logger->debug(MSG_NO_CHUNK_CHECKSUM);
-    return;
-  }
-  int64_t hashOffset = ((int64_t)hashStartIndex)*chunkChecksum->getChecksumLength();
-  int32_t startIndex;
-  int32_t endIndex;
-  Util::indexRange(startIndex, endIndex,
-		   hashOffset,
-		   (hashEndIndex-hashStartIndex+1)*chunkChecksum->getChecksumLength(),
-		   bitfield->getBlockLength());
-  logger->debug("startIndex=%d, endIndex=%d", startIndex, endIndex);
-  if(bitfield->isBitRangeSet(startIndex, endIndex)) {
-    for(int32_t index = hashStartIndex; index <= hashEndIndex; ++index) {
-      int64_t offset = ((int64_t)index)*chunkChecksum->getChecksumLength();
-      int32_t dataLength =
-	offset+chunkChecksum->getChecksumLength() <= totalSize ?
-	chunkChecksum->getChecksumLength() : totalSize-offset;
-      string actualChecksum = MessageDigestHelper::digest(chunkChecksum->getAlgo(), diskWriter, offset, dataLength);
-      if(chunkChecksum->validateChunk(actualChecksum, index)) {
-	logger->info(MSG_GOOD_CHUNK_CHECKSUM, actualChecksum.c_str());
-      } else {
-	logger->info(EX_INVALID_CHUNK_CHECKSUM,
-		     index, Util::llitos(offset, true).c_str(),
-		     chunkChecksum->getChecksum(index).c_str(), actualChecksum.c_str());
-	logger->debug("Unset bit from %d to %d(inclusive)", startIndex, endIndex);
-	bitfield->unsetBitRange(startIndex, endIndex);
-	break;
-      }
-    }
-  }
-}
-  */
-#endif // ENABLE_MESSAGE_DIGEST
-
 SegmentEntryHandle SegmentMan::getSegmentEntryByIndex(int32_t index)
 {
   for(SegmentEntries::const_iterator itr = usedSegmentEntries.begin();

src/SegmentMan.h

@@ -185,16 +185,6 @@ public:
   void markAllPiecesDone();
 
   void markPieceDone(int64_t length);
-
-#ifdef ENABLE_MESSAGE_DIGEST
-
-  void validatePieceHash(const SegmentHandle& segment, const string& pieceHash);
-  /*
-  void tryChunkChecksumValidation(const SegmentHandle& segment, const ChunkChecksumHandle& chunkChecksum);
-
-  bool isChunkChecksumValidationReady(const ChunkChecksumHandle& chunkChecksum) const;
-  */
-#endif // ENABLE_MESSAGE_DIGEST
 };
 
 typedef SharedHandle<SegmentMan> SegmentManHandle;

test/GrowSegmentTest.cc

@@ -8,6 +8,7 @@ class GrowSegmentTest : public CppUnit::TestFixture {
 
   CPPUNIT_TEST_SUITE(GrowSegmentTest);
   CPPUNIT_TEST(testUpdateWrittenLength);
+  CPPUNIT_TEST(testClear);
   CPPUNIT_TEST_SUITE_END();
 private:
 
@@ -15,6 +16,7 @@ public:
   void setUp() {}
 
   void testUpdateWrittenLength();
+  void testClear();
 };
 
 
@@ -29,3 +31,12 @@ void GrowSegmentTest::testUpdateWrittenLength()
   CPPUNIT_ASSERT(!segment.complete());
   CPPUNIT_ASSERT(segment.getPiece()->pieceComplete());
 }
+
+void GrowSegmentTest::testClear()
+{
+  GrowSegment segment(new Piece());
+  segment.updateWrittenLength(32*1024);
+  CPPUNIT_ASSERT_EQUAL((int32_t)32*1024, segment.getWrittenLength());
+  segment.clear();
+  CPPUNIT_ASSERT_EQUAL((int32_t)0, segment.getWrittenLength());  
+}

test/SegmentTest.cc

@@ -11,6 +11,7 @@ class SegmentTest : public CppUnit::TestFixture {
   CPPUNIT_TEST(testUpdateWrittenLength_overflow);
   CPPUNIT_TEST(testUpdateWrittenLength_lastPiece);
   CPPUNIT_TEST(testUpdateWrittenLength_incompleteLastPiece);
+  CPPUNIT_TEST(testClear);
   CPPUNIT_TEST_SUITE_END();
 private:
 
@@ -21,6 +22,7 @@ public:
   void testUpdateWrittenLength_overflow();
   void testUpdateWrittenLength_lastPiece();
   void testUpdateWrittenLength_incompleteLastPiece();
+  void testClear();
 };
 
 
@@ -69,3 +71,15 @@ void SegmentTest::testUpdateWrittenLength_incompleteLastPiece()
   s.updateWrittenLength(1);
   CPPUNIT_ASSERT(p->pieceComplete());
 }
+
+void SegmentTest::testClear()
+{
+  PieceHandle p = new Piece(0, 16*1024*10);
+  PiecedSegment s(16*1024*10, p);
+  s.updateWrittenLength(16*1024*11);
+  CPPUNIT_ASSERT_EQUAL((int32_t)16*1024*10, s.getWrittenLength());
+  CPPUNIT_ASSERT_EQUAL((int32_t)16*1024, s.getOverflowLength());
+  s.clear();
+  CPPUNIT_ASSERT_EQUAL((int32_t)0, s.getWrittenLength());
+  CPPUNIT_ASSERT_EQUAL((int32_t)0, s.getOverflowLength());
+}