2007-11-13 Tatsuhiro Tsujikawa <tujikawa at rednoah dot com>

Recalculates download progress when loading a control file,
	if the piece length of DownloadContext is different from the one 
saved
	in the control file. Currently in-flight pieces are ignored.
	* src/DefaultBtProgressInfoFile.cc
	* test/DefaultBtProgressInfoFileTest.cc
	* src/Util.{h, cc}
	* test/UtilTest.cc

ChangeLog

@@ -1,3 +1,13 @@
+2007-11-13  Tatsuhiro Tsujikawa  <tujikawa at rednoah dot com>
+
+	Recalculates download progress when loading a control file,
+	if the piece length of DownloadContext is different from the one saved
+	in the control file. Currently in-flight pieces are ignored.
+	* src/DefaultBtProgressInfoFile.cc
+	* test/DefaultBtProgressInfoFileTest.cc
+	* src/Util.{h, cc}
+	* test/UtilTest.cc
+
 2007-11-12  Tatsuhiro Tsujikawa  <tujikawa at rednoah dot com>
 
 	Do not rotate tiers in announce-list.

TODO

@@ -56,6 +56,5 @@
   FatalException .... Program should abort.
 
 -- remaining features to be implemented for 0.12.0 release
-* Implement duplicate download checking in Bt
+* Implement duplicate download checking
 * improve --metalink-location field
-* Piece length conversion when loading file

src/DefaultBtProgressInfoFile.cc

@@ -190,13 +190,8 @@ void DefaultBtProgressInfoFile::load()
       savedInfoHash = 0;
     }
 
-    // TODO implement the conversion mechanism between different piece length.
     int32_t pieceLength;
     in.read(reinterpret_cast<char*>(&pieceLength), sizeof(pieceLength));
-    if(pieceLength != _dctx->getPieceLength()) {
-      throw new DlAbortEx("piece length mismatch. expected: %d, actual: %d",
-			  _dctx->getPieceLength(), pieceLength);
-    }
 
     int64_t totalLength;
     in.read(reinterpret_cast<char*>(&totalLength), sizeof(totalLength));
@@ -214,51 +209,61 @@ void DefaultBtProgressInfoFile::load()
     // TODO implement the conversion mechanism between different piece length.
     int32_t bitfieldLength;
     in.read(reinterpret_cast<char*>(&bitfieldLength), sizeof(bitfieldLength));
-    if(_pieceStorage->getBitfieldLength() != bitfieldLength) {
+    int32_t expectedBitfieldLength = ((totalLength+pieceLength-1)/pieceLength+7)/8;
+    if(expectedBitfieldLength != bitfieldLength) {
       throw new DlAbortEx("bitfield length mismatch. expected: %d, actual: %d",
-			  _pieceStorage->getBitfieldLength(),
+			  expectedBitfieldLength,
 			  bitfieldLength);
     }
 
-    // TODO implement the conversion mechanism between different piece length.
     savedBitfield = new unsigned char[bitfieldLength];
     in.read(reinterpret_cast<char*>(savedBitfield), bitfieldLength);
-    _pieceStorage->setBitfield(savedBitfield, bitfieldLength);
-    delete [] savedBitfield;
-    savedBitfield = 0;
 
-    int32_t numInFlightPiece;
+    if(pieceLength == _dctx->getPieceLength()) {
-    in.read(reinterpret_cast<char*>(&numInFlightPiece), sizeof(numInFlightPiece));
+      _pieceStorage->setBitfield(savedBitfield, bitfieldLength);
-    
-    Pieces inFlightPieces;
-    while(numInFlightPiece--) {
-      int32_t index;
-      in.read(reinterpret_cast<char*>(&index), sizeof(index));
-      if(!(0 <= index && index < _dctx->getNumPieces())) {
-	throw new DlAbortEx("piece index out of range: %d", index);
-      }
-      int32_t length;
-      in.read(reinterpret_cast<char*>(&length), sizeof(length));
-      if(!(0 < length && length <=_dctx->getPieceLength())) {
-	throw new DlAbortEx("piece length out of range: %d", length);
-      }
-      PieceHandle piece = new Piece(index, length);
-      int32_t bitfieldLength;
-      in.read(reinterpret_cast<char*>(&bitfieldLength), sizeof(bitfieldLength));
-      if(piece->getBitfieldLength() != bitfieldLength) {
-	throw new DlAbortEx("piece bitfield length mismatch. expected: %d actual: %d",
-			    piece->getBitfieldLength(), bitfieldLength);
-      }
-      savedBitfield = new unsigned char[bitfieldLength];
-      in.read(reinterpret_cast<char*>(savedBitfield), bitfieldLength);
-      piece->setBitfield(savedBitfield, bitfieldLength);
       delete [] savedBitfield;
       savedBitfield = 0;
-      
-      inFlightPieces.push_back(piece);
-    }
-    _pieceStorage->addInFlightPiece(inFlightPieces);
 
+      int32_t numInFlightPiece;
+      in.read(reinterpret_cast<char*>(&numInFlightPiece), sizeof(numInFlightPiece));
+      
+      Pieces inFlightPieces;
+      while(numInFlightPiece--) {
+	int32_t index;
+	in.read(reinterpret_cast<char*>(&index), sizeof(index));
+	if(!(0 <= index && index < _dctx->getNumPieces())) {
+	  throw new DlAbortEx("piece index out of range: %d", index);
+	}
+	int32_t length;
+	in.read(reinterpret_cast<char*>(&length), sizeof(length));
+	if(!(0 < length && length <=_dctx->getPieceLength())) {
+	  throw new DlAbortEx("piece length out of range: %d", length);
+	}
+	PieceHandle piece = new Piece(index, length);
+	int32_t bitfieldLength;
+	in.read(reinterpret_cast<char*>(&bitfieldLength), sizeof(bitfieldLength));
+	if(piece->getBitfieldLength() != bitfieldLength) {
+	  throw new DlAbortEx("piece bitfield length mismatch. expected: %d actual: %d",
+			      piece->getBitfieldLength(), bitfieldLength);
+	}
+	savedBitfield = new unsigned char[bitfieldLength];
+	in.read(reinterpret_cast<char*>(savedBitfield), bitfieldLength);
+	piece->setBitfield(savedBitfield, bitfieldLength);
+	delete [] savedBitfield;
+	savedBitfield = 0;
+	
+	inFlightPieces.push_back(piece);
+      }
+      _pieceStorage->addInFlightPiece(inFlightPieces);
+    } else {
+      BitfieldMan src(pieceLength, totalLength);
+      src.setBitfield(savedBitfield, bitfieldLength);
+      BitfieldMan dest(_dctx->getPieceLength(), totalLength);
+      Util::convertBitfield(&dest, &src);
+      _pieceStorage->setBitfield(dest.getBitfield(), dest.getBitfieldLength());
+      delete [] savedBitfield;
+      savedBitfield = 0;
+    }
     _logger->info(MSG_LOADED_SEGMENT_FILE);
   } catch(ios::failure const& exception) {
     delete [] savedBitfield;

src/Util.cc

@@ -39,6 +39,7 @@
 #include "a2netcompat.h"
 #include "a2time.h"
 #include "DlAbortEx.h"
+#include "BitfieldMan.h"
 #include <ctype.h>
 #include <errno.h>
 #include <fcntl.h>
@@ -782,3 +783,13 @@ void Util::mkdirs(const string& dirpath)
     throw new DlAbortEx(EX_MAKE_DIR, dir.getPath().c_str(), strerror(errno));
   }
 }
+
+void Util::convertBitfield(BitfieldMan* dest, const BitfieldMan* src)
+{
+  for(int32_t index = 0; index < dest->countBlock(); ++index) {
+    if(src->isBitSetOffsetRange((int64_t)index*dest->getBlockLength(),
+				dest->getBlockLength())) {
+      dest->setBit(index);
+    }
+  }
+}

src/Util.h

@@ -44,6 +44,7 @@
 
 class Randomizer;
 extern typedef SharedHandle<Randomizer> RandomizerHandle;
+class BitfieldMan;
 
 #define STRTOLL(X) strtoll(X, (char**)NULL, 10)
 
@@ -153,6 +154,8 @@ public:
   static int32_t alphaToNum(const string& alphabets);
 
   static void mkdirs(const string& dirpath);
+
+  static void convertBitfield(BitfieldMan* dest, const BitfieldMan* src);
 };
 
 #endif // _D_UTIL_H_

test/DefaultBtProgressInfoFileTest.cc

@@ -24,6 +24,7 @@ class DefaultBtProgressInfoFileTest:public CppUnit::TestFixture {
   CPPUNIT_TEST(testSave_nonBt);
   CPPUNIT_TEST(testLoad);
   CPPUNIT_TEST(testLoad_nonBt);
+  CPPUNIT_TEST(testLoad_nonBt_pieceLengthShorter);
   CPPUNIT_TEST_SUITE_END();
 private:
   MockBtContextHandle _btContext;
@@ -35,19 +36,22 @@ public:
 
   void setUp() {
     BtRegistry::unregisterAll();
-
+  }
+   
+  void initializeMembers(int32_t pieceLength, int64_t totalLength)
+  {
     static unsigned char infoHash[] = {
       0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa,
       0xbb, 0xcc, 0xdd, 0xee, 0xff, 0x00, 0xff, 0xff, 0xff, 0xff,
     };
-
+  
     _option = new Option();
     _option->put(PREF_DIR, ".");
 
     _btContext = new MockBtContext();
     _btContext->setInfoHash(infoHash);
 
-    _bitfield = new BitfieldMan(1024, 80*1024);
+    _bitfield = new BitfieldMan(pieceLength, totalLength);
 
     _pieceStorage = new MockPieceStorage();
     _pieceStorage->setBitfield(_bitfield.get());
@@ -75,6 +79,7 @@ public:
   void testLoad();
   void testSave_nonBt();
   void testLoad_nonBt();
+  void testLoad_nonBt_pieceLengthShorter();
 };
 
 #undef BLOCK_LENGTH
@@ -84,7 +89,8 @@ CPPUNIT_TEST_SUITE_REGISTRATION(DefaultBtProgressInfoFileTest);
 
 void DefaultBtProgressInfoFileTest::testLoad()
 {
-  try {
+  initializeMembers(1024, 81920);
+
   _btContext->setName("load");
   _btContext->setPieceLength(1024);
   _btContext->setTotalLength(81920);
@@ -121,16 +127,11 @@ void DefaultBtProgressInfoFileTest::testLoad()
   PieceHandle piece2 = inFlightPieces[1];
   CPPUNIT_ASSERT_EQUAL((int32_t)2, piece2->getIndex());
   CPPUNIT_ASSERT_EQUAL((int32_t)512, piece2->getLength());
-  
-  } catch(Exception* e) {
-    cerr << e->getMsg() << endl;
-    delete e;
-  }
 }
 
 void DefaultBtProgressInfoFileTest::testLoad_nonBt()
 {
-  BtRegistry::unregisterAll();
+  initializeMembers(1024, 81920);
 
   SingleFileDownloadContextHandle dctx =
     new SingleFileDownloadContext(1024, 81920, "load-nonBt");
@@ -165,9 +166,31 @@ void DefaultBtProgressInfoFileTest::testLoad_nonBt()
 
 }
 
+void DefaultBtProgressInfoFileTest::testLoad_nonBt_pieceLengthShorter()
+{
+  initializeMembers(512, 81920);
+
+  SingleFileDownloadContextHandle dctx =
+    new SingleFileDownloadContext(512, 81920, "load-nonBt");
+
+  DefaultBtProgressInfoFile infoFile(dctx, _pieceStorage, _option.get());
+  CPPUNIT_ASSERT_EQUAL(string("./load-nonBt.aria2"), infoFile.getFilename());
+  infoFile.load();
+
+  // check the contents of objects
+
+  // bitfield
+  CPPUNIT_ASSERT_EQUAL(string("fffffffffffffffffffffffffffffffffffffffc"),
+		       Util::toHex(_bitfield->getBitfield(), _bitfield->getBitfieldLength()));
+
+  // the number of in-flight pieces
+  CPPUNIT_ASSERT_EQUAL((int32_t)0,
+		       _pieceStorage->countInFlightPiece());
+}
+
 void DefaultBtProgressInfoFileTest::testSave_nonBt()
 {
-  BtRegistry::unregisterAll();
+  initializeMembers(1024, 81920);
 
   SingleFileDownloadContextHandle dctx =
     new SingleFileDownloadContext(1024, 81920, "save-temp");
@@ -261,6 +284,8 @@ void DefaultBtProgressInfoFileTest::testSave_nonBt()
 
 void DefaultBtProgressInfoFileTest::testSave()
 {
+  initializeMembers(1024, 81920);
+
   _btContext->setName("save-temp");
   _btContext->setPieceLength(1024);
   _btContext->setTotalLength(81920);

test/UtilTest.cc

@@ -1,6 +1,7 @@
 #include "Util.h"
 #include "FixedNumberRandomizer.h"
 #include "DlAbortEx.h"
+#include "BitfieldMan.h"
 #include <string>
 #include <cppunit/extensions/HelperMacros.h>
 
@@ -30,6 +31,7 @@ class UtilTest:public CppUnit::TestFixture {
   CPPUNIT_TEST(testIsUppercase);
   CPPUNIT_TEST(testAlphaToNum);
   CPPUNIT_TEST(testMkdirs);
+  CPPUNIT_TEST(testConvertBitfield);
   CPPUNIT_TEST_SUITE_END();
 private:
 
@@ -58,6 +60,7 @@ public:
   void testIsUppercase();
   void testAlphaToNum();
   void testMkdirs();
+  void testConvertBitfield();
 };
 
 
@@ -390,3 +393,17 @@ void UtilTest::testMkdirs()
     delete ex;
   }
 }
+
+void UtilTest::testConvertBitfield()
+{
+  BitfieldMan srcBitfield(384*1024, 256*1024*256+1);
+  BitfieldMan destBitfield(512*1024, srcBitfield.getTotalLength());
+  srcBitfield.setAllBit();
+  srcBitfield.unsetBit(2);// <- range [768, 1152)
+  // which corresponds to the index [1,2] in destBitfield
+  Util::convertBitfield(&destBitfield, &srcBitfield);
+  
+  CPPUNIT_ASSERT_EQUAL(string("9fffffffffffffffffffffffffffffff80"),
+		       Util::toHex(destBitfield.getBitfield(),
+				   destBitfield.getBitfieldLength()));
+}