Added --stream-piece-selector option.

This option specifies piece selection algorithm used in HTTP/FTP
download. Piece means fixed length segment which is downloaded in
parallel in segmented download. If 'default' is given, aria2 selects
piece so that it reduces the number of establishing connection. This
is reasonable default behaviour because establishing connection is an
expensive operation.  If 'inorder' is given, aria2 selects piece which
has minimum index. Index=0 means first of the file. This will be
useful to view movie while downloading it. --enable-http-pipelining
option may be useful to reduce reconnection overhead.  Please note
that aria2 honors --min-split-size option, so it will be necessary to
specify a reasonable value to --min-split-size option.

src/BitfieldMan.cc

@@ -309,6 +309,73 @@ bool BitfieldMan::getSparseMissingUnusedIndex
   }
 }
 
+namespace {
+template<typename Array>
+bool getInorderMissingUnusedIndex
+(size_t& index,
+ size_t minSplitSize,
+ const Array& bitfield,
+ const unsigned char* useBitfield,
+ size_t blockLength_,
+ size_t blocks)
+{
+  // We always return first piece if it is available.
+  if(!bitfield::test(bitfield, blocks, 0) &&
+     !bitfield::test(useBitfield, blocks, 0)) {
+    index = 0;
+    return true;
+  }
+  for(size_t i = 1; i < blocks;) {
+    if(!bitfield::test(bitfield, blocks, i) &&
+       !bitfield::test(useBitfield, blocks, i)) {
+      // If previous piece has already been retrieved, we can download
+      // from this index.
+      if(!bitfield::test(useBitfield, blocks, i-1) &&
+         bitfield::test(bitfield, blocks, i-1)) {
+        index = i;
+        return true;
+      }
+      // Check free space of minSplitSize.
+      size_t j;
+      for(j = i; j < blocks; ++j) {
+        if(bitfield::test(bitfield, blocks, j) ||
+           bitfield::test(useBitfield, blocks, j)) {
+          break;
+        }
+        if((j-i+1)*blockLength_ >= minSplitSize) {
+          index = j;
+          return true;
+        }
+      }
+      i = j+1;
+    } else {
+      ++i;
+    }
+  }
+  return false;
+}
+} // namespace
+
+bool BitfieldMan::getInorderMissingUnusedIndex
+(size_t& index,
+ size_t minSplitSize,
+ const unsigned char* ignoreBitfield,
+ size_t ignoreBitfieldLength) const
+{
+  if(filterEnabled_) {
+    return aria2::getInorderMissingUnusedIndex
+      (index, minSplitSize,
+       array(ignoreBitfield)|~array(filterBitfield_)|
+       array(bitfield_)|array(useBitfield_),
+       useBitfield_, blockLength_, blocks_);
+  } else {
+    return aria2::getInorderMissingUnusedIndex
+      (index, minSplitSize,
+       array(ignoreBitfield)|array(bitfield_)|array(useBitfield_),
+       useBitfield_, blockLength_, blocks_);
+  }
+}
+
 namespace {
 template<typename Array>
 bool copyBitfield(unsigned char* dst, const Array& src, size_t blocks)

src/BitfieldMan.h

@@ -135,6 +135,16 @@ public:
    const unsigned char* ignoreBitfield,
    size_t ignoreBitfieldLength) const;
 
+  // Stores missing bit index to index. This function selects smallest
+  // index of missing piece, considering minSplitSize.  Set bits in
+  // ignoreBitfield are excluded. Returns true if such bit index is
+  // found. Otherwise returns false.
+  bool getInorderMissingUnusedIndex
+  (size_t& index,
+   size_t minSplitSize,
+   const unsigned char* ignoreBitfield,
+   size_t ignoreBitfieldLength) const;
+
   // affected by filter
   bool getAllMissingIndexes(unsigned char* misbitfield, size_t mislen) const;
 

src/DefaultPieceStorage.cc

@@ -57,6 +57,7 @@
 #include "fmt.h"
 #include "RarestPieceSelector.h"
 #include "DefaultStreamPieceSelector.h"
+#include "InOrderStreamPieceSelector.h"
 #include "array_fun.h"
 #include "PieceStatMan.h"
 #include "wallclock.h"
@@ -77,9 +78,18 @@ DefaultPieceStorage::DefaultPieceStorage
    endGamePieceNum_(END_GAME_PIECE_NUM),
    option_(option),
    pieceStatMan_(new PieceStatMan(downloadContext->getNumPieces(), true)),
-   pieceSelector_(new RarestPieceSelector(pieceStatMan_)),
-   streamPieceSelector_(new DefaultStreamPieceSelector(bitfieldMan_))
-{}
+   pieceSelector_(new RarestPieceSelector(pieceStatMan_))
+{
+  const std::string& pieceSelectorOpt =
+    option_->get(PREF_STREAM_PIECE_SELECTOR);
+  if(pieceSelectorOpt.empty() || pieceSelectorOpt == A2_V_DEFAULT) {
+    streamPieceSelector_ = SharedHandle<StreamPieceSelector>
+      (new DefaultStreamPieceSelector(bitfieldMan_));
+  } else if(pieceSelectorOpt == V_INORDER) {
+    streamPieceSelector_ = SharedHandle<StreamPieceSelector>
+      (new InorderStreamPieceSelector(bitfieldMan_));
+  }
+}
 
 DefaultPieceStorage::~DefaultPieceStorage()
 {

src/InorderStreamPieceSelector.cc

@@ -0,0 +1,57 @@
+/* <!-- copyright */
+/*
+ * aria2 - The high speed download utility
+ *
+ * Copyright (C) 2011 Tatsuhiro Tsujikawa
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of portions of this program with the
+ * OpenSSL library under certain conditions as described in each
+ * individual source file, and distribute linked combinations
+ * including the two.
+ * You must obey the GNU General Public License in all respects
+ * for all of the code used other than OpenSSL.  If you modify
+ * file(s) with this exception, you may extend this exception to your
+ * version of the file(s), but you are not obligated to do so.  If you
+ * do not wish to do so, delete this exception statement from your
+ * version.  If you delete this exception statement from all source
+ * files in the program, then also delete it here.
+ */
+/* copyright --> */
+#include "InorderStreamPieceSelector.h"
+#include "BitfieldMan.h"
+
+namespace aria2 {
+
+InorderStreamPieceSelector::InorderStreamPieceSelector
+(BitfieldMan* bitfieldMan)
+  : bitfieldMan_(bitfieldMan)
+{}
+
+InorderStreamPieceSelector::~InorderStreamPieceSelector() {}
+
+bool InorderStreamPieceSelector::select
+(size_t& index,
+ size_t minSplitSize,
+ const unsigned char* ignoreBitfield,
+ size_t length)
+{
+  return bitfieldMan_->getInorderMissingUnusedIndex
+    (index, minSplitSize, ignoreBitfield, length);
+}
+
+} // namespace aria2

src/InorderStreamPieceSelector.h

@@ -0,0 +1,60 @@
+/* <!-- copyright */
+/*
+ * aria2 - The high speed download utility
+ *
+ * Copyright (C) 2011 Tatsuhiro Tsujikawa
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of portions of this program with the
+ * OpenSSL library under certain conditions as described in each
+ * individual source file, and distribute linked combinations
+ * including the two.
+ * You must obey the GNU General Public License in all respects
+ * for all of the code used other than OpenSSL.  If you modify
+ * file(s) with this exception, you may extend this exception to your
+ * version of the file(s), but you are not obligated to do so.  If you
+ * do not wish to do so, delete this exception statement from your
+ * version.  If you delete this exception statement from all source
+ * files in the program, then also delete it here.
+ */
+/* copyright --> */
+#ifndef D_INORDER_STREAM_PIECE_SELECTOR_H
+#define D_INORDER_STREAM_PIECE_SELECTOR_H
+
+#include "StreamPieceSelector.h"
+
+namespace aria2 {
+
+class BitfieldMan;
+
+class InorderStreamPieceSelector:public StreamPieceSelector {
+public:
+  InorderStreamPieceSelector(BitfieldMan* bitfieldMan);
+  virtual ~InorderStreamPieceSelector();
+
+  virtual bool select
+  (size_t& index,
+   size_t minSplitSize,
+   const unsigned char* ignoreBitfield,
+   size_t length);
+private:
+  BitfieldMan* bitfieldMan_;
+};
+
+} // namespace aria2
+
+#endif // D_INORDER_STREAM_PIECE_SELECTOR_H

src/Makefile.am

@@ -223,7 +223,8 @@ SRCS =  Socket.h\
 	HttpServerResponseCommand.cc HttpServerResponseCommand.h\
 	HttpServer.cc HttpServer.h\
 	StreamPieceSelector.h\
-	DefaultStreamPieceSelector.cc DefaultStreamPieceSelector.h
+	DefaultStreamPieceSelector.cc DefaultStreamPieceSelector.h\
+	InorderStreamPieceSelector.cc InorderStreamPieceSelector.h
 
 if ENABLE_XML_RPC
 SRCS += XmlRpcRequestParserController.cc XmlRpcRequestParserController.h\

src/OptionHandlerFactory.cc

@@ -861,6 +861,17 @@ OptionHandlers OptionHandlerFactory::createOptionHandlers()
     op->hide();
     handlers.push_back(op);
   }
+  {
+    SharedHandle<OptionHandler> op(new ParameterOptionHandler
+                                   (PREF_STREAM_PIECE_SELECTOR,
+                                    TEXT_STREAM_PIECE_SELECTOR,
+                                    A2_V_DEFAULT,
+                                    A2_V_DEFAULT,
+                                    V_INORDER));
+    op->addTag(TAG_FTP);
+    op->addTag(TAG_HTTP);
+    handlers.push_back(op);
+  }
   {
     SharedHandle<OptionHandler> op(new NumberOptionHandler
                                    (PREF_TIMEOUT,

src/prefs.cc

@@ -41,9 +41,11 @@ namespace aria2 {
  */
 const std::string A2_V_TRUE("true");
 const std::string A2_V_FALSE("false");
+const std::string A2_V_DEFAULT("default");
 const std::string V_NONE("none");
 const std::string V_MEM("mem");
 const std::string V_ALL("all");
+
 /**
  * General preferences
  */
@@ -220,6 +222,8 @@ const std::string PREF_RETRY_WAIT("retry-wait");
 const std::string PREF_ASYNC_DNS_SERVER("async-dns-server");
 // value: true | false
 const std::string PREF_SHOW_CONSOLE_READOUT("show-console-readout");
+// value: default | inorder
+const std::string PREF_STREAM_PIECE_SELECTOR("stream-piece-selector");
 
 /**
  * FTP related preferences

src/prefs.h

@@ -45,6 +45,7 @@ namespace aria2 {
  */
 extern const std::string A2_V_TRUE;
 extern const std::string A2_V_FALSE;
+extern const std::string A2_V_DEFAULT;
 extern const std::string V_NONE;
 extern const std::string V_MEM;
 extern const std::string V_ALL;
@@ -224,6 +225,8 @@ extern const std::string PREF_RETRY_WAIT;
 extern const std::string PREF_ASYNC_DNS_SERVER;
 // value: true | false
 extern const std::string PREF_SHOW_CONSOLE_READOUT;
+// value: default | inorder
+extern const std::string PREF_STREAM_PIECE_SELECTOR;
 
 /**
  * FTP related preferences

src/usage_text.h

@@ -790,3 +790,22 @@
     "                              metalink:url and metalink:metaurl element in a\n" \
     "                              metalink file stored in local disk. If URI points\n" \
     "                              to a directory, URI must end with '/'.")
+#define TEXT_STREAM_PIECE_SELECTOR              \
+  _(" --stream-piece-selector=SELECTOR Specify piece selection algorithm\n" \
+    "                              used in HTTP/FTP download. Piece means fixed\n" \
+    "                              length segment which is downloaded in parallel\n" \
+    "                              in segmented download. If 'default' is given,\n" \
+    "                              aria2 selects piece so that it reduces the\n" \
+    "                              number of establishing connection. This is\n" \
+    "                              reasonable default behaviour because\n" \
+    "                              establishing connection is an expensive\n" \
+    "                              operation.\n"                        \
+    "                              If 'inorder' is given, aria2 selects piece which\n" \
+    "                              has minimum index. Index=0 means first of the\n" \
+    "                              file. This will be useful to view movie while\n" \
+    "                              downloading it. --enable-http-pipelining option\n" \
+    "                              may be useful to reduce reconnection overhead.\n" \
+    "                              Please note that aria2 honors\n"     \
+    "                              --min-split-size option, so it will be necessary\n" \
+    "                              to specify a reasonable value to\n"  \
+    "                              --min-split-size option.")

test/BitfieldManTest.cc

@@ -36,6 +36,7 @@ class BitfieldManTest:public CppUnit::TestFixture {
   CPPUNIT_TEST(testCountMissingBlock);
   CPPUNIT_TEST(testZeroLengthFilter);
   CPPUNIT_TEST(testGetFirstNMissingUnusedIndex);
+  CPPUNIT_TEST(testGetInorderMissingUnusedIndex);
   CPPUNIT_TEST_SUITE_END();
 public:
   void testGetBlockSize();
@@ -62,6 +63,7 @@ public:
   void testCountMissingBlock();
   void testZeroLengthFilter();
   void testGetFirstNMissingUnusedIndex();
+  void testGetInorderMissingUnusedIndex();
 };
 
 
@@ -640,4 +642,68 @@ void BitfieldManTest::testGetFirstNMissingUnusedIndex()
   CPPUNIT_ASSERT_EQUAL((size_t)9, out[0]);
 }
 
+void BitfieldManTest::testGetInorderMissingUnusedIndex()
+{
+  BitfieldMan bt(1024, 1024*20);
+  const size_t length = 3;
+  unsigned char ignoreBitfield[length];
+  memset(ignoreBitfield, 0, sizeof(ignoreBitfield));
+  size_t minSplitSize = 1024;
+  size_t index;
+  // 00000|00000|00000|00000
+  CPPUNIT_ASSERT
+    (bt.getInorderMissingUnusedIndex
+     (index, minSplitSize, ignoreBitfield, length));
+  CPPUNIT_ASSERT_EQUAL((size_t)0, index);
+  bt.setUseBit(0);
+  // 10000|00000|00000|00000
+  CPPUNIT_ASSERT
+    (bt.getInorderMissingUnusedIndex
+     (index, minSplitSize, ignoreBitfield, length));
+  CPPUNIT_ASSERT_EQUAL((size_t)1, index);
+  minSplitSize = 1024*2;
+  CPPUNIT_ASSERT
+    (bt.getInorderMissingUnusedIndex
+     (index, minSplitSize, ignoreBitfield, length));
+  CPPUNIT_ASSERT_EQUAL((size_t)2, index);
+  bt.unsetUseBit(0);
+  bt.setBit(0);
+  CPPUNIT_ASSERT
+    (bt.getInorderMissingUnusedIndex
+     (index, minSplitSize, ignoreBitfield, length));
+  CPPUNIT_ASSERT_EQUAL((size_t)1, index);
+  bt.setAllBit();
+  bt.unsetBit(10);
+  // 11111|11111|01111|11111
+  CPPUNIT_ASSERT
+    (bt.getInorderMissingUnusedIndex
+     (index, minSplitSize, ignoreBitfield, length));
+  CPPUNIT_ASSERT_EQUAL((size_t)10, index);
+  bt.setUseBit(10);
+  CPPUNIT_ASSERT
+    (!bt.getInorderMissingUnusedIndex
+     (index, minSplitSize, ignoreBitfield, length));
+  bt.unsetUseBit(10);
+  bt.setAllBit();
+  // 11111|11111|11111|11111
+  CPPUNIT_ASSERT
+    (!bt.getInorderMissingUnusedIndex
+     (index, minSplitSize, ignoreBitfield, length));
+  bt.clearAllBit();
+  // 00000|00000|00000|00000
+  for(int i = 0; i <= 1; ++i) {
+    bitfield::flipBit(ignoreBitfield, length, i);
+  }
+  CPPUNIT_ASSERT
+    (bt.getInorderMissingUnusedIndex
+     (index, minSplitSize, ignoreBitfield, length));
+  CPPUNIT_ASSERT_EQUAL((size_t)2, index);
+  bt.addFilter(1024*3, 1024*3);
+  bt.enableFilter();
+  CPPUNIT_ASSERT
+    (bt.getInorderMissingUnusedIndex
+     (index, minSplitSize, ignoreBitfield, length));
+  CPPUNIT_ASSERT_EQUAL((size_t)3, index);
+}
+
 } // namespace aria2