2007-03-24 Tatsuhiro Tsujikawa <tujikawa at rednoah dot com>

To add the ability to resume downloading a partially downloaded 
file
	which is downloaded from the beginning:
	* src/FileAllocator.h: Made abstract class. New 
DefaultFileAllocator
	takes this role.
	* src/main.cc: Added -c option.
	* src/BitfieldMan.h, src/BitfieldMan.cc (setBitRange): New 
function.
	* src/DiskWriter.h (openExistingFile): Added totalLength 
argument.
	* src/prefs.h (PREF_CONTINUE): New definition.
	* src/SegmentMan.h, src/SegmentMan.cc (markPieceDone): New 
function.
	* src/DefaultDiskWriter.cc
	(createNewDiskWriter): Add GlowFileAllocator to the new object.
	* src/AbstractDiskWriter.h
	(glowFileAllocator): New variable.
	* src/AbstractDiskWriter.cc
	(openExistingFile): Now preallocate file space from the end of 
the
	existing file if totalLength argument is specified and grater 
than 0.
	* src/UrlRequestInfo.cc: If -c option is specified and the file
	to download exists in local, continue the download of the file.
	--allow-overwrite=true is assumed in this context.
	* src/DefaultFileAllocator.h, src/DefaultFileAllocator.cc: New 
class.
	* src/GlowFileAllocator.h, src/GlowFileAllocator.cc: New class.

	Throw exception if --check-integrity=true is specified but chunk
	checksums are not provided:
	* src/UrlRequestInfo.cc

ChangeLog

@@ -1,3 +1,31 @@
+2007-03-24  Tatsuhiro Tsujikawa  <tujikawa at rednoah dot com>
+
+	To add the ability to resume downloading a partially downloaded file
+	which is downloaded from the beginning:
+	* src/FileAllocator.h: Made abstract class. New DefaultFileAllocator
+	takes this role.
+	* src/main.cc: Added -c option.
+	* src/BitfieldMan.h, src/BitfieldMan.cc (setBitRange): New function.
+	* src/DiskWriter.h (openExistingFile): Added totalLength argument.
+	* src/prefs.h (PREF_CONTINUE): New definition.
+	* src/SegmentMan.h, src/SegmentMan.cc (markPieceDone): New function.
+	* src/DefaultDiskWriter.cc
+	(createNewDiskWriter): Add GlowFileAllocator to the new object.
+	* src/AbstractDiskWriter.h
+	(glowFileAllocator): New variable.
+	* src/AbstractDiskWriter.cc
+	(openExistingFile): Now preallocate file space from the end of the
+	existing file if totalLength argument is specified and grater than 0.
+	* src/UrlRequestInfo.cc: If -c option is specified and the file
+	to download exists in local, continue the download of the file.
+	--allow-overwrite=true is assumed in this context.
+	* src/DefaultFileAllocator.h, src/DefaultFileAllocator.cc: New class.
+	* src/GlowFileAllocator.h, src/GlowFileAllocator.cc: New class.
+
+	Throw exception if --check-integrity=true is specified but chunk
+	checksums are not provided:
+	* src/UrlRequestInfo.cc
+	
 2007-03-21  Tatsuhiro Tsujikawa  <tujikawa at rednoah dot com>
 
 	* src/Request.h: Use AuthResolver to get authentication information.

TODO

@@ -28,4 +28,5 @@
 * -c command line option to continue the download of existing file assuming
 that it was downloaded from the beginning.
 * Continue file allocation with existing file
-* keep-alive
+* Netrc, mode 600, enabled in ftp, http, all
+* preallocate file in MultiDiskAdaptor

src/AbstractDiskWriter.cc

@@ -47,6 +47,7 @@
 AbstractDiskWriter::AbstractDiskWriter():
   fd(0),
   fileAllocator(0),
+  glowFileAllocator(0),
   logger(LogFactory::getInstance())
 {}
 
@@ -71,7 +72,7 @@ void AbstractDiskWriter::closeFile() {
   }
 }
 
-void AbstractDiskWriter::openExistingFile(const string& filename) {
+void AbstractDiskWriter::openExistingFile(const string& filename, int64_t totalLength) {
   this->filename = filename;
   File f(filename);
   if(!f.isFile()) {
@@ -81,6 +82,14 @@ void AbstractDiskWriter::openExistingFile(const string& filename) {
   if((fd = open(filename.c_str(), O_RDWR, S_IRUSR|S_IWUSR)) < 0) {
     throw new DlAbortEx(EX_FILE_OPEN, filename.c_str(), strerror(errno));
   }
+  if(f.size() < totalLength) {
+    if(!fileAllocator.isNull()) {
+      logger->notice("Allocating file %s, %s bytes",
+		     filename.c_str(),
+		     Util::ullitos(totalLength).c_str());
+      glowFileAllocator->allocate(fd, totalLength);
+    }
+  }
 }
 
 void AbstractDiskWriter::createFile(const string& filename, int32_t addFlags) {

src/AbstractDiskWriter.h

@@ -44,6 +44,7 @@ protected:
   string filename;
   int32_t fd;
   FileAllocatorHandle fileAllocator;
+  FileAllocatorHandle glowFileAllocator;
   const Logger* logger;
 
   void createFile(const string& filename, int32_t addFlags = 0);
@@ -62,7 +63,7 @@ public:
 
   virtual void closeFile();
 
-  virtual void openExistingFile(const string& filename);
+  virtual void openExistingFile(const string& filename, int64_t totalLength = 0);
 
 #ifdef ENABLE_MESSAGE_DIGEST
   virtual string messageDigest(int64_t offset, int64_t length,
@@ -73,9 +74,15 @@ public:
 
   virtual int32_t readData(char* data, int32_t len, int64_t offset);
 
-  void setFileAllocator(const FileAllocatorHandle& fileAllocator) {
+  void setFileAllocator(const FileAllocatorHandle& fileAllocator)
+  {
     this->fileAllocator = fileAllocator;
   }
+
+  void setGlowFileAllocator(const FileAllocatorHandle& fileAllocator)
+  {
+    this->glowFileAllocator = fileAllocator;
+  }
 };
 
 #endif // _D_ABSTRACT_DISK_WRITER_H_

src/BitfieldMan.cc

@@ -645,6 +645,14 @@ void BitfieldMan::unsetBitRange(int32_t startIndex, int32_t endIndex)
   updateCache();
 }
 
+void BitfieldMan::setBitRange(int32_t startIndex, int32_t endIndex)
+{
+  for(int32_t i = startIndex; i <= endIndex; ++i) {
+    setBit(i);
+  }
+  updateCache();
+}
+
 bool BitfieldMan::isBitSetOffsetRange(int64_t offset, int64_t length) const
 {
   if(length <= 0) {

src/BitfieldMan.h

@@ -255,6 +255,8 @@ public:
 
   void unsetBitRange(int32_t startIndex, int32_t endIndex);
 
+  void setBitRange(int32_t startIndex, int32_t endIndex);
+
   bool isBitSetOffsetRange(int64_t offset, int64_t length) const;
 
   int64_t getMissingUnusedLength(int32_t startingIndex) const;

src/ByteArrayDiskWriter.cc

@@ -68,7 +68,8 @@ void ByteArrayDiskWriter::closeFile() {
   clear();
 }
 
-void ByteArrayDiskWriter::openExistingFile(const string& filename) {
+void ByteArrayDiskWriter::openExistingFile(const string& filename,
+					   int64_t totalLength) {
   openFile(filename);
 }
 

src/ByteArrayDiskWriter.h

@@ -55,7 +55,7 @@ public:
 
   virtual void closeFile();
 
-  virtual void openExistingFile(const string& filename);
+  virtual void openExistingFile(const string& filename, int64_t totalLength = 0);
 
   // position is ignored
   virtual void writeData(const char* data, int32_t len, int64_t position = 0);

src/DefaultDiskWriter.cc

@@ -35,7 +35,8 @@
 #include "DefaultDiskWriter.h"
 #include "DlAbortEx.h"
 #include "message.h"
-#include "FileAllocator.h"
+#include "DefaultFileAllocator.h"
+#include "GlowFileAllocator.h"
 #include "prefs.h"
 #include "Util.h"
 #include <errno.h>
@@ -69,11 +70,16 @@ DefaultDiskWriter* DefaultDiskWriter::createNewDiskWriter(const Option* option)
 {
   DefaultDiskWriter* diskWriter = new DefaultDiskWriter();
   if(option->get(PREF_FILE_ALLOCATION) == V_PREALLOC) {
-    FileAllocatorHandle allocator = new FileAllocator();
+    DefaultFileAllocatorHandle allocator = new DefaultFileAllocator();
     allocator->setFileAllocationMonitor(FileAllocationMonitorFactory::getFactory()->createNewMonitor());
     diskWriter->setFileAllocator(allocator);
+
+    GlowFileAllocatorHandle glowAllocator = new GlowFileAllocator();
+    glowAllocator->setFileAllocationMonitor(FileAllocationMonitorFactory::getFactory()->createNewMonitor());
+    diskWriter->setGlowFileAllocator(glowAllocator);
   } else {
     diskWriter->setFileAllocator(0);
+    diskWriter->setGlowFileAllocator(0);
   }
   return diskWriter;
 }

src/DefaultFileAllocator.cc

@@ -1,4 +1,3 @@
-
 /* <!-- copyright */
 /*
  * aria2 - The high speed download utility
@@ -33,14 +32,14 @@
  * files in the program, then also delete it here.
  */
 /* copyright --> */
-#include "FileAllocator.h"
+#include "DefaultFileAllocator.h"
 #include "DlAbortEx.h"
 #include "TimeA2.h"
 #include <sys/types.h>
 #include <unistd.h>
 #include <errno.h>
 
-void FileAllocator::allocate(int fd, int64_t totalLength)
+void DefaultFileAllocator::allocate(int fd, int64_t totalLength)
 {
   if(0 != lseek(fd, 0, SEEK_SET)) {
     throw new DlAbortEx("Seek failed: %s", strerror(errno));

src/DefaultFileAllocator.h

@@ -0,0 +1,51 @@
+/* <!-- copyright */
+/*
+ * aria2 - The high speed download utility
+ *
+ * Copyright (C) 2006 Tatsuhiro Tsujikawa
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of portions of this program with the
+ * OpenSSL library under certain conditions as described in each
+ * individual source file, and distribute linked combinations
+ * including the two.
+ * You must obey the GNU General Public License in all respects
+ * for all of the code used other than OpenSSL.  If you modify
+ * file(s) with this exception, you may extend this exception to your
+ * version of the file(s), but you are not obligated to do so.  If you
+ * do not wish to do so, delete this exception statement from your
+ * version.  If you delete this exception statement from all source
+ * files in the program, then also delete it here.
+ */
+/* copyright --> */
+#ifndef _D_DEFAULT_FILE_ALLOCATOR_H_
+#define _D_DEFAULT_FILE_ALLOCATOR_H_
+
+#include "FileAllocator.h"
+
+class DefaultFileAllocator : public FileAllocator {
+public:
+  DefaultFileAllocator() {}
+
+  virtual ~DefaultFileAllocator() {}
+
+  virtual void allocate(int fd, int64_t totalLength);
+};
+
+typedef SharedHandle<DefaultFileAllocator> DefaultFileAllocatorHandle;
+
+#endif // _D_DEFAULT_FILE_ALLOCATOR_H_

src/DiskAdaptorWriter.h

@@ -62,7 +62,7 @@ public:
     diskAdaptor->closeFile();
   }
 
-  virtual void openExistingFile(const string& filename)
+  virtual void openExistingFile(const string& filename, int64_t totalLength = 0)
   {
     diskAdaptor->openExistingFile();
   }

src/DiskWriter.h

@@ -70,7 +70,7 @@ public:
    *
    * @param filename the file name to be opened.
    */
-  virtual void openExistingFile(const string& filename) = 0;
+  virtual void openExistingFile(const string& filename, int64_t totalLength = 0) = 0;
 
   /*
    * Writes len bytes from data to this binary stream at offset position.

src/FileAllocator.h

@@ -40,14 +40,14 @@
 #include "NullFileAllocationMonitor.h"
 
 class FileAllocator {
-private:
+protected:
   FileAllocationMonitorHandle fileAllocationMonitor;
 public:
   FileAllocator():fileAllocationMonitor(new NullFileAllocationMonitor()) {}
 
-  ~FileAllocator() {}
+  virtual ~FileAllocator() {}
 
-  void allocate(int fd, int64_t totalLength);
+  virtual void allocate(int fd, int64_t totalLength) = 0;
 
   void setFileAllocationMonitor(const FileAllocationMonitorHandle& monitor)
   {

src/GlowFileAllocator.cc

@@ -0,0 +1,74 @@
+/* <!-- copyright */
+/*
+ * aria2 - The high speed download utility
+ *
+ * Copyright (C) 2006 Tatsuhiro Tsujikawa
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of portions of this program with the
+ * OpenSSL library under certain conditions as described in each
+ * individual source file, and distribute linked combinations
+ * including the two.
+ * You must obey the GNU General Public License in all respects
+ * for all of the code used other than OpenSSL.  If you modify
+ * file(s) with this exception, you may extend this exception to your
+ * version of the file(s), but you are not obligated to do so.  If you
+ * do not wish to do so, delete this exception statement from your
+ * version.  If you delete this exception statement from all source
+ * files in the program, then also delete it here.
+ */
+/* copyright --> */
+#include "GlowFileAllocator.h"
+#include "DlAbortEx.h"
+#include "TimeA2.h"
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <errno.h>
+
+void GlowFileAllocator::allocate(int fd, int64_t totalLength)
+{
+  struct stat fs;
+  if(fstat(fd, &fs) < 0) {
+    throw new DlAbortEx("fstat filed: %s", strerror(errno));
+  }
+  if(fs.st_size != lseek(fd, 0, SEEK_END)) {
+    throw new DlAbortEx("Seek failed: %s", strerror(errno));
+  }
+  int32_t bufSize = 4096;
+  char buf[4096];
+  memset(buf, 0, bufSize);
+  int64_t x = (totalLength-fs.st_size+bufSize-1)/bufSize;
+  fileAllocationMonitor->setMinValue(0);
+  fileAllocationMonitor->setMaxValue(totalLength);
+  fileAllocationMonitor->setCurrentValue(fs.st_size);
+  fileAllocationMonitor->showProgress();
+  Time cp;
+  for(int64_t i = 0; i < x; ++i) {
+    if(write(fd, buf, bufSize) < 0) {
+      throw new DlAbortEx("Allocation failed: %s", strerror(errno));
+    }
+    if(cp.elapsedInMillis(500)) {
+      fileAllocationMonitor->setCurrentValue(i*bufSize);
+      fileAllocationMonitor->showProgress();
+      cp.reset();
+    }
+  }
+  fileAllocationMonitor->setCurrentValue(totalLength);
+  fileAllocationMonitor->showProgress();
+  ftruncate(fd, totalLength);
+}

src/GlowFileAllocator.h

@@ -0,0 +1,51 @@
+/* <!-- copyright */
+/*
+ * aria2 - The high speed download utility
+ *
+ * Copyright (C) 2006 Tatsuhiro Tsujikawa
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of portions of this program with the
+ * OpenSSL library under certain conditions as described in each
+ * individual source file, and distribute linked combinations
+ * including the two.
+ * You must obey the GNU General Public License in all respects
+ * for all of the code used other than OpenSSL.  If you modify
+ * file(s) with this exception, you may extend this exception to your
+ * version of the file(s), but you are not obligated to do so.  If you
+ * do not wish to do so, delete this exception statement from your
+ * version.  If you delete this exception statement from all source
+ * files in the program, then also delete it here.
+ */
+/* copyright --> */
+#ifndef _D_GLOW_FILE_ALLOCATOR_H_
+#define _D_GLOW_FILE_ALLOCATOR_H_
+
+#include "FileAllocator.h"
+
+class GlowFileAllocator : public FileAllocator {
+public:
+  GlowFileAllocator() {}
+
+  virtual ~GlowFileAllocator() {}
+
+  virtual void allocate(int fd, int64_t totalLength);
+};
+
+typedef SharedHandle<GlowFileAllocator> GlowFileAllocatorHandle;
+
+#endif // _D_GLOW_FILE_ALLOCATOR_H_

src/Makefile.am

@@ -61,7 +61,7 @@ SRCS =  Socket.h\
 	BitfieldManFactory.cc BitfieldManFactory.h\
 	Randomizer.h\
 	SimpleRandomizer.cc SimpleRandomizer.h\
-	FileAllocator.cc FileAllocator.h\
+	FileAllocator.h\
 	FileAllocationMonitor.cc FileAllocationMonitor.h\
 	ConsoleFileAllocationMonitor.cc ConsoleFileAllocationMonitor.h\
 	ChunkChecksumValidator.cc ChunkChecksumValidator.h\
@@ -76,7 +76,9 @@ SRCS =  Socket.h\
 	AbstractAuthResolver.h\
 	DefaultAuthResolver.cc DefaultAuthResolver.h\
 	NetrcAuthResolver.cc NetrcAuthResolver.h\
-	RequestFactory.cc RequestFactory.h
+	RequestFactory.cc RequestFactory.h\
+	DefaultFileAllocator.cc DefaultFileAllocator.h\
+	GlowFileAllocator.cc GlowFileAllocator.h
 #	debug_new.cpp
 
 if ENABLE_ASYNC_DNS

src/Makefile.in

@@ -210,18 +210,19 @@ am__libaria2c_a_SOURCES_DIST = Socket.h SocketCore.cc SocketCore.h \
 	UrlRequestInfo.cc UrlRequestInfo.h SpeedCalc.cc SpeedCalc.h \
 	PeerStat.h BitfieldMan.cc BitfieldMan.h BitfieldManFactory.cc \
 	BitfieldManFactory.h Randomizer.h SimpleRandomizer.cc \
-	SimpleRandomizer.h FileAllocator.cc FileAllocator.h \
+	SimpleRandomizer.h FileAllocator.h FileAllocationMonitor.cc \
-	FileAllocationMonitor.cc FileAllocationMonitor.h \
+	FileAllocationMonitor.h ConsoleFileAllocationMonitor.cc \
-	ConsoleFileAllocationMonitor.cc ConsoleFileAllocationMonitor.h \
+	ConsoleFileAllocationMonitor.h ChunkChecksumValidator.cc \
-	ChunkChecksumValidator.cc ChunkChecksumValidator.h \
+	ChunkChecksumValidator.h HttpResponse.cc HttpResponse.h \
-	HttpResponse.cc HttpResponse.h HttpRequest.cc HttpRequest.h \
+	HttpRequest.cc HttpRequest.h Range.h \
-	Range.h AbstractProxyRequestCommand.cc \
+	AbstractProxyRequestCommand.cc AbstractProxyRequestCommand.h \
-	AbstractProxyRequestCommand.h AbstractProxyResponseCommand.cc \
+	AbstractProxyResponseCommand.cc AbstractProxyResponseCommand.h \
-	AbstractProxyResponseCommand.h Netrc.cc Netrc.h AuthConfig.cc \
+	Netrc.cc Netrc.h AuthConfig.cc AuthConfig.h AuthResolver.h \
-	AuthConfig.h AuthResolver.h AbstractAuthResolver.h \
+	AbstractAuthResolver.h DefaultAuthResolver.cc \
-	DefaultAuthResolver.cc DefaultAuthResolver.h \
+	DefaultAuthResolver.h NetrcAuthResolver.cc NetrcAuthResolver.h \
-	NetrcAuthResolver.cc NetrcAuthResolver.h RequestFactory.cc \
+	RequestFactory.cc RequestFactory.h DefaultFileAllocator.cc \
-	RequestFactory.h NameResolver.cc NameResolver.h MetaEntry.h \
+	DefaultFileAllocator.h GlowFileAllocator.cc \
+	GlowFileAllocator.h NameResolver.cc NameResolver.h MetaEntry.h \
 	Data.cc Data.h Dictionary.cc Dictionary.h List.cc List.h \
 	MetaFileUtil.cc MetaFileUtil.h MetaEntryVisitor.h \
 	ShaVisitor.cc ShaVisitor.h PeerConnection.cc PeerConnection.h \
@@ -384,13 +385,14 @@ am__objects_4 = SocketCore.$(OBJEXT) Command.$(OBJEXT) \
 	DownloadEngineFactory.$(OBJEXT) UrlRequestInfo.$(OBJEXT) \
 	SpeedCalc.$(OBJEXT) BitfieldMan.$(OBJEXT) \
 	BitfieldManFactory.$(OBJEXT) SimpleRandomizer.$(OBJEXT) \
-	FileAllocator.$(OBJEXT) FileAllocationMonitor.$(OBJEXT) \
+	FileAllocationMonitor.$(OBJEXT) \
 	ConsoleFileAllocationMonitor.$(OBJEXT) \
 	ChunkChecksumValidator.$(OBJEXT) HttpResponse.$(OBJEXT) \
 	HttpRequest.$(OBJEXT) AbstractProxyRequestCommand.$(OBJEXT) \
 	AbstractProxyResponseCommand.$(OBJEXT) Netrc.$(OBJEXT) \
 	AuthConfig.$(OBJEXT) DefaultAuthResolver.$(OBJEXT) \
 	NetrcAuthResolver.$(OBJEXT) RequestFactory.$(OBJEXT) \
+	DefaultFileAllocator.$(OBJEXT) GlowFileAllocator.$(OBJEXT) \
 	$(am__objects_1) $(am__objects_2) $(am__objects_3)
 am_libaria2c_a_OBJECTS = $(am__objects_4)
 libaria2c_a_OBJECTS = $(am_libaria2c_a_OBJECTS)
@@ -594,18 +596,19 @@ SRCS = Socket.h SocketCore.cc SocketCore.h Command.cc Command.h \
 	UrlRequestInfo.cc UrlRequestInfo.h SpeedCalc.cc SpeedCalc.h \
 	PeerStat.h BitfieldMan.cc BitfieldMan.h BitfieldManFactory.cc \
 	BitfieldManFactory.h Randomizer.h SimpleRandomizer.cc \
-	SimpleRandomizer.h FileAllocator.cc FileAllocator.h \
+	SimpleRandomizer.h FileAllocator.h FileAllocationMonitor.cc \
-	FileAllocationMonitor.cc FileAllocationMonitor.h \
+	FileAllocationMonitor.h ConsoleFileAllocationMonitor.cc \
-	ConsoleFileAllocationMonitor.cc ConsoleFileAllocationMonitor.h \
+	ConsoleFileAllocationMonitor.h ChunkChecksumValidator.cc \
-	ChunkChecksumValidator.cc ChunkChecksumValidator.h \
+	ChunkChecksumValidator.h HttpResponse.cc HttpResponse.h \
-	HttpResponse.cc HttpResponse.h HttpRequest.cc HttpRequest.h \
+	HttpRequest.cc HttpRequest.h Range.h \
-	Range.h AbstractProxyRequestCommand.cc \
+	AbstractProxyRequestCommand.cc AbstractProxyRequestCommand.h \
-	AbstractProxyRequestCommand.h AbstractProxyResponseCommand.cc \
+	AbstractProxyResponseCommand.cc AbstractProxyResponseCommand.h \
-	AbstractProxyResponseCommand.h Netrc.cc Netrc.h AuthConfig.cc \
+	Netrc.cc Netrc.h AuthConfig.cc AuthConfig.h AuthResolver.h \
-	AuthConfig.h AuthResolver.h AbstractAuthResolver.h \
+	AbstractAuthResolver.h DefaultAuthResolver.cc \
-	DefaultAuthResolver.cc DefaultAuthResolver.h \
+	DefaultAuthResolver.h NetrcAuthResolver.cc NetrcAuthResolver.h \
-	NetrcAuthResolver.cc NetrcAuthResolver.h RequestFactory.cc \
+	RequestFactory.cc RequestFactory.h DefaultFileAllocator.cc \
-	RequestFactory.h $(am__append_1) $(am__append_2) \
+	DefaultFileAllocator.h GlowFileAllocator.cc \
+	GlowFileAllocator.h $(am__append_1) $(am__append_2) \
 	$(am__append_3)
 noinst_LIBRARIES = libaria2c.a
 libaria2c_a_SOURCES = $(SRCS)
@@ -743,6 +746,7 @@ distclean-compile:
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DefaultBtProgressInfoFile.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DefaultBtRequestFactory.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DefaultDiskWriter.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DefaultFileAllocator.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DefaultPeerListProcessor.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DefaultPeerStorage.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DefaultPieceStorage.Po@am__quote@
@@ -757,7 +761,6 @@ distclean-compile:
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FeatureConfig.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/File.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FileAllocationMonitor.Po@am__quote@
-@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FileAllocator.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FileEntry.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FtpConnection.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FtpDownloadCommand.Po@am__quote@
@@ -765,6 +768,7 @@ distclean-compile:
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FtpNegotiationCommand.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FtpTunnelRequestCommand.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FtpTunnelResponseCommand.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/GlowFileAllocator.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/HaveEraseCommand.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/HttpConnection.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/HttpDownloadCommand.Po@am__quote@

src/SegmentMan.cc

@@ -450,6 +450,27 @@ void SegmentMan::markAllPiecesDone()
   }
 }
 
+void SegmentMan::markPieceDone(int64_t length)
+{
+  if(bitfield) {
+    if(length == bitfield->getTotalLength()) {
+      bitfield->setAllBit();
+    } else {
+      int32_t numSegment = length/bitfield->getBlockLength();
+      int32_t remainingLength = length%bitfield->getBlockLength();
+      bitfield->setBitRange(0, numSegment-1);
+      if(remainingLength > 0) {
+	SegmentHandle segment = new Segment();
+	segment->index = numSegment;
+	segment->length = bitfield->getBlockLength(numSegment);
+	segment->segmentLength = bitfield->getBlockLength();
+	segment->writtenLength = remainingLength;
+	usedSegmentEntries.push_back(new SegmentEntry(0, segment));
+      }
+    }
+  }
+}
+
 #ifdef ENABLE_MESSAGE_DIGEST
 void SegmentMan::checkIntegrity()
 {

src/SegmentMan.h

@@ -285,6 +285,8 @@ public:
 
   void markAllPiecesDone();
 
+  void markPieceDone(int64_t length);
+
 #ifdef ENABLE_MESSAGE_DIGEST
   void checkIntegrity();
 

src/UrlRequestInfo.cc

@@ -41,6 +41,10 @@
 #include "FatalException.h"
 #include "message.h"
 #include "RequestFactory.h"
+#include "GlowFileAllocator.h"
+#include "File.h"
+#include "DefaultDiskWriter.h"
+#include "DlAbortEx.h"
 
 std::ostream& operator<<(std::ostream& o, const HeadResult& hr) {
   o << "filename = " << hr.filename << ", " << "totalLength = " << hr.totalLength;
@@ -184,11 +188,41 @@ RequestInfos UrlRequestInfo::execute() {
   }
 #endif // ENABLE_MESSAGE_DIGEST
 
-  if(e->segmentMan->segmentFileExists()) {
+  if(op->get(PREF_CONTINUE) == V_TRUE && e->segmentMan->fileExists()) {
-    e->segmentMan->load();
+    if(e->segmentMan->totalSize == 0) {
-    e->segmentMan->diskWriter->openExistingFile(e->segmentMan->getFilePath());
+      logger->notice("Cannot get file length. Download aborted.");
-#ifdef ENABLE_MESSAGE_DIGEST
+      return RequestInfos();
+    }
+    File existingFile(e->segmentMan->getFilePath());
+    if(e->segmentMan->totalSize < existingFile.size()) {
+      logger->notice("The local file length is larger than the remote file size. Download aborted.");
+      return RequestInfos();
+    }
+    e->segmentMan->initBitfield(e->option->getAsInt(PREF_SEGMENT_SIZE),
+				e->segmentMan->totalSize);
+
+    e->segmentMan->diskWriter->openExistingFile(e->segmentMan->getFilePath(),
+						e->segmentMan->totalSize);
     if(e->option->get(PREF_CHECK_INTEGRITY) == V_TRUE) {
+#ifdef ENABLE_MESSAGE_DIGEST
+      if(!e->segmentMan->isChunkChecksumValidationReady()) {
+	throw new DlAbortEx("Chunk checksums are not provided.");
+      }
+      e->segmentMan->markAllPiecesDone();
+      e->segmentMan->checkIntegrity();
+#endif // ENABLE_MESSAGE_DIGEST
+    } else {
+      e->segmentMan->markPieceDone(existingFile.size());
+    }
+  } else if(e->segmentMan->segmentFileExists()) {
+    e->segmentMan->load();
+    e->segmentMan->diskWriter->openExistingFile(e->segmentMan->getFilePath(),
+						e->segmentMan->totalSize);
+#ifdef ENABLE_MESSAGE_DIGEST
+    if(op->get(PREF_CHECK_INTEGRITY) == V_TRUE) {
+      if(!e->segmentMan->isChunkChecksumValidationReady()) {
+	throw new DlAbortEx("Chunk checksums are not provided.");
+      }
       e->segmentMan->checkIntegrity();
     }
 #endif // ENABLE_MESSAGE_DIGEST
@@ -202,7 +236,13 @@ RequestInfos UrlRequestInfo::execute() {
       e->segmentMan->initBitfield(e->option->getAsInt(PREF_SEGMENT_SIZE),
 				  e->segmentMan->totalSize);
       if(e->segmentMan->fileExists() && e->option->get(PREF_CHECK_INTEGRITY) == V_TRUE) {
-	e->segmentMan->diskWriter->openExistingFile(e->segmentMan->getFilePath());
+#ifdef ENABLE_MESSAGE_DIGEST
+	if(!e->segmentMan->isChunkChecksumValidationReady()) {
+	  throw new DlAbortEx("Chunk checksums are not provided.");
+	}
+#endif // ENABLE_MESSAGE_DIGEST
+	e->segmentMan->diskWriter->openExistingFile(e->segmentMan->getFilePath(),
+						    e->segmentMan->totalSize);
 #ifdef ENABLE_MESSAGE_DIGEST
 	e->segmentMan->markAllPiecesDone();
 	e->segmentMan->checkIntegrity();

src/main.cc

@@ -352,6 +352,7 @@ int main(int argc, char* argv[]) {
   op->put(PREF_REALTIME_CHUNK_CHECKSUM, V_TRUE);
   op->put(PREF_CHECK_INTEGRITY, V_FALSE);
   op->put(PREF_NETRC_PATH, Util::getHomeDir()+"/.netrc");
+  op->put(PREF_CONTINUE, V_FALSE);
   while(1) {
     int optIndex = 0;
     int lopt;
@@ -383,7 +384,8 @@ int main(int argc, char* argv[]) {
       { "file-allocation", required_argument, 0, 'a' },
       { "allow-overwrite", required_argument, &lopt, 202 },
       { "check-integrity", required_argument, &lopt, 203 },
-      {" realtime-chunk-checksum", required_argument, &lopt, 204 },
+      { "realtime-chunk-checksum", required_argument, &lopt, 204 },
+      { "continue", no_argument, NULL, 'c' },
 #ifdef ENABLE_BITTORRENT
       { "torrent-file", required_argument, NULL, 'T' },
       { "listen-port", required_argument, &lopt, 15 },
@@ -411,7 +413,7 @@ int main(int argc, char* argv[]) {
       { "help", no_argument, NULL, 'h' },
       { 0, 0, 0, 0 }
     };
-    c = getopt_long(argc, argv, "Dd:o:l:s:pt:m:vhST:M:C:a:", longOpts, &optIndex);
+    c = getopt_long(argc, argv, "Dd:o:l:s:pt:m:vhST:M:C:a:c", longOpts, &optIndex);
     if(c == -1) {
       break;
     }
@@ -717,6 +719,9 @@ int main(int argc, char* argv[]) {
       }
       break;
     }
+    case 'c':
+      op->put(PREF_CONTINUE, V_TRUE);
+      break;
     case 'v':
       showVersion();
       exit(EXIT_SUCCESS);

src/prefs.h

@@ -92,6 +92,8 @@
 #define PREF_CHECK_INTEGRITY "check_integrity"
 // value: string that your file system recognizes as a file name.
 #define PREF_NETRC_PATH "netrc_path"
+// value:
+#define PREF_CONTINUE "continue"
 
 /**
  * FTP related preferences

test/BitfieldManTest.cc

@@ -17,6 +17,7 @@ class BitfieldManTest:public CppUnit::TestFixture {
   CPPUNIT_TEST(testGetSparceMissingUnusedIndex);
   CPPUNIT_TEST(testIsBitSetOffsetRange);
   CPPUNIT_TEST(testGetMissingUnusedLength);
+  CPPUNIT_TEST(testSetBitRange);
   CPPUNIT_TEST_SUITE_END();
 private:
   RandomizerHandle fixedNumberRandomizer;
@@ -39,6 +40,7 @@ public:
   void testGetSparceMissingUnusedIndex();
   void testIsBitSetOffsetRange();
   void testGetMissingUnusedLength();
+  void testSetBitRange();
 };
 
 
@@ -303,3 +305,21 @@ void BitfieldManTest::testGetMissingUnusedLength()
   // from index 1
   CPPUNIT_ASSERT_EQUAL((int64_t)3*blockLength, bf.getMissingUnusedLength(1));
 }
+
+void BitfieldManTest::testSetBitRange()
+{
+  int32_t blockLength = 1024*1024;
+  int64_t totalLength = 10*blockLength;
+
+  BitfieldMan bf(blockLength, totalLength);
+
+  bf.setBitRange(0, 4);
+
+  for(int32_t i = 0; i < 5; ++i) {
+    CPPUNIT_ASSERT(bf.isBitSet(i));
+  }
+  for(int32_t i = 5; i < 10; ++i) {
+    CPPUNIT_ASSERT(!bf.isBitSet(i));
+  }
+  CPPUNIT_ASSERT_EQUAL(int64_t(5*blockLength), bf.getCompletedLength());
+}

test/GlowFileAllocatorTest.cc

@@ -0,0 +1,58 @@
+#include "GlowFileAllocator.h"
+#include "File.h"
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <fstream>
+#include <iomanip>
+#include <cppunit/extensions/HelperMacros.h>
+
+class GlowFileAllocatorTest:public CppUnit::TestFixture {
+
+  CPPUNIT_TEST_SUITE(GlowFileAllocatorTest);
+  CPPUNIT_TEST(testAllocate);
+  CPPUNIT_TEST_SUITE_END();
+private:
+
+public:
+  void setUp() {}
+
+  void testAllocate();
+};
+
+
+CPPUNIT_TEST_SUITE_REGISTRATION( GlowFileAllocatorTest );
+
+void GlowFileAllocatorTest::testAllocate()
+{
+  string fn = "/tmp/aria2_GlowFileAllocatorTest_testAllocate";
+  ofstream of(fn.c_str());
+
+  of << "0123456789";
+
+  of.close();
+
+  File x("/tmp/aria2_GlowFileAllocatorTest_testAllocate");
+  CPPUNIT_ASSERT_EQUAL((int64_t)10, x.size());
+
+  int fd;
+  if((fd = open(fn.c_str(), O_RDWR, S_IRUSR|S_IWUSR)) < 0) {
+    CPPUNIT_FAIL("cannot open file");
+  }
+  GlowFileAllocator allocator;
+  allocator.allocate(fd, 4097);
+
+  if(close(fd) < 0) {
+    CPPUNIT_FAIL("cannot close file");
+  }
+
+  ifstream is(fn.c_str());
+
+  char buf[11];
+  is >> std::setw(sizeof(buf)) >> buf;
+
+  CPPUNIT_ASSERT(strcmp("0123456789", buf) == 0);
+
+  File f(fn);
+  CPPUNIT_ASSERT_EQUAL((int64_t)4097, f.size());
+}

test/Makefile.am

@@ -1,13 +1,15 @@
 TESTS = aria2c
 check_PROGRAMS = $(TESTS)
 aria2c_SOURCES = AllTest.cc\
+	SegmentManTest.cc\
+	BitfieldManTest.cc\
+	GlowFileAllocatorTest.cc\
 	RequestTest.cc\
 	HttpRequestTest.cc\
 	NetrcTest.cc\
 	SingletonHolderTest.cc\
 	HttpHeaderTest.cc\
 	HttpResponseTest.cc\
-	BitfieldManTest.cc\
 	SharedHandleTest.cc\
 	ChunkedEncodingTest.cc\
 	FileTest.cc\
@@ -29,7 +31,6 @@ aria2c_SOURCES = AllTest.cc\
 	FeatureConfigTest.cc\
 	ShareRatioSeedCriteriaTest.cc\
 	TimeSeedCriteriaTest.cc\
-	SegmentManTest.cc\
 	SpeedCalcTest.cc\
 	DefaultPeerListProcessorTest.cc\
 	AnnounceListTest.cc\

test/Makefile.in

@@ -57,10 +57,11 @@ mkinstalldirs = $(SHELL) $(top_srcdir)/mkinstalldirs
 CONFIG_HEADER = $(top_builddir)/config.h
 CONFIG_CLEAN_FILES =
 am__EXEEXT_1 = aria2c$(EXEEXT)
-am_aria2c_OBJECTS = AllTest.$(OBJEXT) RequestTest.$(OBJEXT) \
+am_aria2c_OBJECTS = AllTest.$(OBJEXT) SegmentManTest.$(OBJEXT) \
-	HttpRequestTest.$(OBJEXT) NetrcTest.$(OBJEXT) \
+	BitfieldManTest.$(OBJEXT) GlowFileAllocatorTest.$(OBJEXT) \
-	SingletonHolderTest.$(OBJEXT) HttpHeaderTest.$(OBJEXT) \
+	RequestTest.$(OBJEXT) HttpRequestTest.$(OBJEXT) \
-	HttpResponseTest.$(OBJEXT) BitfieldManTest.$(OBJEXT) \
+	NetrcTest.$(OBJEXT) SingletonHolderTest.$(OBJEXT) \
+	HttpHeaderTest.$(OBJEXT) HttpResponseTest.$(OBJEXT) \
 	SharedHandleTest.$(OBJEXT) ChunkedEncodingTest.$(OBJEXT) \
 	FileTest.$(OBJEXT) OptionTest.$(OBJEXT) Base64Test.$(OBJEXT) \
 	UtilTest.$(OBJEXT) CookieBoxTest.$(OBJEXT) DataTest.$(OBJEXT) \
@@ -71,8 +72,8 @@ am_aria2c_OBJECTS = AllTest.$(OBJEXT) RequestTest.$(OBJEXT) \
 	Xml2MetalinkProcessorTest.$(OBJEXT) MetalinkerTest.$(OBJEXT) \
 	MetalinkEntryTest.$(OBJEXT) FeatureConfigTest.$(OBJEXT) \
 	ShareRatioSeedCriteriaTest.$(OBJEXT) \
-	TimeSeedCriteriaTest.$(OBJEXT) SegmentManTest.$(OBJEXT) \
+	TimeSeedCriteriaTest.$(OBJEXT) SpeedCalcTest.$(OBJEXT) \
-	SpeedCalcTest.$(OBJEXT) DefaultPeerListProcessorTest.$(OBJEXT) \
+	DefaultPeerListProcessorTest.$(OBJEXT) \
 	AnnounceListTest.$(OBJEXT) TrackerWatcherCommandTest.$(OBJEXT) \
 	DefaultBtContextTest.$(OBJEXT) \
 	DefaultPieceStorageTest.$(OBJEXT) \
@@ -258,13 +259,15 @@ sysconfdir = @sysconfdir@
 target_alias = @target_alias@
 TESTS = aria2c
 aria2c_SOURCES = AllTest.cc\
+	SegmentManTest.cc\
+	BitfieldManTest.cc\
+	GlowFileAllocatorTest.cc\
 	RequestTest.cc\
 	HttpRequestTest.cc\
 	NetrcTest.cc\
 	SingletonHolderTest.cc\
 	HttpHeaderTest.cc\
 	HttpResponseTest.cc\
-	BitfieldManTest.cc\
 	SharedHandleTest.cc\
 	ChunkedEncodingTest.cc\
 	FileTest.cc\
@@ -286,7 +289,6 @@ aria2c_SOURCES = AllTest.cc\
 	FeatureConfigTest.cc\
 	ShareRatioSeedCriteriaTest.cc\
 	TimeSeedCriteriaTest.cc\
-	SegmentManTest.cc\
 	SpeedCalcTest.cc\
 	DefaultPeerListProcessorTest.cc\
 	AnnounceListTest.cc\
@@ -423,6 +425,7 @@ distclean-compile:
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DictionaryTest.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FeatureConfigTest.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FileTest.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/GlowFileAllocatorTest.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/HttpHeaderTest.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/HttpRequestTest.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/HttpResponseTest.Po@am__quote@

test/SegmentManTest.cc

@@ -12,6 +12,8 @@ class SegmentManTest:public CppUnit::TestFixture {
   CPPUNIT_TEST(testSaveAndLoad);
   CPPUNIT_TEST(testNullBitfield);
   CPPUNIT_TEST(testCancelSegmentOnNullBitfield);
+  CPPUNIT_TEST(testMarkPieceDone);
+  CPPUNIT_TEST(testMarkPieceDone_usedSegment);
   CPPUNIT_TEST_SUITE_END();
 private:
 
@@ -22,7 +24,8 @@ public:
   void testSaveAndLoad();
   void testNullBitfield();
   void testCancelSegmentOnNullBitfield();
-
+  void testMarkPieceDone();
+  void testMarkPieceDone_usedSegment();
 };
 
 
@@ -109,3 +112,41 @@ void SegmentManTest::testCancelSegmentOnNullBitfield() {
   CPPUNIT_ASSERT(!segmentMan.getSegment(1).isNull());
 }
 
+void SegmentManTest::testMarkPieceDone()
+{
+  SegmentMan segmentMan;
+  int32_t pieceLength = 1024*1024;
+  int64_t totalLength = 10*pieceLength;
+  segmentMan.initBitfield(pieceLength, totalLength);
+  segmentMan.markPieceDone(5*pieceLength);
+
+  for(int32_t i = 0; i < 5; ++i) {
+    CPPUNIT_ASSERT(segmentMan.hasSegment(i));
+  }
+  for(int32_t i = 5; i < 10; ++i) {
+    CPPUNIT_ASSERT(!segmentMan.hasSegment(i));
+  }
+}
+
+void SegmentManTest::testMarkPieceDone_usedSegment()
+{
+  SegmentMan segmentMan;
+  int32_t pieceLength = 1024*1024;
+  int64_t totalLength = 10*pieceLength;
+  segmentMan.initBitfield(pieceLength, totalLength);
+  segmentMan.markPieceDone(5*pieceLength+100);
+
+  for(int32_t i = 0; i < 5; ++i) {
+    CPPUNIT_ASSERT(segmentMan.hasSegment(i));
+  }
+  for(int32_t i = 5; i < 10; ++i) {
+    CPPUNIT_ASSERT(!segmentMan.hasSegment(i));
+  }
+
+  SegmentHandle segment = segmentMan.getSegment(0, 5);
+  CPPUNIT_ASSERT(!segment.isNull());
+  CPPUNIT_ASSERT_EQUAL(5, segment->index);
+  CPPUNIT_ASSERT_EQUAL(pieceLength, segment->length);
+  CPPUNIT_ASSERT_EQUAL(pieceLength, segment->segmentLength);
+  CPPUNIT_ASSERT_EQUAL(100, segment->writtenLength);
+}