2010-09-10 Tatsuhiro Tsujikawa <t-tujikawa@users.sourceforge.net>

Check hash(hash for entire file, not piece hash) if
	--check-integrity option is given and file is
	downloaded(determined by file length). If it fails, re-download
	file.
	* src/AbstractCommand.cc
	* src/ChecksumCheckIntegrityEntry.cc
	* src/ChecksumCheckIntegrityEntry.h
	* src/FtpNegotiationCommand.cc
	* src/HttpResponseCommand.cc
	* src/RequestGroup.cc

ChangeLog

@@ -1,3 +1,16 @@
+2010-09-10  Tatsuhiro Tsujikawa  <t-tujikawa@users.sourceforge.net>
+
+	Check hash(hash for entire file, not piece hash) if
+	--check-integrity option is given and file is
+	downloaded(determined by file length). If it fails, re-download
+	file.
+	* src/AbstractCommand.cc
+	* src/ChecksumCheckIntegrityEntry.cc
+	* src/ChecksumCheckIntegrityEntry.h
+	* src/FtpNegotiationCommand.cc
+	* src/HttpResponseCommand.cc
+	* src/RequestGroup.cc
+
 2010-09-10  Tatsuhiro Tsujikawa  <t-tujikawa@users.sourceforge.net>
 
 	Make finished() return true only when offset == totalLength.

src/AbstractCommand.cc

@@ -780,29 +780,26 @@ std::string AbstractCommand::resolveHostname
 // function call.
 void AbstractCommand::prepareForNextAction(Command* nextCommand)
 {
+  SharedHandle<CheckIntegrityEntry> checkEntry;
 #ifdef ENABLE_MESSAGE_DIGEST
   if(requestGroup_->downloadFinished() &&
      getDownloadContext()->isChecksumVerificationNeeded()) {
     if(getLogger()->info()) {
       getLogger()->info(MSG_HASH_CHECK_NOT_DONE);
     }
-    SharedHandle<CheckIntegrityEntry> entry
+    SharedHandle<ChecksumCheckIntegrityEntry> entry
-      (new ChecksumCheckIntegrityEntry(requestGroup_));
+      (new ChecksumCheckIntegrityEntry(requestGroup_, nextCommand));
-    if(entry->isValidationReady()) {
+    entry->setRedownload(true);
-      delete nextCommand;
+    checkEntry = entry;
-      entry->initValidator();
+  } else
-      entry->cutTrailingGarbage();
-      e_->getCheckIntegrityMan()->pushEntry(entry);
-      return;
-    }
-  }
 #endif // ENABLE_MESSAGE_DIGEST
-  SharedHandle<CheckIntegrityEntry> entry
+    {
+      checkEntry.reset
         (new StreamCheckIntegrityEntry(requestGroup_, nextCommand));
-
+    }
   std::vector<Command*>* commands = new std::vector<Command*>();
   auto_delete_container<std::vector<Command*> > commandsDel(commands);
-  requestGroup_->processCheckIntegrityEntry(*commands, entry, e_);
+  requestGroup_->processCheckIntegrityEntry(*commands, checkEntry, e_);
   e_->addCommand(*commands);
   commands->clear();
   e_->setNoWait(true);

src/ChecksumCheckIntegrityEntry.cc

@@ -42,11 +42,13 @@
 #include "RequestGroupMan.h"
 #include "FileAllocationEntry.h"
 #include "ServerStatMan.h"
+#include "StreamFileAllocationEntry.h"
 
 namespace aria2 {
 
 ChecksumCheckIntegrityEntry::ChecksumCheckIntegrityEntry(RequestGroup* requestGroup, Command* nextCommand):
-  CheckIntegrityEntry(requestGroup, nextCommand) {}
+  CheckIntegrityEntry(requestGroup, nextCommand),
+  redownload_(false) {}
 
 ChecksumCheckIntegrityEntry::~ChecksumCheckIntegrityEntry() {}
 
@@ -75,6 +77,12 @@ ChecksumCheckIntegrityEntry::onDownloadFinished
 void
 ChecksumCheckIntegrityEntry::onDownloadIncomplete
 (std::vector<Command*>& commands, DownloadEngine* e)
-{}
+{
+  if(redownload_) {
+    SharedHandle<FileAllocationEntry> entry
+      (new StreamFileAllocationEntry(getRequestGroup(), popNextCommand()));
+    proceedFileAllocation(commands, entry, e);
+  }
+}
 
 } // namespace aria2

src/ChecksumCheckIntegrityEntry.h

@@ -41,6 +41,8 @@ namespace aria2 {
 
 class ChecksumCheckIntegrityEntry:public CheckIntegrityEntry
 {
+private:
+  bool redownload_;
 public:
   ChecksumCheckIntegrityEntry(RequestGroup* requestGroup, Command* nextCommand = 0);
 
@@ -55,6 +57,11 @@ public:
 
   virtual void onDownloadIncomplete(std::vector<Command*>& commands,
                                     DownloadEngine* e);
+
+  void setRedownload(bool redownload)
+  {
+    redownload_ = redownload;
+  }
 };
 
 } // namespace aria2

src/FtpNegotiationCommand.cc

@@ -454,20 +454,18 @@ bool FtpNegotiationCommand::onFileSizeDetermined(uint64_t totalLength)
     if(!infoFile->exists() &&
        getRequestGroup()->downloadFinishedByFileLength()) {
       getPieceStorage()->markAllPiecesDone();
-      // TODO It would be good to issue ChecksumCheckIntegrity here
+      // See also RequestGroup::createInitialCommand()
-      // instead of just pretending checksum verification is done.
+      if(!getOption()->getAsBool(PREF_CHECK_INTEGRITY) ||
+         !getDownloadContext()->isChecksumVerificationNeeded()) {
         getDownloadContext()->setChecksumVerified(true);
-
         sequence_ = SEQ_DOWNLOAD_ALREADY_COMPLETED;
-      
         getLogger()->notice(MSG_DOWNLOAD_ALREADY_COMPLETED,
                             util::itos(getRequestGroup()->getGID()).c_str(),
                             getRequestGroup()->getFirstFilePath().c_str());
-
         poolConnection();
-      
         return false;
       }
+    }
     getRequestGroup()->loadAndOpenFile(infoFile);
     // We have to make sure that command that has Request object must
     // have segment after PieceStorage is initialized. See

src/HttpResponseCommand.cc

@@ -131,8 +131,7 @@ bool HttpResponseCommand::executeInternal()
       getFileEntry()->setLength(totalLength);
       getRequestGroup()->initPieceStorage();
       getPieceStorage()->markAllPiecesDone();
-      // TODO It would be good to issue ChecksumCheckIntegrity here
+      // Just set checksum verification done.
-      // instead of just pretending checksum verification is done.
       getDownloadContext()->setChecksumVerified(true);
       getLogger()->notice(MSG_DOWNLOAD_ALREADY_COMPLETED,
                           util::itos(getRequestGroup()->getGID()).c_str(),
@@ -268,14 +267,16 @@ bool HttpResponseCommand::handleDefaultEncoding
                                    getOption().get()));
   if(!infoFile->exists() && getRequestGroup()->downloadFinishedByFileLength()) {
     getPieceStorage()->markAllPiecesDone();
-    // TODO It would be good to issue ChecksumCheckIntegrity here
+    // See also RequestGroup::createInitialCommand()
-    // instead of just pretending checksum verification is done.
+    if(!getOption()->getAsBool(PREF_CHECK_INTEGRITY) ||
+       !getDownloadContext()->isChecksumVerificationNeeded()) {
       getDownloadContext()->setChecksumVerified(true);
       getLogger()->notice(MSG_DOWNLOAD_ALREADY_COMPLETED,
                           util::itos(getRequestGroup()->getGID()).c_str(),
                           getRequestGroup()->getFirstFilePath().c_str());
       return true;
     }
+  }
   getRequestGroup()->loadAndOpenFile(infoFile);
   File file(getRequestGroup()->getFirstFilePath());
   // We have to make sure that command that has Request object must
@@ -376,8 +377,7 @@ bool HttpResponseCommand::handleOtherEncoding
   if(getRequestGroup()->downloadFinishedByFileLength()) {
     getRequestGroup()->initPieceStorage();
     getPieceStorage()->markAllPiecesDone();
-    // TODO It would be good to issue ChecksumCheckIntegrity here
+    // This is zero-size file, so hash check is no use.
-    // instead of just pretending checksum verification is done.
     getDownloadContext()->setChecksumVerified(true);
     getLogger()->notice(MSG_DOWNLOAD_ALREADY_COMPLETED,
                         util::itos(getRequestGroup()->getGID()).c_str(),

src/RequestGroup.cc

@@ -424,34 +424,43 @@ void RequestGroup::createInitialCommand
       BtProgressInfoFileHandle infoFile
         (new DefaultBtProgressInfoFile(downloadContext_, pieceStorage_,
                                        option_.get()));
-      if(!infoFile->exists() && downloadFinishedByFileLength()) {
+      bool finishedBySize =
+        !infoFile->exists() && downloadFinishedByFileLength();
+      if(finishedBySize) {
         pieceStorage_->markAllPiecesDone();
-        // TODO It would be good to issue ChecksumCheckIntegrity here
+        if(!option_->getAsBool(PREF_CHECK_INTEGRITY) ||
-        // instead of just pretending checksum verification is done.
+           !downloadContext_->isChecksumVerificationNeeded()) {
+          // If --check-integrity=false and no checksum is provided,
+          // and .aria2 file does not exist, we just report download
+          // finished. We need
+          // DownloadContext::setChecksumVerified(true): without this,
+          // aria2 reports error for this download.
           downloadContext_->setChecksumVerified(true);
           logger_->notice(MSG_DOWNLOAD_ALREADY_COMPLETED,
                           util::itos(gid_).c_str(),
                           downloadContext_->getBasePath().c_str());
         } else {
+          finishedBySize = false;
+        }
+      }
+      if(!finishedBySize) {
         loadAndOpenFile(infoFile);
+        SharedHandle<CheckIntegrityEntry> checkIntegrityEntry;
 #ifdef ENABLE_MESSAGE_DIGEST
         if(downloadFinished() &&
            downloadContext_->isChecksumVerificationNeeded()) {
           if(logger_->info()) {
             logger_->info(MSG_HASH_CHECK_NOT_DONE);
           }
-          SharedHandle<CheckIntegrityEntry> entry
+          SharedHandle<ChecksumCheckIntegrityEntry> entry
             (new ChecksumCheckIntegrityEntry(this));
-          if(entry->isValidationReady()) {
+          entry->setRedownload(true);
-            entry->initValidator();
+          checkIntegrityEntry = entry;
-            entry->cutTrailingGarbage();
+        } else
-            e->getCheckIntegrityMan()->pushEntry(entry);
-            return;
-          }
-        }
 #endif // ENABLE_MESSAGE_DIGEST
-        SharedHandle<CheckIntegrityEntry> checkIntegrityEntry
+          {
-          (new StreamCheckIntegrityEntry(this));
+            checkIntegrityEntry.reset(new StreamCheckIntegrityEntry(this));
+          }
         processCheckIntegrityEntry(commands, checkIntegrityEntry, e);
       }
     }
@@ -505,11 +514,15 @@ void RequestGroup::processCheckIntegrityEntry
  const SharedHandle<CheckIntegrityEntry>& entry,
  DownloadEngine* e)
 {
+  uint64_t actualFileSize = pieceStorage_->getDiskAdaptor()->size();
+  if(actualFileSize > downloadContext_->getTotalLength()) {
+    entry->cutTrailingGarbage();
+  }
 #ifdef ENABLE_MESSAGE_DIGEST
-  if(option_->getAsBool(PREF_CHECK_INTEGRITY) &&
+  if((option_->getAsBool(PREF_CHECK_INTEGRITY) ||
+      downloadContext_->isChecksumVerificationNeeded()) &&
      entry->isValidationReady()) {
     entry->initValidator();
-    entry->cutTrailingGarbage();
     // Don't save control file(.aria2 file) when user presses
     // control-c key while aria2 is checking hashes. If control file
     // doesn't exist when aria2 launched, the completed length in