2008-06-24 Tatsuhiro Tsujikawa <tujikawa at rednoah dot com>

Supported FTP server which don't recognize SIZE raw command.
	If SIZE raw command is failed, aria2 will try to get file size
	from the response of RETR raw command. If both attempts are 
failed,
	then resuming and segmented downloading are disabled.
	* src/FtpConnection.cc
	* src/FtpConnection.h
	* src/FtpNegotiationCommand.cc
	* src/FtpNegotiationCommand.h

ChangeLog

@@ -1,3 +1,14 @@
+2008-06-24  Tatsuhiro Tsujikawa  <tujikawa at rednoah dot com>
+
+	Supported FTP server which don't recognize SIZE raw command.
+	If SIZE raw command is failed, aria2 will try to get file size
+	from the response of RETR raw command. If both attempts are failed,
+	then resuming and segmented downloading are disabled.
+	* src/FtpConnection.cc
+	* src/FtpConnection.h
+	* src/FtpNegotiationCommand.cc
+	* src/FtpNegotiationCommand.h
+
 2008-06-24  Tatsuhiro Tsujikawa  <tujikawa at rednoah dot com>
 
 	* Release 0.14.0+1

src/FtpConnection.cc

@@ -133,7 +133,14 @@ SocketHandle FtpConnection::sendPort() const
 
 void FtpConnection::sendRest(const SegmentHandle& segment) const
 {
-  std::string request = "REST "+Util::itos(segment->getPositionToWrite())+"\r\n";
+  std::string request = "REST ";
+  if(segment.isNull()) {
+    request += "0";
+  } else {
+    request += Util::itos(segment->getPositionToWrite());
+  }
+  request += "\r\n";
+
   logger->info(MSG_SENDING_REQUEST, cuid, request.c_str());
   socket->writeData(request);
 }
@@ -276,4 +283,33 @@ unsigned int FtpConnection::receivePasvResponse(std::pair<std::string, uint16_t>
   }
 }
 
+unsigned int FtpConnection::receiveRetrResponse(uint64_t& size)
+{
+  std::pair<unsigned int, std::string> response;
+  if(bulkReceiveResponse(response)) {
+    if(response.first == 150 || response.first == 125) {
+      // Attempting to get file size from the response.
+      // We assume the response is like:
+      // 150 Opening BINARY mode data connection for aria2.tar.bz2 (12345 bytes)
+      // If the attempt is failed, size is unchanged.
+      std::string& res = response.second;
+      std::string::size_type start;
+      if((start = res.find_first_of("(")) != std::string::npos &&
+	 (start = res.find_first_not_of("( ", start)) != std::string::npos) {
+
+	// now start points to the first byte of the size string.
+	std::string::size_type end =
+	  res.find_first_not_of("0123456789", start);
+
+	if(end != std::string::npos) {
+	  size = Util::parseULLInt(res.substr(start, end-start));
+	}
+      }
+    }
+    return response.first;
+  } else {
+    return 0;
+  }
+}
+
 } // namespace aria2

src/FtpConnection.h

@@ -83,6 +83,7 @@ public:
   unsigned int receiveResponse();
   unsigned int receiveSizeResponse(uint64_t& size);
   unsigned int receivePasvResponse(std::pair<std::string, uint16_t>& dest);
+  unsigned int receiveRetrResponse(uint64_t& size);
 };
 
 } // namespace aria2

src/FtpNegotiationCommand.cc

@@ -54,6 +54,8 @@
 #include "ServerHost.h"
 #include "Socket.h"
 #include "StringFormat.h"
+#include "DiskAdaptor.h"
+#include "SegmentMan.h"
 #include <stdint.h>
 #include <cassert>
 #include <utility>
@@ -79,7 +81,12 @@ bool FtpNegotiationCommand::executeInternal() {
   while(processSequence(_segments.front()));
   if(sequence == SEQ_RETRY) {
     return prepareForRetry(0);
+  } else if(sequence == SEQ_EXIT) {
+    return true;
   } else if(sequence == SEQ_NEGOTIATION_COMPLETED) {
+
+    afterFileAllocation();
+
     FtpDownloadCommand* command =
       new FtpDownloadCommand(cuid, req, _requestGroup, ftp, e, dataSocket, socket);
     command->setMaxDownloadSpeedLimit(e->option->getAsInt(PREF_MAX_DOWNLOAD_LIMIT));
@@ -99,6 +106,13 @@ bool FtpNegotiationCommand::executeInternal() {
       sequence = SEQ_SEND_PORT;
     }
     return false;
+  } else if(sequence == SEQ_FILE_PREPARATION_ON_RETR) {
+    if(e->option->getAsBool(PREF_FTP_PASV)) {
+      sequence = SEQ_NEGOTIATION_COMPLETED;
+    } else {
+      sequence = SEQ_WAIT_CONNECTION;
+    }
+    return false;
   } else {
     e->commands.push_back(this);
     return false;
@@ -206,61 +220,93 @@ bool FtpNegotiationCommand::sendSize() {
   return false;
 }
 
-bool FtpNegotiationCommand::recvSize() {
-  uint64_t size = 0;
-  unsigned int status = ftp->receiveSizeResponse(size);
-  if(status == 0) {
-    return false;
+bool FtpNegotiationCommand::onFileSizeDetermined(uint64_t totalLength)
+{
+  SingleFileDownloadContextHandle dctx =
+    dynamic_pointer_cast<SingleFileDownloadContext>(_requestGroup->getDownloadContext());
+  dctx->setTotalLength(totalLength);
+  dctx->setFilename(Util::urldecode(req->getFile()));
+  _requestGroup->preDownloadProcessing();
+  if(e->_requestGroupMan->isSameFileBeingDownloaded(_requestGroup)) {
+    throw DownloadFailureException
+      (StringFormat(EX_DUPLICATE_FILE_DOWNLOAD,
+		    _requestGroup->getFilePath().c_str()).str());
   }
-  if(status != 213) {
-    poolConnection();
-    throw DlAbortEx(StringFormat(EX_BAD_STATUS, status).str());
-  }
-  if(size > INT64_MAX) {
-    throw DlAbortEx
-      (StringFormat(EX_TOO_LARGE_FILE, Util::uitos(size, true).c_str()).str());
-  }
-  if(_requestGroup->getPieceStorage().isNull()) {
-    SingleFileDownloadContextHandle dctx =
-      dynamic_pointer_cast<SingleFileDownloadContext>(_requestGroup->getDownloadContext());
-    dctx->setTotalLength(size);
-    dctx->setFilename(Util::urldecode(req->getFile()));
-    _requestGroup->preDownloadProcessing();
-    if(e->_requestGroupMan->isSameFileBeingDownloaded(_requestGroup)) {
-      throw DownloadFailureException
-	(StringFormat(EX_DUPLICATE_FILE_DOWNLOAD,
-		      _requestGroup->getFilePath().c_str()).str());
-    }
+  if(totalLength == 0) {
 
     _requestGroup->initPieceStorage();
+    _requestGroup->shouldCancelDownloadForSafety();
+    _requestGroup->getPieceStorage()->getDiskAdaptor()->initAndOpenFile();
+
+  } else {
+    _requestGroup->initPieceStorage();
 
     // TODO Is this really necessary?
     if(req->getMethod() == Request::METHOD_HEAD) {
       sequence = SEQ_HEAD_OK;
       return false;
     }
-
+    
     BtProgressInfoFileHandle infoFile(new DefaultBtProgressInfoFile(_requestGroup->getDownloadContext(), _requestGroup->getPieceStorage(), e->option));
     if(!infoFile->exists() && _requestGroup->downloadFinishedByFileLength()) {
       sequence = SEQ_DOWNLOAD_ALREADY_COMPLETED;
-
+      
       poolConnection();
-
+      
       return false;
     }
     _requestGroup->loadAndOpenFile(infoFile);
 
-    prepareForNextAction(this);
-    
-    sequence = SEQ_FILE_PREPARATION;
+    if(sequence == SEQ_FILE_PREPARATION_ON_RETR) {
+      // See the transfer is starting from 0 byte.
+      SharedHandle<Segment> segment =
+	_requestGroup->getSegmentMan()->getSegment(cuid, 0);
+      if(!segment.isNull() && segment->getPositionToWrite() == 0) {
+	prepareForNextAction(this);
+      } else {
+	// If not, drop connection and connect the server and send REST
+	// with appropriate starting position.
+	logger->debug("Request range is not valid. Re-sending RETR is necessary.");
+	sequence = SEQ_EXIT;
+	prepareForNextAction(0);
+      }
+    } else {
+      // At the time of this writing, when this clause is executed,
+      // sequence should be SEQ_FILE_PREPARATION.
+      // At this point, REST is not sent yet, so checking the starting byte
+      // is not necessary.
+      prepareForNextAction(this);
+    }
     disableReadCheckSocket();
+  }
+  return false;
+}
 
-    //setWriteCheckSocket(dataSocket);
-
-    //e->noWait = true;
+bool FtpNegotiationCommand::recvSize() {
+  uint64_t size = 0;
+  unsigned int status = ftp->receiveSizeResponse(size);
+  if(status == 0) {
     return false;
+  }
+  if(status == 213) {
+
+    if(size > INT64_MAX) {
+      throw DlAbortEx
+	(StringFormat(EX_TOO_LARGE_FILE, Util::uitos(size, true).c_str()).str());
+    }
+    if(_requestGroup->getPieceStorage().isNull()) {
+
+      sequence = SEQ_FILE_PREPARATION;
+      return onFileSizeDetermined(size);
+
+    } else {
+      _requestGroup->validateTotalLength(size);
+    }
+
   } else {
-    _requestGroup->validateTotalLength(size);
+    
+    logger->info("CUID#%d - The remote FTP Server doesn't recognize SIZE command. Continue.", cuid);
+
   }
   if(e->option->getAsBool(PREF_FTP_PASV)) {
     sequence = SEQ_SEND_PASV;
@@ -357,13 +403,24 @@ bool FtpNegotiationCommand::sendRetr() {
 }
 
 bool FtpNegotiationCommand::recvRetr() {
-  unsigned int status = ftp->receiveResponse();
+  uint64_t size = 0;
+  unsigned int status = ftp->receiveRetrResponse(size);
   if(status == 0) {
     return false;
   }
   if(status != 150 && status != 125) {
     throw DlAbortEx(StringFormat(EX_BAD_STATUS, status).str());
   }
+
+  if(_requestGroup->getPieceStorage().isNull()) {
+
+    sequence = SEQ_FILE_PREPARATION_ON_RETR;
+    return onFileSizeDetermined(size);
+
+  } else {
+    _requestGroup->validateTotalLength(size);
+  }
+
   if(e->option->getAsBool(PREF_FTP_PASV)) {
     sequence = SEQ_NEGOTIATION_COMPLETED;
     return false;
@@ -430,6 +487,8 @@ bool FtpNegotiationCommand::processSequence(const SegmentHandle& segment) {
     return recvRetr();
   case SEQ_WAIT_CONNECTION:
     return waitConnection();
+  case SEQ_NEGOTIATION_COMPLETED:
+    return false;
   default:
     abort();
   }

src/FtpNegotiationCommand.h

@@ -70,7 +70,9 @@ public:
     SEQ_RETRY,
     SEQ_HEAD_OK,
     SEQ_DOWNLOAD_ALREADY_COMPLETED,
-    SEQ_FILE_PREPARATION
+    SEQ_FILE_PREPARATION, // File allocation after SIZE command
+    SEQ_FILE_PREPARATION_ON_RETR, // File allocation after RETR command
+    SEQ_EXIT, // Make executeInternal() return true.
   };
 private:
   bool recvGreeting();
@@ -100,6 +102,8 @@ private:
 
   void poolConnection() const;
 
+  bool onFileSizeDetermined(uint64_t totalLength);
+
   SharedHandle<SocketCore> dataSocket;
   SharedHandle<SocketCore> serverSocket;
   Seq sequence;