From 3e12ebf78f95853f9eaf15a7b8fa11850321de2d Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 24 Jun 2008 14:31:23 +0000 Subject: [PATCH] 2008-06-24 Tatsuhiro Tsujikawa Supported FTP server which don't recognize SIZE raw command. If SIZE raw command is failed, aria2 will try to get file size from the response of RETR raw command. If both attempts are failed, then resuming and segmented downloading are disabled. * src/FtpConnection.cc * src/FtpConnection.h * src/FtpNegotiationCommand.cc * src/FtpNegotiationCommand.h --- ChangeLog | 11 +++ src/FtpConnection.cc | 38 ++++++++++- src/FtpConnection.h | 1 + src/FtpNegotiationCommand.cc | 129 +++++++++++++++++++++++++---------- src/FtpNegotiationCommand.h | 6 +- 5 files changed, 148 insertions(+), 37 deletions(-) diff --git a/ChangeLog b/ChangeLog index ae739727..a9d1babd 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,14 @@ +2008-06-24 Tatsuhiro Tsujikawa + + Supported FTP server which don't recognize SIZE raw command. + If SIZE raw command is failed, aria2 will try to get file size + from the response of RETR raw command. If both attempts are failed, + then resuming and segmented downloading are disabled. + * src/FtpConnection.cc + * src/FtpConnection.h + * src/FtpNegotiationCommand.cc + * src/FtpNegotiationCommand.h + 2008-06-24 Tatsuhiro Tsujikawa * Release 0.14.0+1 diff --git a/src/FtpConnection.cc b/src/FtpConnection.cc index d4cf5f33..0eb56946 100644 --- a/src/FtpConnection.cc +++ b/src/FtpConnection.cc @@ -133,7 +133,14 @@ SocketHandle FtpConnection::sendPort() const void FtpConnection::sendRest(const SegmentHandle& segment) const { - std::string request = "REST "+Util::itos(segment->getPositionToWrite())+"\r\n"; + std::string request = "REST "; + if(segment.isNull()) { + request += "0"; + } else { + request += Util::itos(segment->getPositionToWrite()); + } + request += "\r\n"; + logger->info(MSG_SENDING_REQUEST, cuid, request.c_str()); socket->writeData(request); } @@ -276,4 +283,33 @@ unsigned int FtpConnection::receivePasvResponse(std::pair } } +unsigned int FtpConnection::receiveRetrResponse(uint64_t& size) +{ + std::pair response; + if(bulkReceiveResponse(response)) { + if(response.first == 150 || response.first == 125) { + // Attempting to get file size from the response. + // We assume the response is like: + // 150 Opening BINARY mode data connection for aria2.tar.bz2 (12345 bytes) + // If the attempt is failed, size is unchanged. + std::string& res = response.second; + std::string::size_type start; + if((start = res.find_first_of("(")) != std::string::npos && + (start = res.find_first_not_of("( ", start)) != std::string::npos) { + + // now start points to the first byte of the size string. + std::string::size_type end = + res.find_first_not_of("0123456789", start); + + if(end != std::string::npos) { + size = Util::parseULLInt(res.substr(start, end-start)); + } + } + } + return response.first; + } else { + return 0; + } +} + } // namespace aria2 diff --git a/src/FtpConnection.h b/src/FtpConnection.h index 51fe4039..875894d5 100644 --- a/src/FtpConnection.h +++ b/src/FtpConnection.h @@ -83,6 +83,7 @@ public: unsigned int receiveResponse(); unsigned int receiveSizeResponse(uint64_t& size); unsigned int receivePasvResponse(std::pair& dest); + unsigned int receiveRetrResponse(uint64_t& size); }; } // namespace aria2 diff --git a/src/FtpNegotiationCommand.cc b/src/FtpNegotiationCommand.cc index 0818bf74..72873665 100644 --- a/src/FtpNegotiationCommand.cc +++ b/src/FtpNegotiationCommand.cc @@ -54,6 +54,8 @@ #include "ServerHost.h" #include "Socket.h" #include "StringFormat.h" +#include "DiskAdaptor.h" +#include "SegmentMan.h" #include #include #include @@ -79,7 +81,12 @@ bool FtpNegotiationCommand::executeInternal() { while(processSequence(_segments.front())); if(sequence == SEQ_RETRY) { return prepareForRetry(0); + } else if(sequence == SEQ_EXIT) { + return true; } else if(sequence == SEQ_NEGOTIATION_COMPLETED) { + + afterFileAllocation(); + FtpDownloadCommand* command = new FtpDownloadCommand(cuid, req, _requestGroup, ftp, e, dataSocket, socket); command->setMaxDownloadSpeedLimit(e->option->getAsInt(PREF_MAX_DOWNLOAD_LIMIT)); @@ -99,6 +106,13 @@ bool FtpNegotiationCommand::executeInternal() { sequence = SEQ_SEND_PORT; } return false; + } else if(sequence == SEQ_FILE_PREPARATION_ON_RETR) { + if(e->option->getAsBool(PREF_FTP_PASV)) { + sequence = SEQ_NEGOTIATION_COMPLETED; + } else { + sequence = SEQ_WAIT_CONNECTION; + } + return false; } else { e->commands.push_back(this); return false; @@ -206,61 +220,93 @@ bool FtpNegotiationCommand::sendSize() { return false; } -bool FtpNegotiationCommand::recvSize() { - uint64_t size = 0; - unsigned int status = ftp->receiveSizeResponse(size); - if(status == 0) { - return false; +bool FtpNegotiationCommand::onFileSizeDetermined(uint64_t totalLength) +{ + SingleFileDownloadContextHandle dctx = + dynamic_pointer_cast(_requestGroup->getDownloadContext()); + dctx->setTotalLength(totalLength); + dctx->setFilename(Util::urldecode(req->getFile())); + _requestGroup->preDownloadProcessing(); + if(e->_requestGroupMan->isSameFileBeingDownloaded(_requestGroup)) { + throw DownloadFailureException + (StringFormat(EX_DUPLICATE_FILE_DOWNLOAD, + _requestGroup->getFilePath().c_str()).str()); } - if(status != 213) { - poolConnection(); - throw DlAbortEx(StringFormat(EX_BAD_STATUS, status).str()); - } - if(size > INT64_MAX) { - throw DlAbortEx - (StringFormat(EX_TOO_LARGE_FILE, Util::uitos(size, true).c_str()).str()); - } - if(_requestGroup->getPieceStorage().isNull()) { - SingleFileDownloadContextHandle dctx = - dynamic_pointer_cast(_requestGroup->getDownloadContext()); - dctx->setTotalLength(size); - dctx->setFilename(Util::urldecode(req->getFile())); - _requestGroup->preDownloadProcessing(); - if(e->_requestGroupMan->isSameFileBeingDownloaded(_requestGroup)) { - throw DownloadFailureException - (StringFormat(EX_DUPLICATE_FILE_DOWNLOAD, - _requestGroup->getFilePath().c_str()).str()); - } + if(totalLength == 0) { _requestGroup->initPieceStorage(); + _requestGroup->shouldCancelDownloadForSafety(); + _requestGroup->getPieceStorage()->getDiskAdaptor()->initAndOpenFile(); + + } else { + _requestGroup->initPieceStorage(); // TODO Is this really necessary? if(req->getMethod() == Request::METHOD_HEAD) { sequence = SEQ_HEAD_OK; return false; } - + BtProgressInfoFileHandle infoFile(new DefaultBtProgressInfoFile(_requestGroup->getDownloadContext(), _requestGroup->getPieceStorage(), e->option)); if(!infoFile->exists() && _requestGroup->downloadFinishedByFileLength()) { sequence = SEQ_DOWNLOAD_ALREADY_COMPLETED; - + poolConnection(); - + return false; } _requestGroup->loadAndOpenFile(infoFile); - prepareForNextAction(this); - - sequence = SEQ_FILE_PREPARATION; + if(sequence == SEQ_FILE_PREPARATION_ON_RETR) { + // See the transfer is starting from 0 byte. + SharedHandle segment = + _requestGroup->getSegmentMan()->getSegment(cuid, 0); + if(!segment.isNull() && segment->getPositionToWrite() == 0) { + prepareForNextAction(this); + } else { + // If not, drop connection and connect the server and send REST + // with appropriate starting position. + logger->debug("Request range is not valid. Re-sending RETR is necessary."); + sequence = SEQ_EXIT; + prepareForNextAction(0); + } + } else { + // At the time of this writing, when this clause is executed, + // sequence should be SEQ_FILE_PREPARATION. + // At this point, REST is not sent yet, so checking the starting byte + // is not necessary. + prepareForNextAction(this); + } disableReadCheckSocket(); + } + return false; +} - //setWriteCheckSocket(dataSocket); - - //e->noWait = true; +bool FtpNegotiationCommand::recvSize() { + uint64_t size = 0; + unsigned int status = ftp->receiveSizeResponse(size); + if(status == 0) { return false; + } + if(status == 213) { + + if(size > INT64_MAX) { + throw DlAbortEx + (StringFormat(EX_TOO_LARGE_FILE, Util::uitos(size, true).c_str()).str()); + } + if(_requestGroup->getPieceStorage().isNull()) { + + sequence = SEQ_FILE_PREPARATION; + return onFileSizeDetermined(size); + + } else { + _requestGroup->validateTotalLength(size); + } + } else { - _requestGroup->validateTotalLength(size); + + logger->info("CUID#%d - The remote FTP Server doesn't recognize SIZE command. Continue.", cuid); + } if(e->option->getAsBool(PREF_FTP_PASV)) { sequence = SEQ_SEND_PASV; @@ -357,13 +403,24 @@ bool FtpNegotiationCommand::sendRetr() { } bool FtpNegotiationCommand::recvRetr() { - unsigned int status = ftp->receiveResponse(); + uint64_t size = 0; + unsigned int status = ftp->receiveRetrResponse(size); if(status == 0) { return false; } if(status != 150 && status != 125) { throw DlAbortEx(StringFormat(EX_BAD_STATUS, status).str()); } + + if(_requestGroup->getPieceStorage().isNull()) { + + sequence = SEQ_FILE_PREPARATION_ON_RETR; + return onFileSizeDetermined(size); + + } else { + _requestGroup->validateTotalLength(size); + } + if(e->option->getAsBool(PREF_FTP_PASV)) { sequence = SEQ_NEGOTIATION_COMPLETED; return false; @@ -430,6 +487,8 @@ bool FtpNegotiationCommand::processSequence(const SegmentHandle& segment) { return recvRetr(); case SEQ_WAIT_CONNECTION: return waitConnection(); + case SEQ_NEGOTIATION_COMPLETED: + return false; default: abort(); } diff --git a/src/FtpNegotiationCommand.h b/src/FtpNegotiationCommand.h index 962b1999..6fd27c9e 100644 --- a/src/FtpNegotiationCommand.h +++ b/src/FtpNegotiationCommand.h @@ -70,7 +70,9 @@ public: SEQ_RETRY, SEQ_HEAD_OK, SEQ_DOWNLOAD_ALREADY_COMPLETED, - SEQ_FILE_PREPARATION + SEQ_FILE_PREPARATION, // File allocation after SIZE command + SEQ_FILE_PREPARATION_ON_RETR, // File allocation after RETR command + SEQ_EXIT, // Make executeInternal() return true. }; private: bool recvGreeting(); @@ -100,6 +102,8 @@ private: void poolConnection() const; + bool onFileSizeDetermined(uint64_t totalLength); + SharedHandle dataSocket; SharedHandle serverSocket; Seq sequence;