diff --git a/ChangeLog b/ChangeLog index ae739727..a9d1babd 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,14 @@ +2008-06-24 Tatsuhiro Tsujikawa + + Supported FTP server which don't recognize SIZE raw command. + If SIZE raw command is failed, aria2 will try to get file size + from the response of RETR raw command. If both attempts are failed, + then resuming and segmented downloading are disabled. + * src/FtpConnection.cc + * src/FtpConnection.h + * src/FtpNegotiationCommand.cc + * src/FtpNegotiationCommand.h + 2008-06-24 Tatsuhiro Tsujikawa * Release 0.14.0+1 diff --git a/src/FtpConnection.cc b/src/FtpConnection.cc index d4cf5f33..0eb56946 100644 --- a/src/FtpConnection.cc +++ b/src/FtpConnection.cc @@ -133,7 +133,14 @@ SocketHandle FtpConnection::sendPort() const void FtpConnection::sendRest(const SegmentHandle& segment) const { - std::string request = "REST "+Util::itos(segment->getPositionToWrite())+"\r\n"; + std::string request = "REST "; + if(segment.isNull()) { + request += "0"; + } else { + request += Util::itos(segment->getPositionToWrite()); + } + request += "\r\n"; + logger->info(MSG_SENDING_REQUEST, cuid, request.c_str()); socket->writeData(request); } @@ -276,4 +283,33 @@ unsigned int FtpConnection::receivePasvResponse(std::pair } } +unsigned int FtpConnection::receiveRetrResponse(uint64_t& size) +{ + std::pair response; + if(bulkReceiveResponse(response)) { + if(response.first == 150 || response.first == 125) { + // Attempting to get file size from the response. + // We assume the response is like: + // 150 Opening BINARY mode data connection for aria2.tar.bz2 (12345 bytes) + // If the attempt is failed, size is unchanged. + std::string& res = response.second; + std::string::size_type start; + if((start = res.find_first_of("(")) != std::string::npos && + (start = res.find_first_not_of("( ", start)) != std::string::npos) { + + // now start points to the first byte of the size string. + std::string::size_type end = + res.find_first_not_of("0123456789", start); + + if(end != std::string::npos) { + size = Util::parseULLInt(res.substr(start, end-start)); + } + } + } + return response.first; + } else { + return 0; + } +} + } // namespace aria2 diff --git a/src/FtpConnection.h b/src/FtpConnection.h index 51fe4039..875894d5 100644 --- a/src/FtpConnection.h +++ b/src/FtpConnection.h @@ -83,6 +83,7 @@ public: unsigned int receiveResponse(); unsigned int receiveSizeResponse(uint64_t& size); unsigned int receivePasvResponse(std::pair& dest); + unsigned int receiveRetrResponse(uint64_t& size); }; } // namespace aria2 diff --git a/src/FtpNegotiationCommand.cc b/src/FtpNegotiationCommand.cc index 0818bf74..72873665 100644 --- a/src/FtpNegotiationCommand.cc +++ b/src/FtpNegotiationCommand.cc @@ -54,6 +54,8 @@ #include "ServerHost.h" #include "Socket.h" #include "StringFormat.h" +#include "DiskAdaptor.h" +#include "SegmentMan.h" #include #include #include @@ -79,7 +81,12 @@ bool FtpNegotiationCommand::executeInternal() { while(processSequence(_segments.front())); if(sequence == SEQ_RETRY) { return prepareForRetry(0); + } else if(sequence == SEQ_EXIT) { + return true; } else if(sequence == SEQ_NEGOTIATION_COMPLETED) { + + afterFileAllocation(); + FtpDownloadCommand* command = new FtpDownloadCommand(cuid, req, _requestGroup, ftp, e, dataSocket, socket); command->setMaxDownloadSpeedLimit(e->option->getAsInt(PREF_MAX_DOWNLOAD_LIMIT)); @@ -99,6 +106,13 @@ bool FtpNegotiationCommand::executeInternal() { sequence = SEQ_SEND_PORT; } return false; + } else if(sequence == SEQ_FILE_PREPARATION_ON_RETR) { + if(e->option->getAsBool(PREF_FTP_PASV)) { + sequence = SEQ_NEGOTIATION_COMPLETED; + } else { + sequence = SEQ_WAIT_CONNECTION; + } + return false; } else { e->commands.push_back(this); return false; @@ -206,61 +220,93 @@ bool FtpNegotiationCommand::sendSize() { return false; } -bool FtpNegotiationCommand::recvSize() { - uint64_t size = 0; - unsigned int status = ftp->receiveSizeResponse(size); - if(status == 0) { - return false; +bool FtpNegotiationCommand::onFileSizeDetermined(uint64_t totalLength) +{ + SingleFileDownloadContextHandle dctx = + dynamic_pointer_cast(_requestGroup->getDownloadContext()); + dctx->setTotalLength(totalLength); + dctx->setFilename(Util::urldecode(req->getFile())); + _requestGroup->preDownloadProcessing(); + if(e->_requestGroupMan->isSameFileBeingDownloaded(_requestGroup)) { + throw DownloadFailureException + (StringFormat(EX_DUPLICATE_FILE_DOWNLOAD, + _requestGroup->getFilePath().c_str()).str()); } - if(status != 213) { - poolConnection(); - throw DlAbortEx(StringFormat(EX_BAD_STATUS, status).str()); - } - if(size > INT64_MAX) { - throw DlAbortEx - (StringFormat(EX_TOO_LARGE_FILE, Util::uitos(size, true).c_str()).str()); - } - if(_requestGroup->getPieceStorage().isNull()) { - SingleFileDownloadContextHandle dctx = - dynamic_pointer_cast(_requestGroup->getDownloadContext()); - dctx->setTotalLength(size); - dctx->setFilename(Util::urldecode(req->getFile())); - _requestGroup->preDownloadProcessing(); - if(e->_requestGroupMan->isSameFileBeingDownloaded(_requestGroup)) { - throw DownloadFailureException - (StringFormat(EX_DUPLICATE_FILE_DOWNLOAD, - _requestGroup->getFilePath().c_str()).str()); - } + if(totalLength == 0) { _requestGroup->initPieceStorage(); + _requestGroup->shouldCancelDownloadForSafety(); + _requestGroup->getPieceStorage()->getDiskAdaptor()->initAndOpenFile(); + + } else { + _requestGroup->initPieceStorage(); // TODO Is this really necessary? if(req->getMethod() == Request::METHOD_HEAD) { sequence = SEQ_HEAD_OK; return false; } - + BtProgressInfoFileHandle infoFile(new DefaultBtProgressInfoFile(_requestGroup->getDownloadContext(), _requestGroup->getPieceStorage(), e->option)); if(!infoFile->exists() && _requestGroup->downloadFinishedByFileLength()) { sequence = SEQ_DOWNLOAD_ALREADY_COMPLETED; - + poolConnection(); - + return false; } _requestGroup->loadAndOpenFile(infoFile); - prepareForNextAction(this); - - sequence = SEQ_FILE_PREPARATION; + if(sequence == SEQ_FILE_PREPARATION_ON_RETR) { + // See the transfer is starting from 0 byte. + SharedHandle segment = + _requestGroup->getSegmentMan()->getSegment(cuid, 0); + if(!segment.isNull() && segment->getPositionToWrite() == 0) { + prepareForNextAction(this); + } else { + // If not, drop connection and connect the server and send REST + // with appropriate starting position. + logger->debug("Request range is not valid. Re-sending RETR is necessary."); + sequence = SEQ_EXIT; + prepareForNextAction(0); + } + } else { + // At the time of this writing, when this clause is executed, + // sequence should be SEQ_FILE_PREPARATION. + // At this point, REST is not sent yet, so checking the starting byte + // is not necessary. + prepareForNextAction(this); + } disableReadCheckSocket(); + } + return false; +} - //setWriteCheckSocket(dataSocket); - - //e->noWait = true; +bool FtpNegotiationCommand::recvSize() { + uint64_t size = 0; + unsigned int status = ftp->receiveSizeResponse(size); + if(status == 0) { return false; + } + if(status == 213) { + + if(size > INT64_MAX) { + throw DlAbortEx + (StringFormat(EX_TOO_LARGE_FILE, Util::uitos(size, true).c_str()).str()); + } + if(_requestGroup->getPieceStorage().isNull()) { + + sequence = SEQ_FILE_PREPARATION; + return onFileSizeDetermined(size); + + } else { + _requestGroup->validateTotalLength(size); + } + } else { - _requestGroup->validateTotalLength(size); + + logger->info("CUID#%d - The remote FTP Server doesn't recognize SIZE command. Continue.", cuid); + } if(e->option->getAsBool(PREF_FTP_PASV)) { sequence = SEQ_SEND_PASV; @@ -357,13 +403,24 @@ bool FtpNegotiationCommand::sendRetr() { } bool FtpNegotiationCommand::recvRetr() { - unsigned int status = ftp->receiveResponse(); + uint64_t size = 0; + unsigned int status = ftp->receiveRetrResponse(size); if(status == 0) { return false; } if(status != 150 && status != 125) { throw DlAbortEx(StringFormat(EX_BAD_STATUS, status).str()); } + + if(_requestGroup->getPieceStorage().isNull()) { + + sequence = SEQ_FILE_PREPARATION_ON_RETR; + return onFileSizeDetermined(size); + + } else { + _requestGroup->validateTotalLength(size); + } + if(e->option->getAsBool(PREF_FTP_PASV)) { sequence = SEQ_NEGOTIATION_COMPLETED; return false; @@ -430,6 +487,8 @@ bool FtpNegotiationCommand::processSequence(const SegmentHandle& segment) { return recvRetr(); case SEQ_WAIT_CONNECTION: return waitConnection(); + case SEQ_NEGOTIATION_COMPLETED: + return false; default: abort(); } diff --git a/src/FtpNegotiationCommand.h b/src/FtpNegotiationCommand.h index 962b1999..6fd27c9e 100644 --- a/src/FtpNegotiationCommand.h +++ b/src/FtpNegotiationCommand.h @@ -70,7 +70,9 @@ public: SEQ_RETRY, SEQ_HEAD_OK, SEQ_DOWNLOAD_ALREADY_COMPLETED, - SEQ_FILE_PREPARATION + SEQ_FILE_PREPARATION, // File allocation after SIZE command + SEQ_FILE_PREPARATION_ON_RETR, // File allocation after RETR command + SEQ_EXIT, // Make executeInternal() return true. }; private: bool recvGreeting(); @@ -100,6 +102,8 @@ private: void poolConnection() const; + bool onFileSizeDetermined(uint64_t totalLength); + SharedHandle dataSocket; SharedHandle serverSocket; Seq sequence;