diff --git a/ChangeLog b/ChangeLog index ac5e36f5..6fdcad1b 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,14 @@ +2010-04-07 Tatsuhiro Tsujikawa + + Fixed the bug that FTP data connection is not established via + proxy when --ftp-proxy is defined and --ftp-pasv=true and + --proxy-method=tunnel. + * src/AbstractCommand.cc + * src/AbstractCommand.h + * src/FtpNegotiationCommand.cc + * src/FtpNegotiationCommand.h + * src/InitiateConnectionCommand.cc + 2010-04-04 Tatsuhiro Tsujikawa Updated po templates. diff --git a/src/AbstractCommand.cc b/src/AbstractCommand.cc index 85b55475..abe36b8b 100644 --- a/src/AbstractCommand.cc +++ b/src/AbstractCommand.cc @@ -65,6 +65,7 @@ #include "LogFactory.h" #include "DownloadContext.h" #include "wallclock.h" +#include "NameResolver.h" namespace aria2 { @@ -604,8 +605,10 @@ bool AbstractCommand::asyncResolveHostname() { switch(_asyncNameResolver->getStatus()) { case AsyncNameResolver::STATUS_SUCCESS: + disableNameResolverCheck(_asyncNameResolver); return true; case AsyncNameResolver::STATUS_ERROR: + disableNameResolverCheck(_asyncNameResolver); if(!isProxyRequest(req->getProtocol(), getOption())) { e->_requestGroupMan->getOrCreateServerStat (req->getHost(), req->getProtocol())->setError(); @@ -647,6 +650,53 @@ bool AbstractCommand::nameResolveFinished() const { } #endif // ENABLE_ASYNC_DNS +std::string AbstractCommand::resolveHostname +(std::vector& addrs, const std::string& hostname, uint16_t port) +{ + e->findAllCachedIPAddresses(std::back_inserter(addrs), hostname, port); + std::string ipaddr; + if(addrs.empty()) { +#ifdef ENABLE_ASYNC_DNS + if(getOption()->getAsBool(PREF_ASYNC_DNS)) { + if(!isAsyncNameResolverInitialized()) { + initAsyncNameResolver(hostname); + } + if(asyncResolveHostname()) { + addrs = getResolvedAddresses(); + } else { + return A2STR::NIL; + } + } else +#endif // ENABLE_ASYNC_DNS + { + NameResolver res; + res.setSocktype(SOCK_STREAM); + if(e->option->getAsBool(PREF_DISABLE_IPV6)) { + res.setFamily(AF_INET); + } + res.resolve(addrs, hostname); + } + if(logger->info()) { + logger->info(MSG_NAME_RESOLUTION_COMPLETE, util::itos(cuid).c_str(), + hostname.c_str(), + strjoin(addrs.begin(), addrs.end(), ", ").c_str()); + } + for(std::vector::const_iterator i = addrs.begin(), + eoi = addrs.end(); i != eoi; ++i) { + e->cacheIPAddress(hostname, *i, port); + } + ipaddr = e->findCachedIPAddress(hostname, port); + } else { + ipaddr = addrs.front(); + if(logger->info()) { + logger->info(MSG_DNS_CACHE_HIT, + util::itos(cuid).c_str(), hostname.c_str(), + strjoin(addrs.begin(), addrs.end(), ", ").c_str()); + } + } + return ipaddr; +} + void AbstractCommand::prepareForNextAction(Command* nextCommand) { SharedHandle entry diff --git a/src/AbstractCommand.h b/src/AbstractCommand.h index 6f9393f6..8e7e6e98 100644 --- a/src/AbstractCommand.h +++ b/src/AbstractCommand.h @@ -77,6 +77,14 @@ protected: const std::vector& getResolvedAddresses(); #endif // ENABLE_ASYNC_DNS + // Resolves hostname. The resolved addresses are stored in addrs + // and first element is returned. If resolve is not finished, + // return empty string. In this case, call this function with same + // arguments until resolved address is returned. Exception is + // thrown on error. port is used for retrieving cached addresses. + std::string resolveHostname + (std::vector& addrs, const std::string& hostname, uint16_t port); + void tryReserved(); virtual bool prepareForRetry(time_t wait); virtual void onAbort(); @@ -125,7 +133,6 @@ protected: */ SharedHandle createProxyRequest() const; - // Returns proxy method for given protocol. Either V_GET or V_TUNNEL // is returned. For HTTPS, always returns V_TUNNEL. const std::string& resolveProxyMethod(const std::string& protocol) const; @@ -149,7 +156,6 @@ private: void disableNameResolverCheck(const SharedHandle& resolver); bool nameResolveFinished() const; - #endif // ENABLE_ASYNC_DNS public: AbstractCommand(cuid_t cuid, const SharedHandle& req, diff --git a/src/FtpNegotiationCommand.cc b/src/FtpNegotiationCommand.cc index 4fa26fcd..9f89b6b2 100644 --- a/src/FtpNegotiationCommand.cc +++ b/src/FtpNegotiationCommand.cc @@ -65,6 +65,12 @@ #include "AuthConfig.h" #include "a2functional.h" #include "URISelector.h" +#include "HttpConnection.h" +#include "HttpHeader.h" +#include "HttpRequest.h" +#include "HttpResponse.h" +#include "DlRetryEx.h" +#include "CookieStorage.h" namespace aria2 { @@ -539,22 +545,116 @@ bool FtpNegotiationCommand::recvPasv() { if(status != 227) { throw DL_ABORT_EX(StringFormat(EX_BAD_STATUS, status).str()); } - // make a data connection to the server. + // TODO Should we check to see that dest.first is not in noProxy list? + if(isProxyDefined()) { + _dataConnAddr = dest; + sequence = SEQ_RESOLVE_PROXY; + return true; + } else { + // make a data connection to the server. + if(logger->info()) { + logger->info(MSG_CONNECTING_TO_SERVER, util::itos(cuid).c_str(), + dest.first.c_str(), + dest.second); + } + dataSocket.reset(new SocketCore()); + dataSocket->establishConnection(dest.first, dest.second); + disableReadCheckSocket(); + setWriteCheckSocket(dataSocket); + sequence = SEQ_SEND_REST_PASV; + return false; + } +} + +bool FtpNegotiationCommand::resolveProxy() +{ + SharedHandle proxyReq = createProxyRequest(); + std::vector addrs; + _proxyAddr = resolveHostname + (addrs, proxyReq->getHost(), proxyReq->getPort()); + if(_proxyAddr.empty()) { + return false; + } if(logger->info()) { logger->info(MSG_CONNECTING_TO_SERVER, util::itos(cuid).c_str(), - dest.first.c_str(), - dest.second); + _proxyAddr.c_str(), proxyReq->getPort()); } - dataSocket.reset(new SocketCore()); - dataSocket->establishConnection(dest.first, dest.second); - + dataSocket.reset(new SocketCore()); + dataSocket->establishConnection(_proxyAddr, proxyReq->getPort()); disableReadCheckSocket(); setWriteCheckSocket(dataSocket); - - sequence = SEQ_SEND_REST_PASV; + _http.reset(new HttpConnection(cuid, dataSocket, getOption().get())); + sequence = SEQ_SEND_TUNNEL_REQUEST; return false; } +bool FtpNegotiationCommand::sendTunnelRequest() +{ + if(_http->sendBufferIsEmpty()) { + if(dataSocket->isReadable(0)) { + std::string error = socket->getSocketError(); + if(!error.empty()) { + SharedHandle proxyReq = createProxyRequest(); + e->markBadIPAddress(proxyReq->getHost(),_proxyAddr,proxyReq->getPort()); + std::string nextProxyAddr = e->findCachedIPAddress + (proxyReq->getHost(), proxyReq->getPort()); + if(nextProxyAddr.empty()) { + e->removeCachedIPAddress(proxyReq->getHost(), proxyReq->getPort()); + throw DL_RETRY_EX + (StringFormat(MSG_ESTABLISHING_CONNECTION_FAILED, + error.c_str()).str()); + } else { + if(logger->info()) { + logger->info(MSG_CONNECT_FAILED_AND_RETRY, + util::itos(cuid).c_str(), + _proxyAddr.c_str(), proxyReq->getPort()); + } + _proxyAddr = nextProxyAddr; + if(logger->info()) { + logger->info(MSG_CONNECTING_TO_SERVER, util::itos(cuid).c_str(), + _proxyAddr.c_str(), proxyReq->getPort()); + } + dataSocket->establishConnection(_proxyAddr, proxyReq->getPort()); + return false; + } + } + } + SharedHandle httpRequest(new HttpRequest()); + httpRequest->setUserAgent(getOption()->get(PREF_USER_AGENT)); + SharedHandle req(new Request()); + // Construct fake URI in order to use HttpRequest + req->setUri(strconcat("ftp://", _dataConnAddr.first, + A2STR::COLON_C, util::uitos(_dataConnAddr.second))); + httpRequest->setRequest(req); + httpRequest->setProxyRequest(createProxyRequest()); + _http->sendProxyRequest(httpRequest); + } else { + _http->sendPendingData(); + } + if(_http->sendBufferIsEmpty()) { + disableWriteCheckSocket(); + setReadCheckSocket(dataSocket); + sequence = SEQ_RECV_TUNNEL_RESPONSE; + return false; + } else { + setWriteCheckSocket(dataSocket); + return false; + } +} + +bool FtpNegotiationCommand::recvTunnelResponse() +{ + SharedHandle httpResponse = _http->receiveResponse(); + if(httpResponse.isNull()) { + return false; + } + if(httpResponse->getResponseStatus() != HttpHeader::S200) { + throw DL_RETRY_EX(EX_PROXY_CONNECTION_FAILED); + } + sequence = SEQ_SEND_REST_PASV; + return true; +} + bool FtpNegotiationCommand::sendRestPasv(const SharedHandle& segment) { //dataSocket->setBlockingMode(); setReadCheckSocket(socket); @@ -677,6 +777,12 @@ bool FtpNegotiationCommand::processSequence return sendPasv(); case SEQ_RECV_PASV: return recvPasv(); + case SEQ_RESOLVE_PROXY: + return resolveProxy(); + case SEQ_SEND_TUNNEL_REQUEST: + return sendTunnelRequest(); + case SEQ_RECV_TUNNEL_RESPONSE: + return recvTunnelResponse(); case SEQ_SEND_REST_PASV: return sendRestPasv(segment); case SEQ_SEND_REST: diff --git a/src/FtpNegotiationCommand.h b/src/FtpNegotiationCommand.h index 6e829ca4..7eb94795 100644 --- a/src/FtpNegotiationCommand.h +++ b/src/FtpNegotiationCommand.h @@ -41,6 +41,7 @@ namespace aria2 { class FtpConnection; class SocketCore; +class HttpConnection; class FtpNegotiationCommand : public AbstractCommand { public: @@ -65,6 +66,9 @@ public: SEQ_RECV_PORT, SEQ_SEND_PASV, SEQ_RECV_PASV, + SEQ_RESOLVE_PROXY, + SEQ_SEND_TUNNEL_REQUEST, + SEQ_RECV_TUNNEL_RESPONSE, SEQ_SEND_REST_PASV, SEQ_SEND_REST, SEQ_RECV_REST, @@ -99,6 +103,9 @@ private: bool recvPort(); bool sendPasv(); bool recvPasv(); + bool resolveProxy(); + bool sendTunnelRequest(); + bool recvTunnelResponse(); bool sendRest(const SharedHandle& segment); bool sendRestPasv(const SharedHandle& segment); bool recvRest(const SharedHandle& segment); @@ -119,6 +126,12 @@ private: SharedHandle serverSocket; Seq sequence; SharedHandle ftp; + // For tunneling + SharedHandle _http; + // IP, Port pair in pasv response + std::pair _dataConnAddr; + // Resolved address for proxy + std::string _proxyAddr; std::string _connectedHostname; std::string _connectedAddr; diff --git a/src/InitiateConnectionCommand.cc b/src/InitiateConnectionCommand.cc index 68a4356d..8b049a87 100644 --- a/src/InitiateConnectionCommand.cc +++ b/src/InitiateConnectionCommand.cc @@ -40,7 +40,6 @@ #include "message.h" #include "prefs.h" #include "NameResolver.h" -#include "DNSCache.h" #include "SocketCore.h" #include "FileEntry.h" #include "RequestGroup.h" @@ -81,47 +80,10 @@ bool InitiateConnectionCommand::executeInternal() { port = proxyRequest->getPort(); } std::vector addrs; - e->findAllCachedIPAddresses(std::back_inserter(addrs), hostname, port); - std::string ipaddr; - if(addrs.empty()) { -#ifdef ENABLE_ASYNC_DNS - if(getOption()->getAsBool(PREF_ASYNC_DNS)) { - if(!isAsyncNameResolverInitialized()) { - initAsyncNameResolver(hostname); - } - if(asyncResolveHostname()) { - addrs = getResolvedAddresses(); - } else { - e->commands.push_back(this); - return false; - } - } else -#endif // ENABLE_ASYNC_DNS - { - NameResolver res; - res.setSocktype(SOCK_STREAM); - if(e->option->getAsBool(PREF_DISABLE_IPV6)) { - res.setFamily(AF_INET); - } - res.resolve(addrs, hostname); - } - if(logger->info()) { - logger->info(MSG_NAME_RESOLUTION_COMPLETE, util::itos(cuid).c_str(), - hostname.c_str(), - strjoin(addrs.begin(), addrs.end(), ", ").c_str()); - } - for(std::vector::const_iterator i = addrs.begin(), - eoi = addrs.end(); i != eoi; ++i) { - e->cacheIPAddress(hostname, *i, port); - } - ipaddr = e->findCachedIPAddress(hostname, port); - } else { - ipaddr = addrs.front(); - if(logger->info()) { - logger->info(MSG_DNS_CACHE_HIT, - util::itos(cuid).c_str(), hostname.c_str(), - strjoin(addrs.begin(), addrs.end(), ", ").c_str()); - } + std::string ipaddr = resolveHostname(addrs, hostname, port); + if(ipaddr.empty()) { + e->commands.push_back(this); + return false; } try { Command* command = createNextCommand(hostname, ipaddr, port,