2010-04-07 Tatsuhiro Tsujikawa <t-tujikawa@users.sourceforge.net>

Fixed the bug that FTP data connection is not established via
	proxy when --ftp-proxy is defined and --ftp-pasv=true and
	--proxy-method=tunnel.
	* src/AbstractCommand.cc
	* src/AbstractCommand.h
	* src/FtpNegotiationCommand.cc
	* src/FtpNegotiationCommand.h
	* src/InitiateConnectionCommand.cc

ChangeLog

@@ -1,3 +1,14 @@
+2010-04-07  Tatsuhiro Tsujikawa  <t-tujikawa@users.sourceforge.net>
+
+	Fixed the bug that FTP data connection is not established via
+	proxy when --ftp-proxy is defined and --ftp-pasv=true and
+	--proxy-method=tunnel.
+	* src/AbstractCommand.cc
+	* src/AbstractCommand.h
+	* src/FtpNegotiationCommand.cc
+	* src/FtpNegotiationCommand.h
+	* src/InitiateConnectionCommand.cc
+
 2010-04-04  Tatsuhiro Tsujikawa  <t-tujikawa@users.sourceforge.net>
 
 	Updated po templates.

src/AbstractCommand.cc

@@ -65,6 +65,7 @@
 #include "LogFactory.h"
 #include "DownloadContext.h"
 #include "wallclock.h"
+#include "NameResolver.h"
 
 namespace aria2 {
 
@@ -604,8 +605,10 @@ bool AbstractCommand::asyncResolveHostname()
 {
   switch(_asyncNameResolver->getStatus()) {
   case AsyncNameResolver::STATUS_SUCCESS:
+    disableNameResolverCheck(_asyncNameResolver);
     return true;
   case AsyncNameResolver::STATUS_ERROR:
+    disableNameResolverCheck(_asyncNameResolver);
     if(!isProxyRequest(req->getProtocol(), getOption())) {
       e->_requestGroupMan->getOrCreateServerStat
         (req->getHost(), req->getProtocol())->setError();
@@ -647,6 +650,53 @@ bool AbstractCommand::nameResolveFinished() const {
 }
 #endif // ENABLE_ASYNC_DNS
 
+std::string AbstractCommand::resolveHostname
+(std::vector<std::string>& addrs, const std::string& hostname, uint16_t port)
+{
+  e->findAllCachedIPAddresses(std::back_inserter(addrs), hostname, port);
+  std::string ipaddr;
+  if(addrs.empty()) {
+#ifdef ENABLE_ASYNC_DNS
+    if(getOption()->getAsBool(PREF_ASYNC_DNS)) {
+      if(!isAsyncNameResolverInitialized()) {
+        initAsyncNameResolver(hostname);
+      }
+      if(asyncResolveHostname()) {
+        addrs = getResolvedAddresses();
+      } else {
+        return A2STR::NIL;
+      }
+    } else
+#endif // ENABLE_ASYNC_DNS
+      {
+        NameResolver res;
+        res.setSocktype(SOCK_STREAM);
+        if(e->option->getAsBool(PREF_DISABLE_IPV6)) {
+          res.setFamily(AF_INET);
+        }
+        res.resolve(addrs, hostname);
+      }
+    if(logger->info()) {
+      logger->info(MSG_NAME_RESOLUTION_COMPLETE, util::itos(cuid).c_str(),
+                   hostname.c_str(),
+                   strjoin(addrs.begin(), addrs.end(), ", ").c_str());
+    }
+    for(std::vector<std::string>::const_iterator i = addrs.begin(),
+          eoi = addrs.end(); i != eoi; ++i) {
+      e->cacheIPAddress(hostname, *i, port);
+    }
+    ipaddr = e->findCachedIPAddress(hostname, port);
+  } else {
+    ipaddr = addrs.front();
+    if(logger->info()) {
+      logger->info(MSG_DNS_CACHE_HIT,
+                   util::itos(cuid).c_str(), hostname.c_str(),
+                   strjoin(addrs.begin(), addrs.end(), ", ").c_str());
+    }
+  }
+  return ipaddr;
+}
+
 void AbstractCommand::prepareForNextAction(Command* nextCommand)
 {
   SharedHandle<CheckIntegrityEntry> entry

src/AbstractCommand.h

@@ -77,6 +77,14 @@ protected:
   const std::vector<std::string>& getResolvedAddresses();
 #endif // ENABLE_ASYNC_DNS
 
+  // Resolves hostname.  The resolved addresses are stored in addrs
+  // and first element is returned.  If resolve is not finished,
+  // return empty string. In this case, call this function with same
+  // arguments until resolved address is returned.  Exception is
+  // thrown on error. port is used for retrieving cached addresses.
+  std::string resolveHostname
+  (std::vector<std::string>& addrs, const std::string& hostname, uint16_t port);
+
   void tryReserved();
   virtual bool prepareForRetry(time_t wait);
   virtual void onAbort();
@@ -125,7 +133,6 @@ protected:
    */
   SharedHandle<Request> createProxyRequest() const;
 
-
   // Returns proxy method for given protocol. Either V_GET or V_TUNNEL
   // is returned.  For HTTPS, always returns V_TUNNEL.
   const std::string& resolveProxyMethod(const std::string& protocol) const;
@@ -149,7 +156,6 @@ private:
 
   void disableNameResolverCheck(const SharedHandle<AsyncNameResolver>& resolver);
   bool nameResolveFinished() const;
-
 #endif // ENABLE_ASYNC_DNS
 public:
   AbstractCommand(cuid_t cuid, const SharedHandle<Request>& req,

src/FtpNegotiationCommand.cc

@@ -65,6 +65,12 @@
 #include "AuthConfig.h"
 #include "a2functional.h"
 #include "URISelector.h"
+#include "HttpConnection.h"
+#include "HttpHeader.h"
+#include "HttpRequest.h"
+#include "HttpResponse.h"
+#include "DlRetryEx.h"
+#include "CookieStorage.h"
 
 namespace aria2 {
 
@@ -539,22 +545,116 @@ bool FtpNegotiationCommand::recvPasv() {
   if(status != 227) {
     throw DL_ABORT_EX(StringFormat(EX_BAD_STATUS, status).str());
   }
-  // make a data connection to the server.
+  // TODO Should we check to see that dest.first is not in noProxy list?
+  if(isProxyDefined()) {
+    _dataConnAddr = dest;
+    sequence = SEQ_RESOLVE_PROXY;
+    return true;
+  } else {
+    // make a data connection to the server.
+    if(logger->info()) {
+      logger->info(MSG_CONNECTING_TO_SERVER, util::itos(cuid).c_str(),
+                   dest.first.c_str(),
+                   dest.second);
+    }
+    dataSocket.reset(new SocketCore());
+    dataSocket->establishConnection(dest.first, dest.second);
+    disableReadCheckSocket();
+    setWriteCheckSocket(dataSocket);
+    sequence = SEQ_SEND_REST_PASV;
+    return false;
+  }
+}
+
+bool FtpNegotiationCommand::resolveProxy()
+{
+  SharedHandle<Request> proxyReq = createProxyRequest();
+  std::vector<std::string> addrs;
+  _proxyAddr = resolveHostname
+    (addrs, proxyReq->getHost(), proxyReq->getPort());
+  if(_proxyAddr.empty()) {
+    return false;
+  }
   if(logger->info()) {
     logger->info(MSG_CONNECTING_TO_SERVER, util::itos(cuid).c_str(),
-                 dest.first.c_str(),
+                 _proxyAddr.c_str(), proxyReq->getPort());
-                 dest.second);
   }
-  dataSocket.reset(new SocketCore());
+  dataSocket.reset(new SocketCore());                  
-  dataSocket->establishConnection(dest.first, dest.second);
+  dataSocket->establishConnection(_proxyAddr, proxyReq->getPort());
-
   disableReadCheckSocket();
   setWriteCheckSocket(dataSocket);
-
+  _http.reset(new HttpConnection(cuid, dataSocket, getOption().get()));
-  sequence = SEQ_SEND_REST_PASV;
+  sequence = SEQ_SEND_TUNNEL_REQUEST;
   return false;
 }
 
+bool FtpNegotiationCommand::sendTunnelRequest()
+{
+  if(_http->sendBufferIsEmpty()) {
+    if(dataSocket->isReadable(0)) {
+      std::string error = socket->getSocketError();
+      if(!error.empty()) {
+        SharedHandle<Request> proxyReq = createProxyRequest();
+        e->markBadIPAddress(proxyReq->getHost(),_proxyAddr,proxyReq->getPort());
+        std::string nextProxyAddr = e->findCachedIPAddress
+          (proxyReq->getHost(), proxyReq->getPort());
+        if(nextProxyAddr.empty()) {
+          e->removeCachedIPAddress(proxyReq->getHost(), proxyReq->getPort());
+          throw DL_RETRY_EX
+            (StringFormat(MSG_ESTABLISHING_CONNECTION_FAILED,
+                          error.c_str()).str());
+        } else {
+          if(logger->info()) {
+            logger->info(MSG_CONNECT_FAILED_AND_RETRY,
+                         util::itos(cuid).c_str(),
+                         _proxyAddr.c_str(), proxyReq->getPort());
+          }
+          _proxyAddr = nextProxyAddr;
+          if(logger->info()) {
+            logger->info(MSG_CONNECTING_TO_SERVER, util::itos(cuid).c_str(),
+                         _proxyAddr.c_str(), proxyReq->getPort());
+          }
+          dataSocket->establishConnection(_proxyAddr, proxyReq->getPort());
+          return false;
+        }
+      }
+    }      
+    SharedHandle<HttpRequest> httpRequest(new HttpRequest());
+    httpRequest->setUserAgent(getOption()->get(PREF_USER_AGENT));
+    SharedHandle<Request> req(new Request());
+    // Construct fake URI in order to use HttpRequest
+    req->setUri(strconcat("ftp://", _dataConnAddr.first,
+                          A2STR::COLON_C, util::uitos(_dataConnAddr.second)));
+    httpRequest->setRequest(req);
+    httpRequest->setProxyRequest(createProxyRequest());
+    _http->sendProxyRequest(httpRequest);
+  } else {
+    _http->sendPendingData();
+  }
+  if(_http->sendBufferIsEmpty()) {
+    disableWriteCheckSocket();
+    setReadCheckSocket(dataSocket);
+    sequence = SEQ_RECV_TUNNEL_RESPONSE;
+    return false;
+  } else {
+    setWriteCheckSocket(dataSocket);
+    return false;
+  }
+}
+
+bool FtpNegotiationCommand::recvTunnelResponse()
+{
+  SharedHandle<HttpResponse> httpResponse = _http->receiveResponse();
+  if(httpResponse.isNull()) {
+    return false;
+  }
+  if(httpResponse->getResponseStatus() != HttpHeader::S200) {
+    throw DL_RETRY_EX(EX_PROXY_CONNECTION_FAILED);
+  }
+  sequence = SEQ_SEND_REST_PASV;
+  return true;
+}
+
 bool FtpNegotiationCommand::sendRestPasv(const SharedHandle<Segment>& segment) {
   //dataSocket->setBlockingMode();
   setReadCheckSocket(socket);
@@ -677,6 +777,12 @@ bool FtpNegotiationCommand::processSequence
     return sendPasv();
   case SEQ_RECV_PASV:
     return recvPasv();
+  case SEQ_RESOLVE_PROXY:
+    return resolveProxy();
+  case SEQ_SEND_TUNNEL_REQUEST:
+    return sendTunnelRequest();
+  case SEQ_RECV_TUNNEL_RESPONSE:
+    return recvTunnelResponse();
   case SEQ_SEND_REST_PASV:
     return sendRestPasv(segment);
   case SEQ_SEND_REST:

src/FtpNegotiationCommand.h

@@ -41,6 +41,7 @@ namespace aria2 {
 
 class FtpConnection;
 class SocketCore;
+class HttpConnection;
 
 class FtpNegotiationCommand : public AbstractCommand {
 public:
@@ -65,6 +66,9 @@ public:
     SEQ_RECV_PORT,
     SEQ_SEND_PASV,
     SEQ_RECV_PASV,
+    SEQ_RESOLVE_PROXY,
+    SEQ_SEND_TUNNEL_REQUEST,
+    SEQ_RECV_TUNNEL_RESPONSE,
     SEQ_SEND_REST_PASV,
     SEQ_SEND_REST,
     SEQ_RECV_REST,
@@ -99,6 +103,9 @@ private:
   bool recvPort();
   bool sendPasv();
   bool recvPasv();
+  bool resolveProxy();
+  bool sendTunnelRequest();
+  bool recvTunnelResponse();
   bool sendRest(const SharedHandle<Segment>& segment);
   bool sendRestPasv(const SharedHandle<Segment>& segment);
   bool recvRest(const SharedHandle<Segment>& segment);
@@ -119,6 +126,12 @@ private:
   SharedHandle<SocketCore> serverSocket;
   Seq sequence;
   SharedHandle<FtpConnection> ftp;
+  // For tunneling
+  SharedHandle<HttpConnection> _http;
+  // IP, Port pair in pasv response
+  std::pair<std::string, uint16_t> _dataConnAddr;
+  // Resolved address for proxy
+  std::string _proxyAddr;
 
   std::string _connectedHostname;
   std::string _connectedAddr;

src/InitiateConnectionCommand.cc

@@ -40,7 +40,6 @@
 #include "message.h"
 #include "prefs.h"
 #include "NameResolver.h"
-#include "DNSCache.h"
 #include "SocketCore.h"
 #include "FileEntry.h"
 #include "RequestGroup.h"
@@ -81,47 +80,10 @@ bool InitiateConnectionCommand::executeInternal() {
     port = proxyRequest->getPort();
   }
   std::vector<std::string> addrs;
-  e->findAllCachedIPAddresses(std::back_inserter(addrs), hostname, port);
+  std::string ipaddr = resolveHostname(addrs, hostname, port);
-  std::string ipaddr;
+  if(ipaddr.empty()) {
-  if(addrs.empty()) {
+    e->commands.push_back(this);
-#ifdef ENABLE_ASYNC_DNS
+    return false;
-    if(getOption()->getAsBool(PREF_ASYNC_DNS)) {
-      if(!isAsyncNameResolverInitialized()) {
-        initAsyncNameResolver(hostname);
-      }
-      if(asyncResolveHostname()) {
-        addrs = getResolvedAddresses();
-      } else {
-        e->commands.push_back(this);
-        return false;
-      }
-    } else
-#endif // ENABLE_ASYNC_DNS
-      {
-        NameResolver res;
-        res.setSocktype(SOCK_STREAM);
-        if(e->option->getAsBool(PREF_DISABLE_IPV6)) {
-          res.setFamily(AF_INET);
-        }
-        res.resolve(addrs, hostname);
-      }
-    if(logger->info()) {
-      logger->info(MSG_NAME_RESOLUTION_COMPLETE, util::itos(cuid).c_str(),
-                   hostname.c_str(),
-                   strjoin(addrs.begin(), addrs.end(), ", ").c_str());
-    }
-    for(std::vector<std::string>::const_iterator i = addrs.begin(),
-          eoi = addrs.end(); i != eoi; ++i) {
-      e->cacheIPAddress(hostname, *i, port);
-    }
-    ipaddr = e->findCachedIPAddress(hostname, port);
-  } else {
-    ipaddr = addrs.front();
-    if(logger->info()) {
-      logger->info(MSG_DNS_CACHE_HIT,
-                   util::itos(cuid).c_str(), hostname.c_str(),
-                   strjoin(addrs.begin(), addrs.end(), ", ").c_str());
-    }
   }
   try {
     Command* command = createNextCommand(hostname, ipaddr, port,