From 93a49e484065d47b5d8426ab3c5a5f12ec8899d4 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 5 Nov 2008 12:30:22 +0000 Subject: [PATCH] 2008-11-05 Tatsuhiro Tsujikawa Added the ability to pool proxy connection. The conneciton in FTP with proxy-method=GET is not pooled. Proxy-Connection header will not be sent when sending CONNECT method. * src/DownloadEngine.cc * src/DownloadEngine.h * src/FtpFinishDownloadCommand.cc * src/FtpInitiateConnectionCommand.cc * src/FtpNegotiationCommand.cc * src/HttpDownloadCommand.cc * src/HttpInitiateConnectionCommand.cc * src/HttpRequest.cc * src/HttpRequest.h * src/HttpResponse.cc * src/HttpSkipResponseCommand.cc * test/HttpRequestTest.cc * test/HttpResponseTest.cc --- ChangeLog | 19 +++++++++ src/DownloadEngine.cc | 33 +++++++++++++++ src/DownloadEngine.h | 11 +++++ src/FtpFinishDownloadCommand.cc | 6 +-- src/FtpInitiateConnectionCommand.cc | 61 +++++++++++++++++++--------- src/FtpNegotiationCommand.cc | 6 +-- src/HttpDownloadCommand.cc | 18 ++++---- src/HttpInitiateConnectionCommand.cc | 49 ++++++++++++++-------- src/HttpRequest.cc | 16 +++++--- src/HttpRequest.h | 6 +++ src/HttpResponse.cc | 14 +++++-- src/HttpSkipResponseCommand.cc | 6 +-- test/HttpRequestTest.cc | 10 ++--- test/HttpResponseTest.cc | 55 ++++++++++++++++++++++--- 14 files changed, 234 insertions(+), 76 deletions(-) diff --git a/ChangeLog b/ChangeLog index 3569e075..bebd216c 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,22 @@ +2008-11-05 Tatsuhiro Tsujikawa + + Added the ability to pool proxy connection. + The conneciton in FTP with proxy-method=GET is not pooled. + Proxy-Connection header will not be sent when sending CONNECT method. + * src/DownloadEngine.cc + * src/DownloadEngine.h + * src/FtpFinishDownloadCommand.cc + * src/FtpInitiateConnectionCommand.cc + * src/FtpNegotiationCommand.cc + * src/HttpDownloadCommand.cc + * src/HttpInitiateConnectionCommand.cc + * src/HttpRequest.cc + * src/HttpRequest.h + * src/HttpResponse.cc + * src/HttpSkipResponseCommand.cc + * test/HttpRequestTest.cc + * test/HttpResponseTest.cc + 2008-11-05 Tatsuhiro Tsujikawa Handle date before epoch. diff --git a/src/DownloadEngine.cc b/src/DownloadEngine.cc index 14a334c6..2c9ac17b 100644 --- a/src/DownloadEngine.cc +++ b/src/DownloadEngine.cc @@ -65,6 +65,7 @@ #include "DNSCache.h" #include "AuthConfigFactory.h" #include "AuthConfig.h" +#include "Request.h" #include "BtRegistry.h" #include "BtContext.h" @@ -955,6 +956,38 @@ void DownloadEngine::poolSocket poolSocket(ipaddr, port, e); } +void DownloadEngine::poolSocket(const SharedHandle& request, + bool proxyDefined, + const SharedHandle& socket, + time_t timeout) +{ + if(proxyDefined) { + // If proxy is defined, then pool socket with its hostname. + poolSocket(request->getHost(), request->getPort(), socket); + } else { + std::pair peerInfo; + socket->getPeerInfo(peerInfo); + poolSocket(peerInfo.first, peerInfo.second, socket); + } +} + +void DownloadEngine::poolSocket +(const SharedHandle& request, + bool proxyDefined, + const SharedHandle& socket, + const std::map& options, + time_t timeout) +{ + if(proxyDefined) { + // If proxy is defined, then pool socket with its hostname. + poolSocket(request->getHost(), request->getPort(), socket, options); + } else { + std::pair peerInfo; + socket->getPeerInfo(peerInfo); + poolSocket(peerInfo.first, peerInfo.second, socket, options); + } +} + std::multimap::iterator DownloadEngine::findSocketPoolEntry(const std::string& ipaddr, uint16_t port) { diff --git a/src/DownloadEngine.h b/src/DownloadEngine.h index bd9c10c3..046c39be 100644 --- a/src/DownloadEngine.h +++ b/src/DownloadEngine.h @@ -403,6 +403,17 @@ public: const SharedHandle& sock, time_t timeout = 15); + void poolSocket(const SharedHandle& request, + bool proxyDefined, + const SharedHandle& socket, + const std::map& options, + time_t timeout = 15); + + void poolSocket(const SharedHandle& request, + bool proxyDefined, + const SharedHandle& socket, + time_t timeout = 15); + SharedHandle popPooledSocket(const std::string& ipaddr, uint16_t port); diff --git a/src/FtpFinishDownloadCommand.cc b/src/FtpFinishDownloadCommand.cc index 9a6cf3b5..76c42244 100644 --- a/src/FtpFinishDownloadCommand.cc +++ b/src/FtpFinishDownloadCommand.cc @@ -84,12 +84,10 @@ bool FtpFinishDownloadCommand::execute() if(status != 226) { throw DlAbortEx(StringFormat(EX_BAD_STATUS, status).str()); } - if(!isProxyDefined() && e->option->getAsBool(PREF_FTP_REUSE_CONNECTION)) { - std::pair peerInfo; - socket->getPeerInfo(peerInfo); + if(e->option->getAsBool(PREF_FTP_REUSE_CONNECTION)) { std::map options; options["baseWorkingDir"] = _ftpConnection->getBaseWorkingDir(); - e->poolSocket(peerInfo.first, peerInfo.second, socket, options); + e->poolSocket(req, isProxyDefined(), socket, options); } } catch(RecoverableException& e) { logger->info(EX_EXCEPTION_CAUGHT, e); diff --git a/src/FtpInitiateConnectionCommand.cc b/src/FtpInitiateConnectionCommand.cc index a31670d1..f0f65948 100644 --- a/src/FtpInitiateConnectionCommand.cc +++ b/src/FtpInitiateConnectionCommand.cc @@ -68,26 +68,49 @@ Command* FtpInitiateConnectionCommand::createNextCommand { Command* command; if(!proxyRequest.isNull()) { - logger->info(MSG_CONNECTING_TO_SERVER, cuid, - proxyRequest->getHost().c_str(), proxyRequest->getPort()); - socket.reset(new SocketCore()); - socket->establishConnection(resolvedAddresses.front(), - proxyRequest->getPort()); - - if(e->option->get(PREF_PROXY_METHOD) == V_GET) { - SharedHandle hc - (new HttpConnection(cuid, socket, e->option)); - - HttpRequestCommand* c = - new HttpRequestCommand(cuid, req, _requestGroup, hc, e, socket); - c->setProxyRequest(proxyRequest); - command = c; - } else if(e->option->get(PREF_PROXY_METHOD) == V_TUNNEL) { - command = new FtpTunnelRequestCommand(cuid, req, _requestGroup, e, - proxyRequest, socket); + std::map options; + SharedHandle pooledSocket = + e->popPooledSocket(options, req->getHost(), req->getPort()); + if(pooledSocket.isNull()) { + logger->info(MSG_CONNECTING_TO_SERVER, cuid, + proxyRequest->getHost().c_str(), proxyRequest->getPort()); + socket.reset(new SocketCore()); + socket->establishConnection(resolvedAddresses.front(), + proxyRequest->getPort()); + + if(e->option->get(PREF_PROXY_METHOD) == V_GET) { + SharedHandle hc + (new HttpConnection(cuid, socket, e->option)); + + HttpRequestCommand* c = + new HttpRequestCommand(cuid, req, _requestGroup, hc, e, socket); + c->setProxyRequest(proxyRequest); + command = c; + } else if(e->option->get(PREF_PROXY_METHOD) == V_TUNNEL) { + command = new FtpTunnelRequestCommand(cuid, req, _requestGroup, e, + proxyRequest, socket); + } else { + // TODO + throw DlAbortEx("ERROR"); + } } else { - // TODO - throw DlAbortEx("ERROR"); + if(e->option->get(PREF_PROXY_METHOD) == V_TUNNEL) { + command = + new FtpNegotiationCommand(cuid, req, _requestGroup, e, pooledSocket, + FtpNegotiationCommand::SEQ_SEND_CWD, + options["baseWorkingDir"]); + } else if(e->option->get(PREF_PROXY_METHOD) == V_GET) { + SharedHandle hc + (new HttpConnection(cuid, pooledSocket, e->option)); + + HttpRequestCommand* c = + new HttpRequestCommand(cuid, req, _requestGroup, hc, e, pooledSocket); + c->setProxyRequest(proxyRequest); + command = c; + } else { + // TODO + throw DlAbortEx("ERROR"); + } } } else { std::map options; diff --git a/src/FtpNegotiationCommand.cc b/src/FtpNegotiationCommand.cc index e2c48936..4f25804f 100644 --- a/src/FtpNegotiationCommand.cc +++ b/src/FtpNegotiationCommand.cc @@ -618,12 +618,10 @@ bool FtpNegotiationCommand::processSequence(const SegmentHandle& segment) { void FtpNegotiationCommand::poolConnection() const { - if(!isProxyDefined() && e->option->getAsBool(PREF_FTP_REUSE_CONNECTION)) { - std::pair peerInfo; - socket->getPeerInfo(peerInfo); + if(e->option->getAsBool(PREF_FTP_REUSE_CONNECTION)) { std::map options; options["baseWorkingDir"] = ftp->getBaseWorkingDir(); - e->poolSocket(peerInfo.first, peerInfo.second, socket, options); + e->poolSocket(req, isProxyDefined(), socket, options); } } diff --git a/src/HttpDownloadCommand.cc b/src/HttpDownloadCommand.cc index c6870dc2..cfb3e6fd 100644 --- a/src/HttpDownloadCommand.cc +++ b/src/HttpDownloadCommand.cc @@ -63,17 +63,13 @@ bool HttpDownloadCommand::prepareForNextSegment() { e->commands.push_back(command); return true; } else { - if(!isProxyDefined()) { - if(req->isPipeliningEnabled() || - (req->isKeepAliveEnabled() && - ((!_transferEncodingDecoder.isNull() && - _requestGroup->downloadFinished()) || - (uint64_t)_segments.front()->getPositionToWrite() == - _requestGroup->getTotalLength()))) { - std::pair peerInfo; - socket->getPeerInfo(peerInfo); - e->poolSocket(peerInfo.first, peerInfo.second, socket); - } + if(req->isPipeliningEnabled() || + (req->isKeepAliveEnabled() && + ((!_transferEncodingDecoder.isNull() && + _requestGroup->downloadFinished()) || + (uint64_t)_segments.front()->getPositionToWrite() == + _requestGroup->getTotalLength()))) { + e->poolSocket(req, isProxyDefined(), socket); } return DownloadCommand::prepareForNextSegment(); diff --git a/src/HttpInitiateConnectionCommand.cc b/src/HttpInitiateConnectionCommand.cc index e541c1bf..6e67718a 100644 --- a/src/HttpInitiateConnectionCommand.cc +++ b/src/HttpInitiateConnectionCommand.cc @@ -65,23 +65,40 @@ Command* HttpInitiateConnectionCommand::createNextCommand { Command* command; if(!proxyRequest.isNull()) { - logger->info(MSG_CONNECTING_TO_SERVER, cuid, - proxyRequest->getHost().c_str(), proxyRequest->getPort()); - socket.reset(new SocketCore()); - socket->establishConnection(resolvedAddresses.front(), - proxyRequest->getPort()); - if(useProxyTunnel()) { - command = new HttpProxyRequestCommand(cuid, req, _requestGroup, e, - proxyRequest, socket); - } else if(useProxyGet()) { - SharedHandle httpConnection(new HttpConnection(cuid, socket, e->option)); - HttpRequestCommand* c = new HttpRequestCommand(cuid, req, _requestGroup, - httpConnection, e, socket); - c->setProxyRequest(proxyRequest); - command = c; + SharedHandle pooledSocket = + e->popPooledSocket(req->getHost(), req->getPort()); + if(pooledSocket.isNull()) { + logger->info(MSG_CONNECTING_TO_SERVER, cuid, + proxyRequest->getHost().c_str(), proxyRequest->getPort()); + socket.reset(new SocketCore()); + socket->establishConnection(resolvedAddresses.front(), + proxyRequest->getPort()); + + if(useProxyTunnel()) { + command = new HttpProxyRequestCommand(cuid, req, _requestGroup, e, + proxyRequest, socket); + } else if(useProxyGet()) { + SharedHandle httpConnection + (new HttpConnection(cuid, socket, e->option)); + HttpRequestCommand* c = new HttpRequestCommand(cuid, req, _requestGroup, + httpConnection, e, + socket); + c->setProxyRequest(proxyRequest); + command = c; + } else { + // TODO + throw DlAbortEx("ERROR"); + } } else { - // TODO - throw DlAbortEx("ERROR"); + SharedHandle httpConnection + (new HttpConnection(cuid, pooledSocket, e->option)); + HttpRequestCommand* c = new HttpRequestCommand(cuid, req, _requestGroup, + httpConnection, e, + pooledSocket); + if(useProxyGet()) { + c->setProxyRequest(proxyRequest); + } + command = c; } } else { SharedHandle pooledSocket = diff --git a/src/HttpRequest.cc b/src/HttpRequest.cc index 2b80adaf..0a0656f5 100644 --- a/src/HttpRequest.cc +++ b/src/HttpRequest.cc @@ -245,11 +245,12 @@ std::string HttpRequest::createProxyRequest() const std::string(" HTTP/1.1\r\n")+ "User-Agent: "+userAgent+"\r\n"+ "Host: "+getHost()+":"+Util::uitos(getPort())+"\r\n"; - if(request->isKeepAliveEnabled() || request->isPipeliningEnabled()) { - requestLine += "Proxy-Connection: Keep-Alive\r\n"; - }else { - requestLine += "Proxy-Connection: close\r\n"; - } + // TODO Is "Proxy-Connection" needed here? +// if(request->isKeepAliveEnabled() || request->isPipeliningEnabled()) { +// requestLine += "Proxy-Connection: Keep-Alive\r\n"; +// }else { +// requestLine += "Proxy-Connection: close\r\n"; +// } if(!_proxyRequest->getUsername().empty()) { requestLine += getProxyAuthString(); } @@ -354,4 +355,9 @@ void HttpRequest::setProxyRequest(const SharedHandle& proxyRequest) _proxyRequest = proxyRequest; } +bool HttpRequest::isProxyRequestSet() const +{ + return !_proxyRequest.isNull(); +} + } // namespace aria2 diff --git a/src/HttpRequest.h b/src/HttpRequest.h index b80089bb..fcb09be2 100644 --- a/src/HttpRequest.h +++ b/src/HttpRequest.h @@ -176,6 +176,12 @@ public: * object. */ void setProxyRequest(const SharedHandle& proxyRequest); + + /* + * Returns true if non-Null proxy request is set by setProxyRequest(). + * Otherwise, returns false. + */ + bool isProxyRequestSet() const; }; typedef SharedHandle HttpRequestHandle; diff --git a/src/HttpResponse.cc b/src/HttpResponse.cc index e3f76ec9..715effce 100644 --- a/src/HttpResponse.cc +++ b/src/HttpResponse.cc @@ -259,9 +259,17 @@ Time HttpResponse::getLastModifiedTime() const bool HttpResponse::supportsPersistentConnection() const { - return Util::toLower(httpHeader->getFirst(HttpHeader::CONNECTION)). - find(HttpHeader::CLOSE) == std::string::npos - && httpHeader->getVersion() == HttpHeader::HTTP_1_1; + std::string connection = + Util::toLower(httpHeader->getFirst(HttpHeader::CONNECTION)); + std::string version = httpHeader->getVersion(); + + return + connection.find(HttpHeader::CLOSE) == std::string::npos && + (version == HttpHeader::HTTP_1_1 || + connection.find("keep-alive") != std::string::npos) && + (!httpRequest->isProxyRequestSet() || + Util::toLower(httpHeader->getFirst("Proxy-Connection")).find("keep-alive") + != std::string::npos); } } // namespace aria2 diff --git a/src/HttpSkipResponseCommand.cc b/src/HttpSkipResponseCommand.cc index 1cacc0b6..90511203 100644 --- a/src/HttpSkipResponseCommand.cc +++ b/src/HttpSkipResponseCommand.cc @@ -137,10 +137,8 @@ bool HttpSkipResponseCommand::executeInternal() void HttpSkipResponseCommand::poolConnection() const { - if(!isProxyDefined() && req->supportsPersistentConnection()) { - std::pair peerInfo; - socket->getPeerInfo(peerInfo); - e->poolSocket(peerInfo.first, peerInfo.second, socket); + if(req->supportsPersistentConnection()) { + e->poolSocket(req, isProxyDefined(), socket); } } diff --git a/test/HttpRequestTest.cc b/test/HttpRequestTest.cc index 62a60d4c..da6f3a37 100644 --- a/test/HttpRequestTest.cc +++ b/test/HttpRequestTest.cc @@ -471,7 +471,7 @@ void HttpRequestTest::testCreateProxyRequest() std::string expectedText = "CONNECT localhost:80 HTTP/1.1\r\n" "User-Agent: aria2\r\n" "Host: localhost:80\r\n" - "Proxy-Connection: close\r\n" + //"Proxy-Connection: close\r\n" "\r\n"; CPPUNIT_ASSERT_EQUAL(expectedText, httpRequest.createProxyRequest()); @@ -482,7 +482,7 @@ void HttpRequestTest::testCreateProxyRequest() expectedText = "CONNECT localhost:80 HTTP/1.1\r\n" "User-Agent: aria2\r\n" "Host: localhost:80\r\n" - "Proxy-Connection: Keep-Alive\r\n" + //"Proxy-Connection: Keep-Alive\r\n" "\r\n"; CPPUNIT_ASSERT_EQUAL(expectedText, httpRequest.createProxyRequest()); @@ -494,7 +494,7 @@ void HttpRequestTest::testCreateProxyRequest() expectedText = "CONNECT localhost:80 HTTP/1.1\r\n" "User-Agent: aria2\r\n" "Host: localhost:80\r\n" - "Proxy-Connection: Keep-Alive\r\n" + //"Proxy-Connection: Keep-Alive\r\n" "\r\n"; CPPUNIT_ASSERT_EQUAL(expectedText, httpRequest.createProxyRequest()); @@ -506,7 +506,7 @@ void HttpRequestTest::testCreateProxyRequest() expectedText = "CONNECT localhost:80 HTTP/1.1\r\n" "User-Agent: aria2\r\n" "Host: localhost:80\r\n" - "Proxy-Connection: Keep-Alive\r\n" + //"Proxy-Connection: Keep-Alive\r\n" "Proxy-Authorization: Basic YXJpYTJwcm94eXVzZXI6YXJpYTJwcm94eXBhc3N3ZA==\r\n" "\r\n"; @@ -601,7 +601,7 @@ void HttpRequestTest::testUserAgent() std::string expectedTextForProxy = "CONNECT localhost:8080 HTTP/1.1\r\n" "User-Agent: aria2 (Linux)\r\n" "Host: localhost:8080\r\n" - "Proxy-Connection: close\r\n" + //"Proxy-Connection: close\r\n" "\r\n"; CPPUNIT_ASSERT_EQUAL(expectedTextForProxy, httpRequest.createProxyRequest()); diff --git a/test/HttpResponseTest.cc b/test/HttpResponseTest.cc index 4ced6997..81176154 100644 --- a/test/HttpResponseTest.cc +++ b/test/HttpResponseTest.cc @@ -496,19 +496,64 @@ void HttpResponseTest::testSupportsPersistentConnection() HttpResponse httpResponse; SharedHandle httpHeader(new HttpHeader()); httpResponse.setHttpHeader(httpHeader); + SharedHandle httpRequest(new HttpRequest()); + httpResponse.setHttpRequest(httpRequest); httpHeader->setVersion("HTTP/1.1"); CPPUNIT_ASSERT(httpResponse.supportsPersistentConnection()); - - httpHeader->setVersion("HTTP/1.0"); - CPPUNIT_ASSERT(!httpResponse.supportsPersistentConnection()); - - httpHeader->setVersion("HTTP/1.1"); httpHeader->put("Connection", "close"); CPPUNIT_ASSERT(!httpResponse.supportsPersistentConnection()); httpHeader->clearField(); httpHeader->put("Connection", "keep-alive"); CPPUNIT_ASSERT(httpResponse.supportsPersistentConnection()); + httpHeader->clearField(); + + httpHeader->setVersion("HTTP/1.0"); + CPPUNIT_ASSERT(!httpResponse.supportsPersistentConnection()); + httpHeader->put("Connection", "close"); + CPPUNIT_ASSERT(!httpResponse.supportsPersistentConnection()); + httpHeader->clearField(); + httpHeader->put("Connection", "keep-alive"); + CPPUNIT_ASSERT(httpResponse.supportsPersistentConnection()); + httpHeader->clearField(); + + // test proxy connection + SharedHandle proxyRequest(new Request()); + httpRequest->setProxyRequest(proxyRequest); + + httpHeader->setVersion("HTTP/1.1"); + CPPUNIT_ASSERT(!httpResponse.supportsPersistentConnection()); + httpHeader->put("Connection", "close"); + CPPUNIT_ASSERT(!httpResponse.supportsPersistentConnection()); + httpHeader->clearField(); + httpHeader->put("Connection", "keep-alive"); + CPPUNIT_ASSERT(!httpResponse.supportsPersistentConnection()); + httpHeader->clearField(); + httpHeader->put("Proxy-Connection", "keep-alive"); + CPPUNIT_ASSERT(httpResponse.supportsPersistentConnection()); + httpHeader->put("Connection", "close"); + CPPUNIT_ASSERT(!httpResponse.supportsPersistentConnection()); + httpHeader->clearField(); + httpHeader->put("Proxy-Connection", "close"); + CPPUNIT_ASSERT(!httpResponse.supportsPersistentConnection()); + httpHeader->clearField(); + + httpHeader->setVersion("HTTP/1.0"); + CPPUNIT_ASSERT(!httpResponse.supportsPersistentConnection()); + httpHeader->put("Connection", "close"); + CPPUNIT_ASSERT(!httpResponse.supportsPersistentConnection()); + httpHeader->clearField(); + httpHeader->put("Connection", "keep-alive"); + CPPUNIT_ASSERT(!httpResponse.supportsPersistentConnection()); + httpHeader->put("Proxy-Connection", "keep-alive"); + CPPUNIT_ASSERT(httpResponse.supportsPersistentConnection()); + httpHeader->clearField(); + httpHeader->put("Proxy-Connection", "keep-alive"); + CPPUNIT_ASSERT(!httpResponse.supportsPersistentConnection()); + httpHeader->clearField(); + httpHeader->put("Proxy-Connection", "close"); + CPPUNIT_ASSERT(!httpResponse.supportsPersistentConnection()); + httpHeader->clearField(); } } // namespace aria2