From e2700f50a5d8da1596cefccc01d70aa4696e3f4a Mon Sep 17 00:00:00 2001 From: Nils Maier Date: Sun, 15 Sep 2013 21:52:02 +0200 Subject: [PATCH] Code cleanup in Http* --- src/HttpConnection.cc | 7 +- src/HttpConnection.h | 18 +- src/HttpDownloadCommand.cc | 113 +++++----- src/HttpDownloadCommand.h | 2 + src/HttpResponse.cc | 247 +++++++++++----------- src/HttpResponse.h | 14 +- src/HttpResponseCommand.cc | 412 +++++++++++++++++++------------------ src/HttpResponseCommand.h | 11 +- 8 files changed, 427 insertions(+), 397 deletions(-) diff --git a/src/HttpConnection.cc b/src/HttpConnection.cc index 8574338e..5a7ce909 100644 --- a/src/HttpConnection.cc +++ b/src/HttpConnection.cc @@ -144,6 +144,7 @@ std::unique_ptr HttpConnection::receiveResponse() throw DL_RETRY_EX(EX_GOT_EOF); } } + const auto& proc = outstandingHttpRequests_.front()->getHttpHeaderProcessor(); if(proc->parse(socketRecvBuffer_->getBuffer(), socketRecvBuffer_->getBufferLength())) { @@ -158,10 +159,10 @@ std::unique_ptr HttpConnection::receiveResponse() socketRecvBuffer_->shiftBuffer(proc->getLastBytesProcessed()); outstandingHttpRequests_.pop_front(); return httpResponse; - } else { - socketRecvBuffer_->shiftBuffer(proc->getLastBytesProcessed()); - return nullptr; } + + socketRecvBuffer_->shiftBuffer(proc->getLastBytesProcessed()); + return nullptr; } bool HttpConnection::isIssued(const std::shared_ptr& segment) const diff --git a/src/HttpConnection.h b/src/HttpConnection.h index 1a8cef5d..4eadf1f1 100644 --- a/src/HttpConnection.h +++ b/src/HttpConnection.h @@ -58,11 +58,11 @@ class HttpRequestEntry { private: std::unique_ptr httpRequest_; std::unique_ptr proc_; + public: HttpRequestEntry(std::unique_ptr httpRequest); - const std::unique_ptr& getHttpRequest() const - { + const std::unique_ptr& getHttpRequest() const { return httpRequest_; } @@ -83,13 +83,12 @@ private: HttpRequestEntries outstandingHttpRequests_; std::string eraseConfidentialInfo(const std::string& request); - void sendRequest - (std::unique_ptr httpRequest, std::string request); + void sendRequest(std::unique_ptr httpRequest, + std::string request); + public: - HttpConnection - (cuid_t cuid, - const std::shared_ptr& socket, - const std::shared_ptr& socketRecvBuffer); + HttpConnection(cuid_t cuid, const std::shared_ptr& socket, + const std::shared_ptr& socketRecvBuffer); ~HttpConnection(); /** @@ -125,8 +124,7 @@ public: void sendPendingData(); - const std::shared_ptr& getSocketRecvBuffer() const - { + const std::shared_ptr& getSocketRecvBuffer() const { return socketRecvBuffer_; } }; diff --git a/src/HttpDownloadCommand.cc b/src/HttpDownloadCommand.cc index fb60add5..db57aa12 100644 --- a/src/HttpDownloadCommand.cc +++ b/src/HttpDownloadCommand.cc @@ -56,15 +56,14 @@ namespace aria2 { -HttpDownloadCommand::HttpDownloadCommand -(cuid_t cuid, - const std::shared_ptr& req, - const std::shared_ptr& fileEntry, - RequestGroup* requestGroup, - std::unique_ptr httpResponse, - const std::shared_ptr& httpConnection, - DownloadEngine* e, - const std::shared_ptr& socket) +HttpDownloadCommand::HttpDownloadCommand(cuid_t cuid, + const std::shared_ptr& req, + const std::shared_ptr& fileEntry, + RequestGroup* requestGroup, + std::unique_ptr httpResponse, + const std::shared_ptr& httpConnection, + DownloadEngine* e, + const std::shared_ptr& socket) : DownloadCommand(cuid, req, fileEntry, requestGroup, e, socket, httpConnection->getSocketRecvBuffer()), httpResponse_(std::move(httpResponse)), @@ -73,71 +72,73 @@ HttpDownloadCommand::HttpDownloadCommand HttpDownloadCommand::~HttpDownloadCommand() {} -bool HttpDownloadCommand::prepareForNextSegment() { +bool HttpDownloadCommand::prepareForNextSegment() +{ bool downloadFinished = getRequestGroup()->downloadFinished(); - if(getRequest()->isPipeliningEnabled() && !downloadFinished) { + if (getRequest()->isPipeliningEnabled() && !downloadFinished) { auto command = make_unique (getCuid(), getRequest(), getFileEntry(), getRequestGroup(), httpConnection_, getDownloadEngine(), getSocket()); // Set proxy request here. aria2 sends the HTTP request specialized for // proxy. - if(resolveProxyMethod(getRequest()->getProtocol()) == V_GET) { + if (resolveProxyMethod(getRequest()->getProtocol()) == V_GET) { command->setProxyRequest(createProxyRequest()); } getDownloadEngine()->addCommand(std::move(command)); return true; - } else { - const std::string& streamFilterName = getStreamFilter()->getName(); - if(getRequest()->isPipeliningEnabled() || - (getRequest()->isKeepAliveEnabled() && - ( - // Make sure that all filters are finished to pool socket - (!util::endsWith(streamFilterName, SinkStreamFilter::NAME) && - getStreamFilter()->finished()) || - getRequestEndOffset() == - getFileEntry()->gtoloff(getSegments().front()->getPositionToWrite()) - ) - ) - ) { - // TODO What if server sends EOF when non-SinkStreamFilter is - // used and server didn't send Connection: close? We end up to - // pool terminated socket. In HTTP/1.1, keep-alive is default, - // so closing connection without Connection: close header means - // that server is broken or not configured properly. - getDownloadEngine()->poolSocket - (getRequest(), createProxyRequest(), getSocket()); - } - // The request was sent assuming that server supported pipelining, but - // it turned out that server didn't support it. - // We detect this situation by comparing the end byte in range header - // of the response with the end byte of segment. - // If it is the same, HTTP negotiation is necessary for the next request. - if(!getRequest()->isPipeliningEnabled() && - getRequest()->isPipeliningHint() && - !downloadFinished) { - const std::shared_ptr& segment = getSegments().front(); - - int64_t lastOffset =getFileEntry()->gtoloff - (std::min(segment->getPosition()+segment->getLength(), - getFileEntry()->getLastOffset())); - Range range = httpResponse_->getHttpHeader()->getRange(); - if(lastOffset == range.endByte + 1) { - return prepareForRetry(0); - } - } - return DownloadCommand::prepareForNextSegment(); } + + const std::string& streamFilterName = getStreamFilter()->getName(); + if (getRequest()->isPipeliningEnabled() || + (getRequest()->isKeepAliveEnabled() && + ( + // Make sure that all filters are finished to pool socket + (!util::endsWith(streamFilterName, SinkStreamFilter::NAME) && + getStreamFilter()->finished()) || + getRequestEndOffset() == + getFileEntry()->gtoloff(getSegments().front()->getPositionToWrite()) + ) + ) + ) { + // TODO What if server sends EOF when non-SinkStreamFilter is + // used and server didn't send Connection: close? We end up to + // pool terminated socket. In HTTP/1.1, keep-alive is default, + // so closing connection without Connection: close header means + // that server is broken or not configured properly. + getDownloadEngine()->poolSocket(getRequest(), createProxyRequest(), + getSocket()); + } + + // The request was sent assuming that server supported pipelining, but + // it turned out that server didn't support it. + // We detect this situation by comparing the end byte in range header + // of the response with the end byte of segment. + // If it is the same, HTTP negotiation is necessary for the next request. + if (!getRequest()->isPipeliningEnabled() && + getRequest()->isPipeliningHint() && + !downloadFinished) { + const std::shared_ptr& segment = getSegments().front(); + + int64_t lastOffset =getFileEntry()->gtoloff( + std::min(segment->getPosition()+segment->getLength(), + getFileEntry()->getLastOffset())); + auto range = httpResponse_->getHttpHeader()->getRange(); + if (lastOffset == range.endByte + 1) { + return prepareForRetry(0); + } + } + + return DownloadCommand::prepareForNextSegment(); } int64_t HttpDownloadCommand::getRequestEndOffset() const { - int64_t endByte = httpResponse_->getHttpHeader()->getRange().endByte; - if(endByte > 0) { + auto endByte = httpResponse_->getHttpHeader()->getRange().endByte; + if (endByte > 0) { return endByte+1; - } else { - return endByte; } + return endByte; } } // namespace aria2 diff --git a/src/HttpDownloadCommand.h b/src/HttpDownloadCommand.h index b8e70cd2..e9b3801f 100644 --- a/src/HttpDownloadCommand.h +++ b/src/HttpDownloadCommand.h @@ -46,9 +46,11 @@ class HttpDownloadCommand : public DownloadCommand { private: std::unique_ptr httpResponse_; std::shared_ptr httpConnection_; + protected: virtual bool prepareForNextSegment() CXX11_OVERRIDE; virtual int64_t getRequestEndOffset() const CXX11_OVERRIDE; + public: HttpDownloadCommand(cuid_t cuid, const std::shared_ptr& req, diff --git a/src/HttpResponse.cc b/src/HttpResponse.cc index fa330867..c4562532 100644 --- a/src/HttpResponse.cc +++ b/src/HttpResponse.cc @@ -32,6 +32,7 @@ * files in the program, then also delete it here. */ /* copyright --> */ + #include "HttpResponse.h" #include "Request.h" #include "Segment.h" @@ -74,40 +75,43 @@ HttpResponse::HttpResponse() void HttpResponse::validateResponse() const { int statusCode = getStatusCode(); - if(statusCode >= 400) { + if (statusCode >= 400) { return; } - if(statusCode == 304) { - if(!httpRequest_->conditionalRequest()) { + + if (statusCode == 304) { + if (!httpRequest_->conditionalRequest()) { throw DL_ABORT_EX2("Got 304 without If-Modified-Since or If-None-Match", error_code::HTTP_PROTOCOL_ERROR); } - } else if(statusCode == 301 || - statusCode == 302 || - statusCode == 303 || - statusCode == 307) { - if(!httpHeader_->defined(HttpHeader::LOCATION)) { + } + else if (statusCode == 301 || + statusCode == 302 || + statusCode == 303 || + statusCode == 307) { + if (!httpHeader_->defined(HttpHeader::LOCATION)) { throw DL_ABORT_EX2(fmt(EX_LOCATION_HEADER_REQUIRED, statusCode), error_code::HTTP_PROTOCOL_ERROR); } return; - } else if(statusCode == 200 || statusCode == 206) { - if(!httpHeader_->defined(HttpHeader::TRANSFER_ENCODING)) { + } + else if (statusCode == 200 || statusCode == 206) { + if (!httpHeader_->defined(HttpHeader::TRANSFER_ENCODING)) { // compare the received range against the requested range - Range responseRange = httpHeader_->getRange(); - if(!httpRequest_->isRangeSatisfied(responseRange)) { - throw DL_ABORT_EX2 - (fmt(EX_INVALID_RANGE_HEADER, - httpRequest_->getStartByte(), - httpRequest_->getEndByte(), - httpRequest_->getEntityLength(), - responseRange.startByte, - responseRange.endByte, - responseRange.entityLength), - error_code::CANNOT_RESUME); + auto responseRange = httpHeader_->getRange(); + if (!httpRequest_->isRangeSatisfied(responseRange)) { + throw DL_ABORT_EX2(fmt(EX_INVALID_RANGE_HEADER, + httpRequest_->getStartByte(), + httpRequest_->getEndByte(), + httpRequest_->getEntityLength(), + responseRange.startByte, + responseRange.endByte, + responseRange.entityLength), + error_code::CANNOT_RESUME); } } - } else { + } + else { throw DL_ABORT_EX2(fmt("Unexpected status %d", statusCode), error_code::HTTP_PROTOCOL_ERROR); } @@ -115,18 +119,17 @@ void HttpResponse::validateResponse() const std::string HttpResponse::determinFilename() const { - std::string contentDisposition = - util::getContentDispositionFilename - (httpHeader_->find(HttpHeader::CONTENT_DISPOSITION)); - if(contentDisposition.empty()) { - std::string file = - util::percentDecode(httpRequest_->getFile().begin(), - httpRequest_->getFile().end()); - if(file.empty()) { + std::string contentDisposition = util::getContentDispositionFilename( + httpHeader_->find(HttpHeader::CONTENT_DISPOSITION)); + if (contentDisposition.empty()) { + auto file = httpRequest_->getFile(); + file = util::percentDecode(file.begin(), file.end()); + if (file.empty()) { return "index.html"; } return file; } + A2_LOG_INFO(fmt(MSG_CONTENT_DISPOSITION_DETECTED, cuid_, contentDisposition.c_str())); @@ -136,39 +139,36 @@ std::string HttpResponse::determinFilename() const void HttpResponse::retrieveCookie() { Time now; - std::pair::const_iterator, - std::multimap::const_iterator> r = - httpHeader_->equalRange(HttpHeader::SET_COOKIE); - for(; r.first != r.second; ++r.first) { - httpRequest_->getCookieStorage()->parseAndStore - ((*r.first).second, httpRequest_->getHost(), httpRequest_->getDir(), - now.getTime()); + auto r = httpHeader_->equalRange(HttpHeader::SET_COOKIE); + for (; r.first != r.second; ++r.first) { + httpRequest_->getCookieStorage()->parseAndStore( + (*r.first).second, + httpRequest_->getHost(), + httpRequest_->getDir(), + now.getTime()); } } bool HttpResponse::isRedirect() const { - int statusCode = getStatusCode(); - return (301 == statusCode || - 302 == statusCode || - 303 == statusCode || - 307 == statusCode) && - httpHeader_->defined(HttpHeader::LOCATION); + auto code = getStatusCode(); + return (301 == code || 302 == code || 303 == code || 307 == code) && + httpHeader_->defined(HttpHeader::LOCATION); } void HttpResponse::processRedirect() { - if(httpRequest_->getRequest()->redirectUri - (util::percentEncodeMini(getRedirectURI()))) { - A2_LOG_INFO(fmt(MSG_REDIRECT, - cuid_, - httpRequest_->getRequest()->getCurrentUri().c_str())); - } else { - throw DL_RETRY_EX - (fmt("CUID#%" PRId64 " - Redirect to %s failed. It may not be a valid URI.", - cuid_, - httpRequest_->getRequest()->getCurrentUri().c_str())); + const auto& req = httpRequest_->getRequest(); + if (!req->redirectUri(util::percentEncodeMini(getRedirectURI()))) { + throw DL_RETRY_EX(fmt("CUID#%" PRId64 + " - Redirect to %s failed. It may not be a valid URI.", + cuid_, + req->getCurrentUri().c_str())); } + + A2_LOG_INFO(fmt(MSG_REDIRECT, + cuid_, + httpRequest_->getRequest()->getCurrentUri().c_str())); } const std::string& HttpResponse::getRedirectURI() const @@ -192,8 +192,8 @@ HttpResponse::getTransferEncodingStreamFilter() const { // TODO Transfer-Encoding header field can contains multiple tokens. We should // parse the field and retrieve each token. - if(isTransferEncodingSpecified()) { - if(util::strieq(getTransferEncoding(), "chunked")) { + if (isTransferEncodingSpecified()) { + if (util::strieq(getTransferEncoding(), "chunked")) { return make_unique(); } } @@ -219,6 +219,7 @@ HttpResponse::getContentEncodingStreamFilter() const return make_unique(); } #endif // HAVE_ZLIB + return nullptr; } @@ -226,30 +227,30 @@ int64_t HttpResponse::getContentLength() const { if(!httpHeader_) { return 0; - } else { - return httpHeader_->getRange().getContentLength(); } + + return httpHeader_->getRange().getContentLength(); } int64_t HttpResponse::getEntityLength() const { if(!httpHeader_) { return 0; - } else { - return httpHeader_->getRange().entityLength; } + + return httpHeader_->getRange().entityLength; } std::string HttpResponse::getContentType() const { if(!httpHeader_) { return A2STR::NIL; - } else { - const std::string& ctype = httpHeader_->find(HttpHeader::CONTENT_TYPE); - std::string::const_iterator i = std::find(ctype.begin(), ctype.end(), ';'); - Scip p = util::stripIter(ctype.begin(), i); - return std::string(p.first, p.second); } + + const auto& ctype = httpHeader_->find(HttpHeader::CONTENT_TYPE); + auto i = std::find(ctype.begin(), ctype.end(), ';'); + Scip p = util::stripIter(ctype.begin(), i); + return std::string(p.first, p.second); } void HttpResponse::setHttpHeader(std::unique_ptr httpHeader) @@ -283,81 +284,93 @@ bool HttpResponse::supportsPersistentConnection() const } namespace { + bool parseMetalinkHttpLink(MetalinkHttpEntry& result, const std::string& s) { - std::string::const_iterator first = std::find(s.begin(), s.end(), '<'); - if(first == s.end()) { + const auto first = std::find(s.begin(), s.end(), '<'); + if (first == s.end()) { return false; } - std::string::const_iterator last = std::find(first, s.end(), '>'); + + auto last = std::find(first, s.end(), '>'); if(last == s.end()) { return false; } - std::pair p = util::stripIter(first+1, last); - if(p.first == p.second) { + + auto p = util::stripIter(first+1, last); + if (p.first == p.second) { return false; } + result.uri.assign(p.first, p.second); last = std::find(last, s.end(), ';'); - if(last != s.end()) { + if (last != s.end()) { ++last; } bool ok = false; - while(1) { + while (1) { std::string name, value; - std::pair r = - util::nextParam(name, value, last, s.end(), ';'); + auto r = util::nextParam(name, value, last, s.end(), ';'); last = r.first; - if(!r.second) { + + if (!r.second) { break; } + if(value.empty()) { if(name == "pref") { result.pref = true; } - } else { - if(name == "rel") { - if(value == "duplicate") { - ok = true; - } else { - ok = false; - } - } else if(name == "pri") { - int32_t priValue; - if(util::parseIntNoThrow(priValue, value)) { - if(1 <= priValue && priValue <= 999999) { - result.pri = priValue; - } - } - } else if(name == "geo") { - util::lowercase(value); - result.geo = value; + continue; + } + + if(name == "rel") { + if(value == "duplicate") { + ok = true; + } else { + ok = false; } + continue; + } + + if (name == "pri") { + int32_t priValue; + if(util::parseIntNoThrow(priValue, value)) { + if(1 <= priValue && priValue <= 999999) { + result.pri = priValue; + } + } + continue; + } + + if (name == "geo") { + util::lowercase(value); + result.geo = value; + continue; } } return ok; } + } // namespace // Metalink/HTTP is defined by http://tools.ietf.org/html/rfc6249. // Link header field is defined by http://tools.ietf.org/html/rfc5988. -void HttpResponse::getMetalinKHttpEntries -(std::vector& result, - const std::shared_ptr