Merge 83cb6c15d3 into d372b788ec

src/ChunkedDecodingStreamFilter.cc

@@ -84,10 +84,31 @@ ChunkedDecodingStreamFilter::transform(const std::shared_ptr<BinaryStream>& out,
                                        const unsigned char* inbuf, size_t inlen)
 {
   ssize_t outlen = 0;
-  size_t i;
   bytesProcessed_ = 0;
-  for (i = 0; i < inlen; ++i) {
+  while (bytesProcessed_ < inlen) {
-    unsigned char c = inbuf[i];
+    if (state_ == CHUNKS_COMPLETE) {
+      break;
+    }
+    if (state_ == CHUNK) {
+      int64_t readlen = std::min(chunkRemaining_,
+                                 static_cast<int64_t>(inlen - bytesProcessed_));
+      outlen += getDelegate()->transform(out, segment, inbuf + bytesProcessed_,
+                                         readlen);
+      int64_t processedlen = getDelegate()->getBytesProcessed();
+      bytesProcessed_ += processedlen;
+      chunkRemaining_ -= processedlen;
+      if (chunkRemaining_ == 0) {
+        state_ = PREV_CHUNK_CR;
+      }
+      if (processedlen < readlen) {
+        // segment download finished
+        break;
+      }
+      continue;
+    }
+    // The following states consume single char
+    unsigned char c = inbuf[bytesProcessed_];
+    bytesProcessed_++;
     switch (state_) {
     case PREV_CHUNK_SIZE:
       if (util::isHexDigit(c)) {
@@ -136,17 +157,6 @@ ChunkedDecodingStreamFilter::transform(const std::shared_ptr<BinaryStream>& out,
                           "missing LF at the end of chunk size");
       }
       break;
-    case CHUNK: {
-      int64_t readlen =
-          std::min(chunkRemaining_, static_cast<int64_t>(inlen - i));
-      outlen += getDelegate()->transform(out, segment, inbuf + i, readlen);
-      chunkRemaining_ -= readlen;
-      i += readlen - 1;
-      if (chunkRemaining_ == 0) {
-        state_ = PREV_CHUNK_CR;
-      }
-      break;
-    }
     case PREV_CHUNK_CR:
       if (c == '\r') {
         state_ = PREV_CHUNK_LF;
@@ -203,15 +213,12 @@ ChunkedDecodingStreamFilter::transform(const std::shared_ptr<BinaryStream>& out,
                           "missing LF at the end of chunks");
       }
       break;
-    case CHUNKS_COMPLETE:
-      goto fin;
     default:
       // unreachable
       assert(0);
     }
   }
-fin:
+
-  bytesProcessed_ += i;
   return outlen;
 }
 

src/GZipDecodingStreamFilter.cc

@@ -47,6 +47,7 @@ GZipDecodingStreamFilter::GZipDecodingStreamFilter(
     std::unique_ptr<StreamFilter> delegate)
     : StreamFilter{std::move(delegate)},
       strm_{nullptr},
+      outbuf_(),
       finished_{false},
       bytesProcessed_{0}
 {
@@ -57,6 +58,8 @@ GZipDecodingStreamFilter::~GZipDecodingStreamFilter() { release(); }
 void GZipDecodingStreamFilter::init()
 {
   finished_ = false;
+  outbuf_.reserve(OUTBUF_CAPACITY);
+  outbuf_.resize(0);
   release();
   strm_ = new z_stream();
   strm_->zalloc = Z_NULL;
@@ -87,42 +90,51 @@ GZipDecodingStreamFilter::transform(const std::shared_ptr<BinaryStream>& out,
 {
   bytesProcessed_ = 0;
   ssize_t outlen = 0;
-  if (inlen == 0) {
-    return outlen;
-  }
 
   strm_->avail_in = inlen;
   strm_->next_in = const_cast<unsigned char*>(inbuf);
 
-  unsigned char outbuf[OUTBUF_LENGTH];
+  while (bytesProcessed_ < inlen) {
-  while (1) {
+    // inflate into outbuf_, if empty!
-    strm_->avail_out = OUTBUF_LENGTH;
+    if (outbuf_.empty()) {
-    strm_->next_out = outbuf;
+      outbuf_.resize(OUTBUF_CAPACITY);
+      strm_->avail_out = outbuf_.size();
+      strm_->next_out = outbuf_.data();
 
-    int ret = ::inflate(strm_, Z_NO_FLUSH);
+      int ret = ::inflate(strm_, Z_NO_FLUSH);
+      if (ret == Z_STREAM_END) {
+        finished_ = true;
+      }
+      else if (ret != Z_OK && ret != Z_BUF_ERROR) {
+        throw DL_ABORT_EX(fmt("libz::inflate() failed. cause:%s", strm_->msg));
+      }
 
-    if (ret == Z_STREAM_END) {
+      assert(inlen >= strm_->avail_in);
-      finished_ = true;
+      bytesProcessed_ = strm_->next_in - inbuf;
-    }
+      outbuf_.resize(strm_->next_out - outbuf_.data());
-    else if (ret != Z_OK && ret != Z_BUF_ERROR) {
+      if (outbuf_.empty())
-      throw DL_ABORT_EX(fmt("libz::inflate() failed. cause:%s", strm_->msg));
+        break;
     }
 
-    size_t produced = OUTBUF_LENGTH - strm_->avail_out;
+    // flush outbuf_
-
+    outlen += getDelegate()->transform(out, segment, outbuf_.data(),
-    outlen += getDelegate()->transform(out, segment, outbuf, produced);
+                                       outbuf_.size());
-    if (strm_->avail_out > 0) {
+    size_t processedlen = getDelegate()->getBytesProcessed();
+    if (processedlen == outbuf_.size()) {
+      outbuf_.clear();
+    }
+    else {
+      // segment download finished
+      outbuf_.erase(outbuf_.begin(), outbuf_.begin() + processedlen);
       break;
     }
   }
-  assert(inlen >= strm_->avail_in);
-  bytesProcessed_ = inlen - strm_->avail_in;
   return outlen;
 }
 
 bool GZipDecodingStreamFilter::finished()
 {
-  return finished_ && getDelegate()->finished();
+  return finished_ && outbuf_.empty() && getDelegate()->finished();
 }
 
 const std::string& GZipDecodingStreamFilter::getName() const { return NAME; }

src/GZipDecodingStreamFilter.h

@@ -37,6 +37,7 @@
 
 #include "StreamFilter.h"
 #include <zlib.h>
+#include <vector>
 
 #include "a2functional.h"
 
@@ -47,11 +48,13 @@ class GZipDecodingStreamFilter : public StreamFilter {
 private:
   z_stream* strm_;
 
+  std::vector<unsigned char> outbuf_;
+
   bool finished_;
 
   size_t bytesProcessed_;
 
-  static const size_t OUTBUF_LENGTH = 16_k;
+  static const size_t OUTBUF_CAPACITY = 16_k;
 
 public:
   GZipDecodingStreamFilter(std::unique_ptr<StreamFilter> delegate = nullptr);

src/HttpHeaderProcessor.cc

@@ -452,12 +452,9 @@ fin:
   // are present, delete content-length and content-range.  RFC 7230
   // says that sender must not send both transfer-encoding and
   // content-length.  If both present, transfer-encoding overrides
-  // content-length.  There is no text about transfer-encoding and
+  // content-length.
-  // content-range.  But there is no reason to send transfer-encoding
-  // when range is set.
   if (result_->defined(HttpHeader::TRANSFER_ENCODING)) {
     result_->remove(HttpHeader::CONTENT_LENGTH);
-    result_->remove(HttpHeader::CONTENT_RANGE);
   }
 
   return true;

src/HttpRequest.cc

@@ -213,6 +213,9 @@ std::string HttpRequest::createRequest()
     }
     builtinHds.emplace_back("Range:", rangeHeader);
   }
+  else if (!segment_ && getMethod() == "GET") {
+    builtinHds.emplace_back("Range:", "bytes=0-");
+  }
   if (proxyRequest_) {
     if (request_->isKeepAliveEnabled() || request_->isPipeliningEnabled()) {
       builtinHds.emplace_back("Connection:", "Keep-Alive");

src/HttpResponse.cc

@@ -74,7 +74,7 @@ void HttpResponse::validateResponse() const
   switch (statusCode) {
   case 200: // OK
   case 206: // Partial Content
-    if (!httpHeader_->defined(HttpHeader::TRANSFER_ENCODING)) {
+    {
       // compare the received range against the requested range
       auto responseRange = httpHeader_->getRange();
       if (!httpRequest_->isRangeSatisfied(responseRange)) {

src/HttpResponseCommand.cc

@@ -272,9 +272,6 @@ bool HttpResponseCommand::executeInternal()
     // update last modified time
     updateLastModifiedTime(httpResponse->getLastModifiedTime());
 
-    // If both transfer-encoding and total length is specified, we
-    // should have ignored total length.  In this case, we can not do
-    // segmented downloading
     if (totalLength == 0 || shouldInflateContentEncoding(httpResponse.get())) {
       // we ignore content-length when inflate is required
       fe->setLength(0);

test/HttpHeaderProcessorTest.cc

@@ -224,7 +224,7 @@ void HttpHeaderProcessorTest::testGetHttpResponseHeader_teAndCl()
   CPPUNIT_ASSERT_EQUAL(std::string("chunked"),
                        httpHeader->find(HttpHeader::TRANSFER_ENCODING));
   CPPUNIT_ASSERT(!httpHeader->defined(HttpHeader::CONTENT_LENGTH));
-  CPPUNIT_ASSERT(!httpHeader->defined(HttpHeader::CONTENT_RANGE));
+  CPPUNIT_ASSERT(httpHeader->defined(HttpHeader::CONTENT_RANGE));
 }
 
 void HttpHeaderProcessorTest::testBeyondLimit()

test/HttpRequestTest.cc

@@ -527,6 +527,7 @@ void HttpRequestTest::testCreateRequest_query()
       "Pragma: no-cache\r\n"
       "Cache-Control: no-cache\r\n"
       "Connection: close\r\n"
+      "Range: bytes=0-\r\n"
       "\r\n";
 
   CPPUNIT_ASSERT_EQUAL(expectedText, httpRequest.createRequest());
@@ -624,6 +625,7 @@ void HttpRequestTest::testCreateRequest_wantDigest()
                              "Pragma: no-cache\r\n"
                              "Cache-Control: no-cache\r\n"
                              "Connection: close\r\n"
+                             "Range: bytes=0-\r\n"
                              "Want-Digest: " +
                              wantDigest +
                              "\r\n"
@@ -781,6 +783,7 @@ void HttpRequestTest::testUserAgent()
                              "Pragma: no-cache\r\n"
                              "Cache-Control: no-cache\r\n"
                              "Connection: close\r\n"
+                             "Range: bytes=0-\r\n"
                              "\r\n";
 
   CPPUNIT_ASSERT_EQUAL(expectedText, httpRequest.createRequest());
@@ -818,6 +821,7 @@ void HttpRequestTest::testAddHeader()
                              "Pragma: no-cache\r\n"
                              "Cache-Control: no-cache\r\n"
                              "Connection: close\r\n"
+                             "Range: bytes=0-\r\n"
                              "X-ARIA2: v0.13\r\n"
                              "X-ARIA2-DISTRIBUTE: enabled\r\n"
                              "Accept: text/html\r\n"
@@ -847,6 +851,7 @@ void HttpRequestTest::testAcceptMetalink()
       "Pragma: no-cache\r\n"
       "Cache-Control: no-cache\r\n"
       "Connection: close\r\n"
+      "Range: bytes=0-\r\n"
       "\r\n";
 
   CPPUNIT_ASSERT_EQUAL(expectedText, httpRequest.createRequest());
@@ -876,6 +881,7 @@ void HttpRequestTest::testEnableAcceptEncoding()
                                  "Pragma: no-cache\r\n"
                                  "Cache-Control: no-cache\r\n"
                                  "Connection: close\r\n"
+                                 "Range: bytes=0-\r\n"
                                  "\r\n";
 
   std::string expectedText = expectedTextHead;

test/HttpResponseTest.cc

@@ -451,12 +451,12 @@ void HttpResponseTest::testValidateResponse_chunked()
                                     "bytes 0-10485760/10485761");
   httpResponse.getHttpHeader()->put(HttpHeader::TRANSFER_ENCODING, "chunked");
 
-  // if transfer-encoding is specified, then range validation is skipped.
+  // if transfer-encoding is specified, range validation is still necessary.
   try {
     httpResponse.validateResponse();
+    CPPUNIT_FAIL("exception must be thrown.");
   }
   catch (Exception& e) {
-    CPPUNIT_FAIL("exception must not be thrown.");
   }
 }