transfer-encoding: support segmented download

Partial content responses (code 206) with content-range works at higher
level than content-length and transfer-encoding and is totally
compatible with both. In other words, transfer-encoding specifies the
encoding in which the request part is sent  while content-range species
the position of this part within the whole "representation".

Also, do range validation test for the response.

This parially reverts 5ccd5b6.

Fixes GH-1576.

src/HttpHeaderProcessor.cc

@@ -452,12 +452,9 @@ fin:
   // are present, delete content-length and content-range.  RFC 7230
   // says that sender must not send both transfer-encoding and
   // content-length.  If both present, transfer-encoding overrides
-  // content-length.  There is no text about transfer-encoding and
+  // content-length.
-  // content-range.  But there is no reason to send transfer-encoding
-  // when range is set.
   if (result_->defined(HttpHeader::TRANSFER_ENCODING)) {
     result_->remove(HttpHeader::CONTENT_LENGTH);
-    result_->remove(HttpHeader::CONTENT_RANGE);
   }
 
   return true;

src/HttpResponse.cc

@@ -74,7 +74,7 @@ void HttpResponse::validateResponse() const
   switch (statusCode) {
   case 200: // OK
   case 206: // Partial Content
-    if (!httpHeader_->defined(HttpHeader::TRANSFER_ENCODING)) {
+    {
       // compare the received range against the requested range
       auto responseRange = httpHeader_->getRange();
       if (!httpRequest_->isRangeSatisfied(responseRange)) {

src/HttpResponseCommand.cc

@@ -272,9 +272,6 @@ bool HttpResponseCommand::executeInternal()
     // update last modified time
     updateLastModifiedTime(httpResponse->getLastModifiedTime());
 
-    // If both transfer-encoding and total length is specified, we
-    // should have ignored total length.  In this case, we can not do
-    // segmented downloading
     if (totalLength == 0 || shouldInflateContentEncoding(httpResponse.get())) {
       // we ignore content-length when inflate is required
       fe->setLength(0);

test/HttpHeaderProcessorTest.cc

@@ -224,7 +224,7 @@ void HttpHeaderProcessorTest::testGetHttpResponseHeader_teAndCl()
   CPPUNIT_ASSERT_EQUAL(std::string("chunked"),
                        httpHeader->find(HttpHeader::TRANSFER_ENCODING));
   CPPUNIT_ASSERT(!httpHeader->defined(HttpHeader::CONTENT_LENGTH));
-  CPPUNIT_ASSERT(!httpHeader->defined(HttpHeader::CONTENT_RANGE));
+  CPPUNIT_ASSERT(httpHeader->defined(HttpHeader::CONTENT_RANGE));
 }
 
 void HttpHeaderProcessorTest::testBeyondLimit()

test/HttpResponseTest.cc

@@ -451,12 +451,12 @@ void HttpResponseTest::testValidateResponse_chunked()
                                     "bytes 0-10485760/10485761");
   httpResponse.getHttpHeader()->put(HttpHeader::TRANSFER_ENCODING, "chunked");
 
-  // if transfer-encoding is specified, then range validation is skipped.
+  // if transfer-encoding is specified, range validation is still necessary.
   try {
     httpResponse.validateResponse();
+    CPPUNIT_FAIL("exception must be thrown.");
   }
   catch (Exception& e) {
-    CPPUNIT_FAIL("exception must not be thrown.");
   }
 }