Merge branch 'Jimmy-Z-master'

doc/manual-src/en/aria2c.rst

@@ -1211,6 +1211,12 @@ Advanced Options
   Set log level to output to console.  LEVEL is either ``debug``,
   ``info``, ``notice``, ``warn`` or ``error``.  Default: ``notice``
 
+.. option:: --content-disposition-default-utf8[=true|false]
+
+  Handle quoted string in Content-Disposition header as UTF-8 instead
+  of ISO-8859-1, for example, the filename parameter, but not the
+  extended version filename*.  Default: ``false``
+
 .. option:: -D, --daemon[=true|false]
 
   Run as daemon. The current working directory will be changed to ``/``
@@ -2104,6 +2110,7 @@ of URIs. These optional lines must start with white space(s).
   * :option:`checksum <--checksum>`
   * :option:`conditional-get <--conditional-get>`
   * :option:`connect-timeout <--connect-timeout>`
+  * :option:`content-disposition-default-utf8 <--content-disposition-default-utf8>`
   * :option:`continue <-c>`
   * :option:`dir <-d>`
   * :option:`dry-run <--dry-run>`

src/HttpResponse.cc

@@ -112,10 +112,11 @@ void HttpResponse::validateResponse() const
                      error_code::HTTP_PROTOCOL_ERROR);
 }
 
-std::string HttpResponse::determineFilename() const
+std::string HttpResponse::determineFilename(bool contentDispositionUTF8) const
 {
   std::string contentDisposition = util::getContentDispositionFilename(
-      httpHeader_->find(HttpHeader::CONTENT_DISPOSITION));
+      httpHeader_->find(HttpHeader::CONTENT_DISPOSITION),
+      contentDispositionUTF8);
   if (contentDisposition.empty()) {
     auto file = httpRequest_->getFile();
     file = util::percentDecode(file.begin(), file.end());

src/HttpResponse.h

@@ -70,7 +70,7 @@ public:
    * this function returns the filename from it.
    * If it is not there, returns the part of filename from the request URL.
    */
-  std::string determineFilename() const;
+  std::string determineFilename(bool contentDispositionUTF8) const;
 
   void retrieveCookie();
 

src/HttpResponseCommand.cc

@@ -261,8 +261,8 @@ bool HttpResponseCommand::executeInternal()
     int64_t totalLength = httpResponse->getEntityLength();
     fe->setLength(totalLength);
     if (fe->getPath().empty()) {
-      auto suffixPath = util::createSafePath(httpResponse->determineFilename());
+      auto suffixPath = util::createSafePath(httpResponse->determineFilename(
-
+          getOption()->getAsBool(PREF_CONTENT_DISPOSITION_DEFAULT_UTF8)));
       fe->setPath(util::applyDir(getOption()->get(PREF_DIR), suffixPath));
       fe->setSuffixPath(suffixPath);
     }

src/OptionHandlerFactory.cc

@@ -1054,6 +1054,18 @@ std::vector<OptionHandler*> OptionHandlerFactory::createOptionHandlers()
     op->addTag(TAG_HTTPS);
     handlers.push_back(op);
   }
+  {
+    OptionHandler* op(
+        new BooleanOptionHandler(PREF_CONTENT_DISPOSITION_DEFAULT_UTF8,
+                                 TEXT_CONTENT_DISPOSITION_DEFAULT_UTF8,
+                                 A2_V_FALSE, OptionHandler::OPT_ARG));
+    op->addTag(TAG_ADVANCED);
+    op->addTag(TAG_HTTP);
+    op->setInitialOption(true);
+    op->setChangeGlobalOption(true);
+    op->setChangeOptionForReserved(true);
+    handlers.push_back(op);
+  }
   {
     OptionHandler* op(new BooleanOptionHandler(
         PREF_ENABLE_HTTP_KEEP_ALIVE, TEXT_ENABLE_HTTP_KEEP_ALIVE, A2_V_TRUE,

src/prefs.cc

@@ -427,6 +427,9 @@ PrefPtr PREF_HTTP_AUTH_CHALLENGE = makePref("http-auth-challenge");
 PrefPtr PREF_HTTP_NO_CACHE = makePref("http-no-cache");
 // value: true | false
 PrefPtr PREF_HTTP_ACCEPT_GZIP = makePref("http-accept-gzip");
+// value: true | false
+PrefPtr PREF_CONTENT_DISPOSITION_DEFAULT_UTF8 =
+    makePref("content-disposition-default-utf8");
 
 /**
  * Proxy related preferences

src/prefs.h

@@ -380,6 +380,8 @@ extern PrefPtr PREF_HTTP_AUTH_CHALLENGE;
 extern PrefPtr PREF_HTTP_NO_CACHE;
 // value: true | false
 extern PrefPtr PREF_HTTP_ACCEPT_GZIP;
+// value: true | false
+extern PrefPtr PREF_CONTENT_DISPOSITION_DEFAULT_UTF8;
 
 /**;
  * Proxy related preferences

src/usage_text.h

@@ -537,6 +537,11 @@
 #define TEXT_USE_HEAD                                                   \
   _(" --use-head[=true|false]      Use HEAD method for the first request to the HTTP\n" \
     "                              server.")
+#define TEXT_CONTENT_DISPOSITION_DEFAULT_UTF8                          \
+  _(" --content-disposition-default-utf8[=true|false] Handle quoted string in\n" \
+    "                              Content-Disposition header as UTF-8 instead of\n" \
+    "                              ISO-8859-1, for example, the filename parameter,\n" \
+    "                              but not the extended version filename*.")
 #define TEXT_EVENT_POLL                                                 \
   _(" --event-poll=POLL            Specify the method for polling events.")
 #define TEXT_BT_EXTERNAL_IP                                             \

src/util.cc

@@ -879,7 +879,7 @@ typedef enum {
 
 ssize_t parse_content_disposition(char* dest, size_t destlen,
                                   const char** charsetp, size_t* charsetlenp,
-                                  const char* in, size_t len)
+                                  const char* in, size_t len, bool defaultUTF8)
 {
   const char *p = in, *eop = in + len, *mark_first = nullptr,
              *mark_last = nullptr;
@@ -891,7 +891,7 @@ ssize_t parse_content_disposition(char* dest, size_t destlen,
   /* To suppress warnings */
   char* dp = dest;
   size_t dlen = destlen;
-  uint32_t dfa_state = 0;
+  uint32_t dfa_state = UTF8_ACCEPT;
   uint32_t dfa_code = 0;
   uint8_t pctval = 0;
 
@@ -981,6 +981,10 @@ ssize_t parse_content_disposition(char* dest, size_t destlen,
       if (*p == '"') {
         quoted_seen = 0;
         state = CD_QUOTED_STRING;
+        if (defaultUTF8) {
+          dfa_state = UTF8_ACCEPT;
+          dfa_code = 0;
+        }
       }
       else if (inRFC2616HttpToken(*p)) {
         if (in_file_parm) {
@@ -1011,16 +1015,25 @@ ssize_t parse_content_disposition(char* dest, size_t destlen,
         quoted_seen = 1;
       }
       else if (*p == '"' && quoted_seen == 0) {
+        if (defaultUTF8 && dfa_state != UTF8_ACCEPT) {
+          return -1;
+        }
         if (in_file_parm) {
           flags |= CD_FILENAME_FOUND;
         }
         state = CD_AFTER_VALUE;
       }
       else {
-        /* TEXT which is OCTET except CTLs, but including LWS. We only
+        /* TEXT which is OCTET except CTLs, but including LWS. Accept
-           accept ISO-8859-1 chars. */
+           ISO-8859-1 chars, or UTF-8 if defaultUTF8 is set */
         quoted_seen = 0;
-        if (!isIso8859p1(*p)) {
+        if (defaultUTF8) {
+          if (utf8dfa(&dfa_state, &dfa_code, (unsigned char)*p) ==
+              UTF8_REJECT) {
+            return -1;
+          }
+        }
+        else if (!isIso8859p1(*p)) {
           return -1;
         }
         if (in_file_parm) {
@@ -1204,7 +1217,8 @@ ssize_t parse_content_disposition(char* dest, size_t destlen,
   }
 }
 
-std::string getContentDispositionFilename(const std::string& header)
+std::string getContentDispositionFilename(const std::string& header,
+                                          bool defaultUTF8)
 {
   std::array<char, 1_k> cdval;
   size_t cdvallen = cdval.size();
@@ -1212,13 +1226,14 @@ std::string getContentDispositionFilename(const std::string& header)
   size_t charsetlen;
   ssize_t rv =
       parse_content_disposition(cdval.data(), cdvallen, &charset, &charsetlen,
-                                header.c_str(), header.size());
+                                header.c_str(), header.size(), defaultUTF8);
   if (rv == -1) {
     return "";
   }
 
   std::string res;
-  if (!charset || strieq(charset, charset + charsetlen, "iso-8859-1")) {
+  if ((charset && strieq(charset, charset + charsetlen, "iso-8859-1")) ||
+      (!charset && !defaultUTF8)) {
     res = iso8859p1ToUtf8(cdval.data(), rv);
   }
   else {

src/util.h

@@ -322,11 +322,15 @@ std::string iso8859p1ToUtf8(const std::string& src);
 // succeeds, or -1. If there is enough room to store filename in
 // |dest|, this function returns -1. If this function returns -1, the
 // |dest|, |*charsetp| and |*charsetlenp| are undefined.
+//
+// It will handle quoted string(as in RFC 7230 section 3.2.6) as
+// ISO-8859-1 by default, or UTF-8 if defaultUTF8 == true.
 ssize_t parse_content_disposition(char* dest, size_t destlen,
                                   const char** charsetp, size_t* charsetlenp,
-                                  const char* in, size_t len);
+                                  const char* in, size_t len, bool defaultUTF8);
 
-std::string getContentDispositionFilename(const std::string& header);
+std::string getContentDispositionFilename(const std::string& header,
+                                          bool defaultUTF8);
 
 std::string toUpper(std::string src);
 

test/HttpResponseTest.cc

@@ -148,7 +148,7 @@ void HttpResponseTest::testDetermineFilename_without_ContentDisposition()
   httpResponse.setHttpRequest(std::move(httpRequest));
 
   CPPUNIT_ASSERT_EQUAL(std::string("aria2-1.0.0.tar.bz2"),
-                       httpResponse.determineFilename());
+                       httpResponse.determineFilename(false));
 }
 
 void HttpResponseTest::
@@ -166,7 +166,7 @@ void HttpResponseTest::
   httpResponse.setHttpRequest(std::move(httpRequest));
 
   CPPUNIT_ASSERT_EQUAL(std::string("aria2-1.0.0.tar.bz2"),
-                       httpResponse.determineFilename());
+                       httpResponse.determineFilename(false));
 }
 
 void HttpResponseTest::testDetermineFilename_with_ContentDisposition()
@@ -184,7 +184,7 @@ void HttpResponseTest::testDetermineFilename_with_ContentDisposition()
   httpResponse.setHttpRequest(std::move(httpRequest));
 
   CPPUNIT_ASSERT_EQUAL(std::string("aria2-current.tar.bz2"),
-                       httpResponse.determineFilename());
+                       httpResponse.determineFilename(false));
 }
 
 void HttpResponseTest::testGetRedirectURI_without_Location()