2008-10-09 Tatsuhiro Tsujikawa <tujikawa at rednoah dot com>

Fixed the bug that referer header field contains fragment.
	Now Request class removes fragment part of URI and encodes it
	in Request::setUri() and setReferer().
	* src/Request.h
	* src/Request.cc
	* test/RequestTest.cc

ChangeLog

@@ -1,3 +1,12 @@
+2008-10-09  Tatsuhiro Tsujikawa  <tujikawa at rednoah dot com>
+
+	Fixed the bug that referer header field contains fragment.
+	Now Request class removes fragment part of URI and encodes it
+	in Request::setUri() and setReferer().
+	* src/Request.h
+	* src/Request.cc
+	* test/RequestTest.cc
+
 2008-10-08  Tatsuhiro Tsujikawa  <tujikawa at rednoah dot com>
 
 	Added missing m4 macros.

src/Request.cc

@@ -33,12 +33,23 @@
  */
 /* copyright --> */
 #include "Request.h"
+
+#include <utility>
+
 #include "Util.h"
 #include "FeatureConfig.h"
 #include "RecoverableException.h"
 #include "StringFormat.h"
 #include "A2STR.h"
-#include <utility>
+
+#define SAFE_CHARS "abcdefghijklmnopqrstuvwxyz"\
+"ABCDEFGHIJKLMNOPQRSTUVWXYZ"\
+"0123456789"\
+":/?[]@"\
+"!$&'()*+,;="\
+"-._~"\
+"%"\
+"#"
 
 namespace aria2 {
 
@@ -63,9 +74,51 @@ Request::Request():
 
 Request::~Request() {}
 
+static std::string removeFragment(const std::string url)
+{
+  std::string::size_type sharpIndex = url.find("#");
+  if(sharpIndex == std::string::npos) {
+    return url;
+  } else {
+    return url.substr(0, sharpIndex);
+  }
+}
+
+static bool isHexNumber(const char c)
+{
+  return ('0' <= c && c <= '9') || ('A' <= c && c <= 'F') ||
+    ('a' <= c && c <= 'f');
+}
+
+static std::string urlencode(const std::string& src)
+{
+  std::string result = src;
+  if(src.empty()) {
+    return result;
+  }
+  size_t lastIndex = src.size()-1;
+  result += "  ";
+  size_t index = lastIndex;
+  while(index-- > 0) {
+    const unsigned char c = result[index];
+    // '/' is not urlencoded because src is expected to be a path.
+    if(Util::shouldUrlencode(c)) {
+      if(c == '%') {
+	if(!isHexNumber(result[index+1]) || !isHexNumber(result[index+2])) {
+	  result.replace(index, 1, "%25");
+	}
+      } else {
+	result.replace(index, 1, StringFormat("%%%02x", c).str());
+      }
+    }
+  }
+  result.erase(result.size()-2);
+  return result;
+}
+
 bool Request::setUrl(const std::string& url) {
-  this->url = url;
+  this->url = urlencode(removeFragment(url));
-  return parseUrl(url);
+  return parseUrl(this->url);
 }
 
 bool Request::resetUrl() {
@@ -73,6 +126,11 @@ bool Request::resetUrl() {
   return parseUrl(url);
 }
 
+void Request::setReferer(const std::string& url)
+{
+  referer = previousUrl = urlencode(removeFragment(url));
+}
+
 bool Request::redirectUrl(const std::string& url) {
   previousUrl = A2STR::NIL;
   _supportsPersistentConnection = true;
@@ -93,14 +151,8 @@ bool Request::redirectUrl(const std::string& url) {
 }
 
 bool Request::parseUrl(const std::string& url) {
-  std::string tempUrl;
+  currentUrl = url;
-  std::string::size_type sharpIndex = url.find("#");
+  std::string tempUrl = url;
-  if(sharpIndex != std::string::npos) {
-    urlencode(tempUrl, url.substr(0, sharpIndex));
-  } else {
-    urlencode(tempUrl, url);
-  }
-  currentUrl = tempUrl;
   std::string query;
   host = A2STR::NIL;
   port = 0;
@@ -179,36 +231,6 @@ bool Request::parseUrl(const std::string& url) {
   return true;
 }
 
-bool Request::isHexNumber(const char c) const
-{
-  return ('0' <= c && c <= '9') || ('A' <= c && c <= 'F') || ('a' <= c && c <= 'f');
-}
-
-void Request::urlencode(std::string& result, const std::string& src) const
-{
-  if(src.empty()) {
-    result = A2STR::NIL;
-    return;
-  }
-  size_t lastIndex = src.size()-1;
-  result = src+"  ";
-  size_t index = lastIndex;
-  while(index-- > 0) {
-    const unsigned char c = result[index];
-    // '/' is not urlencoded because src is expected to be a path.
-    if(Util::shouldUrlencode(c)) {
-      if(c == '%') {
-	if(!isHexNumber(result[index+1]) || !isHexNumber(result[index+2])) {
-	  result.replace(index, 1, "%25");
-	}
-      } else {
-	result.replace(index, 1, StringFormat("%%%02x", c).str());
-      }
-    }
-  }
-  result.erase(result.size()-2);
-}
-
 void Request::resetRedirectCount()
 {
   _redirectCount = 0;

src/Request.h

@@ -35,18 +35,11 @@
 #ifndef _D_REQUEST_H_
 #define _D_REQUEST_H_
 #include "common.h"
-#include "SharedHandle.h"
+
 #include <string>
 #include <deque>
 
-#define SAFE_CHARS "abcdefghijklmnopqrstuvwxyz"\
+#include "SharedHandle.h"
-"ABCDEFGHIJKLMNOPQRSTUVWXYZ"\
-"0123456789"\
-":/?[]@"\
-"!$&'()*+,;="\
-"-._~"\
-"%"\
-"#"
 
 namespace aria2 {
 
@@ -87,10 +80,6 @@ private:
   std::string _password;
 
   bool parseUrl(const std::string& url);
-
-  bool isHexNumber(const char c) const;
-
-  void urlencode(std::string& result, const std::string& src) const;
 public:
   Request();
   virtual ~Request();
@@ -116,7 +105,7 @@ public:
   const std::string& getCurrentUrl() const { return currentUrl; }
   const std::string& getPreviousUrl() const { return previousUrl; }
   const std::string& getReferer() const { return referer; }
-  void setReferer(const std::string& url) { referer = previousUrl = url; }
+  void setReferer(const std::string& url);
   const std::string& getProtocol() const { return protocol; }
   const std::string& getHost() const { return host; }
   uint16_t getPort() const { return port; }

test/RequestTest.cc

@@ -1,8 +1,10 @@
 #include "Request.h"
+
+#include <cppunit/extensions/HelperMacros.h>
+
 #include "Netrc.h"
 #include "DefaultAuthResolver.h"
 #include "NetrcAuthResolver.h"
-#include <cppunit/extensions/HelperMacros.h>
 
 namespace aria2 {
 
@@ -33,6 +35,7 @@ class RequestTest:public CppUnit::TestFixture {
   CPPUNIT_TEST(testRedirectUrl2);
   CPPUNIT_TEST(testResetUrl);
   CPPUNIT_TEST(testInnerLink);
+  CPPUNIT_TEST(testInnerLinkInReferer);
   CPPUNIT_TEST_SUITE_END();
   
 public:
@@ -60,6 +63,7 @@ public:
   void testRedirectUrl2();
   void testResetUrl();
   void testInnerLink();
+  void testInnerLinkInReferer();
 };
 
 
@@ -267,6 +271,10 @@ void RequestTest::testSetUrl17()
   CPPUNIT_ASSERT_EQUAL(std::string("/file%3cwith%252%20%20space"), req.getDir());
   CPPUNIT_ASSERT_EQUAL(std::string("file%20with%20space;param"), req.getFile());
   CPPUNIT_ASSERT_EQUAL(std::string("?a=/?"), req.getQuery());
+  CPPUNIT_ASSERT_EQUAL(std::string("http://host:80/file%3cwith%252%20%20space"
+				   "/file%20with%20space;param?a=/?"),
+		       req.getCurrentUrl());
+  CPPUNIT_ASSERT_EQUAL(req.getCurrentUrl(), req.getUrl());
 }
 
 void RequestTest::testRedirectUrl() {
@@ -357,10 +365,21 @@ void RequestTest::testInnerLink() {
   Request req;
   bool v = req.setUrl("http://aria.rednoah.com/index.html#download");
   CPPUNIT_ASSERT(v);
+  CPPUNIT_ASSERT_EQUAL(std::string("http://aria.rednoah.com/index.html"),
+		       req.getUrl());
+  CPPUNIT_ASSERT_EQUAL(std::string("http://aria.rednoah.com/index.html"),
+		       req.getCurrentUrl());
   CPPUNIT_ASSERT_EQUAL(std::string("index.html"), req.getFile());
   CPPUNIT_ASSERT_EQUAL(std::string(""), req.getQuery());
 }
 
+void RequestTest::testInnerLinkInReferer() {
+  Request req;
+  req.setReferer("http://aria.rednoah.com/home.html#top");
+  CPPUNIT_ASSERT_EQUAL(std::string("http://aria.rednoah.com/home.html"),
+		       req.getReferer());
+}
+
 void RequestTest::testSetUrl_zeroUsername()
 {
   Request req;