2009-01-31 Tatsuhiro Tsujikawa <t-tujikawa@users.sourceforge.net>

Send HTTP Authorization header only if it is requested by the
	server(response 401). There are some exceptions. The
	authorization header for proxy is always sent if it is available	
	without request	by the server.  If username/password is specified
	 in the URI, it is immediately sent to the server without
	request by the server.
	
	AbstractAuthResolver::_defaultAuthConfig is now initialized as
	SharedHandle<AuthConfig>().
	NetrcAuthResolver::resolveAuthConfig() returns
	SharedHandle<AuthConfig>() if no AuthConfig is available.
	* src/AbstractAuthResolver.cc
	* src/AbstractProxyRequestCommand.cc
	* src/AuthConfigFactory.cc
	* src/AuthConfigFactory.h
	* src/HttpConnection.cc
	* src/HttpRequest.cc
	* src/HttpRequest.h
	* src/HttpResponse.cc
	* src/HttpResponseCommand.cc
	* src/HttpSkipResponseCommand.cc
	* test/AuthConfigFactoryTest.cc
	* test/HttpRequestTest.cc
	* test/HttpResponseTest.cc

ChangeLog

@@ -1,3 +1,29 @@
+2009-01-31  Tatsuhiro Tsujikawa  <t-tujikawa@users.sourceforge.net>
+
+	Send HTTP Authorization header only if it is requested by the
+	server(response 401). There are some exceptions. The authorization
+	header for proxy is always sent if it is available without request
+	by the server.  If username/password is specified in the URI, it
+	is immediately sent to the server without request by the server.
+	
+	AbstractAuthResolver::_defaultAuthConfig is now initialized as
+	SharedHandle<AuthConfig>().
+	NetrcAuthResolver::resolveAuthConfig() returns
+	SharedHandle<AuthConfig>() if no AuthConfig is available.
+	* src/AbstractAuthResolver.cc
+	* src/AbstractProxyRequestCommand.cc
+	* src/AuthConfigFactory.cc
+	* src/AuthConfigFactory.h
+	* src/HttpConnection.cc
+	* src/HttpRequest.cc
+	* src/HttpRequest.h
+	* src/HttpResponse.cc
+	* src/HttpResponseCommand.cc
+	* src/HttpSkipResponseCommand.cc
+	* test/AuthConfigFactoryTest.cc
+	* test/HttpRequestTest.cc
+	* test/HttpResponseTest.cc
+
 2009-01-27  Tatsuhiro Tsujikawa  <t-tujikawa@users.sourceforge.net>
 
 	Remove leading "--" from help keyword and a substring after "="

src/AbstractAuthResolver.cc

@@ -37,8 +37,7 @@
 
 namespace aria2 {
 
-AbstractAuthResolver::AbstractAuthResolver():
-  _defaultAuthConfig(new AuthConfig()) {}
+AbstractAuthResolver::AbstractAuthResolver() {}
 
 AbstractAuthResolver::~AbstractAuthResolver() {}
 

src/AbstractProxyRequestCommand.cc

@@ -44,6 +44,7 @@
 #include "Socket.h"
 #include "CookieStorage.h"
 #include "AuthConfigFactory.h"
+#include "AuthConfig.h"
 
 namespace aria2 {
 
@@ -72,7 +73,6 @@ bool AbstractProxyRequestCommand::executeInternal() {
     HttpRequestHandle httpRequest(new HttpRequest());
     httpRequest->setUserAgent(e->option->get(PREF_USER_AGENT));
     httpRequest->setRequest(req);
-    httpRequest->setAuthConfigFactory(e->getAuthConfigFactory());
     httpRequest->setProxyRequest(_proxyRequest);
 
     httpConnection->sendProxyRequest(httpRequest);

src/AuthConfigFactory.cc

@@ -33,6 +33,9 @@
  */
 /* copyright --> */
 #include "AuthConfigFactory.h"
+
+#include <algorithm>
+
 #include "Option.h"
 #include "AuthConfig.h"
 #include "Netrc.h"
@@ -40,6 +43,7 @@
 #include "NetrcAuthResolver.h"
 #include "prefs.h"
 #include "Request.h"
+#include "Util.h"
 
 namespace aria2 {
 
@@ -53,10 +57,25 @@ AuthConfigFactory::AuthConfigFactory(const Option* option):
 AuthConfigFactory::~AuthConfigFactory() {}
 
 AuthConfigHandle
-AuthConfigFactory::createAuthConfig(const RequestHandle& request) const
+AuthConfigFactory::createAuthConfig(const RequestHandle& request)
 {
   if(request->getProtocol() == Request::PROTO_HTTP ||
      request->getProtocol() == Request::PROTO_HTTPS) {
+
+    if(!request->getUsername().empty()) {
+      // TODO setting "/" as path. Should we use request->getDir() instead?
+      updateBasicCred(BasicCred(request->getUsername(), request->getPassword(),
+				request->getHost(), "/", true));
+      return createAuthConfig(request->getUsername(), request->getPassword());
+    }
+    std::deque<BasicCred>::const_iterator i =
+      findBasicCred(request->getHost(), request->getDir());
+    if(i == _basicCreds.end()) {
+      return SharedHandle<AuthConfig>();
+    } else {
+      return createAuthConfig((*i)._user, (*i)._password);
+    }
+
     return createHttpAuthResolver()->resolveAuthConfig(request->getHost());
   } else if(request->getProtocol() == Request::PROTO_FTP) {
     if(!request->getUsername().empty()) {
@@ -65,7 +84,7 @@ AuthConfigFactory::createAuthConfig(const RequestHandle& request) const
       return createFtpAuthResolver()->resolveAuthConfig(request->getHost());
     }
   } else {
-    return SharedHandle<AuthConfig>(new AuthConfig());
+    return SharedHandle<AuthConfig>();
   }
 }
 
@@ -73,7 +92,7 @@ AuthConfigHandle
 AuthConfigFactory::createAuthConfig(const std::string& user, const std::string& password) const
 {
   SharedHandle<AuthConfig> ac;
-  if(user.length() > 0) {
+  if(!user.empty()) {
     ac.reset(new AuthConfig(user, password));
   }
   return ac;
@@ -117,4 +136,94 @@ void AuthConfigFactory::setNetrc(const NetrcHandle& netrc)
   _netrc = netrc;
 }
 
+void AuthConfigFactory::updateBasicCred(const BasicCred& basicCred)
+{
+  std::deque<BasicCred>::iterator i =
+    std::lower_bound(_basicCreds.begin(), _basicCreds.end(), basicCred);
+
+  if(i != _basicCreds.end() && (*i) == basicCred) {
+    (*i) = basicCred;
+  } else {
+    _basicCreds.insert(i, basicCred);
+  }
+}
+
+bool AuthConfigFactory::activateBasicCred
+(const std::string& host, const std::string& path)
+{
+
+  std::deque<BasicCred>::iterator i =
+    findBasicCred(host, path);
+  if(i == _basicCreds.end()) {
+    SharedHandle<AuthConfig> authConfig =
+      createHttpAuthResolver()->resolveAuthConfig(host);
+    if(authConfig.isNull()) {
+      return false;
+    } else {
+      BasicCred bc("", "", host, path);
+      i = std::lower_bound(_basicCreds.begin(), _basicCreds.end(), bc);
+      // TODO setting "/" as path. Should we use path instead?
+      _basicCreds.insert
+	(i, BasicCred(authConfig->getUser(), authConfig->getPassword(),
+		      host, "/", true));
+      return true;
+    }
+  } else {
+    (*i).activate();
+    return true;
+  }
+}
+
+AuthConfigFactory::BasicCred::BasicCred
+(const std::string& user, const std::string& password,
+ const std::string& host, const std::string& path,
+ bool activated):
+  _user(user), _password(password),
+  _host(host), _path(path), _activated(activated)
+{
+  if(!Util::endsWith(_path, "/")) {
+    _path += "/";
+  }
+}
+
+void AuthConfigFactory::BasicCred::activate()
+{
+  _activated = true;
+}
+
+bool AuthConfigFactory::BasicCred::isActivated() const
+{
+  return _activated;
+}
+
+bool AuthConfigFactory::BasicCred::operator==(const BasicCred& cred) const
+{
+  return _host == cred._host && _path == cred._path;
+}
+
+bool AuthConfigFactory::BasicCred::operator<(const BasicCred& cred) const
+{
+  int c = _host.compare(cred._host);
+  if(c == 0) {
+    return _path > cred._path;
+  } else {
+    return c < 0;
+  }
+}
+
+std::deque<AuthConfigFactory::BasicCred>::iterator
+AuthConfigFactory::findBasicCred(const std::string& host,
+				 const std::string& path)
+{
+  BasicCred bc("", "", host, path);
+  std::deque<BasicCred>::iterator i =
+    std::lower_bound(_basicCreds.begin(), _basicCreds.end(), bc);
+  for(; i != _basicCreds.end() && (*i)._host == host; ++i) {
+    if(Util::startsWith(bc._path, (*i)._path)) {
+      return i;
+    }
+  }
+  return _basicCreds.end();
+}
+
 } // namespace aria2

src/AuthConfigFactory.h

@@ -38,6 +38,7 @@
 #include "common.h"
 
 #include <string>
+#include <deque>
 
 #include "SharedHandle.h"
 #include "SingletonHolder.h"
@@ -62,7 +63,29 @@ private:
   SharedHandle<AuthResolver> createHttpAuthResolver() const;
   
   SharedHandle<AuthResolver> createFtpAuthResolver() const;
+public:
+  class BasicCred {
+  public:
+    std::string _user;
+    std::string _password;
+    std::string _host;
+    std::string _path;
+    bool _activated;
 
+    BasicCred(const std::string& user, const std::string& password,
+	      const std::string& host, const std::string& path,
+	      bool activated = false);
+
+    void activate();
+
+    bool isActivated() const;
+
+    bool operator==(const BasicCred& cred) const;
+
+    bool operator<(const BasicCred& cred) const;
+  };
+private:
+  std::deque<BasicCred> _basicCreds;
 public:
   
   AuthConfigFactory(const Option* option);
@@ -70,10 +93,25 @@ public:
   ~AuthConfigFactory();
 
   SharedHandle<AuthConfig> createAuthConfig
-  (const SharedHandle<Request>& request) const;
+  (const SharedHandle<Request>& request);
 
   void setNetrc(const SharedHandle<Netrc>& netrc);
 
+  // Find a BasicCred using findBasicCred() and activate it then
+  // return true.  If matching BasicCred is not found, then return
+  // false.
+  bool activateBasicCred(const std::string& host, const std::string& path);
+
+  // Find a BasicCred using host and path and return the iterator
+  // pointing to it. If not found, then return _basicCreds.end().
+  std::deque<AuthConfigFactory::BasicCred>::iterator
+  findBasicCred(const std::string& host, const std::string& path);
+
+  // If the same BasicCred is already added, then it is replaced with
+  // given basicCred. Otherwise, insert given basicCred to
+  // _basicCreds.
+  void updateBasicCred(const BasicCred& basicCred);
+
   static const std::string ANONYMOUS;
 
   static const std::string ARIA2USER_AT;

src/HttpConnection.cc

@@ -53,6 +53,7 @@
 #include "Option.h"
 #include "CookieStorage.h"
 #include "AuthConfigFactory.h"
+#include "AuthConfig.h"
 
 namespace aria2 {
 

src/HttpRequest.cc

@@ -132,18 +132,17 @@ std::string HttpRequest::getHostText(const std::string& host, uint16_t port) con
   return  host+(port == 80 || port == 443 ? "" : ":"+Util::uitos(port));
 }
 
-std::string HttpRequest::createRequest() const
+std::string HttpRequest::createRequest()
 {
-  SharedHandle<AuthConfig> authConfig =
-    _authConfigFactory->createAuthConfig(request);
+  _authConfig = _authConfigFactory->createAuthConfig(request);
   std::string requestLine = request->getMethod()+" ";
   if(!_proxyRequest.isNull()) {
     if(getProtocol() == Request::PROTO_FTP &&
-       request->getUsername().empty() && !authConfig->getUser().empty()) {
+       request->getUsername().empty() && !_authConfig.isNull()) {
       // Insert user into URI, like ftp://USER@host/
       std::string uri = getCurrentURI();
       assert(uri.size() >= 6);
-      uri.insert(6, Util::urlencode(authConfig->getUser())+"@");
+      uri.insert(6, Util::urlencode(_authConfig->getUser())+"@");
       requestLine += uri;
     } else {
       requestLine += getCurrentURI();
@@ -204,9 +203,9 @@ std::string HttpRequest::createRequest() const
   if(!_proxyRequest.isNull() && !_proxyRequest->getUsername().empty()) {
     requestLine += getProxyAuthString();
   }
-  if(!authConfig->getUser().empty()) {
+  if(!_authConfig.isNull()) {
     requestLine += "Authorization: Basic "+
-      Base64::encode(authConfig->getAuthText())+"\r\n";
+      Base64::encode(_authConfig->getAuthText())+"\r\n";
   }
   if(getPreviousURI().size()) {
     requestLine += "Referer: "+getPreviousURI()+"\r\n";
@@ -360,4 +359,14 @@ bool HttpRequest::isProxyRequestSet() const
   return !_proxyRequest.isNull();
 }
 
+bool HttpRequest::authenticationUsed() const
+{
+  return !_authConfig.isNull();
+}
+
+const SharedHandle<AuthConfig>& HttpRequest::getAuthConfig() const
+{
+  return _authConfig;
+}
+
 } // namespace aria2

src/HttpRequest.h

@@ -50,6 +50,7 @@ class Range;
 class Option;
 class CookieStorage;
 class AuthConfigFactory;
+class AuthConfig;
 
 class HttpRequest {
 private:
@@ -74,6 +75,8 @@ private:
 
   SharedHandle<AuthConfigFactory> _authConfigFactory;
 
+  SharedHandle<AuthConfig> _authConfig;
+
   SharedHandle<Request> _proxyRequest;
 
   std::string getHostText(const std::string& host, uint16_t port) const;
@@ -134,10 +137,13 @@ public:
   off_t getEndByte() const;
 
   /**
-   * Returns string representation of http request.
-   * It usually starts with "GET ..." and ends with "\r\n".
+   * Returns string representation of http request.  It usually starts
+   * with "GET ..." and ends with "\r\n".  The AuthConfig for this
+   * request is resolved using _authConfigFactory and stored in
+   * _authConfig.  getAuthConfig() returns AuthConfig used in the last
+   * invocation of createRequest().
    */
-  std::string createRequest() const;
+  std::string createRequest();
 
   /**
    * Returns string representation of http tunnel request.
@@ -182,6 +188,14 @@ public:
    * Otherwise, returns false.
    */
   bool isProxyRequestSet() const;
+
+  // Returns true if authentication was used in the last
+  // createRequest().
+  bool authenticationUsed() const;
+
+  // Returns AuthConfig used in the last invocation of
+  // createRequest().
+  const SharedHandle<AuthConfig>& getAuthConfig() const;
 };
 
 typedef SharedHandle<HttpRequest> HttpRequestHandle;

src/HttpResponse.cc

@@ -56,6 +56,7 @@
 #endif // HAVE_LIBZ
 #include "CookieStorage.h"
 #include "AuthConfigFactory.h"
+#include "AuthConfig.h"
 
 namespace aria2 {
 

src/HttpResponseCommand.cc

@@ -64,6 +64,7 @@
 #include "LogFactory.h"
 #include "CookieStorage.h"
 #include "AuthConfigFactory.h"
+#include "AuthConfig.h"
 
 namespace aria2 {
 

src/HttpSkipResponseCommand.cc

@@ -52,6 +52,7 @@
 #include "Option.h"
 #include "CookieStorage.h"
 #include "AuthConfigFactory.h"
+#include "AuthConfig.h"
 
 namespace aria2 {
 
@@ -158,7 +159,13 @@ bool HttpSkipResponseCommand::processResponse()
     return prepareForRetry(_httpResponse->getRetryAfter());
   } else if(_httpResponse->getResponseStatus() >= HttpHeader::S400) {
     if(_httpResponse->getResponseStatus() == HttpHeader::S401) {
-      throw DlAbortEx(EX_AUTH_FAILED);
+      if(!_httpResponse->getHttpRequest()->authenticationUsed() &&
+	 e->getAuthConfigFactory()->activateBasicCred
+	 (req->getHost(), req->getDir())) {
+	return prepareForRetry(0);
+      } else {
+	throw DlAbortEx(EX_AUTH_FAILED);
+      }
     }else if(_httpResponse->getResponseStatus() == HttpHeader::S404) {
       throw DlAbortEx(MSG_RESOURCE_NOT_FOUND,
 		      DownloadResult::RESOURCE_NOT_FOUND);

test/AuthConfigFactoryTest.cc

@@ -15,11 +15,13 @@ class AuthConfigFactoryTest:public CppUnit::TestFixture {
   CPPUNIT_TEST_SUITE(AuthConfigFactoryTest);
   CPPUNIT_TEST(testCreateAuthConfig_http);
   CPPUNIT_TEST(testCreateAuthConfig_ftp);
+  CPPUNIT_TEST(testUpdateBasicCred);
   CPPUNIT_TEST_SUITE_END();
   
 public:
   void testCreateAuthConfig_http();
   void testCreateAuthConfig_ftp();
+  void testUpdateBasicCred();
 };
 
 
@@ -36,8 +38,7 @@ void AuthConfigFactoryTest::testCreateAuthConfig_http()
   AuthConfigFactory factory(&option);
 
   // without auth info
-  CPPUNIT_ASSERT_EQUAL(std::string(":"),
-		       factory.createAuthConfig(req)->getAuthText());
+  CPPUNIT_ASSERT(factory.createAuthConfig(req).isNull());
 
   // with Netrc
   SharedHandle<Netrc> netrc(new Netrc());
@@ -50,24 +51,36 @@ void AuthConfigFactoryTest::testCreateAuthConfig_http()
     (SharedHandle<Authenticator>(new DefaultAuthenticator("default", "defaultpassword", "defaultaccount")));
   factory.setNetrc(netrc);
 
+  // not activated
+  CPPUNIT_ASSERT(factory.createAuthConfig(req).isNull());
+
+  CPPUNIT_ASSERT(factory.activateBasicCred("localhost", "/"));
+
   CPPUNIT_ASSERT_EQUAL(std::string("localhostuser:localhostpass"),
 		       factory.createAuthConfig(req)->getAuthText());
 
   // See default token in netrc is ignored.
   SharedHandle<Request> mirrorReq(new Request());
   req->setUrl("http://mirror/");
-  CPPUNIT_ASSERT_EQUAL(std::string(":"),
-		       factory.createAuthConfig(mirrorReq)->getAuthText());
+
+  CPPUNIT_ASSERT(!factory.activateBasicCred("mirror", "/"));
+
+  CPPUNIT_ASSERT(factory.createAuthConfig(mirrorReq).isNull());
 
   // with Netrc + user defined
   option.put(PREF_HTTP_USER, "userDefinedUser");
   option.put(PREF_HTTP_PASSWD, "userDefinedPassword");
+
+  CPPUNIT_ASSERT(factory.createAuthConfig(req).isNull());
+
+  CPPUNIT_ASSERT(factory.activateBasicCred("mirror", "/"));
+
   CPPUNIT_ASSERT_EQUAL(std::string("userDefinedUser:userDefinedPassword"),
 		       factory.createAuthConfig(req)->getAuthText());
 
-  // username and password in URI: disabled by default.
+  // username and password in URI
   req->setUrl("http://aria2user:aria2password@localhost/download/aria2-1.0.0.tar.bz2");
-  CPPUNIT_ASSERT_EQUAL(std::string("userDefinedUser:userDefinedPassword"),
+  CPPUNIT_ASSERT_EQUAL(std::string("aria2user:aria2password"),
 		       factory.createAuthConfig(req)->getAuthText());  
 }
 
@@ -111,4 +124,45 @@ void AuthConfigFactoryTest::testCreateAuthConfig_ftp()
  		       factory.createAuthConfig(req)->getAuthText());
 }
 
+void AuthConfigFactoryTest::testUpdateBasicCred()
+{
+  Option option;
+  option.put(PREF_NO_NETRC, V_FALSE);
+
+  AuthConfigFactory factory(&option);
+
+  factory.updateBasicCred
+    (AuthConfigFactory::BasicCred("myname", "mypass", "localhost", "/", true));
+  factory.updateBasicCred
+    (AuthConfigFactory::BasicCred("price","j38jdc","localhost","/download", true));
+  factory.updateBasicCred
+    (AuthConfigFactory::BasicCred("alice","ium8","localhost","/documents", true));
+  factory.updateBasicCred
+    (AuthConfigFactory::BasicCred("jack", "jackx","mirror", "/doc", true));
+
+  SharedHandle<Request> req(new Request());
+  req->setUrl("http://localhost/download/v2.6/Changelog");
+  
+  CPPUNIT_ASSERT_EQUAL(std::string("price:j38jdc"),
+		       factory.createAuthConfig(req)->getAuthText());
+
+  req->setUrl("http://localhost/documents/reference.html");
+  CPPUNIT_ASSERT_EQUAL(std::string("alice:ium8"),
+		       factory.createAuthConfig(req)->getAuthText());
+
+  req->setUrl("http://localhost/documents2/manual.html");
+  CPPUNIT_ASSERT_EQUAL(std::string("myname:mypass"),
+		       factory.createAuthConfig(req)->getAuthText());
+
+  req->setUrl("http://localhost/doc/readme.txt");
+  CPPUNIT_ASSERT_EQUAL(std::string("myname:mypass"),
+		       factory.createAuthConfig(req)->getAuthText());
+
+  req->setUrl("http://local/");
+  CPPUNIT_ASSERT(factory.createAuthConfig(req).isNull());
+
+  req->setUrl("http://mirror/");
+  CPPUNIT_ASSERT(factory.createAuthConfig(req).isNull());
+}
+
 } // namespace aria2

test/HttpRequestTest.cc

@@ -14,6 +14,7 @@
 #include "array_fun.h"
 #include "CookieStorage.h"
 #include "Util.h"
+#include "AuthConfig.h"
 
 namespace aria2 {
 
@@ -224,6 +225,8 @@ void HttpRequestTest::testCreateRequest()
   _option->put(PREF_HTTP_USER, "aria2user");
   _option->put(PREF_HTTP_PASSWD, "aria2passwd");
 
+  CPPUNIT_ASSERT(_authConfigFactory->activateBasicCred("localhost", "/"));
+
   expectedText = "GET /archives/aria2-1.0.0.tar.bz2 HTTP/1.1\r\n"
     "User-Agent: aria2\r\n"
     "Accept: */*\r\n"

test/HttpResponseTest.cc

@@ -15,6 +15,7 @@
 #include "DlRetryEx.h"
 #include "CookieStorage.h"
 #include "AuthConfigFactory.h"
+#include "AuthConfig.h"
 
 namespace aria2 {