2010-07-16 Tatsuhiro Tsujikawa <t-tujikawa@users.sourceforge.net>

Added --conditional-get option.  Download file only when the local
	file is older than remote file.  This function only works with
	HTTP(S) downloads only. It does not work if file size is specified
	in Metalink. It also ignores Content-Disposition header. If a
	control file exists, this option will be ignored. This function
	uses If-Modified-Since header to get only newer file
	conditionally. When getting modification time of local file, it
	uses user supplied filename(see --out option) or filename part in
	URI if --out is not specified.
	* doc/aria2c.1.txt
	* src/HttpHeader.cc
	* src/HttpHeader.h
	* src/HttpRequest.cc
	* src/HttpRequest.h
	* src/HttpRequestCommand.cc
	* src/HttpResponse.cc
	* src/HttpResponseCommand.cc
	* src/OptionHandlerFactory.cc
	* src/TimeA2.cc
	* src/TimeA2.h
	* src/prefs.cc
	* src/prefs.h
	* src/usage_text.h
	* test/HttpResponseTest.cc
	* test/TimeTest.cc

ChangeLog

@@ -1,3 +1,31 @@
+2010-07-16  Tatsuhiro Tsujikawa  <t-tujikawa@users.sourceforge.net>
+
+	Added --conditional-get option.  Download file only when the local
+	file is older than remote file.  This function only works with
+	HTTP(S) downloads only. It does not work if file size is specified
+	in Metalink. It also ignores Content-Disposition header. If a
+	control file exists, this option will be ignored. This function
+	uses If-Modified-Since header to get only newer file
+	conditionally. When getting modification time of local file, it
+	uses user supplied filename(see --out option) or filename part in
+	URI if --out is not specified.
+	* doc/aria2c.1.txt
+	* src/HttpHeader.cc
+	* src/HttpHeader.h
+	* src/HttpRequest.cc
+	* src/HttpRequest.h
+	* src/HttpRequestCommand.cc
+	* src/HttpResponse.cc
+	* src/HttpResponseCommand.cc
+	* src/OptionHandlerFactory.cc
+	* src/TimeA2.cc
+	* src/TimeA2.h
+	* src/prefs.cc
+	* src/prefs.h
+	* src/usage_text.h
+	* test/HttpResponseTest.cc
+	* test/TimeTest.cc
+
 2010-07-15  Tatsuhiro Tsujikawa  <t-tujikawa@users.sourceforge.net>
 
 	FeedbackURISelector now tries to select URI whose host is least

doc/aria2c.1

@@ -2,12 +2,12 @@
 .\"     Title: aria2c
 .\"    Author: [FIXME: author] [see http://docbook.sf.net/el/author]
 .\" Generator: DocBook XSL Stylesheets v1.75.2 <http://docbook.sf.net/>
-.\"      Date: 07/15/2010
+.\"      Date: 07/16/2010
 .\"    Manual: Aria2 Manual
 .\"    Source: Aria2 1.9.5
 .\"  Language: English
 .\"
-.TH "ARIA2C" "1" "07/15/2010" "Aria2 1\&.9\&.5" "Aria2 Manual"
+.TH "ARIA2C" "1" "07/16/2010" "Aria2 1\&.9\&.5" "Aria2 Manual"
 .\" -----------------------------------------------------------------
 .\" * Define some portability stuff
 .\" -----------------------------------------------------------------
@@ -1092,6 +1092,16 @@ to
 \fI60\fR
 .RE
 .PP
+\fB\-\-conditional\-get\fR[=\fItrue\fR|\fIfalse\fR]
+.RS 4
+Download file only when the local file is older than remote file\&. This function only works with HTTP(S) downloads only\&. It does not work if file size is specified in Metalink\&. It also ignores Content\-Disposition header\&. If a control file exists, this option will be ignored\&. This function uses If\-Modified\-Since header to get only newer file conditionally\&. When getting modification time of local file, it uses user supplied filename(see
+\fB\-\-out\fR
+option) or filename part in URI if
+\fB\-\-out\fR
+is not specified\&. Default:
+\fIfalse\fR
+.RE
+.PP
 \fB\-\-conf\-path\fR=PATH
 .RS 4
 Change the configuration file path to PATH\&. Default:

doc/aria2c.1.html

@@ -1912,6 +1912,21 @@ writes the piece to the appropriate files.</td>
 </p>
 </dd>
 <dt class="hdlist1">
+<strong>--conditional-get</strong>[=<em>true</em>|<em>false</em>]
+</dt>
+<dd>
+<p>
+  Download file only when the local file is older than remote
+  file. This function only works with HTTP(S) downloads only.  It does
+  not work if file size is specified in Metalink. It also ignores
+  Content-Disposition header.  If a control file exists, this option
+  will be ignored.  This function uses If-Modified-Since header to get
+  only newer file conditionally. When getting modification time of
+  local file, it uses user supplied filename(see <strong>--out</strong> option) or
+  filename part in URI if <strong>--out</strong> is not specified.  Default: <em>false</em>
+</p>
+</dd>
+<dt class="hdlist1">
 <strong>--conf-path</strong>=PATH
 </dt>
 <dd>
@@ -4215,7 +4230,7 @@ files in the program, then also delete it here.</p></div>
 <div id="footnotes"><hr /></div>
 <div id="footer">
 <div id="footer-text">
-Last updated 2010-07-15 20:42:15 JST
+Last updated 2010-07-16 22:59:29 JST
 </div>
 </div>
 </body>

doc/aria2c.1.txt

@@ -748,6 +748,17 @@ Advanced Options
   The possible values are between '0' to '600'.
   Default: '60'
 
+*--conditional-get*[='true'|'false']::
+
+  Download file only when the local file is older than remote
+  file. This function only works with HTTP(S) downloads only.  It does
+  not work if file size is specified in Metalink. It also ignores
+  Content-Disposition header.  If a control file exists, this option
+  will be ignored.  This function uses If-Modified-Since header to get
+  only newer file conditionally. When getting modification time of
+  local file, it uses user supplied filename(see *--out* option) or
+  filename part in URI if *--out* is not specified.  Default: 'false'
+
 *--conf-path*=PATH::
   Change the configuration file path to PATH.
   Default: '$HOME/.aria2/aria2.conf'

src/HttpHeader.cc

@@ -78,8 +78,20 @@ const std::string HttpHeader::HTTP_1_1("HTTP/1.1");
 
 const std::string HttpHeader::S200("200");
 
+const std::string HttpHeader::S206("206");
+
 const std::string HttpHeader::S300("300");
 
+const std::string HttpHeader::S301("301");
+
+const std::string HttpHeader::S302("302");
+
+const std::string HttpHeader::S303("303");
+
+const std::string HttpHeader::S304("304");
+
+const std::string HttpHeader::S307("307");
+
 const std::string HttpHeader::S400("400");
 
 const std::string HttpHeader::S401("401");

src/HttpHeader.h

@@ -146,8 +146,20 @@ public:
 
   static const std::string S200;
 
+  static const std::string S206;
+
   static const std::string S300;
 
+  static const std::string S301;
+
+  static const std::string S302;
+
+  static const std::string S303;
+
+  static const std::string S304;
+
+  static const std::string S307;
+
   static const std::string S400;
 
   static const std::string S401;

src/HttpRequest.cc

@@ -238,6 +238,10 @@ std::string HttpRequest::createRequest()
       builtinHds.push_back(std::make_pair("Cookie:", cookiesValue));
     }
   }
+  if(!ifModSinceHeader_.empty()) {
+    builtinHds.push_back
+      (std::make_pair("If-Modified-Since:", ifModSinceHeader_));
+  }
   for(std::vector<std::pair<std::string, std::string> >::const_iterator i =
         builtinHds.begin(), eoi = builtinHds.end(); i != eoi; ++i) {
     std::vector<std::string>::const_iterator j = headers_.begin();

src/HttpRequest.h

@@ -89,6 +89,8 @@ private:
 
   off_t endOffsetOverride_;
 
+  std::string ifModSinceHeader_;
+
   std::pair<std::string, std::string> getProxyAuthString() const;
 public:
   HttpRequest();
@@ -279,6 +281,16 @@ public:
   {
     endOffsetOverride_ = offset;
   }
+
+  void setIfModifiedSinceHeader(const std::string& hd)
+  {
+    ifModSinceHeader_ = hd;
+  }
+
+  const std::string& getIfModifiedSinceHeader() const
+  {
+    return ifModSinceHeader_;
+  }
 };
 
 } // namespace aria2

src/HttpRequestCommand.cc

@@ -58,6 +58,8 @@
 #include "CheckIntegrityEntry.h"
 #include "ServerStatMan.h"
 #include "PieceStorage.h"
+#include "DefaultBtProgressInfoFile.h"
+#include "Logger.h"
 
 namespace aria2 {
 
@@ -144,6 +146,31 @@ bool HttpRequestCommand::executeInternal() {
                            getDownloadEngine()->getCookieStorage(),
                            getDownloadEngine()->getAuthConfigFactory(),
                            proxyRequest_));
+      if(getOption()->getAsBool(PREF_CONDITIONAL_GET) &&
+         (getRequest()->getProtocol() == Request::PROTO_HTTP ||
+          getRequest()->getProtocol() == Request::PROTO_HTTPS)) {
+        if(getFileEntry()->getPath().empty() &&
+           getRequest()->getFile().empty()) {
+          if(getLogger()->debug()) {
+            getLogger()->debug("conditional-get is disabled because file name"
+                               "is not available.");
+          }
+        } else {
+          if(getFileEntry()->getPath().empty()) {
+            getFileEntry()->setPath
+              (util::applyDir
+               (getDownloadContext()->getDir(),
+                util::fixTaintedBasename(getRequest()->getFile())));
+          }
+          File ctrlfile(getFileEntry()->getPath()+
+                        DefaultBtProgressInfoFile::getSuffix());
+          File file(getFileEntry()->getPath());
+          if(!ctrlfile.exists() && file.exists()) {
+            httpRequest->setIfModifiedSinceHeader
+              (file.getModifiedTime().toHTTPDate());
+          }
+        }
+      }
       httpConnection_->sendRequest(httpRequest);
     } else {
       for(std::vector<SharedHandle<Segment> >::const_iterator itr =

src/HttpResponse.cc

@@ -69,7 +69,14 @@ void HttpResponse::validateResponse() const
   if(status >= HttpHeader::S400) {
     return;
   }
-  if(status >= HttpHeader::S300) {
+  if(status == HttpHeader::S304 &&
+     httpRequest_->getIfModifiedSinceHeader().empty()) {
+    throw DL_ABORT_EX("Got 304 without If-Modified-Since");
+  }
+  if(status == HttpHeader::S301 ||
+     status == HttpHeader::S302 ||
+     status == HttpHeader::S303 ||
+     status == HttpHeader::S307) {
     if(!httpHeader_->defined(HttpHeader::LOCATION)) {
       throw DL_ABORT_EX
         (StringFormat(EX_LOCATION_HEADER_REQUIRED,
@@ -128,7 +135,10 @@ void HttpResponse::retrieveCookie()
 bool HttpResponse::isRedirect() const
 {
   const std::string& status = getResponseStatus();
-  return HttpHeader::S300 <= status && status < HttpHeader::S400 &&
+  return (HttpHeader::S301 == status ||
+          HttpHeader::S302 == status ||
+          HttpHeader::S303 == status ||
+          HttpHeader::S307 == status) &&
     httpHeader_->defined(HttpHeader::LOCATION);
 }
 

src/HttpResponseCommand.cc

@@ -119,7 +119,26 @@ bool HttpResponseCommand::executeInternal()
       (getOption()->getAsInt(PREF_MAX_HTTP_PIPELINING));
   }
 
-  if(httpResponse->getResponseStatus() >= HttpHeader::S300) {
+  if(!httpResponse->getHttpRequest()->getIfModifiedSinceHeader().empty()) {
+    if(httpResponse->getResponseStatus() == HttpHeader::S304) {
+      uint64_t totalLength = httpResponse->getEntityLength();
+      getFileEntry()->setLength(totalLength);
+      getRequestGroup()->initPieceStorage();
+      getPieceStorage()->markAllPiecesDone();
+      getLogger()->notice(MSG_DOWNLOAD_ALREADY_COMPLETED,
+                          util::itos(getRequestGroup()->getGID()).c_str(),
+                          getRequestGroup()->getFirstFilePath().c_str());
+      poolConnection();
+      getFileEntry()->poolRequest(getRequest());
+      return true;
+    } else if(httpResponse->getResponseStatus() == HttpHeader::S200 ||
+              httpResponse->getResponseStatus() == HttpHeader::S206) {
+      // Remote file is newer than local file. We allow overwrite.
+      getOption()->put(PREF_ALLOW_OVERWRITE, V_TRUE);
+    }
+  }
+  if(httpResponse->getResponseStatus() >= HttpHeader::S300 &&
+     httpResponse->getResponseStatus() != HttpHeader::S304) {
     if(httpResponse->getResponseStatus() == HttpHeader::S404) {
       getRequestGroup()->increaseAndValidateFileNotFoundCount();
     }

src/OptionHandlerFactory.cc

@@ -122,6 +122,16 @@ OptionHandlers OptionHandlerFactory::createOptionHandlers()
     handlers.push_back(op);
   }
 #endif // ENABLE_MESSAGE_DIGEST
+  {
+    SharedHandle<OptionHandler> op(new BooleanOptionHandler
+                                   (PREF_CONDITIONAL_GET,
+                                    TEXT_CONDITIONAL_GET,
+                                    V_FALSE,
+                                    OptionHandler::OPT_ARG));
+    op->addTag(TAG_ADVANCED);
+    op->addTag(TAG_HTTP);
+    handlers.push_back(op);
+  }
   {
     SharedHandle<OptionHandler> op(new DefaultOptionHandler
                                    (PREF_CONF_PATH,

src/TimeA2.cc

@@ -172,6 +172,15 @@ bool Time::bad() const
   return !good_;
 }
 
+std::string Time::toHTTPDate() const
+{
+  char buf[32];
+  time_t t = getTime();
+  struct tm* tms = gmtime(&t); // returned struct is statically allocated.
+  size_t r = strftime(buf, sizeof(buf), "%a, %d %b %Y %H:%M:%S GMT", tms);
+  return std::string(&buf[0], &buf[r]);
+}
+
 Time Time::parse(const std::string& datetime, const std::string& format)
 {
   struct tm tm;

src/TimeA2.h

@@ -112,6 +112,8 @@ public:
   // Returns !good()
   bool bad() const;
 
+  std::string toHTTPDate() const;
+
   // Currently timezone is assumed as GMT.
   static Time parse(const std::string& datetime, const std::string& format);
 

src/prefs.cc

@@ -194,6 +194,8 @@ const std::string PREF_SAVE_SESSION("save-session");
 const std::string PREF_MAX_CONNECTION_PER_SERVER("max-connection-per-server");
 // value: 1*digit
 const std::string PREF_MIN_SPLIT_SIZE("min-split-size");
+// value: true | false
+const std::string PREF_CONDITIONAL_GET("conditional-get");
 
 /**
  * FTP related preferences

src/prefs.h

@@ -198,6 +198,8 @@ extern const std::string PREF_SAVE_SESSION;
 extern const std::string PREF_MAX_CONNECTION_PER_SERVER;
 // value: 1*digit
 extern const std::string PREF_MIN_SPLIT_SIZE;
+// value: true | false
+extern const std::string PREF_CONDITIONAL_GET;
 
 /**
  * FTP related preferences

src/usage_text.h

@@ -694,3 +694,7 @@
     "                              using 2 sources(if --split >= 2, of course).\n" \
     "                              If SIZE is 15M, since 2*15M > 20MiB, aria2 does\n" \
     "                              not split file and download it using 1 source.")
+#define TEXT_CONDITIONAL_GET                    \
+  _(" --conditional-get[=true|false] Download file only when the local file is older\n" \
+    "                                than remote file. Currently, this function has\n" \
+    "                                many limitations. See man page for details.")

test/HttpResponseTest.cc

@@ -44,6 +44,7 @@ class HttpResponseTest : public CppUnit::TestFixture {
   CPPUNIT_TEST(testValidateResponse_good_range);
   CPPUNIT_TEST(testValidateResponse_bad_range);
   CPPUNIT_TEST(testValidateResponse_chunked);
+  CPPUNIT_TEST(testValidateResponse_withIfModifiedSince);
   CPPUNIT_TEST(testHasRetryAfter);
   CPPUNIT_TEST(testProcessRedirect);
   CPPUNIT_TEST(testRetrieveCookie);
@@ -75,6 +76,7 @@ public:
   void testValidateResponse_good_range();
   void testValidateResponse_bad_range();
   void testValidateResponse_chunked();
+  void testValidateResponse_withIfModifiedSince();
   void testHasRetryAfter();
   void testProcessRedirect();
   void testRetrieveCookie();
@@ -217,7 +219,7 @@ void HttpResponseTest::testIsRedirect()
 
   CPPUNIT_ASSERT(!httpResponse.isRedirect());
 
-  httpHeader->setResponseStatus("304");
+  httpHeader->setResponseStatus("301");
 
   CPPUNIT_ASSERT(httpResponse.isRedirect());  
 }
@@ -333,7 +335,7 @@ void HttpResponseTest::testValidateResponse()
   SharedHandle<HttpHeader> httpHeader(new HttpHeader());
   httpResponse.setHttpHeader(httpHeader);
 
-  httpHeader->setResponseStatus("304");
+  httpHeader->setResponseStatus("301");
 
   try {
     httpResponse.validateResponse();
@@ -430,6 +432,23 @@ void HttpResponseTest::testValidateResponse_chunked()
   }
 }
 
+void HttpResponseTest::testValidateResponse_withIfModifiedSince()
+{
+  HttpResponse httpResponse;
+  SharedHandle<HttpHeader> httpHeader(new HttpHeader());
+  httpResponse.setHttpHeader(httpHeader);
+  httpHeader->setResponseStatus("304");
+  SharedHandle<HttpRequest> httpRequest(new HttpRequest());
+  httpResponse.setHttpRequest(httpRequest);
+  try {
+    httpResponse.validateResponse();
+    CPPUNIT_FAIL("exception must be thrown.");
+  } catch(Exception& e) {
+  }
+  httpRequest->setIfModifiedSinceHeader("Fri, 16 Jul 2010 12:56:59 GMT");
+  httpResponse.validateResponse();
+}
+
 void HttpResponseTest::testHasRetryAfter()
 {
   HttpResponse httpResponse;

test/TimeTest.cc

@@ -18,6 +18,7 @@ class TimeTest:public CppUnit::TestFixture {
   CPPUNIT_TEST(testParseHTTPDate);
   CPPUNIT_TEST(testOperatorLess);
   CPPUNIT_TEST(testElapsed);
+  CPPUNIT_TEST(testToHTTPDate);
   CPPUNIT_TEST_SUITE_END();
 public:
   void setUp() {}
@@ -30,6 +31,7 @@ public:
   void testParseHTTPDate();
   void testOperatorLess();
   void testElapsed();
+  void testToHTTPDate();
 };
 
 
@@ -86,6 +88,13 @@ void TimeTest::testOperatorLess()
   CPPUNIT_ASSERT(Time(tv2) < Time(tv1));
 }
 
+void TimeTest::testToHTTPDate()
+{
+  Time t(1220714793);
+  CPPUNIT_ASSERT_EQUAL(std::string("Sat, 06 Sep 2008 15:26:33 GMT"),
+                       t.toHTTPDate());
+}
+
 void TimeTest::testElapsed()
 {
   struct timeval now;