diff --git a/TODO b/TODO index 3c165d21..d92e87f0 100644 --- a/TODO +++ b/TODO @@ -1,6 +1,5 @@ * Add HTTP POST support * Add expires handling for Cookie -* Add Referer support * Fix no wait retry in HttpInitiateConnectionCommand, HttpRequestCommand, HttpResponseCommand, except for redirection. * Add FTP support * Add SSL server cert verification diff --git a/src/HttpConnection.cc b/src/HttpConnection.cc index 2307f8f3..b4b54ce6 100644 --- a/src/HttpConnection.cc +++ b/src/HttpConnection.cc @@ -56,7 +56,7 @@ string HttpConnection::createRequest(const Request* req, const Segment& segment) req->getCurrentUrl()+ //(req->getDir() == "/" ? "/" : req->getDir()+"/")+req->getFile()+ string(" HTTP/1.1\r\n")+ - "Referer: \r\n"+ + "Referer: "+req->getPreviousUrl()+"\r\n"+ "User-Agent: aria2\r\n"+ "Connection: close\r\n"+ "Accept: */*\r\n"+ diff --git a/src/Request.cc b/src/Request.cc index e5f29927..51f454a6 100644 --- a/src/Request.cc +++ b/src/Request.cc @@ -45,10 +45,12 @@ bool Request::setUrl(string url) { } bool Request::resetUrl() { + previousUrl = referer; return setUrl(url); } bool Request::redirectUrl(string url) { + previousUrl = currentUrl; return parseUrl(url); } diff --git a/src/Request.h b/src/Request.h index 7ecbd359..503aa541 100644 --- a/src/Request.h +++ b/src/Request.h @@ -41,6 +41,14 @@ class Request { private: string url; string currentUrl; + /** + * URL previously requested to the server. This is used as Referer + */ + string previousUrl; + /** + * URL used as Referer in the initial request + */ + string referer; string protocol; string host; int port; @@ -70,6 +78,10 @@ public: string getUrl() const { return url; } string getCurrentUrl() const { return currentUrl; } + string getPreviousUrl() const { return previousUrl; } + void setPreviousUrl(string url) { previousUrl = url; } + string getReferer() const { return referer; } + void setReferer(string url) { referer = previousUrl = url; } string getProtocol() const { return protocol; } string getHost() const { return host; } int getPort() const { return port; } diff --git a/src/main.cc b/src/main.cc index 48ed5060..89aaba16 100644 --- a/src/main.cc +++ b/src/main.cc @@ -60,8 +60,9 @@ void handler(int signal) { exit(0); } -void addCommand(int cuid, const char* url, vector requests) { +void addCommand(int cuid, const char* url, string referer, vector requests) { Request* req = new Request(); + req->setReferer(referer); if(req->setUrl(url)) { e->commands.push(InitiateConnectionCommandFactory::createInitiateConnectionCommand(cuid, req, e)); requests.push_back(req); @@ -112,7 +113,10 @@ void showUsage() { cout << " --http-proxy-user=USER Set HTTP proxy user. This affects to all URLs" << endl; cout << " --http-proxy-passwd=PASSWD Set HTTP proxy password. This affects to all URLs." << endl; cout << " --http-auth-scheme=SCHEME Set HTTP authentication scheme. Currently, BASIC" << endl; - cout << " is the only supported scheme." << endl; + cout << " is the only supported scheme. You MUST specify" << endl; + cout << " this option in order to use HTTP authentication" << endl; + cout << " as well as --http-proxy option." << endl; + cout << " --referer Set Referer. This affects to all URLs." << endl; cout << " -v, --version Print the version number and exit." << endl; cout << " -h, --help Print this message and exit." << endl; cout << "URL:" << endl; @@ -135,6 +139,7 @@ int main(int argc, char* argv[]) { string ufilename; int split = 0; bool daemonMode = false; + string referer; int c; Option* op = new Option(); @@ -154,6 +159,7 @@ int main(int argc, char* argv[]) { { "http-proxy-user", required_argument, &lopt, 4 }, { "http-proxy-passwd", required_argument, &lopt, 5 }, { "http-auth-scheme", required_argument, &lopt, 6 }, + { "referer", required_argument, &lopt, 7 }, { "version", no_argument, NULL, 'v' }, { "help", no_argument, NULL, 'h' }, { 0, 0, 0, 0 } @@ -200,6 +206,9 @@ int main(int argc, char* argv[]) { cerr << "Currently, supported authentication scheme is BASIC." << endl; } break; + case 7: + referer = optarg; + break; } break; } @@ -274,11 +283,11 @@ int main(int argc, char* argv[]) { vector requests; if(split > 0) { for(int i = 1; i <= split; i++) { - addCommand(i, argv[optind], requests); + addCommand(i, argv[optind], referer, requests); } } else { for(int i = 1; optind < argc; i++) { - addCommand(i, argv[optind++], requests); + addCommand(i, argv[optind++], referer, requests); } } diff --git a/test/RequestTest.cc b/test/RequestTest.cc index 4c96770f..ede11933 100644 --- a/test/RequestTest.cc +++ b/test/RequestTest.cc @@ -16,6 +16,7 @@ class RequestTest:public CppUnit::TestFixture { CPPUNIT_TEST(testSetUrl9); CPPUNIT_TEST(testSetUrl10); CPPUNIT_TEST(testRedirectUrl); + CPPUNIT_TEST(testRedirectUrl2); CPPUNIT_TEST(testResetUrl); CPPUNIT_TEST(testSafeChar); CPPUNIT_TEST_SUITE_END(); @@ -32,6 +33,7 @@ public: void testSetUrl9(); void testSetUrl10(); void testRedirectUrl(); + void testRedirectUrl2(); void testResetUrl(); void testSafeChar(); }; @@ -46,6 +48,7 @@ void RequestTest::testSetUrl1() { CPPUNIT_ASSERT(v); CPPUNIT_ASSERT_EQUAL(string("http://aria.rednoah.com/"), req.getUrl()); CPPUNIT_ASSERT_EQUAL(string("http://aria.rednoah.com/"), req.getCurrentUrl()); + CPPUNIT_ASSERT_EQUAL(string(""), req.getPreviousUrl()); CPPUNIT_ASSERT_EQUAL(string("http"), req.getProtocol()); CPPUNIT_ASSERT_EQUAL(80, req.getPort()); CPPUNIT_ASSERT_EQUAL(string("aria.rednoah.com"), req.getHost()); @@ -56,8 +59,14 @@ void RequestTest::testSetUrl1() { void RequestTest::testSetUrl2() { Request req; bool v = req.setUrl("http://aria.rednoah.com:8080/index.html"); + req.setReferer("http://aria.rednoah.com:8080"); CPPUNIT_ASSERT(v); + + // referer is unchaged + CPPUNIT_ASSERT_EQUAL(string("http://aria.rednoah.com:8080"), req.getReferer()); + // previousUrl must equal to referer; + CPPUNIT_ASSERT_EQUAL(req.getReferer(), req.getPreviousUrl()); CPPUNIT_ASSERT_EQUAL(string("http"), req.getProtocol()); CPPUNIT_ASSERT_EQUAL(8080, req.getPort()); CPPUNIT_ASSERT_EQUAL(string("aria.rednoah.com"), req.getHost()); @@ -152,6 +161,8 @@ void RequestTest::testRedirectUrl() { req.getUrl()); // currentUrl must be updated CPPUNIT_ASSERT_EQUAL(string("http://aria.rednoah.co.jp/"), req.getCurrentUrl()); + // previousUrl must be updated + CPPUNIT_ASSERT_EQUAL(string("http://aria.rednoah.com:8080/aria2/index.html"), req.getPreviousUrl()); CPPUNIT_ASSERT_EQUAL(string("http"), req.getProtocol()); CPPUNIT_ASSERT_EQUAL(string("aria.rednoah.co.jp"), req.getHost()); CPPUNIT_ASSERT_EQUAL(80, req.getPort()); @@ -159,10 +170,30 @@ void RequestTest::testRedirectUrl() { CPPUNIT_ASSERT_EQUAL(string(""), req.getFile()); } +void RequestTest::testRedirectUrl2() { + Request req; + bool v = req.setUrl("http://aria.rednoah.com/download.html"); + CPPUNIT_ASSERT_EQUAL(string(""), req.getPreviousUrl()); + req.setReferer("http://aria.rednoah.com/"); + // previousUrl is updated when referer is specified + CPPUNIT_ASSERT_EQUAL(string("http://aria.rednoah.com/"), req.getPreviousUrl()); + bool v2 = req.redirectUrl("http://aria.rednoah.com/403.html"); + + // previousUrl is updated + CPPUNIT_ASSERT_EQUAL(string("http://aria.rednoah.com/download.html"), req.getPreviousUrl()); + // referer is unchagned + CPPUNIT_ASSERT_EQUAL(string("http://aria.rednoah.com/"), req.getReferer()); + + bool v3 = req.redirectUrl("http://aria.rednoah.com/error.html"); + + // previousUrl is update + CPPUNIT_ASSERT_EQUAL(string("http://aria.rednoah.com/403.html"), req.getPreviousUrl()); +} + void RequestTest::testResetUrl() { Request req; bool v = req.setUrl("http://aria.rednoah.com:8080/aria2/index.html"); - + req.setReferer("http://aria.rednoah.com:8080/"); bool v2 = req.redirectUrl("ftp://aria.rednoah.co.jp/"); bool v3 = req.resetUrl(); @@ -170,6 +201,10 @@ void RequestTest::testResetUrl() { // currentUrl must equal to url CPPUNIT_ASSERT_EQUAL(string("http://aria.rednoah.com:8080/aria2/index.html"), req.getUrl()); CPPUNIT_ASSERT_EQUAL(req.getUrl(), req.getCurrentUrl()); + // previousUrl must equal to referer + CPPUNIT_ASSERT_EQUAL(string("http://aria.rednoah.com:8080/"), req.getPreviousUrl()); + // referer is unchanged + CPPUNIT_ASSERT_EQUAL(string("http://aria.rednoah.com:8080/"), req.getReferer()); CPPUNIT_ASSERT_EQUAL(string("http"), req.getProtocol()); CPPUNIT_ASSERT_EQUAL(8080, req.getPort()); CPPUNIT_ASSERT_EQUAL(string("aria.rednoah.com"), req.getHost());