diff --git a/ChangeLog b/ChangeLog index 446f4b70..35434dd2 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,12 @@ +2006-02-18 Tatsuhiro Tsujikawa + + * Added HTTPS support. + * Added SocketCore. Socket is now handle class for SocketCore. + * Fixed bug in ChunkedEncoding: expanding buffer size is wrong + * Fixed bug in DownloadCommand: In Chunked Encoding, it wrongly + adds to Segment.ds buff length from the socket. + 2006-02-17 Tatsuhiro Tsujikawa - *Release 0.1.0 + + * Release 0.1.0 \ No newline at end of file diff --git a/TODO b/TODO index b021a990..3c165d21 100644 --- a/TODO +++ b/TODO @@ -1,4 +1,7 @@ * Add HTTP POST support -* Add HTTPS support using OpenSSL * Add expires handling for Cookie -* Add FTP support \ No newline at end of file +* Add Referer support +* Fix no wait retry in HttpInitiateConnectionCommand, HttpRequestCommand, HttpResponseCommand, except for redirection. +* Add FTP support +* Add SSL server cert verification +* Add SSL client cert support \ No newline at end of file diff --git a/config.h.in b/config.h.in index 65035e58..721ce9b4 100644 --- a/config.h.in +++ b/config.h.in @@ -3,6 +3,9 @@ /* Define to 1 if you have the header file. */ #undef HAVE_ARPA_INET_H +/* Define to 1 if you don't have `vprintf' but do have `_doprnt.' */ +#undef HAVE_DOPRNT + /* Define to 1 if you have the header file. */ #undef HAVE_FCNTL_H @@ -18,6 +21,9 @@ /* Define to 1 if you have the header file. */ #undef HAVE_INTTYPES_H +/* Define to 1 if you have the `ssl' library (-lssl). */ +#undef HAVE_LIBSSL + /* Define to 1 if you have the header file. */ #undef HAVE_MEMORY_H @@ -91,6 +97,9 @@ /* Define to 1 if you have the header file. */ #undef HAVE_UNISTD_H +/* Define to 1 if you have the `vprintf' function. */ +#undef HAVE_VPRINTF + /* Define to 1 if the system has the type `_Bool'. */ #undef HAVE__BOOL diff --git a/configure b/configure index 383fab67..8966e04a 100755 --- a/configure +++ b/configure @@ -5336,6 +5336,80 @@ fi done + +echo "$as_me:$LINENO: checking for SSL_library_init in -lssl" >&5 +echo $ECHO_N "checking for SSL_library_init in -lssl... $ECHO_C" >&6 +if test "${ac_cv_lib_ssl_SSL_library_init+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lssl $LIBS" +cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ + +/* Override any gcc2 internal prototype to avoid an error. */ +#ifdef __cplusplus +extern "C" +#endif +/* We use char because int might match the return type of a gcc2 + builtin and then its argument prototype would still apply. */ +char SSL_library_init (); +int +main () +{ +SSL_library_init (); + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext conftest$ac_exeext +if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5 + (eval $ac_link) 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && + { ac_try='test -z "$ac_c_werror_flag" || test ! -s conftest.err' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; } && + { ac_try='test -s conftest$ac_exeext' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; }; then + ac_cv_lib_ssl_SSL_library_init=yes +else + echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + +ac_cv_lib_ssl_SSL_library_init=no +fi +rm -f conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +echo "$as_me:$LINENO: result: $ac_cv_lib_ssl_SSL_library_init" >&5 +echo "${ECHO_T}$ac_cv_lib_ssl_SSL_library_init" >&6 +if test $ac_cv_lib_ssl_SSL_library_init = yes; then + cat >>confdefs.h <<_ACEOF +#define HAVE_LIBSSL 1 +_ACEOF + + LIBS="-lssl $LIBS" + +fi + + ac_config_files="$ac_config_files Makefile src/Makefile test/Makefile" cat >confcache <<\_ACEOF diff --git a/configure.in b/configure.in index f6f31c67..2b2326e5 100644 --- a/configure.in +++ b/configure.in @@ -31,5 +31,7 @@ AC_FUNC_STAT AC_FUNC_VPRINTF AC_CHECK_FUNCS([getpagesize gethostbyname gettimeofday memset mkdir rmdir select socket strcasecmp strerror strstr strtol]) +AC_CHECK_LIB([ssl], [SSL_library_init]) + AC_CONFIG_FILES([Makefile src/Makefile test/Makefile]) AC_OUTPUT diff --git a/src/AbstractCommand.h b/src/AbstractCommand.h index 81e221f3..23c538c5 100644 --- a/src/AbstractCommand.h +++ b/src/AbstractCommand.h @@ -26,6 +26,7 @@ #include "Request.h" #include "DownloadEngine.h" #include "SegmentMan.h" +#include "common.h" class AbstractCommand : public Command { private: diff --git a/src/ChunkedEncoding.cc b/src/ChunkedEncoding.cc index 6783e942..80670601 100644 --- a/src/ChunkedEncoding.cc +++ b/src/ChunkedEncoding.cc @@ -25,7 +25,7 @@ #include #include -#define MAX_BUFSIZE 8192 +#define MAX_BUFSIZE (1024*1024) ChunkedEncoding::ChunkedEncoding() { strbufSize = 4096; @@ -161,11 +161,11 @@ void ChunkedEncoding::addBuffer(const char* inbuf, int inlen) { if(strlen(strbuf)+inlen+1 > MAX_BUFSIZE) { throw new DlAbortEx(EX_TOO_LARGE_CHUNK, strlen(strbuf)+inlen+1); } - char* temp = new char[strlen(strbuf)+inlen+1]; + strbufSize = strlen(strbuf)+inlen+1; + char* temp = new char[strbufSize]; memcpy(temp, strbuf, strlen(strbuf)+1); delete [] strbuf; strbuf = temp; - strbufSize = strlen(strbuf); } int origlen = strlen(strbuf); memcpy(strbuf+origlen, inbuf, inlen); diff --git a/src/DownloadCommand.cc b/src/DownloadCommand.cc index a2ede11b..4be67c1b 100644 --- a/src/DownloadCommand.cc +++ b/src/DownloadCommand.cc @@ -48,10 +48,11 @@ bool DownloadCommand::executeInternal(Segment seg) { char infbuf[infbufSize]; te->inflate(infbuf, infbufSize, buf, bufSize); e->diskWriter->writeData(infbuf, infbufSize, seg.sp+seg.ds); + seg.ds += infbufSize; } else { e->diskWriter->writeData(buf, bufSize, seg.sp+seg.ds); + seg.ds += bufSize; } - seg.ds += bufSize; if(te != NULL && te->finished() || te == NULL && seg.ds >= seg.ep-seg.sp+1 diff --git a/src/HttpConnection.cc b/src/HttpConnection.cc index 03ccb861..2307f8f3 100644 --- a/src/HttpConnection.cc +++ b/src/HttpConnection.cc @@ -48,12 +48,15 @@ void HttpConnection::sendProxyRequest(const Request* req) { } string HttpConnection::getHost(const string& host, int port) { - return host+(port == 80 ? "" : ":"+Util::llitos(port)); + return host+(port == 80 || port == 443 ? "" : ":"+Util::llitos(port)); } string HttpConnection::createRequest(const Request* req, const Segment& segment) { - string request = string("GET ")+req->getCurrentUrl()+string(" HTTP/1.1\r\n")+ - "Referer:\r\n"+ + string request = string("GET ")+ + req->getCurrentUrl()+ + //(req->getDir() == "/" ? "/" : req->getDir()+"/")+req->getFile()+ + string(" HTTP/1.1\r\n")+ + "Referer: \r\n"+ "User-Agent: aria2\r\n"+ "Connection: close\r\n"+ "Accept: */*\r\n"+ diff --git a/src/HttpRequestCommand.cc b/src/HttpRequestCommand.cc index 6405f78d..078b11f1 100644 --- a/src/HttpRequestCommand.cc +++ b/src/HttpRequestCommand.cc @@ -36,6 +36,12 @@ HttpRequestCommand::~HttpRequestCommand() {} bool HttpRequestCommand::executeInternal(Segment seg) { socket->setNonBlockingMode(); +#ifdef HAVE_LIBSSL + // for SSL + if(req->getProtocol() == "https") { + socket->initiateSecureConnection(); + } +#endif // HAVE_LIBSSL HttpConnection httpConnection(cuid, socket, e->option, e->logger); // set seg to request in order to remember the request range req->seg = seg; diff --git a/src/HttpResponseCommand.cc b/src/HttpResponseCommand.cc index 97dd1654..c8347fe0 100644 --- a/src/HttpResponseCommand.cc +++ b/src/HttpResponseCommand.cc @@ -133,6 +133,7 @@ void HttpResponseCommand::createHttpDownloadCommand(string transferEncoding) { throw new DlAbortEx(EX_TRANSFER_ENCODING_NOT_SUPPORTED, transferEncoding.c_str()); } else { if(enc != NULL) { + command->transferEncoding = transferEncoding; enc->init(); } e->commands.push(command); diff --git a/src/InitiateConnectionCommandFactory.cc b/src/InitiateConnectionCommandFactory.cc index 7a145ac4..3ba177bd 100644 --- a/src/InitiateConnectionCommandFactory.cc +++ b/src/InitiateConnectionCommandFactory.cc @@ -23,7 +23,12 @@ #include "HttpInitiateConnectionCommand.h" Command* InitiateConnectionCommandFactory::createInitiateConnectionCommand(int cuid, Request* req, DownloadEngine* e) { - if(req->getProtocol() == "http") { + if(req->getProtocol() == "http" +#ifdef HAVE_LIBSSL + // for SSL + || req->getProtocol() == "https" +#endif // HAVE_LIBSSL + ) { return new HttpInitiateConnectionCommand(cuid, req, e); } else { // these protocols are not supported yet diff --git a/src/InitiateConnectionCommandFactory.h b/src/InitiateConnectionCommandFactory.h index 2bd9bcfd..dea10ed4 100644 --- a/src/InitiateConnectionCommandFactory.h +++ b/src/InitiateConnectionCommandFactory.h @@ -24,6 +24,7 @@ #include "Request.h" #include "DownloadEngine.h" +#include "common.h" class InitiateConnectionCommandFactory { public: diff --git a/src/Request.cc b/src/Request.cc index 503c1286..e5f29927 100644 --- a/src/Request.cc +++ b/src/Request.cc @@ -26,6 +26,10 @@ Request::Request():port(0), retryCount(0) { defaultPorts["http"] = 80; +#ifdef HAVE_LIBSSL + // for SSL + defaultPorts["https"] = 443; +#endif // HAVE_LIBSSL seg.sp = 0; seg.ep = 0; seg.ds = 0; diff --git a/src/Request.h b/src/Request.h index 69ce4ba5..7ecbd359 100644 --- a/src/Request.h +++ b/src/Request.h @@ -25,6 +25,7 @@ #include #include #include "CookieBox.h" +#include "common.h" using namespace std; diff --git a/src/Socket.cc b/src/Socket.cc index f14fc296..9a53cd67 100644 --- a/src/Socket.cc +++ b/src/Socket.cc @@ -80,3 +80,10 @@ void Socket::readData(char* data, int& len, int timeout) { void Socket::peekData(char* data, int& len, int timeout) { core->peekData(data, len, timeout); } + +#ifdef HAVE_LIBSSL +// for SSL +void Socket::initiateSecureConnection() { + core->initiateSecureConnection(); +} +#endif // HAVE_LIBSSL diff --git a/src/Socket.h b/src/Socket.h index e6254f17..45f2d298 100644 --- a/src/Socket.h +++ b/src/Socket.h @@ -24,6 +24,7 @@ #include #include "SocketCore.h" +#include "common.h" using namespace std; @@ -74,6 +75,11 @@ public: // Reads up to len bytes from this socket, but bytes are not removed from // this socket. void peekData(char* data, int& len, int timeout = 5); + +#ifdef HAVE_LIBSSL + // for SSL + void initiateSecureConnection(); +#endif // HAVE_LIBSSL }; #endif // _D_SOCKET_H_ diff --git a/src/SocketCore.cc b/src/SocketCore.cc index e89e1d53..07b4b3c1 100644 --- a/src/SocketCore.cc +++ b/src/SocketCore.cc @@ -32,8 +32,14 @@ #include "DlRetryEx.h" #include "DlAbortEx.h" #include +#include "message.h" -SocketCore::SocketCore():sockfd(-1), use(1) {} +SocketCore::SocketCore():sockfd(-1), use(1), secure(false) +#ifdef HAVE_LIBSSL + // for SSL + , sslCtx(NULL), ssl(NULL) +#endif // HAVE_LIBSSL +{} SocketCore::~SocketCore() { closeConnection(); @@ -87,10 +93,25 @@ void SocketCore::setNonBlockingMode() { } void SocketCore::closeConnection() { +#ifdef HAVE_LIBSSL + // for SSL + if(secure) { + SSL_shutdown(ssl); + } +#endif // HAVE_LIBSSL if(sockfd != -1) { close(sockfd); sockfd = -1; } +#ifdef HAVE_LIBSSL + // for SSL + if(secure) { + SSL_free(ssl); + SSL_CTX_free(sslCtx); + ssl = NULL; + sslCtx = NULL; + } +#endif // HAVE_LIBSSL } bool SocketCore::isWritable(int timeout) { @@ -134,19 +155,65 @@ bool SocketCore::isReadable(int timeout) { } void SocketCore::writeData(const char* data, int len, int timeout) { - if(!isWritable(timeout) || send(sockfd, data, (size_t)len, 0) != len) { + if(!isWritable(timeout) || + !secure && send(sockfd, data, (size_t)len, 0) != len +#ifdef HAVE_LIBSSL + // for SSL + // TODO handling len == 0 case required + || secure && SSL_write(ssl, data, len) != len +#endif // HAVE_LIBSSL + ) { throw new DlRetryEx(strerror(errno)); } } void SocketCore::readData(char* data, int& len, int timeout) { - if(!isReadable(timeout) || (len = recv(sockfd, data, (size_t)len, 0)) < 0) { + if(!isReadable(timeout) || + !secure && (len = recv(sockfd, data, (size_t)len, 0)) < 0 +#ifdef HAVE_LIBSSL + // for SSL + // TODO handling len == 0 case required + || secure && (len = SSL_read(ssl, data, len)) < 0 +#endif // HAVE_LIBSSL + ) { throw new DlRetryEx(strerror(errno)); } } void SocketCore::peekData(char* data, int& len, int timeout) { - if(!isReadable(timeout) || (len = recv(sockfd, data, (size_t)len, MSG_PEEK)) < 0) { + if(!isReadable(timeout) || + !secure && (len = recv(sockfd, data, (size_t)len, MSG_PEEK)) < 0 +#ifdef HAVE_LIBSSL + // for SSL + // TODO handling len == 0 case required + || secure && (len == SSL_peek(ssl, data, len)) < 0 +#endif // HAVE_LIBSSL + ) { throw new DlRetryEx(strerror(errno)); } } + +#ifdef HAVE_LIBSSL +// for SSL +void SocketCore::initiateSecureConnection() { + if(!secure) { + sslCtx = SSL_CTX_new(SSLv23_client_method()); + if(sslCtx == NULL) { + throw new DlAbortEx(EX_SSL_INIT_FAILURE); + } + SSL_CTX_set_mode(sslCtx, SSL_MODE_AUTO_RETRY); + ssl = SSL_new(sslCtx); + if(ssl == NULL) { + throw new DlAbortEx(EX_SSL_INIT_FAILURE); + } + if(SSL_set_fd(ssl, sockfd) == 0) { + throw new DlAbortEx(EX_SSL_INIT_FAILURE); + } + // TODO handling return value == 0 case required + if(SSL_connect(ssl) <= 0) { + throw new DlAbortEx(EX_SSL_INIT_FAILURE); + } + secure = true; + } +} +#endif // HAVE_LIBSSL diff --git a/src/SocketCore.h b/src/SocketCore.h index 2c4cd185..f7a75b43 100644 --- a/src/SocketCore.h +++ b/src/SocketCore.h @@ -23,6 +23,12 @@ #define _D_SOCKET_CORE_H_ #include +#include "common.h" + +#ifdef HAVE_LIBSSL +// for SSL +# include +#endif // HAVE_LIBSSL using namespace std; @@ -33,6 +39,12 @@ private: int sockfd; // reference counter for this object. int use; + bool secure; +#ifdef HAVE_LIBSSL + // for SSL + SSL_CTX* sslCtx; + SSL* ssl; +#endif // HAVE_LIBSSL public: SocketCore(); ~SocketCore(); @@ -71,6 +83,13 @@ public: // Reads up to len bytes from this socket, but bytes are not removed from // this socket. void peekData(char* data, int& len, int timeout = 5); + +#ifdef HAVE_LIBSSL + /** + * Makes this socket SSL endpoint + */ + void initiateSecureConnection(); +#endif // HAVE_LIB_SSL }; #endif // _D_SOCKET_CORE_H_ diff --git a/src/main.cc b/src/main.cc index ce8755bd..48ed5060 100644 --- a/src/main.cc +++ b/src/main.cc @@ -37,6 +37,12 @@ extern char* optarg; extern int optind, opterr, optopt; #include +#ifdef HAVE_LIBSSL +// for SSL +# include +# include +#endif // HAVE_LIBSSL + using namespace std; void clearRequest(Request* req) { @@ -243,6 +249,11 @@ int main(int argc, char* argv[]) { exit(1); } } +#ifdef HAVE_LIBSSL + // for SSL initialization + SSL_load_error_strings(); + SSL_library_init(); +#endif // HAVE_LIBSSL SimpleLogger* logger; if(stdoutLog) { logger = new SimpleLogger(stdout); diff --git a/src/message.h b/src/message.h index 0c5aaf0e..445828fa 100644 --- a/src/message.h +++ b/src/message.h @@ -51,5 +51,5 @@ #define EX_BAD_STATUS "The response status is not successful. status = %d" #define EX_TOO_LARGE_FILE "Too large file size. size = %d" #define EX_TRANSFER_ENCODING_NOT_SUPPORTED "Transfer encoding %s is not supported." - +#define EX_SSL_INIT_FAILURE "SSL initialization failed." #endif // _D_MESSAGE_H_