diff --git a/ChangeLog b/ChangeLog index 99575106..9ba3afd9 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,108 @@ -2007-02-06 Tatsuhiro Tsujikawa +2007-03-15 Tatsuhiro Tsujikawa + + To handle Segment as SegmentHandle: + * src/AbstractCommand.cc (execute): Rewritten. + * src/SegmentMan.h: Segment -> SegmentHandle + + Introducded HttpResponse class, HttpRequest class to improve code + extensiveness and make it clear: + * src/HttpDownloadCommand.cc: transfer encoders are now managed by + HttpResponse class. + * src/HttpRequest.h, src/HttpRequest.cc: New class. + * src/HttpResponse.h, src/HttpResponse.cc: New class. + * src/HttpConnection.cc: Contruction of http request were moved to + HttpRequest class. + * src/HttpResponseCommand.h, src/HttpResponseCommand.cc: Refactored. + * src/HttpRequestCommand.cc (executeInternal): Rewritten. + * src/HttpAuthConfig.h: New class. + * src/Range.h: New class. + + To make FtpTunnel{Request, Response}Command and + HttpProxy{Request, Response}Command derived from + AbstractProxy{Request, Response}Command: + * src/FtpTunnelResponseCommand.h, src/FtpTunnelResponseCommand.cc: + Derived from AbstractProxyRequestCommand class. + * src/FtpTunnelRequestCommand.h, src/FtpTunnelRequestCommand.cc: + Derived from AbstractProxyResponseCommand class. + * src/HttpProxyRequestCommand.h, src/HttpProxyRequestCommand.cc: + Derived from AbstractProxyRequestCommand class. + * src/HttpProxyResponseCommand.h, src/HttpProxyResponseCommand.cc: + Derived from AbstractProxyResponseCommand class. + * src/AbstractProxyRequestCommand.h, src/AbstractProxyRequestCommand.cc + : New class. + * src/AbstractProxyResponseCommand.h, + src/AbstractProxyResponseCommand.cc: New class. + + To add netrc support: + * src/Netrc.h, src/Netrc.cc: New class. + * src/Util.h, src/Util.cc (split): New function. + + * src/HttpHeader.cc (getRange): Fixed so that it inspects + "Content-Range" header instead of "Range" header. + * src/HttpHeader.h + (getStatus): Removed. + (setStatus): Removed. + + * src/Segment.h + (getPositionToWrite): New function. + +2007-03-05 Tatsuhiro Tsujikawa + + * src/HttpHeader.h + (Range.h): New include. + (status): New variable. + (HttpHeader): Initialized status with 0. + (getStatus): New function. + (setStatus): New function. + (getRange): New function. + (HttpHeaderHandle): New function. + * src/HttpHeader.cc + (getRange): New function. + + * src/Request.h + (RequestWeakHandle): New definition. + + * src/HttpConnection.h + (HttpConnectionHandle): New type definition. + * src/HttpConnection.cc + (receiveResponse): Set HTTP status to headers. + + * src/main.cc + (showUsage): Fixed typo. + + * src/Segment.h + (SegmentHandle): New type definition. + + * src/BitfieldMan.h + (getMissingUnusedLength): New function. + * src/BitfieldMan.cc + (getMissingUnusedLength): New function. + +2007-02-12 Tatsuhiro Tsujikawa + + To fix static initialization order problem: + + * src/BitfieldManFactory.h + (defaultRandomizer): Removed. + (factory): New variable. + (getNewFactory): Removed. + (getFactoryInstance): New function. + (setDefaultRandomizer): Rewritten. + (getDefaultRandomizer): Rewritten. + * src/BitfieldManFactory.cc + (defaultRandomizer): Removed. + (factory): Initialized to 0. + (BitfieldManFactory): Initialized randomizer to 0. + * src/DefaultPieceStorage.cc + (DefaultPieceStorage): getNewFactory() -> getFactoryInstance() + * src/Peer.cc + (Peer): getNewFactory() -> getFactoryInstance() + * src/SegmentMan.cc + (initBitfield): getNewFactory() -> getFactoryInstance() + * src/Piece.cc + (Piece): getNewFactory() -> getFactoryInstance() + +2007-02-06 Tatsuhiro Tsujikawa To fix the bug that causes crash on Max OS X: @@ -148,9 +252,9 @@ (recvSize): LONG_LONG_MAX -> INT64_MAX * src/main.cc - (showUsage): Added --check-integiry and --realtime-chunk-checksum + (showUsage): Added --check-integriy and --realtime-chunk-checksum command-line option. - (main): Added --check-integiry and --realtime-chunk-checksum + (main): Added --check-integriy and --realtime-chunk-checksum command-line option. --force-truncate -> --allow-overwrite Set initial value of --check-integrity option to false. diff --git a/TODO b/TODO index f655b333..2db8af0a 100644 --- a/TODO +++ b/TODO @@ -24,3 +24,4 @@ * remove blockIndex * Add an ability of seeding * Continue file allocation with existing file +* Rewrite HttpConnection::receiveResponse() using {i,o}stringstream diff --git a/doc/aria2c.1.txt b/doc/aria2c.1.txt index 61427323..4da90a51 100644 --- a/doc/aria2c.1.txt +++ b/doc/aria2c.1.txt @@ -140,8 +140,8 @@ OPTIONS exist. Default: 'false' - --check-integiry=true|false:: - Check file integiry by validating piece hash. + --check-integriy=true|false:: + Check file integriy by validating piece hash. This option makes effect in BitTorrent download and Metalink with chunk checksums. Use this option to redownload a damaged portion of diff --git a/src/AbstractCommand.cc b/src/AbstractCommand.cc index 45f17bff..d2014542 100644 --- a/src/AbstractCommand.cc +++ b/src/AbstractCommand.cc @@ -42,7 +42,7 @@ #include "prefs.h" AbstractCommand::AbstractCommand(int cuid, - const RequestHandle req, + const RequestHandle& req, DownloadEngine* e, const SocketHandle& s): Command(cuid), req(req), e(e), socket(s), @@ -81,14 +81,17 @@ bool AbstractCommand::execute() { #endif // ENABLE_ASYNC_DNS !checkSocketIsReadable && !checkSocketIsWritable && !nameResolverCheck) { checkPoint.reset(); - Segment segment; if(e->segmentMan->downloadStarted) { - if(!e->segmentMan->getSegment(segment, cuid)) { - logger->info(MSG_NO_SEGMENT_AVAILABLE, cuid); - return prepareForRetry(1); + // TODO Segment::isNull(), Change method name, it is very confusing. + if(segment->isNull()) { + segment = e->segmentMan->getSegment(cuid); + if(segment.isNull()) { + logger->info(MSG_NO_SEGMENT_AVAILABLE, cuid); + return prepareForRetry(1); + } } } - return executeInternal(segment); + return executeInternal(); } else { if(checkPoint.elapsed(timeout)) { diff --git a/src/AbstractCommand.h b/src/AbstractCommand.h index 4b003e4f..345a4a75 100644 --- a/src/AbstractCommand.h +++ b/src/AbstractCommand.h @@ -50,11 +50,12 @@ protected: RequestHandle req; DownloadEngine* e; SocketHandle socket; + SegmentHandle segment; void tryReserved(); virtual bool prepareForRetry(int wait); virtual void onAbort(RecoverableException* ex); - virtual bool executeInternal(Segment& segment) = 0; + virtual bool executeInternal() = 0; void setReadCheckSocket(const SocketHandle& socket); void setWriteCheckSocket(const SocketHandle& socket); @@ -74,7 +75,7 @@ private: SocketHandle writeCheckTarget; bool nameResolverCheck; public: - AbstractCommand(int cuid, const RequestHandle req, DownloadEngine* e, const SocketHandle& s = SocketHandle()); + AbstractCommand(int cuid, const RequestHandle& req, DownloadEngine* e, const SocketHandle& s = SocketHandle()); virtual ~AbstractCommand(); bool execute(); }; diff --git a/src/AbstractProxyRequestCommand.cc b/src/AbstractProxyRequestCommand.cc new file mode 100644 index 00000000..9c01ddbb --- /dev/null +++ b/src/AbstractProxyRequestCommand.cc @@ -0,0 +1,62 @@ +/* */ +#include "AbstractProxyRequestCommand.h" +#include "HttpConnection.h" + +AbstractProxyRequestCommand::AbstractProxyRequestCommand(int cuid, + const RequestHandle& req, + DownloadEngine* e, + const SocketHandle& s) + :AbstractCommand(cuid, req, e, s), httpConnection(0) { + disableReadCheckSocket(); + setWriteCheckSocket(socket); +} + +AbstractProxyRequestCommand::~AbstractProxyRequestCommand() {} + +bool AbstractProxyRequestCommand::executeInternal() { + socket->setBlockingMode(); + + HttpRequestHandle httpRequest = new HttpRequest(); + httpRequest->setRequest(req); + httpRequest->configure(e->option); + + httpConnection= new HttpConnection(cuid, socket, e->option); + + httpConnection->sendProxyRequest(httpRequest); + + e->commands.push_back(getNextCommand()); + return true; +} diff --git a/src/AbstractProxyRequestCommand.h b/src/AbstractProxyRequestCommand.h new file mode 100644 index 00000000..7b2c6feb --- /dev/null +++ b/src/AbstractProxyRequestCommand.h @@ -0,0 +1,56 @@ +/* */ +#ifndef _D_ABSTRACT_PROXY_REQUEST_COMMAND_H_ +#define _D_ABSTRACT_PROXY_REQUEST_COMMAND_H_ + +#include "AbstractCommand.h" +#include "HttpConnection.h" + +class AbstractProxyRequestCommand : public AbstractCommand { +protected: + HttpConnectionHandle httpConnection; + + virtual bool executeInternal(); +public: + AbstractProxyRequestCommand(int cuid, + const RequestHandle& req, + DownloadEngine* e, + const SocketHandle& s); + virtual ~AbstractProxyRequestCommand(); + + virtual Command* getNextCommand() = 0; +}; + +#endif // _D_ABSTRACT_PROXY_REQUEST_COMMAND_H_ diff --git a/src/AbstractProxyResponseCommand.cc b/src/AbstractProxyResponseCommand.cc new file mode 100644 index 00000000..f35fd706 --- /dev/null +++ b/src/AbstractProxyResponseCommand.cc @@ -0,0 +1,62 @@ +/* */ +#include "AbstractProxyResponseCommand.h" +#include "HttpRequestCommand.h" +#include "DlRetryEx.h" +#include "message.h" + +AbstractProxyResponseCommand::AbstractProxyResponseCommand(int cuid, + const RequestHandle& req, + const HttpConnectionHandle& httpConnection, + DownloadEngine* e, + const SocketHandle& s) + :AbstractCommand(cuid, req, e, s), + httpConnection(httpConnection) {} + +AbstractProxyResponseCommand::~AbstractProxyResponseCommand() {} + +bool AbstractProxyResponseCommand::executeInternal() { + HttpResponseHandle httpResponse = httpConnection->receiveResponse(); + if(httpResponse.isNull()) { + // the server has not responded our request yet. + e->commands.push_back(this); + return false; + } + if(httpResponse->getStatus() != 200) { + throw new DlRetryEx(EX_PROXY_CONNECTION_FAILED); + } + e->commands.push_back(getNextCommand()); + return true; +} diff --git a/src/AbstractProxyResponseCommand.h b/src/AbstractProxyResponseCommand.h new file mode 100644 index 00000000..99ef02d2 --- /dev/null +++ b/src/AbstractProxyResponseCommand.h @@ -0,0 +1,57 @@ +/* */ +#ifndef _D_ABSTRACT_PROXY_RESPONSE_COMMAND_H_ +#define _D_ABSTRACT_PROXY_RESPONSE_COMMAND_H_ + +#include "AbstractCommand.h" +#include "HttpConnection.h" + +class AbstractProxyResponseCommand : public AbstractCommand { +protected: + HttpConnectionHandle httpConnection; + + virtual bool executeInternal(); +public: + AbstractProxyResponseCommand(int cuid, + const RequestHandle& req, + const HttpConnectionHandle& httpConnection, + DownloadEngine* e, + const SocketHandle& s); + virtual ~AbstractProxyResponseCommand(); + + virtual Command* getNextCommand() = 0; +}; + +#endif // _D_ABSTRACT_PROXY_RESPONSE_COMMAND_H_ diff --git a/src/DownloadCommand.cc b/src/DownloadCommand.cc index 33c54143..dd905f61 100644 --- a/src/DownloadCommand.cc +++ b/src/DownloadCommand.cc @@ -46,7 +46,10 @@ DownloadCommand::DownloadCommand(int cuid, const RequestHandle req, DownloadEngine* e, const SocketHandle& s): - AbstractCommand(cuid, req, e, s), lastSize(0), peerStat(0) { + AbstractCommand(cuid, req, e, s), + peerStat(0), + transferDecoder(0) +{ peerStat = this->e->segmentMan->getPeerStat(cuid); if(!peerStat.get()) { peerStat = new PeerStat(cuid); @@ -60,34 +63,30 @@ DownloadCommand::~DownloadCommand() { peerStat->downloadStop(); } -bool DownloadCommand::executeInternal(Segment& segment) { +bool DownloadCommand::executeInternal() { if(maxDownloadSpeedLimit > 0 && maxDownloadSpeedLimit < e->segmentMan->calculateDownloadSpeed()) { usleep(1); e->commands.push_back(this); return false; } - TransferEncoding* te = NULL; - if(transferEncoding.size()) { - te = getTransferEncoding(transferEncoding); - assert(te != NULL); - } - int bufSize = 16*1024;//4096; + int32_t bufSize = 16*1024; char buf[bufSize]; socket->readData(buf, bufSize); - if(te != NULL) { - int infbufSize = 16*1024;//4096; - char infbuf[infbufSize]; - te->inflate(infbuf, infbufSize, buf, bufSize); - e->segmentMan->diskWriter->writeData(infbuf, infbufSize, - segment.getPosition()+segment.writtenLength); - segment.writtenLength += infbufSize; - peerStat->updateDownloadLength(infbufSize); - } else { + + if(transferDecoder.isNull()) { e->segmentMan->diskWriter->writeData(buf, bufSize, - segment.getPosition()+segment.writtenLength); - segment.writtenLength += bufSize; + segment->getPositionToWrite()); + segment->writtenLength += bufSize; peerStat->updateDownloadLength(bufSize); + } else { + int32_t infbufSize = 16*1024; + char infbuf[infbufSize]; + transferDecoder->inflate(infbuf, infbufSize, buf, bufSize); + e->segmentMan->diskWriter->writeData(infbuf, infbufSize, + segment->getPositionToWrite()); + segment->writtenLength += infbufSize; + peerStat->updateDownloadLength(infbufSize); } // calculate downloading speed if(peerStat->getDownloadStartTime().elapsed(startupIdleTime)) { @@ -102,10 +101,10 @@ bool DownloadCommand::executeInternal(Segment& segment) { if(e->segmentMan->totalSize != 0 && bufSize == 0) { throw new DlRetryEx(EX_GOT_EOF); } - if(te != NULL && te->finished() - || te == NULL && segment.complete() + if(!transferDecoder.isNull() && transferDecoder->finished() + || transferDecoder.isNull() && segment->complete() || bufSize == 0) { - if(te != NULL) te->end(); + if(!transferDecoder.isNull()) transferDecoder->end(); logger->info(MSG_DOWNLOAD_COMPLETED, cuid); e->segmentMan->completeSegment(cuid, segment); #ifdef ENABLE_MESSAGE_DIGEST @@ -114,38 +113,39 @@ bool DownloadCommand::executeInternal(Segment& segment) { } #endif // ENABLE_MESSAGE_DIGEST // this unit is going to download another segment. - return prepareForNextSegment(segment); + return prepareForNextSegment(); } else { - e->segmentMan->updateSegment(cuid, segment); e->commands.push_back(this); return false; } } -bool DownloadCommand::prepareForNextSegment(const Segment& currentSegment) { +bool DownloadCommand::prepareForNextSegment() { if(e->segmentMan->finished()) { return true; } else { // Merge segment with next segment, if segment.index+1 == nextSegment.index - Segment tempSegment = currentSegment; + SegmentHandle tempSegment = segment; while(1) { - Segment nextSegment; - if(e->segmentMan->getSegment(nextSegment, cuid, tempSegment.index+1)) { - if(nextSegment.writtenLength > 0) { + SegmentHandle nextSegment = e->segmentMan->getSegment(cuid, + tempSegment->index+1); + cerr << nextSegment.isNull() << endl; + if(nextSegment.isNull()) { + break; + } else { + if(nextSegment->writtenLength > 0) { return prepareForRetry(0); } - nextSegment.writtenLength = tempSegment.writtenLength-tempSegment.length; - if(nextSegment.complete()) { + nextSegment->writtenLength = tempSegment->writtenLength-tempSegment->length; + if(nextSegment->complete()) { e->segmentMan->completeSegment(cuid, nextSegment); tempSegment = nextSegment; } else { - e->segmentMan->updateSegment(cuid, nextSegment); + segment = nextSegment; e->commands.push_back(this); return false; } - } else { - break; } } return prepareForRetry(0); diff --git a/src/DownloadCommand.h b/src/DownloadCommand.h index 66179336..7654e579 100644 --- a/src/DownloadCommand.h +++ b/src/DownloadCommand.h @@ -44,23 +44,25 @@ using namespace std; class DownloadCommand : public AbstractCommand { private: - long long int lastSize; int32_t maxDownloadSpeedLimit; int32_t startupIdleTime; int32_t lowestDownloadSpeedLimit; PeerStatHandle peerStat; protected: - bool executeInternal(Segment& segment); + TransferEncodingHandle transferDecoder; - virtual bool prepareForNextSegment(const Segment& currentSegment); + virtual bool executeInternal(); + + virtual bool prepareForNextSegment(); public: DownloadCommand(int cuid, const RequestHandle req, DownloadEngine* e, const SocketHandle& s); virtual ~DownloadCommand(); - virtual TransferEncoding* getTransferEncoding(const string& transferEncoding) = 0; - - string transferEncoding; + void setTransferDecoder(const TransferEncodingHandle& transferDecoder) + { + this->transferDecoder = transferDecoder; + } void setMaxDownloadSpeedLimit(int32_t maxDownloadSpeedLimit) { this->maxDownloadSpeedLimit = maxDownloadSpeedLimit; diff --git a/src/FtpConnection.cc b/src/FtpConnection.cc index c8db3561..48bd416a 100644 --- a/src/FtpConnection.cc +++ b/src/FtpConnection.cc @@ -109,8 +109,8 @@ SocketHandle FtpConnection::sendPort() const { return serverSocket; } -void FtpConnection::sendRest(const Segment& segment) const { - string request = "REST "+Util::llitos(segment.getPosition()+segment.writtenLength)+"\r\n"; +void FtpConnection::sendRest(const SegmentHandle& segment) const { + string request = "REST "+Util::llitos(segment->getPositionToWrite())+"\r\n"; logger->info(MSG_SENDING_REQUEST, cuid, request.c_str()); socket->writeData(request); } diff --git a/src/FtpConnection.h b/src/FtpConnection.h index 9a6c6be4..ed3697a6 100644 --- a/src/FtpConnection.h +++ b/src/FtpConnection.h @@ -69,7 +69,7 @@ public: void sendSize() const; void sendPasv() const; SocketHandle sendPort() const; - void sendRest(const Segment& segment) const; + void sendRest(const SegmentHandle& segment) const; void sendRetr() const; int receiveResponse(); diff --git a/src/FtpDownloadCommand.cc b/src/FtpDownloadCommand.cc index c5377a0b..c13abeb1 100644 --- a/src/FtpDownloadCommand.cc +++ b/src/FtpDownloadCommand.cc @@ -43,7 +43,3 @@ FtpDownloadCommand::FtpDownloadCommand(int cuid, ctrlSocket(ctrlSocket) {} FtpDownloadCommand::~FtpDownloadCommand() {} - -TransferEncoding* FtpDownloadCommand::getTransferEncoding(const string& name) { - return NULL; -} diff --git a/src/FtpDownloadCommand.h b/src/FtpDownloadCommand.h index 6e969ef4..fdf67c91 100644 --- a/src/FtpDownloadCommand.h +++ b/src/FtpDownloadCommand.h @@ -44,9 +44,7 @@ public: FtpDownloadCommand(int cuid, const RequestHandle req, DownloadEngine* e, const SocketHandle& dataSocket, const SocketHandle& ctrlSocket); - ~FtpDownloadCommand(); - - TransferEncoding* getTransferEncoding(const string& name); + virtual ~FtpDownloadCommand(); }; #endif // _D_FTP_DOWNLOAD_COMMAND_H_ diff --git a/src/FtpInitiateConnectionCommand.cc b/src/FtpInitiateConnectionCommand.cc index f8a3fcec..ae464c16 100644 --- a/src/FtpInitiateConnectionCommand.cc +++ b/src/FtpInitiateConnectionCommand.cc @@ -42,7 +42,7 @@ #include "Util.h" FtpInitiateConnectionCommand::FtpInitiateConnectionCommand(int cuid, - const RequestHandle req, + const RequestHandle& req, DownloadEngine* e) :AbstractCommand(cuid, req, e) { @@ -57,7 +57,7 @@ FtpInitiateConnectionCommand::~FtpInitiateConnectionCommand() { #endif // ENABLE_ASYNC_DNS } -bool FtpInitiateConnectionCommand::executeInternal(Segment& segment) { +bool FtpInitiateConnectionCommand::executeInternal() { string hostname; if(useHttpProxy()) { hostname = e->option->get(PREF_HTTP_PROXY_HOST); diff --git a/src/FtpInitiateConnectionCommand.h b/src/FtpInitiateConnectionCommand.h index 52a3fc02..d6af8630 100644 --- a/src/FtpInitiateConnectionCommand.h +++ b/src/FtpInitiateConnectionCommand.h @@ -52,10 +52,10 @@ private: } #endif // ENABLE_ASYNC_DNS protected: - bool executeInternal(Segment& segment); + virtual bool executeInternal(); public: - FtpInitiateConnectionCommand(int cuid, const RequestHandle req, DownloadEngine* e); - ~FtpInitiateConnectionCommand(); + FtpInitiateConnectionCommand(int cuid, const RequestHandle& req, DownloadEngine* e); + virtual ~FtpInitiateConnectionCommand(); }; #endif // _D_FTP_INITIATE_CONNECTION_COMMAND_H_ diff --git a/src/FtpNegotiationCommand.cc b/src/FtpNegotiationCommand.cc index 15214ce3..3b4526e2 100644 --- a/src/FtpNegotiationCommand.cc +++ b/src/FtpNegotiationCommand.cc @@ -41,7 +41,8 @@ #include "Util.h" #include "FatalException.h" -FtpNegotiationCommand::FtpNegotiationCommand(int cuid, const RequestHandle req, +FtpNegotiationCommand::FtpNegotiationCommand(int cuid, + const RequestHandle& req, DownloadEngine* e, const SocketHandle& s): AbstractCommand(cuid, req, e, s), sequence(SEQ_RECV_GREETING) @@ -55,7 +56,7 @@ FtpNegotiationCommand::~FtpNegotiationCommand() { delete ftp; } -bool FtpNegotiationCommand::executeInternal(Segment& segment) { +bool FtpNegotiationCommand::executeInternal() { while(processSequence(segment)); if(sequence == SEQ_RETRY) { return prepareForRetry(0); @@ -262,14 +263,14 @@ bool FtpNegotiationCommand::recvPasv() { return false; } -bool FtpNegotiationCommand::sendRestPasv(const Segment& segment) { +bool FtpNegotiationCommand::sendRestPasv(const SegmentHandle& segment) { dataSocket->setBlockingMode(); setReadCheckSocket(socket); disableWriteCheckSocket(); return sendRest(segment); } -bool FtpNegotiationCommand::sendRest(const Segment& segment) { +bool FtpNegotiationCommand::sendRest(const SegmentHandle& segment) { ftp->sendRest(segment); sequence = SEQ_RECV_REST; return false; @@ -311,7 +312,7 @@ bool FtpNegotiationCommand::recvRetr() { return false; } -bool FtpNegotiationCommand::processSequence(const Segment& segment) { +bool FtpNegotiationCommand::processSequence(const SegmentHandle& segment) { bool doNextSequence = true; switch(sequence) { case SEQ_RECV_GREETING: diff --git a/src/FtpNegotiationCommand.h b/src/FtpNegotiationCommand.h index 138cfac8..971faba6 100644 --- a/src/FtpNegotiationCommand.h +++ b/src/FtpNegotiationCommand.h @@ -80,23 +80,25 @@ private: bool recvPort(); bool sendPasv(); bool recvPasv(); - bool sendRest(const Segment& segment); - bool sendRestPasv(const Segment& segment); + bool sendRest(const SegmentHandle& segment); + bool sendRestPasv(const SegmentHandle& segment); bool recvRest(); bool sendRetr(); bool recvRetr(); - bool processSequence(const Segment& segment); + bool processSequence(const SegmentHandle& segment); SocketHandle dataSocket; SocketHandle serverSocket; int sequence; FtpConnection* ftp; protected: - bool executeInternal(Segment& segment); + virtual bool executeInternal(); public: - FtpNegotiationCommand(int cuid, const RequestHandle req, DownloadEngine* e, + FtpNegotiationCommand(int cuid, + const RequestHandle& req, + DownloadEngine* e, const SocketHandle& s); - ~FtpNegotiationCommand(); + virtual ~FtpNegotiationCommand(); }; #endif // _D_FTP_NEGOTIATION_COMMAND_H_ diff --git a/src/FtpTunnelRequestCommand.cc b/src/FtpTunnelRequestCommand.cc index c1051d20..88e0a85f 100644 --- a/src/FtpTunnelRequestCommand.cc +++ b/src/FtpTunnelRequestCommand.cc @@ -34,26 +34,16 @@ /* copyright --> */ #include "FtpTunnelRequestCommand.h" #include "FtpTunnelResponseCommand.h" -#include "HttpConnection.h" FtpTunnelRequestCommand::FtpTunnelRequestCommand(int cuid, - const RequestHandle req, + const RequestHandle& req, DownloadEngine* e, const SocketHandle& s) - :AbstractCommand(cuid, req, e, s) { - disableReadCheckSocket(); - disableWriteCheckSocket(); -} + :AbstractProxyRequestCommand(cuid, req, e, s) {} FtpTunnelRequestCommand::~FtpTunnelRequestCommand() {} -bool FtpTunnelRequestCommand::executeInternal(Segment& segment) { - socket->setBlockingMode(); - HttpConnection httpConnection(cuid, socket, req, e->option); - httpConnection.sendProxyRequest(); - - FtpTunnelResponseCommand* command - = new FtpTunnelResponseCommand(cuid, req, e, socket); - e->commands.push_back(command); - return true; +Command* FtpTunnelRequestCommand::getNextCommand() +{ + return new FtpTunnelResponseCommand(cuid, req, httpConnection, e, socket); } diff --git a/src/FtpTunnelRequestCommand.h b/src/FtpTunnelRequestCommand.h index 92126597..a1c42a11 100644 --- a/src/FtpTunnelRequestCommand.h +++ b/src/FtpTunnelRequestCommand.h @@ -35,15 +35,17 @@ #ifndef _D_FTP_TUNNEL_REQUEST_COMMAND_H_ #define _D_FTP_TUNNEL_REQUEST_COMMAND_H_ -#include "AbstractCommand.h" +#include "AbstractProxyRequestCommand.h" -class FtpTunnelRequestCommand : public AbstractCommand { -protected: - bool executeInternal(Segment& segment); +class FtpTunnelRequestCommand : public AbstractProxyRequestCommand { public: - FtpTunnelRequestCommand(int cuid, const RequestHandle req, DownloadEngine* e, + FtpTunnelRequestCommand(int cuid, + const RequestHandle& req, + DownloadEngine* e, const SocketHandle& s); - ~FtpTunnelRequestCommand(); + virtual ~FtpTunnelRequestCommand(); + + virtual Command* getNextCommand(); }; #endif // _D_FTP_TUNNEL_REQUEST_COMMAND_H_ diff --git a/src/FtpTunnelResponseCommand.cc b/src/FtpTunnelResponseCommand.cc index 4f09a649..8411e452 100644 --- a/src/FtpTunnelResponseCommand.cc +++ b/src/FtpTunnelResponseCommand.cc @@ -34,34 +34,17 @@ /* copyright --> */ #include "FtpTunnelResponseCommand.h" #include "FtpNegotiationCommand.h" -#include "DlRetryEx.h" -#include "message.h" FtpTunnelResponseCommand::FtpTunnelResponseCommand(int cuid, - const RequestHandle req, + const RequestHandle& req, + const HttpConnectionHandle& httpConnection, DownloadEngine* e, const SocketHandle& s) - :AbstractCommand(cuid, req, e, s) { - http = new HttpConnection(cuid, socket, req, e->option); -} + :AbstractProxyResponseCommand(cuid, req, httpConnection,e, s) {} -FtpTunnelResponseCommand::~FtpTunnelResponseCommand() { - delete http; -} +FtpTunnelResponseCommand::~FtpTunnelResponseCommand() {} -bool FtpTunnelResponseCommand::executeInternal(Segment& segment) { - HttpHeader headers; - int status = http->receiveResponse(headers); - if(status == 0) { - // we didn't receive all of headers yet. - e->commands.push_back(this); - return false; - } - if(status != 200) { - throw new DlRetryEx(EX_PROXY_CONNECTION_FAILED); - } - FtpNegotiationCommand* command - = new FtpNegotiationCommand(cuid, req, e, socket); - e->commands.push_back(command); - return true; +Command* FtpTunnelResponseCommand::getNextCommand() +{ + return new FtpNegotiationCommand(cuid, req, e, socket); } diff --git a/src/FtpTunnelResponseCommand.h b/src/FtpTunnelResponseCommand.h index 3dc19d0f..d33e97b8 100644 --- a/src/FtpTunnelResponseCommand.h +++ b/src/FtpTunnelResponseCommand.h @@ -35,18 +35,18 @@ #ifndef _D_FTP_TUNNEL_RESPONSE_COMMAND_H_ #define _D_FTP_TUNNEL_RESPONSE_COMMAND_H_ -#include "AbstractCommand.h" -#include "HttpConnection.h" +#include "AbstractProxyResponseCommand.h" -class FtpTunnelResponseCommand : public AbstractCommand { -private: - HttpConnection* http; -protected: - bool executeInternal(Segment& segment); +class FtpTunnelResponseCommand : public AbstractProxyResponseCommand { public: - FtpTunnelResponseCommand(int cuid, const RequestHandle req, DownloadEngine* e, + FtpTunnelResponseCommand(int cuid, + const RequestHandle& req, + const HttpConnectionHandle& httpConnection, + DownloadEngine* e, const SocketHandle& s); - ~FtpTunnelResponseCommand(); + virtual ~FtpTunnelResponseCommand(); + + virtual Command* getNextCommand(); }; #endif // _D_FTP_TUNNEL_RESPONSE_COMMAND_H_ diff --git a/src/HttpAuthConfig.h b/src/HttpAuthConfig.h new file mode 100644 index 00000000..4367a78e --- /dev/null +++ b/src/HttpAuthConfig.h @@ -0,0 +1,58 @@ +/* */ +#ifndef _D_HTTP_AUTH_CONFIG_H_ +#define _D_HTTP_AUTH_CONFIG_H_ + +#include "common.h" + +class HttpAuthConfig { +private: + string authScheme; + string authUser; + string authPassword; +public: + + HttpAuthConfig(const string& authUser, const string& authPassword): + authUser(authUser), authPassword(authPassword) {} + + string getAuthText() const + { + return authUser+":"+authPassword; + } +}; + +typedef SharedHandle HttpAuthConfigHandle; + +#endif // _D_HTTP_AUTH_CONFIG_H_ diff --git a/src/HttpConnection.cc b/src/HttpConnection.cc index 095fae5c..40de31be 100644 --- a/src/HttpConnection.cc +++ b/src/HttpConnection.cc @@ -40,93 +40,27 @@ #include "prefs.h" #include "LogFactory.h" -HttpConnection::HttpConnection(int cuid, const SocketHandle& socket, - const RequestHandle req, const Option* op): - cuid(cuid), socket(socket), req(req), option(op), headerBufLength(0) { +HttpConnection::HttpConnection(int cuid, + const SocketHandle& socket, + const Option* op): + cuid(cuid), socket(socket), option(op), headerBufLength(0) { logger = LogFactory::getInstance(); } -void HttpConnection::sendRequest(const Segment& segment) const { - string request = createRequest(segment); +void HttpConnection::sendRequest(const HttpRequestHandle& httpRequest) +{ + string request = httpRequest->createRequest(); logger->info(MSG_SENDING_REQUEST, cuid, request.c_str()); socket->writeData(request.c_str(), request.size()); + outstandingHttpRequests.push_back(httpRequest); } -void HttpConnection::sendProxyRequest() const { - string request = - string("CONNECT ")+req->getHost()+":"+Util::llitos(req->getPort())+ - string(" HTTP/1.1\r\n")+ - "User-Agent: "+USER_AGENT+"\r\n"+ - "Proxy-Connection: close\r\n"+ - "Host: "+getHost(req->getHost(), req->getPort())+"\r\n"; - if(useProxyAuth()) { - request += getProxyAuthString(); - } - request += "\r\n"; +void HttpConnection::sendProxyRequest(const HttpRequestHandle& httpRequest) +{ + string request = httpRequest->createProxyRequest(); logger->info(MSG_SENDING_REQUEST, cuid, request.c_str()); socket->writeData(request.c_str(), request.size()); -} - -string HttpConnection::getProxyAuthString() const { - return "Proxy-Authorization: Basic "+ - Base64::encode(option->get(PREF_HTTP_PROXY_USER)+":"+ - option->get(PREF_HTTP_PROXY_PASSWD))+"\r\n"; -} - -string HttpConnection::getHost(const string& host, int port) const { - return host+(port == 80 || port == 443 ? "" : ":"+Util::llitos(port)); -} - -string HttpConnection::createRequest(const Segment& segment) const { - string request = string("GET ")+ - (req->getProtocol() == "ftp" || useProxy() && useProxyGet() ? - req->getCurrentUrl() : - ((req->getDir() == "/" ? "/" : req->getDir()+"/")+req->getFile()))+ - string(" HTTP/1.1\r\n")+ - "User-Agent: "+USER_AGENT+"\r\n"+ - // use persistent connection - //"Connection: close\r\n"+ - "Accept: */*\r\n"+ /* */ - "Host: "+getHost(req->getHost(), req->getPort())+"\r\n"+ - "Pragma: no-cache\r\n"+ - "Cache-Control: no-cache\r\n"; - if(!req->isKeepAlive()) { - request += "Connection: close\r\n"; - } - if(segment.length > 0) { - request += "Range: bytes="+ - Util::llitos(segment.getPosition()+segment.writtenLength); - request += "-"; - if(req->isKeepAlive()) { - request += Util::llitos(segment.getPosition()+segment.length-1); - } - request += "\r\n"; - } - if(useProxy() && useProxyAuth() && useProxyGet()) { - request += "Proxy-Connection: close\r\n"; - request += getProxyAuthString(); - } - if(option->get(PREF_HTTP_AUTH_ENABLED) == V_TRUE) { - if(option->get(PREF_HTTP_AUTH_SCHEME) == V_BASIC) { - request += "Authorization: Basic "+ - Base64::encode(option->get(PREF_HTTP_USER)+":"+ - option->get(PREF_HTTP_PASSWD))+"\r\n"; - } - } - if(req->getPreviousUrl().size()) { - request += "Referer: "+req->getPreviousUrl()+"\r\n"; - } - - string cookiesValue; - Cookies cookies = req->cookieBox->criteriaFind(req->getHost(), req->getDir(), req->getProtocol() == "https" ? true : false); - for(Cookies::const_iterator itr = cookies.begin(); itr != cookies.end(); itr++) { - cookiesValue += (*itr).toString()+";"; - } - if(cookiesValue.size()) { - request += string("Cookie: ")+cookiesValue+"\r\n"; - } - request += "\r\n"; - return request; + outstandingHttpRequests.push_back(httpRequest); } int HttpConnection::findEndOfHeader(const char* buf, const char* substr, int bufLength) const { @@ -140,7 +74,7 @@ int HttpConnection::findEndOfHeader(const char* buf, const char* substr, int buf return -1; } -int HttpConnection::receiveResponse(HttpHeader& headers) { +HttpResponseHandle HttpConnection::receiveResponse() { //char buf[512]; string header; int delimiterSwitch = 0; @@ -194,26 +128,22 @@ int HttpConnection::receiveResponse(HttpHeader& headers) { } string status = header.substr(9, 3); p = np+2; + HttpHeaderHandle httpHeader = new HttpHeader(); // retreive status name-value pairs, then push these into map while((np = header.find(delimiters[delimiterSwitch], p)) != string::npos && np != p) { string line = header.substr(p, np-p); p = np+2; pair hp; Util::split(hp, line, ':'); - headers.put(hp.first, hp.second); + httpHeader->put(hp.first, hp.second); } - headers.setStatus(strtol(status.c_str(), 0, 10)); - return headers.getStatus(); -} + HttpResponseHandle httpResponse = new HttpResponse(); + httpResponse->setCuid(cuid); + httpResponse->setStatus(strtol(status.c_str(), 0, 10)); + httpResponse->setHttpHeader(httpHeader); + httpResponse->setHttpRequest(outstandingHttpRequests.front()); -bool HttpConnection::useProxy() const { - return option->get(PREF_HTTP_PROXY_ENABLED) == V_TRUE; -} + outstandingHttpRequests.pop_front(); -bool HttpConnection::useProxyAuth() const { - return option->get(PREF_HTTP_PROXY_AUTH_ENABLED) == V_TRUE; -} - -bool HttpConnection::useProxyGet() const { - return option->get(PREF_HTTP_PROXY_METHOD) == V_GET; + return httpResponse; } diff --git a/src/HttpConnection.h b/src/HttpConnection.h index 1c60f95c..d94ddc27 100644 --- a/src/HttpConnection.h +++ b/src/HttpConnection.h @@ -40,33 +40,29 @@ #include "Request.h" #include "Option.h" #include "Logger.h" -#include "HttpHeader.h" #include "common.h" #include "Logger.h" +#include "HttpResponse.h" +#include #include -using namespace std; - #define HEADERBUF_SIZE 4096 class HttpConnection { private: - string getHost(const string& host, int port) const; - string createRequest(const Segment& segment) const; - int findEndOfHeader(const char* buf, const char* substr, int bufLength) const; - bool useProxy() const; - bool useProxyAuth() const; - bool useProxyGet() const; - string getProxyAuthString() const; int cuid; SocketHandle socket; - RequestHandle req; const Option* option; const Logger* logger; char headerBuf[HEADERBUF_SIZE+1]; int headerBufLength; + + HttpRequests outstandingHttpRequests; + + int findEndOfHeader(const char* buf, const char* substr, int bufLength) const; public: - HttpConnection(int cuid, const SocketHandle& socket, const RequestHandle req, + HttpConnection(int cuid, + const SocketHandle& socket, const Option* op); /** @@ -76,25 +72,34 @@ public: * HTTP proxy(GET method). * @param segment indicates starting postion of the file for downloading */ - void sendRequest(const Segment& segment) const; + void sendRequest(const HttpRequestHandle& httpRequest); /** * Sends Http proxy request using CONNECT method. */ - void sendProxyRequest() const; + void sendProxyRequest(const HttpRequestHandle& httpRequest); /** - * Receives HTTP response from the server and store the response header - * into the variable headers. - * If response header is not fully received, received header is buffured - * in this object and headers is undefined and this method returns 0. + * Receives HTTP response from the server and returns HttpResponseHandle + * object which contains response header and HttpRequestHandle object + * for this response. + * If a response is not fully received, received header is buffured + * in this object and returns 0. * You should continue to call this method until whole response header is - * received and this method returns non-zero value. + * received and this method returns non-null HttpResponseHandle object. * - * @param headers holder to store HTTP response header - * @return HTTP status or 0 if whole response header is not received + * @return HttpResponse or 0 if whole response header is not received */ - int receiveResponse(HttpHeader& headers); + HttpResponseHandle receiveResponse(); + + HttpRequestHandle getFirstHttpRequest() const + { + if(outstandingHttpRequests.size() > 0) { + return outstandingHttpRequests.front(); + } else { + return 0; + } + } }; typedef SharedHandle HttpConnectionHandle; diff --git a/src/HttpDownloadCommand.cc b/src/HttpDownloadCommand.cc index 9ea89ef4..969addf9 100644 --- a/src/HttpDownloadCommand.cc +++ b/src/HttpDownloadCommand.cc @@ -33,38 +33,19 @@ */ /* copyright --> */ #include "HttpDownloadCommand.h" -#include "DlRetryEx.h" #include "HttpRequestCommand.h" #include "Util.h" -#include "ChunkedEncoding.h" #include "message.h" -#include -#include -#include -#include -using namespace std; - -HttpDownloadCommand::HttpDownloadCommand(int cuid, const RequestHandle req, +HttpDownloadCommand::HttpDownloadCommand(int cuid, + const RequestHandle req, DownloadEngine* e, const SocketHandle& socket) - :DownloadCommand(cuid, req, e, socket) -{ - ChunkedEncoding* ce = new ChunkedEncoding(); - transferEncodings["chunked"] = ce; -} + :DownloadCommand(cuid, req, e, socket) {} -HttpDownloadCommand::~HttpDownloadCommand() { - for(map::iterator itr = transferEncodings.begin(); itr != transferEncodings.end(); itr++) { - delete((*itr).second); - } -} +HttpDownloadCommand::~HttpDownloadCommand() {} -TransferEncoding* HttpDownloadCommand::getTransferEncoding(const string& name) { - return transferEncodings[name]; -} - -bool HttpDownloadCommand::prepareForNextSegment(const Segment& currentSegment) { +bool HttpDownloadCommand::prepareForNextSegment() { if(e->segmentMan->finished()) { return true; } else { @@ -73,7 +54,7 @@ bool HttpDownloadCommand::prepareForNextSegment(const Segment& currentSegment) { e->commands.push_back(command); return true; } else { - return DownloadCommand::prepareForNextSegment(currentSegment); + return DownloadCommand::prepareForNextSegment(); } } } diff --git a/src/HttpDownloadCommand.h b/src/HttpDownloadCommand.h index ecb2e398..0b6edb63 100644 --- a/src/HttpDownloadCommand.h +++ b/src/HttpDownloadCommand.h @@ -36,27 +36,14 @@ #define _D_HTTP_DOWNLOAD_COMMAND_H_ #include "DownloadCommand.h" -#include "DownloadEngine.h" -#include "Socket.h" -#include "Request.h" -#include "common.h" -#include "TransferEncoding.h" -#include -#include -using namespace std; - -class HttpDownloadCommand:public DownloadCommand { -private: - map transferEncodings; +class HttpDownloadCommand : public DownloadCommand { protected: - virtual bool prepareForNextSegment(const Segment& currentSegment); + virtual bool prepareForNextSegment(); public: HttpDownloadCommand(int cuid, const RequestHandle req, DownloadEngine* e, const SocketHandle& s); - ~HttpDownloadCommand(); - - TransferEncoding* getTransferEncoding(const string& transferEncoding); + virtual ~HttpDownloadCommand(); }; #endif // _D_HTTP_DOWNLOAD_COMMAND_H_ diff --git a/src/HttpHeader.cc b/src/HttpHeader.cc index adf74ae3..9cf6b002 100644 --- a/src/HttpHeader.cc +++ b/src/HttpHeader.cc @@ -70,26 +70,34 @@ long long int HttpHeader::getFirstAsLLInt(const string& name) const { if(value == "") { return 0; } else { - return strtoll(value.c_str(), NULL, 10); + return strtoll(value.c_str(), 0, 10); } } RangeHandle HttpHeader::getRange() const { - string rangeStr = getFirst("Range"); + string rangeStr = getFirst("Content-Range"); if(rangeStr == "") { - return 0; + string contentLengthStr = getFirst("Content-Length"); + if(contentLengthStr == "") { + return new Range(0, 0, 0); + } else { + int64_t contentLength = strtoll(contentLengthStr.c_str(), 0, 10); + return new Range(0, contentLength-1, contentLength); + } + } + string::size_type rangeSpecIndex = rangeStr.find("bytes "); + if(rangeSpecIndex == string::npos) { + return new Range(0, 0, 0); } pair rangePair; - Util::split(rangePair, rangeStr, '/'); + Util::split(rangePair, rangeStr.substr(rangeSpecIndex+6), '/'); pair startEndBytePair; Util::split(startEndBytePair, rangePair.first, '-'); int64_t startByte = STRTOLL(startEndBytePair.first.c_str()); int64_t endByte = STRTOLL(startEndBytePair.second.c_str()); - int64_t contentLength = STRTOLL(rangePair.second.c_str()); + int64_t entityLength = STRTOLL(rangePair.second.c_str()); - RangeHandle range = new Range(startByte, endByte, contentLength); - - return range; + return new Range(startByte, endByte, entityLength); } diff --git a/src/HttpHeader.h b/src/HttpHeader.h index 363f207d..d2cacce8 100644 --- a/src/HttpHeader.h +++ b/src/HttpHeader.h @@ -43,10 +43,9 @@ class HttpHeader { private: - int32_t status; multimap table; public: - HttpHeader():status(0) {} + HttpHeader() {} ~HttpHeader() {} void put(const string& name, const string& value); @@ -57,16 +56,6 @@ public: long long int getFirstAsLLInt(const string& name) const; RangeHandle getRange() const; - - int32_t getStatus() const - { - return status; - } - - void setStatus(int32_t status) - { - this->status = status; - } }; typedef SharedHandle HttpHeaderHandle; diff --git a/src/HttpInitiateConnectionCommand.cc b/src/HttpInitiateConnectionCommand.cc index 4d1492f2..e89e7801 100644 --- a/src/HttpInitiateConnectionCommand.cc +++ b/src/HttpInitiateConnectionCommand.cc @@ -42,7 +42,7 @@ #include "prefs.h" HttpInitiateConnectionCommand::HttpInitiateConnectionCommand(int cuid, - const RequestHandle req, + const RequestHandle& req, DownloadEngine* e): AbstractCommand(cuid, req, e) { @@ -57,7 +57,7 @@ HttpInitiateConnectionCommand::~HttpInitiateConnectionCommand() { #endif // ENABLE_ASYNC_DNS } -bool HttpInitiateConnectionCommand::executeInternal(Segment& segment) { +bool HttpInitiateConnectionCommand::executeInternal() { string hostname; if(useProxy()) { hostname = e->option->get(PREF_HTTP_PROXY_HOST); diff --git a/src/HttpInitiateConnectionCommand.h b/src/HttpInitiateConnectionCommand.h index 8f9b32e4..1c6e7895 100644 --- a/src/HttpInitiateConnectionCommand.h +++ b/src/HttpInitiateConnectionCommand.h @@ -54,7 +54,7 @@ protected: * Whether or not the connection is established successfully is * evaluated by RequestCommand. */ - bool executeInternal(Segment& segment); + virtual bool executeInternal(); #ifdef ENABLE_ASYNC_DNS virtual bool nameResolveFinished() const { return nameResolver->getStatus() == NameResolver::STATUS_SUCCESS || @@ -62,8 +62,8 @@ protected: } #endif // ENABLE_ASYNC_DNS public: - HttpInitiateConnectionCommand(int cuid, const RequestHandle req, DownloadEngine* e); - ~HttpInitiateConnectionCommand(); + HttpInitiateConnectionCommand(int cuid, const RequestHandle& req, DownloadEngine* e); + virtual ~HttpInitiateConnectionCommand(); }; #endif // _D_HTTP_INITIATE_CONNECTION_COMMAND_H_ diff --git a/src/HttpProxyRequestCommand.cc b/src/HttpProxyRequestCommand.cc index 6f6f486e..34ac623c 100644 --- a/src/HttpProxyRequestCommand.cc +++ b/src/HttpProxyRequestCommand.cc @@ -33,26 +33,17 @@ */ /* copyright --> */ #include "HttpProxyRequestCommand.h" -#include "HttpConnection.h" #include "HttpProxyResponseCommand.h" HttpProxyRequestCommand::HttpProxyRequestCommand(int cuid, - const RequestHandle req, + const RequestHandle& req, DownloadEngine* e, const SocketHandle& s) - :AbstractCommand(cuid, req, e, s) { - disableReadCheckSocket(); - setWriteCheckSocket(socket); -} + :AbstractProxyRequestCommand(cuid, req, e, s) {} HttpProxyRequestCommand::~HttpProxyRequestCommand() {} -bool HttpProxyRequestCommand::executeInternal(Segment& segment) { - socket->setBlockingMode(); - HttpConnection httpConnection(cuid, socket, req, e->option); - httpConnection.sendProxyRequest(); - - HttpProxyResponseCommand* command = new HttpProxyResponseCommand(cuid, req, e, socket); - e->commands.push_back(command); - return true; +Command* HttpProxyRequestCommand::getNextCommand() +{ + return new HttpProxyResponseCommand(cuid, req, httpConnection, e, socket); } diff --git a/src/HttpProxyRequestCommand.h b/src/HttpProxyRequestCommand.h index ffec1746..e2aa052a 100644 --- a/src/HttpProxyRequestCommand.h +++ b/src/HttpProxyRequestCommand.h @@ -35,15 +35,17 @@ #ifndef _D_HTTP_PROXY_REQUEST_COMMAND_H_ #define _D_HTTP_PROXY_REQUEST_COMMAND_H_ -#include "AbstractCommand.h" +#include "AbstractProxyRequestCommand.h" -class HttpProxyRequestCommand : public AbstractCommand { -protected: - bool executeInternal(Segment& segment); +class HttpProxyRequestCommand : public AbstractProxyRequestCommand { public: - HttpProxyRequestCommand(int cuid, const RequestHandle req, DownloadEngine* e, + HttpProxyRequestCommand(int cuid, + const RequestHandle& req, + DownloadEngine* e, const SocketHandle& s); - ~HttpProxyRequestCommand(); + virtual ~HttpProxyRequestCommand(); + + virtual Command* getNextCommand(); }; #endif // _D_HTTP_PROXY_REQUEST_COMMAND_H_ diff --git a/src/HttpProxyResponseCommand.cc b/src/HttpProxyResponseCommand.cc index e611acc0..23c1fe47 100644 --- a/src/HttpProxyResponseCommand.cc +++ b/src/HttpProxyResponseCommand.cc @@ -34,34 +34,17 @@ /* copyright --> */ #include "HttpProxyResponseCommand.h" #include "HttpRequestCommand.h" -#include "DlRetryEx.h" -#include "message.h" HttpProxyResponseCommand::HttpProxyResponseCommand(int cuid, - const RequestHandle req, + const RequestHandle& req, + const HttpConnectionHandle& httpConnection, DownloadEngine* e, const SocketHandle& s) - :AbstractCommand(cuid, req, e, s) { - http = new HttpConnection(cuid, socket, req, e->option); -} + :AbstractProxyResponseCommand(cuid, req, httpConnection, e, s) {} -HttpProxyResponseCommand::~HttpProxyResponseCommand() { - delete http; -} +HttpProxyResponseCommand::~HttpProxyResponseCommand() {} -bool HttpProxyResponseCommand::executeInternal(Segment& segment) { - HttpHeader headers; - int status = http->receiveResponse(headers); - if(status == 0) { - // we didn't receive all of headers yet. - e->commands.push_back(this); - return false; - } - if(status != 200) { - throw new DlRetryEx(EX_PROXY_CONNECTION_FAILED); - } - HttpRequestCommand* command = new HttpRequestCommand(cuid, req, e, socket); - e->commands.push_back(command); - return true; +Command* HttpProxyResponseCommand::getNextCommand() +{ + return new HttpRequestCommand(cuid, req, e, socket); } - diff --git a/src/HttpProxyResponseCommand.h b/src/HttpProxyResponseCommand.h index f5b11af6..4cd6471a 100644 --- a/src/HttpProxyResponseCommand.h +++ b/src/HttpProxyResponseCommand.h @@ -35,18 +35,18 @@ #ifndef _D_HTTP_PROXY_RESPONSE_COMMAND_H_ #define _D_HTTP_PROXY_RESPONSE_COMMAND_H_ -#include "AbstractCommand.h" -#include "HttpConnection.h" +#include "AbstractProxyResponseCommand.h" -class HttpProxyResponseCommand : public AbstractCommand { -private: - HttpConnection* http; -protected: - bool executeInternal(Segment& segment); +class HttpProxyResponseCommand : public AbstractProxyResponseCommand { public: - HttpProxyResponseCommand(int cuid, const RequestHandle req, DownloadEngine* e, + HttpProxyResponseCommand(int cuid, + const RequestHandle& req, + const HttpConnectionHandle& httpConnection, + DownloadEngine* e, const SocketHandle& s); - ~HttpProxyResponseCommand(); + virtual ~HttpProxyResponseCommand(); + + virtual Command* getNextCommand(); }; #endif // _D_HTTP_PROXY_RESPONSE_COMMAND_H_ diff --git a/src/HttpRequest.cc b/src/HttpRequest.cc new file mode 100644 index 00000000..b746f612 --- /dev/null +++ b/src/HttpRequest.cc @@ -0,0 +1,162 @@ +/* */ +#include "HttpRequest.h" +#include "Util.h" +#include "Base64.h" +#include "prefs.h" + +RangeHandle HttpRequest::getRange() const +{ + // content-length is always 0 + if(segment->isNull()) { + return new Range(0, 0, 0); + } else { + return new Range(getStartByte(), getEndByte(), entityLength); + } +} + +bool HttpRequest::isRangeSatisfied(const RangeHandle& range) const +{ + if(segment->isNull()) { + return true; + } + if(getStartByte() == range->getStartByte() && + (getEndByte() == 0 || + getEndByte() > 0 && getEndByte() == range->getEndByte()) && + (entityLength == 0 || + entityLength > 0 && entityLength == range->getEntityLength())) { + return true; + } else { + return false; + } +} + +string HttpRequest::getHostText(const string& host, in_port_t port) const +{ + return host+(port == 80 || port == 443 ? "" : ":"+Util::itos(port)); +} + +string HttpRequest::createRequest() const +{ + string requestLine = "GET "; + if(getProtocol() == "ftp" || proxyEnabled) { + requestLine += getCurrentURI(); + } else { + if(getDir() == "/") { + requestLine += getDir(); + } else { + requestLine += getDir()+"/"; + } + requestLine += getFile(); + } + requestLine += + string(" HTTP/1.1\r\n")+ + "User-Agent: "+userAgent+"\r\n"+ + "Accept: */*\r\n"+ /* */ + "Host: "+getHostText(getHost(), getPort())+"\r\n"+ + "Pragma: no-cache\r\n"+ + "Cache-Control: no-cache\r\n"; + if(!request->isKeepAlive()) { + requestLine += "Connection: close\r\n"; + } + if(segment->length > 0) { + requestLine += "Range: bytes="+Util::llitos(getStartByte()); + requestLine += "-"; + if(request->isKeepAlive()) { + requestLine += Util::llitos(getEndByte()); + } + requestLine += "\r\n"; + } + if(proxyEnabled) { + requestLine += "Proxy-Connection: close\r\n"; + } + if(proxyEnabled && proxyAuthEnabled) { + requestLine += getProxyAuthString(); + } + if(authEnabled) { + requestLine += "Authorization: Basic "+ + Base64::encode(authConfig->getAuthText())+"\r\n"; + } + if(getPreviousURI().size()) { + requestLine += "Referer: "+getPreviousURI()+"\r\n"; + } + + string cookiesValue; + Cookies cookies = request->cookieBox->criteriaFind(getHost(), + getDir(), + getProtocol() == "https" ? + true : false); + for(Cookies::const_iterator itr = cookies.begin(); itr != cookies.end(); itr++) { + cookiesValue += (*itr).toString()+";"; + } + if(cookiesValue.size()) { + requestLine += string("Cookie: ")+cookiesValue+"\r\n"; + } + requestLine += "\r\n"; + return requestLine; +} + +string HttpRequest::createProxyRequest() const +{ + string requestLine = + string("CONNECT ")+getHost()+":"+Util::llitos(getPort())+ + string(" HTTP/1.1\r\n")+ + "User-Agent: "+Util::urlencode(userAgent)+"\r\n"+ + "Proxy-Connection: close\r\n"+ + "Host: "+getHostText(getHost(), getPort())+"\r\n"; + if(proxyAuthEnabled) { + requestLine += getProxyAuthString(); + } + requestLine += "\r\n"; + return requestLine; +} + +string HttpRequest::getProxyAuthString() const { + return "Proxy-Authorization: Basic "+ + Base64::encode(proxyAuthConfig->getAuthText())+"\r\n"; +} + +void HttpRequest::configure(const Option* option) +{ + authEnabled = option->get(PREF_HTTP_AUTH_ENABLED) == V_TRUE; + proxyEnabled = + option->get(PREF_HTTP_PROXY_ENABLED) == V_TRUE && + option->get(PREF_HTTP_PROXY_METHOD) == V_GET; + proxyAuthEnabled = option->get(PREF_HTTP_PROXY_AUTH_ENABLED) == V_TRUE; + authConfig = new HttpAuthConfig(option->get(PREF_HTTP_USER), + option->get(PREF_HTTP_PASSWD)); + proxyAuthConfig = new HttpAuthConfig(option->get(PREF_HTTP_PROXY_USER), + option->get(PREF_HTTP_PROXY_PASSWD)); +} diff --git a/src/HttpRequest.h b/src/HttpRequest.h new file mode 100644 index 00000000..d6f4ca4f --- /dev/null +++ b/src/HttpRequest.h @@ -0,0 +1,243 @@ +/* */ +#ifndef _D_HTTP_REQUEST_H_ +#define _D_HTTP_REQUEST_H_ + +#include "common.h" +#include "Segment.h" +#include "Range.h" +#include "Request.h" +#include "HttpAuthConfig.h" +#include "Option.h" +#include + +class HttpRequest { +private: + + RequestHandle request; + + SegmentHandle segment; + + int64_t entityLength; + + bool authEnabled; + + HttpAuthConfigHandle authConfig; + + bool proxyEnabled; + + bool proxyAuthEnabled; + + HttpAuthConfigHandle proxyAuthConfig; + + string userAgent; + + string getHostText(const string& host, in_port_t port) const; + + string getProxyAuthString() const; + +public: + HttpRequest():request(0), + segment(0), + entityLength(0), + authEnabled(false), + authConfig(0), + proxyEnabled(false), + proxyAuthEnabled(false), + proxyAuthConfig(0), + userAgent(USER_AGENT) + {} + + SegmentHandle getSegment() const + { + return segment; + } + + void setSegment(const SegmentHandle& segment) + { + this->segment = segment; + } + + void setRequest(const RequestHandle& request) + { + this->request = request; + } + + /** + * entityLength is used in isRangeSatisfied() method. + */ + void setEntityLength(int64_t entityLength) + { + this->entityLength = entityLength; + } + + int64_t getEntityLength() const + { + return entityLength; + } + + string getHost() const + { + return request->getHost(); + } + + in_port_t getPort() const + { + return request->getPort(); + } + + string getMethod() const + { + return request->getMethod(); + } + + string getProtocol() const + { + return request->getProtocol(); + } + + string getCurrentURI() const + { + return request->getCurrentUrl(); + } + + string getDir() const + { + return request->getDir(); + } + + string getFile() const + { + return request->getFile(); + } + + string getPreviousURI() const + { + return request->getPreviousUrl(); + } + + RangeHandle getRange() const; + + /** + * Inspects whether the specified response range is satisfiable + * with request range. + */ + bool isRangeSatisfied(const RangeHandle& range) const; + + RequestHandle getRequest() const + { + return request; + } + + int64_t getStartByte() const + { + if(segment.isNull()) { + return 0; + } else { + return segment->getPositionToWrite(); + } + } + + int64_t getEndByte() const + { + if(segment.isNull() || request.isNull()) { + return 0; + } else { + if(request->isKeepAlive()) { + return segment->getPosition()+segment->length-1; + } else { + return 0; + } + } + } + + /** + * Returns string representation of http request. + * It usually starts with "GET ..." and ends with "\r\n". + */ + string createRequest() const; + + /** + * Returns string representation of http tunnel request. + * It usually starts with "CONNECT ..." and ends with "\r\n". + */ + string createProxyRequest() const; + + /** + * Configures this object with option. + * Following values are evaluated: + * PREF_HTTP_AUTH_ENABLED, PREF_HTTP_PROXY_ENABLED, + * PREF_HTTP_PROXY_METHOD, PREF_HTTP_PROXY_AUTH_ENABLED, + * PREF_HTTP_USER, PREF_HTTP_PASSWD, + * PREF_HTTP_PROXY_USER, PREF_HTTP_PROXY_PASSWD + * The evaluation results are stored in instance variables. + */ + void configure(const Option* option); + + void setProxyEnabled(bool proxyEnabled) + { + this->proxyEnabled = proxyEnabled; + } + + void setProxyAuthEnabled(bool proxyAuthEnabled) + { + this->proxyAuthEnabled = proxyAuthEnabled; + } + + void setAuthEnabled(bool authEnabled) + { + this->authEnabled = authEnabled; + } + + void setAuthConfig(const HttpAuthConfigHandle& authConfig) + { + this->authConfig = authConfig; + } + + void setProxyAuthConfig(const HttpAuthConfigHandle& proxyAuthConfig) + { + this->proxyAuthConfig = proxyAuthConfig; + } + + void setUserAgent(const string& userAgent) + { + this->userAgent = userAgent; + } +}; + +typedef SharedHandle HttpRequestHandle; +typedef deque HttpRequests; + +#endif // _D_HTTP_REQUEST_H_ diff --git a/src/HttpRequestCommand.cc b/src/HttpRequestCommand.cc index 7775fc30..9f1621b1 100644 --- a/src/HttpRequestCommand.cc +++ b/src/HttpRequestCommand.cc @@ -38,7 +38,7 @@ #include "prefs.h" HttpRequestCommand::HttpRequestCommand(int cuid, - const RequestHandle req, + const RequestHandle& req, DownloadEngine* e, const SocketHandle& s) :AbstractCommand(cuid, req, e, s) { @@ -48,7 +48,7 @@ HttpRequestCommand::HttpRequestCommand(int cuid, HttpRequestCommand::~HttpRequestCommand() {} -bool HttpRequestCommand::executeInternal(Segment& segment) { +bool HttpRequestCommand::executeInternal() { socket->setBlockingMode(); if(req->getProtocol() == "https") { socket->initiateSecureConnection(); @@ -56,15 +56,17 @@ bool HttpRequestCommand::executeInternal(Segment& segment) { if(!e->option->getAsBool(PREF_HTTP_KEEP_ALIVE)) { req->setKeepAlive(false); } - HttpConnection http(cuid, socket, req, e->option); - req->segment = segment; - http.sendRequest(segment); + HttpRequestHandle httpRequest = new HttpRequest(); + httpRequest->setRequest(req); + httpRequest->setSegment(segment); + httpRequest->setEntityLength(e->segmentMan->totalSize); + httpRequest->configure(e->option); - Command* command = getNextCommand(); + HttpConnectionHandle httpConnection = new HttpConnection(cuid, socket, e->option); + + httpConnection->sendRequest(httpRequest); + + Command* command = new HttpResponseCommand(cuid, req, httpConnection, e, socket); e->commands.push_back(command); return true; } - -Command* HttpRequestCommand::getNextCommand() const { - return new HttpResponseCommand(cuid, req, e, socket); -} diff --git a/src/HttpRequestCommand.h b/src/HttpRequestCommand.h index 41a17099..1063b255 100644 --- a/src/HttpRequestCommand.h +++ b/src/HttpRequestCommand.h @@ -39,12 +39,11 @@ class HttpRequestCommand:public AbstractCommand { protected: - bool executeInternal(Segment& segment); - Command* getNextCommand() const; + virtual bool executeInternal(); public: - HttpRequestCommand(int cuid, const RequestHandle req, DownloadEngine* e, + HttpRequestCommand(int cuid, const RequestHandle& req, DownloadEngine* e, const SocketHandle& s); - ~HttpRequestCommand(); + virtual ~HttpRequestCommand(); }; #endif // _D_HTTP_REQUEST_COMMAND_H_ diff --git a/src/HttpResponse.cc b/src/HttpResponse.cc new file mode 100644 index 00000000..2886f30f --- /dev/null +++ b/src/HttpResponse.cc @@ -0,0 +1,160 @@ +/* */ +#include "HttpResponse.h" +#include "DlAbortEx.h" +#include "DlRetryEx.h" +#include "ChunkedEncoding.h" +#include "Util.h" +#include "message.h" + +void HttpResponse::validateResponse() const +{ + if(status == 401) { + throw new DlAbortEx(EX_AUTH_FAILED); + } + if(status >= 400) { + throw new DlRetryEx(EX_BAD_STATUS, status); + } + if(status >= 300) { + if(!httpHeader->defined("Location")) { + throw new DlRetryEx("Got %d status, but no location header provided.", + status); + } + } else { + if(!httpHeader->defined("Transfer-Encoding")) { + // compare the received range against the requested range + RangeHandle responseRange = httpHeader->getRange(); + if(!httpRequest->isRangeSatisfied(responseRange)) { + throw new DlRetryEx("Invalid range header. Request: %lld-%lld/%lld, Response: %lld-%lld/%lld", + httpRequest->getStartByte(), + httpRequest->getEndByte(), + httpRequest->getEntityLength(), + responseRange->getStartByte(), + responseRange->getEndByte(), + responseRange->getEntityLength()); + } + } + } +} + +string HttpResponse::determinFilename() const +{ + string contentDisposition = + Util::getContentDispositionFilename(httpHeader->getFirst("Content-Disposition")); + if(contentDisposition.empty()) { + return Util::urldecode(httpRequest->getRequest()->getFile()); + } else { + logger->info("CUID#%d - Content-Disposition Detected. Use %s as filename", + cuid, contentDisposition.c_str()); + return Util::urldecode(contentDisposition); + } +} + +void HttpResponse::retrieveCookie() +{ + Strings v = httpHeader->get("Set-Cookie"); + for(Strings::const_iterator itr = v.begin(); itr != v.end(); itr++) { + Cookie c; + httpRequest->getRequest()->cookieBox->parse(c, *itr); + httpRequest->getRequest()->cookieBox->add(c); + } +} + +bool HttpResponse::isRedirect() const +{ + return 300 <= status && status < 400 && httpHeader->defined("Location"); +} + +void HttpResponse::processRedirect() +{ + httpRequest->getRequest()->redirectUrl(getRedirectURI()); + +} + +string HttpResponse::getRedirectURI() const +{ + return httpHeader->getFirst("Location"); +} + +bool HttpResponse::isTransferEncodingSpecified() const +{ + return httpHeader->defined("Transfer-Encoding"); +} + +string HttpResponse::getTransferEncoding() const +{ + return httpHeader->getFirst("Transfer-Encoding"); +} + +TransferEncodingHandle HttpResponse::getTransferDecoder() const +{ + if(isTransferEncodingSpecified()) { + if(getTransferEncoding() == "chunked") { + return new ChunkedEncoding(); + } + } + return 0; +} + +int64_t HttpResponse::getContentLength() const +{ + if(httpHeader.isNull()) { + return 0; + } else { + return httpHeader->getRange()->getContentLength(); + } +} + +int64_t HttpResponse::getEntityLength() const +{ + if(httpHeader.isNull()) { + return 0; + } else { + return httpHeader->getRange()->getEntityLength(); + } +} + +void HttpResponse::validateFilename(const string& expectedFilename) const +{ + if(expectedFilename.size() == 0) { + return; + } + string actualFilename = determinFilename(); + if(expectedFilename != actualFilename) { + throw new DlAbortEx(EX_FILENAME_MISMATCH, + actualFilename.c_str(), + expectedFilename.c_str()); + } +} diff --git a/src/HttpResponse.h b/src/HttpResponse.h new file mode 100644 index 00000000..6b5dcfe7 --- /dev/null +++ b/src/HttpResponse.h @@ -0,0 +1,136 @@ +/* */ +#ifndef _D_HTTP_RESPONSE_H_ +#define _D_HTTP_RESPONSE_H_ + +#include "common.h" +#include "HttpRequest.h" +#include "HttpHeader.h" +#include "TransferEncoding.h" +#include "LogFactory.h" + +class HttpResponse { +private: + int32_t cuid; + int32_t status; + HttpRequestHandle httpRequest; + HttpHeaderHandle httpHeader; + const Logger* logger; +public: + HttpResponse():cuid(0), + status(0), + httpRequest(0), + httpHeader(0), + logger(LogFactory::getInstance()) + {} + + ~HttpResponse() {} + + void validateResponse() const; + + /** + * Returns filename. + * If content-disposition header is privided in response header, + * this function returns the filename from it. + * If it is not there, returns the part of filename from the request URL. + */ + string determinFilename() const; + + void retrieveCookie(); + + /** + * Returns true if the response header indicates redirection. + */ + bool isRedirect() const; + + void processRedirect(); + + string getRedirectURI() const; + + bool isTransferEncodingSpecified() const; + + string getTransferEncoding() const; + + TransferEncodingHandle getTransferDecoder() const; + + int64_t getContentLength() const; + + int64_t getEntityLength() const; + + /** + * Compares actual filename with specified expectedFilename. + * The actual filename is the string returned from determinFilename(). + */ + void validateFilename(const string& expectedFilename) const; + + void setHttpHeader(const HttpHeaderHandle& httpHeader) + { + this->httpHeader = httpHeader; + } + + HttpHeaderHandle getHttpHeader() const + { + return httpHeader; + } + + void setStatus(int32_t status) + { + this->status = status; + } + + int32_t getStatus() const + { + return status; + } + + void setHttpRequest(const HttpRequestHandle& httpRequest) + { + this->httpRequest = httpRequest; + } + + HttpRequestHandle getHttpRequest() const + { + return httpRequest; + } + + void setCuid(int32_t cuid) + { + this->cuid = cuid; + } +}; + +typedef SharedHandle HttpResponseHandle; + +#endif // _D_HTTP_RESPONSE_H_ diff --git a/src/HttpResponseCommand.cc b/src/HttpResponseCommand.cc index 030d8460..cc92e0a5 100644 --- a/src/HttpResponseCommand.cc +++ b/src/HttpResponseCommand.cc @@ -36,143 +36,96 @@ #include "DlAbortEx.h" #include "DlRetryEx.h" #include "HttpDownloadCommand.h" -#include "HttpInitiateConnectionCommand.h" #include "message.h" #include "Util.h" #include "prefs.h" #include "File.h" -#include "FatalException.h" #include #include -HttpResponseCommand::HttpResponseCommand(int cuid, - const RequestHandle req, +HttpResponseCommand::HttpResponseCommand(int32_t cuid, + const RequestHandle& req, + const HttpConnectionHandle& httpConnection, DownloadEngine* e, const SocketHandle& s) - :AbstractCommand(cuid, req, e, s) { - http = new HttpConnection(cuid, socket, req, e->option); -} + :AbstractCommand(cuid, req, e, s), + httpConnection(httpConnection) {} -HttpResponseCommand::~HttpResponseCommand() { - delete http; -} +HttpResponseCommand::~HttpResponseCommand() {} -bool HttpResponseCommand::executeInternal(Segment& segment) { - if(req->segment != segment) { +bool HttpResponseCommand::executeInternal() +{ + HttpRequestHandle httpRequest = httpConnection->getFirstHttpRequest(); + if(!(httpRequest->getSegment() == segment)) { logger->info(MSG_SEGMENT_CHANGED, cuid); return prepareForRetry(0); } - HttpHeader headers; - int status = http->receiveResponse(headers); - if(status == 0) { - // didn't receive header fully + HttpResponseHandle httpResponse = httpConnection->receiveResponse(); + if(httpResponse.isNull()) { + // The server has not responded to our request yet. e->commands.push_back(this); return false; } // check HTTP status number - checkResponse(status, segment); - retrieveCookie(headers); + httpResponse->validateResponse(); + httpResponse->retrieveCookie(); // check whether the server supports persistent connections. + /* if(Util::toLower(headers.getFirst("Connection")).find("close") != string::npos) { req->setKeepAlive(false); } + */ // check whether Location header exists. If it does, update request object // with redirected URL. // then establish a connection to the new host and port - if(headers.defined("Location")) { - return handleRedirect(headers.getFirst("Location"), headers); + if(httpResponse->isRedirect()) { + httpResponse->processRedirect(); + logger->info(MSG_REDIRECT, cuid, httpResponse->getRedirectURI().c_str()); + e->noWait = true; + return prepareForRetry(0); } - if(!e->segmentMan->downloadStarted) { - string transferEncoding; - if(headers.defined("Transfer-Encoding")) { - return handleOtherEncoding(headers.getFirst("Transfer-Encoding"), - headers); + httpResponse->validateFilename(e->segmentMan->filename); + if(e->segmentMan->downloadStarted) { + createHttpDownloadCommand(httpResponse); + return true; + } else { + if(httpResponse->isTransferEncodingSpecified()) { + return handleOtherEncoding(httpResponse); } else { - return handleDefaultEncoding(headers); + return handleDefaultEncoding(httpResponse); } - } else { - string filenameInHeader = determinFilename(headers); - if(filenameInHeader != e->segmentMan->filename) { - throw new DlAbortEx(EX_FILENAME_MISMATCH, - filenameInHeader.c_str(), - e->segmentMan->filename.c_str()); - } - createHttpDownloadCommand(); - return true; } } -void HttpResponseCommand::checkResponse(int status, const Segment& segment) { - if(status == 401) { - throw new DlAbortEx(EX_AUTH_FAILED); - } - if(!(300 <= status && status < 400 || - (segment.getPosition()+segment.writtenLength == 0 && (status == 200 || status == 206)) || - (segment.getPosition()+segment.writtenLength > 0 && status == 206))) { - throw new DlRetryEx(EX_BAD_STATUS, status); - } -} - -bool HttpResponseCommand::handleRedirect(const string& url, const HttpHeader& headers) { - req->redirectUrl(url); - logger->info(MSG_REDIRECT, cuid, url.c_str()); - e->noWait = true; - return prepareForRetry(0); -} - -string HttpResponseCommand::determinFilename(const HttpHeader& headers) { - string contentDisposition = - Util::getContentDispositionFilename(headers.getFirst("Content-Disposition")); - if(contentDisposition.empty()) { - return Util::urldecode(req->getFile()); - } else { - logger->info("CUID#%d - Content-Disposition Detected. Use %s as filename", - cuid, contentDisposition.c_str()); - return Util::urldecode(contentDisposition); - } -} - -bool HttpResponseCommand::handleDefaultEncoding(const HttpHeader& headers) { +bool HttpResponseCommand::handleDefaultEncoding(const HttpResponseHandle& httpResponse) +{ + HttpRequestHandle httpRequest = httpResponse->getHttpRequest(); // TODO quick and dirty way - if(req->isTorrent) { - long long int size = headers.getFirstAsLLInt("Content-Length"); - e->segmentMan->totalSize = size; - if(size > 0) { - e->segmentMan->initBitfield(e->option->getAsInt(PREF_SEGMENT_SIZE), - e->segmentMan->totalSize); - } - // disable keep-alive - req->setKeepAlive(false); - e->segmentMan->isSplittable = false; - e->segmentMan->downloadStarted = true; - e->segmentMan->diskWriter->initAndOpenFile("/tmp/aria2"+Util::itos((int32_t)getpid())); - createHttpDownloadCommand(); - return true; + if(httpRequest->getRequest()->isTorrent) { + return doTorrentStuff(httpResponse); } - - long long int size = headers.getFirstAsLLInt("Content-Length"); + int64_t size = httpResponse->getEntityLength(); if(size == INT64_MAX || size < 0) { throw new DlAbortEx(EX_TOO_LARGE_FILE, size); } e->segmentMan->isSplittable = !(size == 0); - e->segmentMan->filename = determinFilename(headers); + e->segmentMan->filename = httpResponse->determinFilename(); + e->segmentMan->downloadStarted = true; + e->segmentMan->totalSize = size; // quick hack for method 'head' - if(req->getMethod() == Request::METHOD_HEAD) { - e->segmentMan->downloadStarted = true; - e->segmentMan->totalSize = size; - e->segmentMan->isSplittable = false; // TODO because we don't want segment file to be saved. + if(httpRequest->getMethod() == Request::METHOD_HEAD) { + // TODO because we don't want segment file to be saved. + e->segmentMan->isSplittable = false; return true; } bool segFileExists = e->segmentMan->segmentFileExists(); - e->segmentMan->downloadStarted = true; if(segFileExists) { e->segmentMan->load(); e->segmentMan->diskWriter->openExistingFile(e->segmentMan->getFilePath()); // send request again to the server with Range header return prepareForRetry(0); } else { - e->segmentMan->totalSize = size; e->segmentMan->initBitfield(e->option->getAsInt(PREF_SEGMENT_SIZE), e->segmentMan->totalSize); e->segmentMan->diskWriter->initAndOpenFile(e->segmentMan->getFilePath(), @@ -181,58 +134,58 @@ bool HttpResponseCommand::handleDefaultEncoding(const HttpHeader& headers) { } } -bool HttpResponseCommand::handleOtherEncoding(const string& transferEncoding, const HttpHeader& headers) { - // quick hack for method 'head' - if(req->getMethod() == Request::METHOD_HEAD) { - e->segmentMan->downloadStarted = true; - e->segmentMan->isSplittable = false; - e->segmentMan->filename = determinFilename(headers); - e->segmentMan->totalSize = 0; - return true; - } - if(e->segmentMan->shouldCancelDownloadForSafety()) { - throw new FatalException(EX_FILE_ALREADY_EXISTS, - e->segmentMan->getFilePath().c_str(), - e->segmentMan->getSegmentFilePath().c_str()); - } +bool HttpResponseCommand::handleOtherEncoding(const HttpResponseHandle& httpResponse) { + HttpRequestHandle httpRequest = httpResponse->getHttpRequest(); // we ignore content-length when transfer-encoding is set e->segmentMan->downloadStarted = true; e->segmentMan->isSplittable = false; - e->segmentMan->filename = determinFilename(headers); + e->segmentMan->filename = httpResponse->determinFilename(); e->segmentMan->totalSize = 0; + // quick hack for method 'head' + if(httpRequest->getMethod() == Request::METHOD_HEAD) { + return true; + } // disable keep-alive req->setKeepAlive(false); - Segment segment; - e->segmentMan->getSegment(segment, cuid); + segment = e->segmentMan->getSegment(cuid); e->segmentMan->diskWriter->initAndOpenFile(e->segmentMan->getFilePath()); - createHttpDownloadCommand(transferEncoding); + createHttpDownloadCommand(httpResponse); return true; } -void HttpResponseCommand::createHttpDownloadCommand(const string& transferEncoding) { +void HttpResponseCommand::createHttpDownloadCommand(const HttpResponseHandle& httpResponse) +{ + TransferEncodingHandle enc = 0; + if(httpResponse->isTransferEncodingSpecified()) { + enc = httpResponse->getTransferDecoder(); + if(enc.isNull()) { + throw new DlAbortEx(EX_TRANSFER_ENCODING_NOT_SUPPORTED, + httpResponse->getTransferEncoding().c_str()); + } + enc->init(); + } HttpDownloadCommand* command = new HttpDownloadCommand(cuid, req, e, socket); command->setMaxDownloadSpeedLimit(e->option->getAsInt(PREF_MAX_DOWNLOAD_LIMIT)); command->setStartupIdleTime(e->option->getAsInt(PREF_STARTUP_IDLE_TIME)); command->setLowestDownloadSpeedLimit(e->option->getAsInt(PREF_LOWEST_SPEED_LIMIT)); - TransferEncoding* enc = NULL; - if(transferEncoding.size() && (enc = command->getTransferEncoding(transferEncoding)) == NULL) { - delete(command); - throw new DlAbortEx(EX_TRANSFER_ENCODING_NOT_SUPPORTED, transferEncoding.c_str()); - } else { - if(enc != NULL) { - command->transferEncoding = transferEncoding; - enc->init(); - } - e->commands.push_back(command); - } + command->setTransferDecoder(enc); + + e->commands.push_back(command); } -void HttpResponseCommand::retrieveCookie(const HttpHeader& headers) { - Strings v = headers.get("Set-Cookie"); - for(Strings::const_iterator itr = v.begin(); itr != v.end(); itr++) { - Cookie c; - req->cookieBox->parse(c, *itr); - req->cookieBox->add(c); +bool HttpResponseCommand::doTorrentStuff(const HttpResponseHandle& httpResponse) +{ + int64_t size = httpResponse->getEntityLength(); + e->segmentMan->totalSize = size; + if(size > 0) { + e->segmentMan->initBitfield(e->option->getAsInt(PREF_SEGMENT_SIZE), + e->segmentMan->totalSize); } + // disable keep-alive + httpResponse->getHttpRequest()->getRequest()->setKeepAlive(false); + e->segmentMan->isSplittable = false; + e->segmentMan->downloadStarted = true; + e->segmentMan->diskWriter->initAndOpenFile("/tmp/aria2"+Util::itos((int32_t)getpid())); + createHttpDownloadCommand(httpResponse); + return true; } - diff --git a/src/HttpResponseCommand.h b/src/HttpResponseCommand.h index 35b99697..00ec83c0 100644 --- a/src/HttpResponseCommand.h +++ b/src/HttpResponseCommand.h @@ -40,21 +40,19 @@ class HttpResponseCommand : public AbstractCommand { private: - void checkResponse(int status, const Segment& segment); - bool handleRedirect(const string& url, const HttpHeader& headers); - bool handleDefaultEncoding(const HttpHeader& headers); - bool handleOtherEncoding(const string& transferEncoding, const HttpHeader& headers); - void createHttpDownloadCommand(const string& transferEncoding = ""); - void retrieveCookie(const HttpHeader& headers); - /** - * Returned filename is URL-decoded. - */ - string determinFilename(const HttpHeader& headers); - HttpConnection* http; + HttpConnectionHandle httpConnection; + + bool handleDefaultEncoding(const HttpResponseHandle& httpResponse); + bool handleOtherEncoding(const HttpResponseHandle& httpResponse); + void createHttpDownloadCommand(const HttpResponseHandle& httpResponse); + bool doTorrentStuff(const HttpResponseHandle& httpResponse); protected: - bool executeInternal(Segment& segment); + bool executeInternal(); public: - HttpResponseCommand(int cuid, const RequestHandle req, DownloadEngine* e, + HttpResponseCommand(int32_t cuid, + const RequestHandle& req, + const HttpConnectionHandle& httpConnection, + DownloadEngine* e, const SocketHandle& s); ~HttpResponseCommand(); }; diff --git a/src/Makefile.am b/src/Makefile.am index 1a88e859..07384a69 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -64,7 +64,14 @@ SRCS = Socket.h\ FileAllocator.cc FileAllocator.h\ FileAllocationMonitor.cc FileAllocationMonitor.h\ ConsoleFileAllocationMonitor.cc ConsoleFileAllocationMonitor.h\ - ChunkChecksumValidator.cc ChunkChecksumValidator.h + ChunkChecksumValidator.cc ChunkChecksumValidator.h\ + HttpResponse.cc HttpResponse.h\ + HttpRequest.cc HttpRequest.h\ + Range.h\ + AbstractProxyRequestCommand.cc AbstractProxyRequestCommand.h\ + AbstractProxyResponseCommand.cc AbstractProxyResponseCommand.h\ + HttpAuthConfig.h\ + Netrc.cc Netrc.h # debug_new.cpp if ENABLE_ASYNC_DNS diff --git a/src/Makefile.in b/src/Makefile.in index e0372f03..d9a5b604 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -214,12 +214,16 @@ am__libaria2c_a_SOURCES_DIST = Socket.h SocketCore.cc SocketCore.h \ FileAllocationMonitor.cc FileAllocationMonitor.h \ ConsoleFileAllocationMonitor.cc ConsoleFileAllocationMonitor.h \ ChunkChecksumValidator.cc ChunkChecksumValidator.h \ - NameResolver.cc NameResolver.h MetaEntry.h Data.cc Data.h \ - Dictionary.cc Dictionary.h List.cc List.h MetaFileUtil.cc \ - MetaFileUtil.h MetaEntryVisitor.h ShaVisitor.cc ShaVisitor.h \ - PeerConnection.cc PeerConnection.h PeerMessageUtil.cc \ - PeerMessageUtil.h PeerAbstractCommand.cc PeerAbstractCommand.h \ - PeerInitiateConnectionCommand.cc \ + HttpResponse.cc HttpResponse.h HttpRequest.cc HttpRequest.h \ + Range.h AbstractProxyRequestCommand.cc \ + AbstractProxyRequestCommand.h AbstractProxyResponseCommand.cc \ + AbstractProxyResponseCommand.h HttpAuthConfig.h Netrc.cc \ + Netrc.h NameResolver.cc NameResolver.h MetaEntry.h Data.cc \ + Data.h Dictionary.cc Dictionary.h List.cc List.h \ + MetaFileUtil.cc MetaFileUtil.h MetaEntryVisitor.h \ + ShaVisitor.cc ShaVisitor.h PeerConnection.cc PeerConnection.h \ + PeerMessageUtil.cc PeerMessageUtil.h PeerAbstractCommand.cc \ + PeerAbstractCommand.h PeerInitiateConnectionCommand.cc \ PeerInitiateConnectionCommand.h PeerInteractionCommand.cc \ PeerInteractionCommand.h Peer.cc Peer.h \ TorrentDownloadEngine.cc TorrentDownloadEngine.h \ @@ -379,8 +383,10 @@ am__objects_4 = SocketCore.$(OBJEXT) Command.$(OBJEXT) \ BitfieldManFactory.$(OBJEXT) SimpleRandomizer.$(OBJEXT) \ FileAllocator.$(OBJEXT) FileAllocationMonitor.$(OBJEXT) \ ConsoleFileAllocationMonitor.$(OBJEXT) \ - ChunkChecksumValidator.$(OBJEXT) $(am__objects_1) \ - $(am__objects_2) $(am__objects_3) + ChunkChecksumValidator.$(OBJEXT) HttpResponse.$(OBJEXT) \ + HttpRequest.$(OBJEXT) AbstractProxyRequestCommand.$(OBJEXT) \ + AbstractProxyResponseCommand.$(OBJEXT) Netrc.$(OBJEXT) \ + $(am__objects_1) $(am__objects_2) $(am__objects_3) am_libaria2c_a_OBJECTS = $(am__objects_4) libaria2c_a_OBJECTS = $(am_libaria2c_a_OBJECTS) am__installdirs = "$(DESTDIR)$(bindir)" @@ -587,7 +593,11 @@ SRCS = Socket.h SocketCore.cc SocketCore.h Command.cc Command.h \ FileAllocationMonitor.cc FileAllocationMonitor.h \ ConsoleFileAllocationMonitor.cc ConsoleFileAllocationMonitor.h \ ChunkChecksumValidator.cc ChunkChecksumValidator.h \ - $(am__append_1) $(am__append_2) $(am__append_3) + HttpResponse.cc HttpResponse.h HttpRequest.cc HttpRequest.h \ + Range.h AbstractProxyRequestCommand.cc \ + AbstractProxyRequestCommand.h AbstractProxyResponseCommand.cc \ + AbstractProxyResponseCommand.h HttpAuthConfig.h Netrc.cc \ + Netrc.h $(am__append_1) $(am__append_2) $(am__append_3) noinst_LIBRARIES = libaria2c.a libaria2c_a_SOURCES = $(SRCS) aria2c_LDADD = libaria2c.a @LIBINTL@ @ALLOCA@ @LIBGNUTLS_LIBS@\ @@ -677,6 +687,8 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@$(DEPDIR)/alloca.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/AbstractCommand.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/AbstractDiskWriter.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/AbstractProxyRequestCommand.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/AbstractProxyResponseCommand.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/AbstractSingleDiskAdaptor.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/AnnounceList.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Base64.Po@am__quote@ @@ -749,7 +761,9 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/HttpInitiateConnectionCommand.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/HttpProxyRequestCommand.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/HttpProxyResponseCommand.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/HttpRequest.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/HttpRequestCommand.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/HttpResponse.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/HttpResponseCommand.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/InitiateConnectionCommandFactory.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/List.Po@am__quote@ @@ -761,6 +775,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Metalinker.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/MultiDiskAdaptor.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/NameResolver.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Netrc.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Option.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Peer.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/PeerAbstractCommand.Po@am__quote@ diff --git a/src/Netrc.cc b/src/Netrc.cc new file mode 100644 index 00000000..e2959835 --- /dev/null +++ b/src/Netrc.cc @@ -0,0 +1,110 @@ +/* */ +#include "Netrc.h" +#include "Util.h" +#include "RecoverableException.h" +#include + +void Netrc::parse(const string& path) +{ + authenticatables.clear(); + ifstream f(path.c_str()); + + if(!f) { + throw new RecoverableException("File not found: %s", path.c_str()); + } + + int32_t lineNum = 0; + string line; + AuthenticatorHandle authenticator = 0; + while(getline(f, line)) { + ++lineNum; + if(Util::trim(line).empty()) { + continue; + } + pair nameValuePair = Util::split(line, "\r\n\t "); + if(nameValuePair.first == "machine") { + storeAuthenticatable(authenticator); + authenticator = new Authenticator(); + authenticator->setMachine(nameValuePair.second); + } else if(nameValuePair.first == "default") { + storeAuthenticatable(authenticator); + authenticator = new DefaultAuthenticator(); + } else { + if(authenticator.isNull()) { + throw new RecoverableException("Malformed netrc file: line %d", lineNum); + } + if(nameValuePair.first == "login") { + authenticator->setLogin(nameValuePair.second); + } else if(nameValuePair.first == "password") { + authenticator->setPassword(nameValuePair.second); + } else if(nameValuePair.first == "account") { + authenticator->setAccount(nameValuePair.second); + } + } + } + storeAuthenticatable(authenticator); +} + +void Netrc::storeAuthenticatable(const AuthenticatableHandle& authenticatable) +{ + if(!authenticatable.isNull()) { + authenticatables.push_back(authenticatable); + } +} + +class AuthHostMatch { +private: + string hostname; +public: + AuthHostMatch(const string& hostname):hostname(hostname) {} + + bool operator()(const AuthenticatableHandle& authenticatable) + { + return authenticatable->match(hostname); + } +}; + +AuthenticatableHandle Netrc::findAuthenticatable(const string& hostname) const +{ + Authenticatables::const_iterator itr = + find_if(authenticatables.begin(), authenticatables.end(), + AuthHostMatch(hostname)); + if(itr == authenticatables.end()) { + return 0; + } else { + return *itr; + } +} diff --git a/src/Netrc.h b/src/Netrc.h new file mode 100644 index 00000000..45a8ab02 --- /dev/null +++ b/src/Netrc.h @@ -0,0 +1,150 @@ +/* */ +#ifndef _D_NETRC_H_ +#define _D_NETRC_H_ + +#include "common.h" + +class Authenticatable { +public: + virtual ~Authenticatable() {} + + virtual bool match(const string& hostname) const = 0; +}; + +typedef SharedHandle AuthenticatableHandle; +typedef deque Authenticatables; + +class Authenticator : public Authenticatable { +private: + string machine; + string login; + string password; + string account; +public: + Authenticator() {} + + Authenticator(const string& machine, + const string& login, + const string& password, + const string& account) + :machine(machine), + login(login), + password(password), + account(account) {} + + virtual ~Authenticator() {} + + virtual bool match(const string& hostname) const + { + return hostname == machine; + } + + const string& getMachine() const + { + return machine; + } + + void setMachine(const string& machine) { this->machine = machine; } + + const string& getLogin() const + { + return login; + } + + void setLogin(const string& login) { this->login = login; } + + const string& getPassword() const + { + return password; + } + + void setPassword(const string& password) { this->password = password; } + + const string& getAccount() const + { + return account; + } + + void setAccount(const string& account) { this->account = account; } +}; + +typedef SharedHandle AuthenticatorHandle; + +class DefaultAuthenticator : public Authenticator { +public: + DefaultAuthenticator() {} + + DefaultAuthenticator(const string& login, + const string& password, + const string& account) + :Authenticator("", login, password, account) {} + + virtual ~DefaultAuthenticator() {} + + virtual bool match(const string& hostname) const + { + return true; + } +}; + +typedef SharedHandle DefaultAuthenticatorHandle; + +class Netrc { +private: + Authenticatables authenticatables; + + void storeAuthenticatable(const AuthenticatableHandle& authenticatable); +public: + Netrc() {} + + void parse(const string& path); + + AuthenticatableHandle findAuthenticatable(const string& hostname) const; + + const Authenticatables& getAuthenticatables() const + { + return authenticatables; + } + + void addAuthenticatable(const AuthenticatableHandle& authenticatable) + { + authenticatables.push_back(authenticatable); + } +}; + +typedef SharedHandle NetrcHandle; + +#endif // _D_NETRC_H_ diff --git a/src/Range.h b/src/Range.h new file mode 100644 index 00000000..ccc6cc39 --- /dev/null +++ b/src/Range.h @@ -0,0 +1,90 @@ +/* */ +#ifndef _D_RANGE_H_ +#define _D_RANGE_H_ + +#include "common.h" + +class Range { +private: + int64_t startByte; + int64_t endByte; + int64_t entityLength; +public: + Range():startByte(0), endByte(0), entityLength(0) {} + + Range(int64_t startByte, int64_t endByte, int64_t entityLength): + startByte(startByte), endByte(endByte), entityLength(entityLength) {} + + bool operator==(const Range& range) const + { + return startByte == range.startByte && + endByte == range.endByte && + entityLength == range.entityLength; + } + + bool operator!=(const Range& range) const + { + return !(*this == range); + } + + int64_t getStartByte() const + { + return startByte; + } + + int64_t getEndByte() const + { + return endByte; + } + + int64_t getEntityLength() const + { + return entityLength; + } + + int64_t getContentLength() const + { + if(endByte >= startByte) { + return endByte-startByte+1; + } else { + return 0; + } + } +}; + +typedef SharedHandle RangeHandle; + +#endif // _D_RANGE_H_ diff --git a/src/Segment.h b/src/Segment.h index eed5cf0e..20e72a58 100644 --- a/src/Segment.h +++ b/src/Segment.h @@ -61,8 +61,13 @@ public: return index == -1; } - long long int getPosition() const { - return ((long long int)index)*segmentLength; + int64_t getPosition() const { + return ((int64_t)index)*segmentLength; + } + + int64_t getPositionToWrite() const + { + return getPosition()+writtenLength; } bool operator==(const Segment& segment) const { diff --git a/src/SegmentMan.cc b/src/SegmentMan.cc index a09859de..df055317 100644 --- a/src/SegmentMan.cc +++ b/src/SegmentMan.cc @@ -136,7 +136,7 @@ void SegmentMan::save() const { } for(SegmentEntries::const_iterator itr = usedSegmentEntries.begin(); itr != usedSegmentEntries.end(); itr++) { - if(fwrite(&(*itr)->segment, sizeof(Segment), 1, segFile) < 1) { + if(fwrite((*itr)->segment.get(), sizeof(Segment), 1, segFile) < 1) { throw string("writeError"); } } @@ -187,8 +187,8 @@ void SegmentMan::read(FILE* file) { throw string("readError"); } while(segmentCount--) { - Segment seg; - if(fread(&seg, sizeof(Segment), 1, file) < 1) { + SegmentHandle seg; + if(fread(seg.get(), sizeof(Segment), 1, file) < 1) { throw string("readError"); } usedSegmentEntries.push_back(SegmentEntryHandle(new SegmentEntry(0, seg))); @@ -236,19 +236,19 @@ void SegmentMan::init() { } -void SegmentMan::initBitfield(int segmentLength, long long int totalLength) { +void SegmentMan::initBitfield(int32_t segmentLength, int64_t totalLength) { delete bitfield; this->bitfield = BitfieldManFactory::getFactoryInstance()->createBitfieldMan(segmentLength, totalLength); } -Segment SegmentMan::checkoutSegment(int cuid, int index) { +SegmentHandle SegmentMan::checkoutSegment(int32_t cuid, int32_t index) { logger->debug("Attach segment#%d to CUID#%d.", index, cuid); bitfield->setUseBit(index); SegmentEntryHandle segmentEntry = getSegmentEntryByIndex(index); - Segment segment; + SegmentHandle segment(0); if(segmentEntry.isNull()) { - segment = Segment(index, bitfield->getBlockLength(index), - bitfield->getBlockLength()); + segment = new Segment(index, bitfield->getBlockLength(index), + bitfield->getBlockLength()); SegmentEntryHandle entry = new SegmentEntry(cuid, segment); usedSegmentEntries.push_back(entry); } else { @@ -256,23 +256,22 @@ Segment SegmentMan::checkoutSegment(int cuid, int index) { segment = segmentEntry->segment; } logger->debug("index=%d, length=%d, segmentLength=%d, writtenLength=%d", - segment.index, segment.length, segment.segmentLength, - segment.writtenLength); + segment->index, segment->length, segment->segmentLength, + segment->writtenLength); return segment; } -bool SegmentMan::onNullBitfield(Segment& segment, int cuid) { +SegmentHandle SegmentMan::onNullBitfield(int32_t cuid) { if(usedSegmentEntries.size() == 0) { - segment = Segment(0, 0, 0); + SegmentHandle segment = new Segment(0, 0, 0); usedSegmentEntries.push_back(SegmentEntryHandle(new SegmentEntry(cuid, segment))); - return true; + return segment; } else { SegmentEntryHandle segmentEntry = getSegmentEntryByCuid(cuid); if(segmentEntry.isNull()) { - return false; + return 0; } else { - segment = segmentEntry->segment; - return true; + return segmentEntry->segment; } } } @@ -301,57 +300,52 @@ SegmentEntryHandle SegmentMan::findSlowerSegmentEntry(const PeerStatHandle& peer return slowSegmentEntry; } -bool SegmentMan::getSegment(Segment& segment, int cuid) { +SegmentHandle SegmentMan::getSegment(int32_t cuid) { if(!bitfield) { - return onNullBitfield(segment, cuid); + return onNullBitfield(cuid); } - SegmentEntryHandle segmentEntry = getSegmentEntryByCuid(cuid); if(!segmentEntry.isNull()) { - segment = segmentEntry->segment; - return true; + return segmentEntry->segment; } int index = bitfield->getSparseMissingUnusedIndex(); if(index == -1) { PeerStatHandle myPeerStat = getPeerStat(cuid); if(!myPeerStat.get()) { - return false; + return 0; } SegmentEntryHandle slowSegmentEntry = findSlowerSegmentEntry(myPeerStat); if(slowSegmentEntry.get()) { logger->info("CUID#%d cancels segment index=%d. CUID#%d handles it instead.", slowSegmentEntry->cuid, - slowSegmentEntry->segment.index, + slowSegmentEntry->segment->index, cuid); PeerStatHandle slowPeerStat = getPeerStat(slowSegmentEntry->cuid); slowPeerStat->requestIdle(); cancelSegment(slowSegmentEntry->cuid); - segment = checkoutSegment(cuid, slowSegmentEntry->segment.index); - return true; + return checkoutSegment(cuid, slowSegmentEntry->segment->index); } else { - return false; + return 0; } } else { - segment = checkoutSegment(cuid, index); - return true; + return checkoutSegment(cuid, index); } } -bool SegmentMan::getSegment(Segment& segment, int cuid, int index) { +SegmentHandle SegmentMan::getSegment(int32_t cuid, int32_t index) { if(!bitfield) { - return onNullBitfield(segment, cuid); + return onNullBitfield(cuid); } if(index < 0 || (int32_t)bitfield->countBlock() <= index) { - return false; + return 0; } if(bitfield->isBitSet(index) || bitfield->isUseBitSet(index)) { - return false; + return 0; } else { - segment = checkoutSegment(cuid, index); - return true; + return checkoutSegment(cuid, index); } } - +/* bool SegmentMan::updateSegment(int cuid, const Segment& segment) { if(segment.isNull()) { return false; @@ -364,41 +358,32 @@ bool SegmentMan::updateSegment(int cuid, const Segment& segment) { return true; } } +*/ -class CancelSegment { -private: - int cuid; - BitfieldMan* bitfield; -public: - CancelSegment(int cuid, BitfieldMan* bitfield):cuid(cuid), - bitfield(bitfield) {} - - void operator()(SegmentEntryHandle& entry) { - if(entry->cuid == cuid) { - bitfield->unsetUseBit(entry->segment.index); - entry->cuid = 0; - } - } -}; - -void SegmentMan::cancelSegment(int cuid) { +void SegmentMan::cancelSegment(int32_t cuid) { if(bitfield) { - for_each(usedSegmentEntries.begin(), usedSegmentEntries.end(), - CancelSegment(cuid, bitfield)); + for(SegmentEntries::iterator itr = usedSegmentEntries.begin(); + itr != usedSegmentEntries.end(); ++itr) { + if((*itr)->cuid == cuid) { + bitfield->unsetUseBit((*itr)->segment->index); + (*itr)->cuid = 0; + break; + } + } } else { usedSegmentEntries.clear(); } } -bool SegmentMan::completeSegment(int cuid, const Segment& segment) { - if(segment.isNull()) { +bool SegmentMan::completeSegment(int32_t cuid, const SegmentHandle& segment) { + if(segment->isNull()) { return false; } if(bitfield) { - bitfield->unsetUseBit(segment.index); - bitfield->setBit(segment.index); + bitfield->unsetUseBit(segment->index); + bitfield->setBit(segment->index); } else { - initBitfield(option->getAsInt(PREF_SEGMENT_SIZE), segment.writtenLength); + initBitfield(option->getAsInt(PREF_SEGMENT_SIZE), segment->writtenLength); bitfield->setAllBit(); } SegmentEntries::iterator itr = getSegmentEntryIteratorByCuid(cuid); @@ -410,7 +395,7 @@ bool SegmentMan::completeSegment(int cuid, const Segment& segment) { } } -bool SegmentMan::hasSegment(int index) const { +bool SegmentMan::hasSegment(int32_t index) const { if(bitfield) { return bitfield->isBitSet(index); } else { @@ -418,14 +403,14 @@ bool SegmentMan::hasSegment(int index) const { } } -long long int SegmentMan::getDownloadLength() const { - long long int dlLength = 0; +int64_t SegmentMan::getDownloadLength() const { + int64_t dlLength = 0; if(bitfield) { dlLength += bitfield->getCompletedLength(); } for(SegmentEntries::const_iterator itr = usedSegmentEntries.begin(); itr != usedSegmentEntries.end(); itr++) { - dlLength += (*itr)->segment.writtenLength; + dlLength += (*itr)->segment->writtenLength; } return dlLength; } @@ -486,7 +471,7 @@ bool SegmentMan::isChunkChecksumValidationReady() const { #endif // ENABLE_MESSAGE_DIGEST #ifdef ENABLE_MESSAGE_DIGEST -void SegmentMan::tryChunkChecksumValidation(const Segment& segment) +void SegmentMan::tryChunkChecksumValidation(const SegmentHandle& segment) { if(!isChunkChecksumValidationReady()) { return; @@ -494,8 +479,8 @@ void SegmentMan::tryChunkChecksumValidation(const Segment& segment) int32_t hashStartIndex; int32_t hashEndIndex; Util::indexRange(hashStartIndex, hashEndIndex, - segment.getPosition(), - segment.writtenLength, + segment->getPosition(), + segment->writtenLength, chunkHashLength); if(!bitfield->isBitSetOffsetRange((int64_t)hashStartIndex*chunkHashLength, chunkHashLength)) { diff --git a/src/SegmentMan.h b/src/SegmentMan.h index 9478f38d..16b7a6e1 100644 --- a/src/SegmentMan.h +++ b/src/SegmentMan.h @@ -51,9 +51,9 @@ using namespace std; class SegmentEntry { public: int cuid; - Segment segment; + SegmentHandle segment; public: - SegmentEntry(int cuid, const Segment& segment) + SegmentEntry(int cuid, const SegmentHandle& segment) :cuid(cuid), segment(segment) {} ~SegmentEntry() {} }; @@ -74,14 +74,14 @@ private: void read(FILE* file); FILE* openSegFile(const string& segFilename, const string& mode) const; - bool onNullBitfield(Segment& segment, int cuid); - Segment checkoutSegment(int cuid, int index); + SegmentHandle onNullBitfield(int32_t cuid); + SegmentHandle checkoutSegment(int32_t cuid, int32_t index); SegmentEntryHandle findSlowerSegmentEntry(const PeerStatHandle& peerStat) const; SegmentEntryHandle getSegmentEntryByIndex(int index) { for(SegmentEntries::const_iterator itr = usedSegmentEntries.begin(); itr != usedSegmentEntries.end(); ++itr) { const SegmentEntryHandle& segmentEntry = *itr; - if(segmentEntry->segment.index == index) { + if(segmentEntry->segment->index == index) { return segmentEntry; } } @@ -116,7 +116,7 @@ public: * If Transfer-Encoding is Chunked or Content-Length header is not provided, * then this value is set to be 0. */ - long long int totalSize; + int64_t totalSize; /** * Represents whether this download is splittable. * In Split download(or segmented download), http client establishes @@ -220,39 +220,39 @@ public: * If there is no vacant segment, then returns a segment instance whose * isNull call is true. */ - bool getSegment(Segment& segment, int cuid); + SegmentHandle getSegment(int32_t cuid); /** * Returns a segment whose index is index. * If it has already assigned * to another cuid or has been downloaded, then returns a segment instance * whose isNull call is true. */ - bool getSegment(Segment& segment, int cuid, int index); + SegmentHandle getSegment(int32_t cuid, int32_t index); /** * Updates download status. */ - bool updateSegment(int cuid, const Segment& segment); + //bool updateSegment(int cuid, const Segment& segment); /** * Cancels all the segment which the command having given cuid * uses. */ - void cancelSegment(int cuid); + void cancelSegment(int32_t cuid); /** * Tells SegmentMan that the segment has been downloaded successfully. */ - bool completeSegment(int cuid, const Segment& segment); + bool completeSegment(int32_t cuid, const SegmentHandle& segment); /** * Initializes bitfield with the provided length parameters. */ - void initBitfield(int segmentLength, long long int totalLength); + void initBitfield(int32_t segmentLength, int64_t totalLength); /** * Returns true if the segment whose index is index has been downloaded. */ - bool hasSegment(int index) const; + bool hasSegment(int32_t index) const; /** * Returns the length of bytes downloaded. */ - long long int getDownloadLength() const; + int64_t getDownloadLength() const; /** * Registers given peerStat if it has not been registerd. @@ -288,7 +288,7 @@ public: #ifdef ENABLE_MESSAGE_DIGEST void checkIntegrity(); - void tryChunkChecksumValidation(const Segment& segment); + void tryChunkChecksumValidation(const SegmentHandle& segment); bool isChunkChecksumValidationReady() const; #endif // ENABLE_MESSAGE_DIGEST diff --git a/src/TransferEncoding.h b/src/TransferEncoding.h index 00baae06..080c9b36 100644 --- a/src/TransferEncoding.h +++ b/src/TransferEncoding.h @@ -46,5 +46,7 @@ public: virtual void end() = 0; }; +typedef SharedHandle TransferEncodingHandle; + #endif // _D_TRANSFER_ENCODING_H_ diff --git a/src/UrlRequestInfo.cc b/src/UrlRequestInfo.cc index 5bc69b45..d33924fb 100644 --- a/src/UrlRequestInfo.cc +++ b/src/UrlRequestInfo.cc @@ -120,7 +120,7 @@ void UrlRequestInfo::printUrls(const Strings& urls) const { } } -HeadResult UrlRequestInfo::getHeadResult() { +HeadResultHandle UrlRequestInfo::getHeadResult() { Requests requests; for_each(urls.begin(), urls.end(), CreateRequest(&requests, @@ -128,27 +128,22 @@ HeadResult UrlRequestInfo::getHeadResult() { 1, Request::METHOD_HEAD)); if(requests.size() == 0) { - fail = true; - return HeadResult(); + return 0; } Requests reserved(requests.begin()+1, requests.end()); requests.erase(requests.begin()+1, requests.end()); SharedHandle e(DownloadEngineFactory::newConsoleEngine(op, requests, reserved)); - HeadResult hr; + HeadResultHandle hr = 0; try { e->run(); - if(e->segmentMan->errors > 0) { - fail = true; - } else { - hr.filename = e->segmentMan->filename; - hr.totalLength = e->segmentMan->totalSize; - } + hr = new HeadResult(); + hr->filename = e->segmentMan->filename; + hr->totalLength = e->segmentMan->totalSize; } catch(RecoverableException *ex) { logger->error("Exception caught", ex); delete ex; - fail = true; } return hr; } @@ -158,10 +153,7 @@ RequestInfos UrlRequestInfo::execute() { Requests requests; Requests reserved; printUrls(urls); - HeadResult hr = getHeadResult(); - if(fail) { - return RequestInfos(); - } + HeadResultHandle hr = getHeadResult(); for_each(urls.begin(), urls.end(), CreateRequest(&requests, @@ -169,16 +161,18 @@ RequestInfos UrlRequestInfo::execute() { op->getAsInt(PREF_SPLIT))); logger->info("Head result: filename=%s, total length=%s", - hr.filename.c_str(), Util::ullitos(hr.totalLength, true).c_str()); + hr->filename.c_str(), Util::ullitos(hr->totalLength, true).c_str()); adjustRequestSize(requests, reserved, maxConnections); SharedHandle e(DownloadEngineFactory::newConsoleEngine(op, requests, reserved)); - if(hr.totalLength > 0) { - e->segmentMan->filename = hr.filename; - e->segmentMan->totalSize = hr.totalLength; + + e->segmentMan->filename = hr->filename; + e->segmentMan->totalSize = hr->totalLength; + if(hr->totalLength > 0) { e->segmentMan->downloadStarted = true; } + #ifdef ENABLE_MESSAGE_DIGEST if(chunkChecksumLength > 0) { e->segmentMan->digestAlgo = digestAlgo; diff --git a/src/UrlRequestInfo.h b/src/UrlRequestInfo.h index 14eacf7f..94e1680b 100644 --- a/src/UrlRequestInfo.h +++ b/src/UrlRequestInfo.h @@ -47,6 +47,8 @@ public: std::ostream& operator<<(std::ostream& o, const HeadResult& hr); +typedef SharedHandle HeadResultHandle; + class UrlRequestInfo : public RequestInfo { private: Strings urls; @@ -62,7 +64,7 @@ private: Requests& reserved, int maxConnections) const; void printUrls(const Strings& urls) const; - HeadResult getHeadResult(); + HeadResultHandle getHeadResult(); public: UrlRequestInfo(const Strings& urls, int maxConnections, Option* op): RequestInfo(op), diff --git a/src/Util.cc b/src/Util.cc index 0bf3b95b..89d241fd 100644 --- a/src/Util.cc +++ b/src/Util.cc @@ -133,6 +133,22 @@ void Util::split(pair& hp, const string& src, char delim) { } } +pair Util::split(const string& src, const string& delims) +{ + pair hp; + hp.first = ""; + hp.second = ""; + string::size_type p = src.find_first_of(delims); + if(p == string::npos) { + hp.first = src; + hp.second = ""; + } else { + hp.first = trim(src.substr(0, p)); + hp.second = trim(src.substr(p+1)); + } + return hp; +} + long long int Util::difftv(struct timeval tv1, struct timeval tv2) { if(tv1.tv_sec < tv2.tv_sec || tv1.tv_sec == tv2.tv_sec && tv1.tv_usec < tv2.tv_usec) { return 0; diff --git a/src/Util.h b/src/Util.h index 70ffb8f2..fd1f1cd1 100644 --- a/src/Util.h +++ b/src/Util.h @@ -52,6 +52,7 @@ using namespace std; class Util { public: static void split(pair& hp, const string& src, char delim); + static pair split(const string& src, const string& delims); static string llitos(int64_t value, bool comma = false); static string ullitos(uint64_t value, bool comma = false); static string itos(int32_t value, bool comma = false); @@ -82,6 +83,11 @@ public: static string urlencode(const unsigned char* target, int len); + static string urlencode(const string& target) + { + return urlencode((const unsigned char*)target.c_str(), target.size()); + } + static string urldecode(const string& target); static string torrentUrlencode(const unsigned char* target, int len); diff --git a/test/BitfieldManTest.cc b/test/BitfieldManTest.cc index 60187cc9..8091b7b4 100644 --- a/test/BitfieldManTest.cc +++ b/test/BitfieldManTest.cc @@ -16,6 +16,7 @@ class BitfieldManTest:public CppUnit::TestFixture { CPPUNIT_TEST(testGetMissingIndex); CPPUNIT_TEST(testGetSparceMissingUnusedIndex); CPPUNIT_TEST(testIsBitSetOffsetRange); + CPPUNIT_TEST(testGetMissingUnusedLength); CPPUNIT_TEST_SUITE_END(); private: RandomizerHandle fixedNumberRandomizer; @@ -37,6 +38,7 @@ public: void testGetMissingIndex(); void testGetSparceMissingUnusedIndex(); void testIsBitSetOffsetRange(); + void testGetMissingUnusedLength(); }; @@ -267,3 +269,37 @@ void BitfieldManTest::testIsBitSetOffsetRange() CPPUNIT_ASSERT(!bitfield.isBitSetOffsetRange(pieceLength*100, pieceLength*3)); } + +void BitfieldManTest::testGetMissingUnusedLength() +{ + int64_t totalLength = 1024*10+10; + int32_t blockLength = 1024; + + BitfieldMan bf(blockLength, totalLength); + + // from index 0 and all blocks are unused and not acquired. + CPPUNIT_ASSERT_EQUAL((int64_t)totalLength, bf.getMissingUnusedLength(0)); + + // from index 10 and all blocks are unused and not acquired. + CPPUNIT_ASSERT_EQUAL((int64_t)10, bf.getMissingUnusedLength(10)); + + // from index -1 + CPPUNIT_ASSERT_EQUAL((int64_t)0, bf.getMissingUnusedLength(-1)); + + // from index 11 + CPPUNIT_ASSERT_EQUAL((int64_t)0, bf.getMissingUnusedLength(11)); + + // from index 12 + CPPUNIT_ASSERT_EQUAL((int64_t)0, bf.getMissingUnusedLength(12)); + + // from index 0 and 5th block is used. + bf.setUseBit(5); + CPPUNIT_ASSERT_EQUAL((int64_t)5*blockLength, bf.getMissingUnusedLength(0)); + + // from index 0 and 4th block is acquired. + bf.setBit(4); + CPPUNIT_ASSERT_EQUAL((int64_t)4*blockLength, bf.getMissingUnusedLength(0)); + + // from index 1 + CPPUNIT_ASSERT_EQUAL((int64_t)3*blockLength, bf.getMissingUnusedLength(1)); +} diff --git a/test/HttpHeaderTest.cc b/test/HttpHeaderTest.cc new file mode 100644 index 00000000..88646439 --- /dev/null +++ b/test/HttpHeaderTest.cc @@ -0,0 +1,28 @@ +#include "HttpHeader.h" +#include + +class HttpHeaderTest:public CppUnit::TestFixture { + + CPPUNIT_TEST_SUITE(HttpHeaderTest); + CPPUNIT_TEST(testGetRange); + CPPUNIT_TEST_SUITE_END(); + +public: + void testGetRange(); + +}; + + +CPPUNIT_TEST_SUITE_REGISTRATION( HttpHeaderTest ); + +void HttpHeaderTest::testGetRange() +{ + HttpHeader httpHeader; + httpHeader.put("Content-Range", "bytes 1-499/1234"); + + RangeHandle range = httpHeader.getRange(); + + CPPUNIT_ASSERT_EQUAL((int64_t)1, range->getStartByte()); + CPPUNIT_ASSERT_EQUAL((int64_t)499, range->getEndByte()); + CPPUNIT_ASSERT_EQUAL((int64_t)1234, range->getEntityLength()); +} diff --git a/test/HttpRequestTest.cc b/test/HttpRequestTest.cc new file mode 100644 index 00000000..a777a0ea --- /dev/null +++ b/test/HttpRequestTest.cc @@ -0,0 +1,474 @@ +#include "HttpRequest.h" +#include "prefs.h" +#include + +using namespace std; + +class HttpRequestTest : public CppUnit::TestFixture { + + CPPUNIT_TEST_SUITE(HttpRequestTest); + CPPUNIT_TEST(testGetStartByte); + CPPUNIT_TEST(testGetEndByte); + CPPUNIT_TEST(testCreateRequest); + CPPUNIT_TEST(testCreateRequest_ftp); + CPPUNIT_TEST(testCreateRequest_with_cookie); + CPPUNIT_TEST(testCreateProxyRequest); + CPPUNIT_TEST(testIsRangeSatisfied); + CPPUNIT_TEST_SUITE_END(); +private: + +public: + void setUp() { + } + + void testGetStartByte(); + void testGetEndByte(); + void testCreateRequest(); + void testCreateRequest_ftp(); + void testCreateRequest_with_cookie(); + void testCreateProxyRequest(); + void testIsRangeSatisfied(); +}; + + +CPPUNIT_TEST_SUITE_REGISTRATION( HttpRequestTest ); + +void HttpRequestTest::testGetStartByte() +{ + HttpRequest httpRequest; + SegmentHandle segment = new Segment(1, 1024*1024, 1024*1024, 0); + + CPPUNIT_ASSERT_EQUAL((int64_t)0, httpRequest.getStartByte()); + + httpRequest.setSegment(segment); + + CPPUNIT_ASSERT_EQUAL((int64_t)1024*1024, httpRequest.getStartByte()); + +} + +void HttpRequestTest::testGetEndByte() +{ + int32_t index = 1; + int32_t length = 1024*1024-1024; + int32_t segmentLength = 1024*1024; + int32_t writtenLength = 1024; + + HttpRequest httpRequest; + SegmentHandle segment = new Segment(index, length, segmentLength, writtenLength); + + CPPUNIT_ASSERT_EQUAL((int64_t)0, httpRequest.getEndByte()); + + httpRequest.setSegment(segment); + + CPPUNIT_ASSERT_EQUAL((int64_t)0, + httpRequest.getEndByte()); + + RequestHandle request = new Request(); + request->setKeepAlive(true); + + httpRequest.setRequest(request); + + CPPUNIT_ASSERT_EQUAL((int64_t)segmentLength*index+length-1, + httpRequest.getEndByte()); + + + request->setKeepAlive(false); + + CPPUNIT_ASSERT_EQUAL((int64_t)0, httpRequest.getEndByte()); +} + +void HttpRequestTest::testCreateRequest() +{ + RequestHandle request = new Request(); + request->setUrl("http://localhost:8080/archives/aria2-1.0.0.tar.bz2"); + SegmentHandle segment = new Segment(); + + HttpRequest httpRequest; + + httpRequest.setRequest(request); + httpRequest.setSegment(segment); + + string expectedText = "GET /archives/aria2-1.0.0.tar.bz2 HTTP/1.1\r\n" + "User-Agent: aria2\r\n" + "Accept: */*\r\n" + "Host: localhost:8080\r\n" + "Pragma: no-cache\r\n" + "Cache-Control: no-cache\r\n" + "\r\n"; + + CPPUNIT_ASSERT_EQUAL(expectedText, httpRequest.createRequest()); + + request->setKeepAlive(false); + + expectedText = "GET /archives/aria2-1.0.0.tar.bz2 HTTP/1.1\r\n" + "User-Agent: aria2\r\n" + "Accept: */*\r\n" + "Host: localhost:8080\r\n" + "Pragma: no-cache\r\n" + "Cache-Control: no-cache\r\n" + "Connection: close\r\n" + "\r\n"; + + CPPUNIT_ASSERT_EQUAL(expectedText, httpRequest.createRequest()); + + segment->index = 1; + segment->length = 1024*1024; + segment->segmentLength = 1024*1024; + segment->writtenLength = 0; + + expectedText = "GET /archives/aria2-1.0.0.tar.bz2 HTTP/1.1\r\n" + "User-Agent: aria2\r\n" + "Accept: */*\r\n" + "Host: localhost:8080\r\n" + "Pragma: no-cache\r\n" + "Cache-Control: no-cache\r\n" + "Connection: close\r\n" + "Range: bytes=1048576-\r\n" + "\r\n"; + + CPPUNIT_ASSERT_EQUAL(expectedText, httpRequest.createRequest()); + + request->setKeepAlive(true); + + expectedText = "GET /archives/aria2-1.0.0.tar.bz2 HTTP/1.1\r\n" + "User-Agent: aria2\r\n" + "Accept: */*\r\n" + "Host: localhost:8080\r\n" + "Pragma: no-cache\r\n" + "Cache-Control: no-cache\r\n" + "Range: bytes=1048576-2097151\r\n" + "\r\n"; + + CPPUNIT_ASSERT_EQUAL(expectedText, httpRequest.createRequest()); + + httpRequest.setSegment(new Segment()); + + request->redirectUrl("http://localhost:8080/archives/download/aria2-1.0.0.tar.bz2"); + + expectedText = "GET /archives/download/aria2-1.0.0.tar.bz2 HTTP/1.1\r\n" + "User-Agent: aria2\r\n" + "Accept: */*\r\n" + "Host: localhost:8080\r\n" + "Pragma: no-cache\r\n" + "Cache-Control: no-cache\r\n" + "Referer: http://localhost:8080/archives/aria2-1.0.0.tar.bz2\r\n" + "\r\n"; + + CPPUNIT_ASSERT_EQUAL(expectedText, httpRequest.createRequest()); + + request->resetUrl(); + + SharedHandle