github
/
aria2
mirror of https://github.com/aria2/aria2
1
0
Fork 0
Code Issues Releases Wiki Activity

2010-09-06 Tatsuhiro Tsujikawa <t-tujikawa@users.sourceforge.net> 

Data from remote server in HTTP/FTP download are now written to
	the disk(or memory) through StreamFilter. Decoding chunked and
	gziped streams are done cascading StreamFilter.
	Removed inefficient 1byte read code.
	* src/ChunkedDecodingStreamFilter.cc
	* src/ChunkedDecodingStreamFilter.h
	* src/DownloadCommand.cc
	* src/DownloadCommand.h
	* src/GZipDecodingStreamFilter.cc
	* src/GZipDecodingStreamFilter.h
	* src/HttpConnection.cc
	* src/HttpDownloadCommand.cc
	* src/HttpResponse.cc
	* src/HttpResponse.h
	* src/HttpResponseCommand.cc
	* src/HttpResponseCommand.h
	* src/HttpSkipResponseCommand.cc
	* src/HttpSkipResponseCommand.h
	* src/Makefile.am
	* src/NullSinkStreamFilter.cc
	* src/NullSinkStreamFilter.h
	* src/RequestGroup.cc
	* src/SinkStreamFilter.cc
	* src/SinkStreamFilter.h
	* src/StreamFilter.cc
	* src/StreamFilter.h
	* test/ChunkedDecodingStreamFilterTest.cc
	* test/GZipDecodingStreamFilterTest.cc
	* test/HttpResponseTest.cc
	* test/Makefile.am
	* test/MockSegment.h
pull/1/head
Tatsuhiro Tsujikawa 2010-09-06 14:29:36 +00:00
parent 552c753dbb
commit efbfe4c006
30 changed files with 1767 additions and 231 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
ChangeLog
View File

@ -1,3 +1,37 @@
2010-09-06 Tatsuhiro Tsujikawa <t-tujikawa@users.sourceforge.net>
Data from remote server in HTTP/FTP download are now written to
the disk(or memory) through StreamFilter. Decoding chunked and
gziped streams are done cascading StreamFilter.
Removed inefficient 1byte read code.
* src/ChunkedDecodingStreamFilter.cc
* src/ChunkedDecodingStreamFilter.h
* src/DownloadCommand.cc
* src/DownloadCommand.h
* src/GZipDecodingStreamFilter.cc
* src/GZipDecodingStreamFilter.h
* src/HttpConnection.cc
* src/HttpDownloadCommand.cc
* src/HttpResponse.cc
* src/HttpResponse.h
* src/HttpResponseCommand.cc
* src/HttpResponseCommand.h
* src/HttpSkipResponseCommand.cc
* src/HttpSkipResponseCommand.h
* src/Makefile.am
* src/NullSinkStreamFilter.cc
* src/NullSinkStreamFilter.h
* src/RequestGroup.cc
* src/SinkStreamFilter.cc
* src/SinkStreamFilter.h
* src/StreamFilter.cc
* src/StreamFilter.h
* test/ChunkedDecodingStreamFilterTest.cc
* test/GZipDecodingStreamFilterTest.cc
* test/HttpResponseTest.cc
* test/Makefile.am
* test/MockSegment.h
2010-09-01 Tatsuhiro Tsujikawa <t-tujikawa@users.sourceforge.net>
Release 1.10.2

223
src/ChunkedDecodingStreamFilter.cc Normal file
View File

@ -0,0 +1,223 @@
/* <!-- copyright */
/*
* aria2 - The high speed download utility
*
* Copyright (C) 2010 Tatsuhiro Tsujikawa
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*
* In addition, as a special exception, the copyright holders give
* permission to link the code of portions of this program with the
* OpenSSL library under certain conditions as described in each
* individual source file, and distribute linked combinations
* including the two.
* You must obey the GNU General Public License in all respects
* for all of the code used other than OpenSSL. If you modify
* file(s) with this exception, you may extend this exception to your
* version of the file(s), but you are not obligated to do so. If you
* do not wish to do so, delete this exception statement from your
* version. If you delete this exception statement from all source
* files in the program, then also delete it here.
*/
/* copyright --> */
#include "ChunkedDecodingStreamFilter.h"
#include <cassert>
#include "util.h"
#include "message.h"
#include "DlAbortEx.h"
#include "StringFormat.h"
#include "A2STR.h"
namespace aria2 {
const std::string ChunkedDecodingStreamFilter::NAME
("ChunkedDecodingStreamFilter");
size_t ChunkedDecodingStreamFilter::MAX_BUF_SIZE = 1024*1024;
ChunkedDecodingStreamFilter::ReadChunkSizeStateHandler*
ChunkedDecodingStreamFilter::readChunkSizeStateHandler_ =
new ChunkedDecodingStreamFilter::ReadChunkSizeStateHandler();
ChunkedDecodingStreamFilter::ReadTrailerStateHandler*
ChunkedDecodingStreamFilter::readTrailerStateHandler_ =
new ChunkedDecodingStreamFilter::ReadTrailerStateHandler();
ChunkedDecodingStreamFilter::ReadDataStateHandler*
ChunkedDecodingStreamFilter::readDataStateHandler_ =
new ChunkedDecodingStreamFilter::ReadDataStateHandler();
ChunkedDecodingStreamFilter::ReadDataEndStateHandler*
ChunkedDecodingStreamFilter::readDataEndStateHandler_ =
new ChunkedDecodingStreamFilter::ReadDataEndStateHandler();
ChunkedDecodingStreamFilter::StreamEndStatehandler*
ChunkedDecodingStreamFilter::streamEndStateHandler_ =
new ChunkedDecodingStreamFilter::StreamEndStatehandler();
ChunkedDecodingStreamFilter::ChunkedDecodingStreamFilter
(const SharedHandle<StreamFilter>& delegate):
StreamFilter(delegate),
state_(readChunkSizeStateHandler_),
chunkSize_(0),
bytesProcessed_(0) {}
ChunkedDecodingStreamFilter::~ChunkedDecodingStreamFilter() {}
void ChunkedDecodingStreamFilter::init() {}
bool ChunkedDecodingStreamFilter::readChunkSize
(size_t& inbufOffset, const unsigned char* inbuf, size_t inlen)
{
size_t pbufSize = buf_.size();
buf_.append(&inbuf[inbufOffset], &inbuf[inlen]);
std::string::size_type crlfPos = buf_.find(A2STR::CRLF);
if(crlfPos == std::string::npos) {
if(buf_.size() > MAX_BUF_SIZE) {
throw DL_ABORT_EX("Could not find chunk size before buffer got full.");
}
inbufOffset = inlen;
return false;
}
std::string::size_type extPos = buf_.find(A2STR::SEMICOLON_C);
if(extPos == std::string::npos || crlfPos < extPos) {
extPos = crlfPos;
}
chunkSize_ = util::parseULLInt(buf_.substr(0, extPos), 16);
assert(crlfPos+2 > pbufSize);
inbufOffset += crlfPos+2-pbufSize;
buf_.clear();
if(chunkSize_ == 0) {
state_ = readTrailerStateHandler_;
} else {
state_ = readDataStateHandler_;
}
return true;
}
bool ChunkedDecodingStreamFilter::readTrailer
(size_t& inbufOffset, const unsigned char* inbuf, size_t inlen)
{
size_t pbufSize = buf_.size();
buf_.append(&inbuf[inbufOffset], &inbuf[inlen]);
std::string::size_type crlfcrlfPos = buf_.find("\r\n\r\n");
if(crlfcrlfPos != std::string::npos) {
// TODO crlfcrlfPos == 0 case?
inbufOffset += crlfcrlfPos+4-pbufSize;
inbufOffset = inlen;
buf_.clear();
state_ = streamEndStateHandler_;
return true;
} else {
std::string::size_type crlfPos = buf_.find(A2STR::CRLF);
if(crlfPos == std::string::npos) {
if(buf_.size() > MAX_BUF_SIZE) {
throw DL_ABORT_EX
("Could not find end of stream before buffer got full.");
}
inbufOffset = inlen;
return false;
} else if(crlfPos == 0) {
inbufOffset += crlfPos+2-pbufSize;
buf_.clear();
state_ = streamEndStateHandler_;
return true;
} else {
if(buf_.size() > MAX_BUF_SIZE) {
throw DL_ABORT_EX
("Could not find end of stream before buffer got full.");
}
inbufOffset = inlen;
return false;
}
}
}
bool ChunkedDecodingStreamFilter::readData
(ssize_t& outlen,
size_t& inbufOffset,
const unsigned char* inbuf,
size_t inlen,
const SharedHandle<BinaryStream>& out,
const SharedHandle<Segment>& segment)
{
uint64_t readlen =
std::min(chunkSize_, static_cast<uint64_t>(inlen-inbufOffset));
outlen += getDelegate()->transform(out, segment, inbuf+inbufOffset, readlen);
chunkSize_ -= readlen;
inbufOffset += readlen;
if(chunkSize_ == 0) {
state_ = readDataEndStateHandler_;
return true;
} else {
return false;
}
}
bool ChunkedDecodingStreamFilter::readDataEnd
(size_t& inbufOffset, const unsigned char* inbuf, size_t inlen)
{
size_t pbufSize = buf_.size();
buf_.append(&inbuf[inbufOffset], &inbuf[inlen]);
if(buf_.size() >= 2) {
if(util::startsWith(buf_, A2STR::CRLF)) {
inbufOffset += 2-pbufSize;
buf_.clear();
state_ = readChunkSizeStateHandler_;
return true;
} else {
throw DL_ABORT_EX("No CRLF at the end of chunk.");
}
} else {
inbufOffset = inlen;
return false;
}
}
ssize_t ChunkedDecodingStreamFilter::transform
(const SharedHandle<BinaryStream>& out,
const SharedHandle<Segment>& segment,
const unsigned char* inbuf, size_t inlen)
{
size_t inbufOffset = 0;
ssize_t outlen = 0;
while(inbufOffset < inlen) {
ssize_t olen = 0;
bool r = state_->execute
(this, olen, inbufOffset, inbuf, inlen, out, segment);
outlen += olen;
if(!r) {
break;
}
}
bytesProcessed_ = inbufOffset;
return outlen;
}
bool ChunkedDecodingStreamFilter::finished()
{
return state_ == streamEndStateHandler_ && getDelegate()->finished();
}
void ChunkedDecodingStreamFilter::release() {}
const std::string& ChunkedDecodingStreamFilter::getName() const
{
return NAME;
}
} // namespace aria2

194
src/ChunkedDecodingStreamFilter.h Normal file
View File

@ -0,0 +1,194 @@
/* <!-- copyright */
/*
* aria2 - The high speed download utility
*
* Copyright (C) 2010 Tatsuhiro Tsujikawa
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*
* In addition, as a special exception, the copyright holders give
* permission to link the code of portions of this program with the
* OpenSSL library under certain conditions as described in each
* individual source file, and distribute linked combinations
* including the two.
* You must obey the GNU General Public License in all respects
* for all of the code used other than OpenSSL. If you modify
* file(s) with this exception, you may extend this exception to your
* version of the file(s), but you are not obligated to do so. If you
* do not wish to do so, delete this exception statement from your
* version. If you delete this exception statement from all source
* files in the program, then also delete it here.
*/
/* copyright --> */
#ifndef D_CHUNKED_DECODING_STREAM_FILTER_H
#define D_CHUNKED_DECODING_STREAM_FILTER_H
#include "StreamFilter.h"
namespace aria2 {
class ChunkedDecodingStreamFilter : public StreamFilter {
private:
class StateHandler {
public:
virtual ~StateHandler() {}
virtual bool execute
(ChunkedDecodingStreamFilter* filter,
ssize_t& outlen,
size_t& inbufOffset,
const unsigned char* inbuf,
size_t inlen,
const SharedHandle<BinaryStream>& out,
const SharedHandle<Segment>& segment) const = 0;
};
StateHandler* state_;
std::string buf_;
uint64_t chunkSize_;
size_t bytesProcessed_;
static size_t MAX_BUF_SIZE;
bool readChunkSize
(size_t& inbufOffset, const unsigned char* inbuf, size_t inlen);
bool readTrailer
(size_t& inbufOffset, const unsigned char* inbuf, size_t inlen);
bool readData
(ssize_t& outlen,
size_t& inbufOffset,
const unsigned char* inbuf,
size_t inlen,
const SharedHandle<BinaryStream>& out,
const SharedHandle<Segment>& segment);
bool readDataEnd
(size_t& inbufOffset, const unsigned char* inbuf, size_t inlen);
class ReadChunkSizeStateHandler:public StateHandler {
public:
virtual bool execute
(ChunkedDecodingStreamFilter* filter,
ssize_t& outlen,
size_t& inbufOffset,
const unsigned char* inbuf,
size_t inlen,
const SharedHandle<BinaryStream>& out,
const SharedHandle<Segment>& segment) const
{
return filter->readChunkSize(inbufOffset, inbuf, inlen);
}
};
class ReadTrailerStateHandler:public StateHandler {
public:
virtual bool execute
(ChunkedDecodingStreamFilter* filter,
ssize_t& outlen,
size_t& inbufOffset,
const unsigned char* inbuf,
size_t inlen,
const SharedHandle<BinaryStream>& out,
const SharedHandle<Segment>& segment) const
{
return filter->readTrailer(inbufOffset, inbuf, inlen);
}
};
class ReadDataStateHandler:public StateHandler {
public:
virtual bool execute
(ChunkedDecodingStreamFilter* filter,
ssize_t& outlen,
size_t& inbufOffset,
const unsigned char* inbuf,
size_t inlen,
const SharedHandle<BinaryStream>& out,
const SharedHandle<Segment>& segment) const
{
return filter->readData(outlen, inbufOffset, inbuf, inlen, out, segment);
}
};
class ReadDataEndStateHandler:public StateHandler {
public:
virtual bool execute
(ChunkedDecodingStreamFilter* filter,
ssize_t& outlen,
size_t& inbufOffset,
const unsigned char* inbuf,
size_t inlen,
const SharedHandle<BinaryStream>& out,
const SharedHandle<Segment>& segment) const
{
return filter->readDataEnd(inbufOffset, inbuf, inlen);
}
};
class StreamEndStatehandler:public StateHandler {
public:
virtual bool execute
(ChunkedDecodingStreamFilter* filter,
ssize_t& outlen,
size_t& inbufOffset,
const unsigned char* inbuf,
size_t inlen,
const SharedHandle<BinaryStream>& out,
const SharedHandle<Segment>& segment) const
{
return false;
}
};
static ReadChunkSizeStateHandler* readChunkSizeStateHandler_;
static ReadTrailerStateHandler* readTrailerStateHandler_;
static ReadDataStateHandler* readDataStateHandler_;
static ReadDataEndStateHandler* readDataEndStateHandler_;
static StreamEndStatehandler* streamEndStateHandler_;
public:
ChunkedDecodingStreamFilter
(const SharedHandle<StreamFilter>& delegate = SharedHandle<StreamFilter>());
virtual ~ChunkedDecodingStreamFilter();
virtual void init();
virtual ssize_t transform
(const SharedHandle<BinaryStream>& out,
const SharedHandle<Segment>& segment,
const unsigned char* inbuf, size_t inlen);
virtual bool finished();
virtual void release();
virtual const std::string& getName() const;
virtual size_t getBytesProcessed() const
{
return bytesProcessed_;
}
static const std::string NAME;
};
} // namespace aria2
#endif // D_CHUNKED_DECODING_STREAM_FILTER_H

148
src/DownloadCommand.cc
View File

@ -56,11 +56,11 @@
#include "message.h"
#include "prefs.h"
#include "StringFormat.h"
#include "Decoder.h"
#include "RequestGroupMan.h"
#include "wallclock.h"
#include "ServerStatMan.h"
#include "FileAllocationEntry.h"
#include "SinkStreamFilter.h"
#ifdef ENABLE_MESSAGE_DIGEST
# include "MessageDigestHelper.h"
#endif // ENABLE_MESSAGE_DIGEST
@ -104,6 +104,10 @@ DownloadCommand::DownloadCommand(cuid_t cuid,
peerStat_ = req->initPeerStat();
peerStat_->downloadStart();
getSegmentMan()->registerPeerStat(peerStat_);
streamFilter_.reset(new SinkStreamFilter(pieceHashValidationEnabled_));
streamFilter_->init();
sinkFilterOnly_ = true;
}
DownloadCommand::~DownloadCommand() {
@ -120,80 +124,45 @@ bool DownloadCommand::executeInternal() {
return false;
}
setReadCheckSocket(getSocket());
SharedHandle<Segment> segment = getSegments().front();
size_t bufSize;
if(segment->getLength() > 0) {
if(static_cast<uint64_t>(segment->getPosition()+segment->getLength()) <=
static_cast<uint64_t>(getFileEntry()->getLastOffset())) {
bufSize = std::min(segment->getLength()-segment->getWrittenLength(),
BUFSIZE);
} else {
bufSize =
std::min
(static_cast<size_t>
(getFileEntry()->getLastOffset()-segment->getPositionToWrite()),
BUFSIZE);
}
} else {
bufSize = BUFSIZE;
}
// It is possible that segment is completed but we have some bytes
// of stream to read. For example, chunked encoding has "0"+CRLF
// after data. After we read data(at this moment segment is
// completed), we need another 3bytes(or more if it has extension).
if(bufSize == 0 &&
((!transferEncodingDecoder_.isNull() &&
!transferEncodingDecoder_->finished()) ||
(!contentEncodingDecoder_.isNull() &&
!contentEncodingDecoder_->finished()))) {
bufSize = 1;
}
getSocket()->readData(buf_, bufSize);
const SharedHandle<DiskAdaptor>& diskAdaptor =
getPieceStorage()->getDiskAdaptor();
const unsigned char* bufFinal;
size_t bufSizeFinal;
std::string decoded;
if(transferEncodingDecoder_.isNull()) {
bufFinal = buf_;
bufSizeFinal = bufSize;
SharedHandle<Segment> segment = getSegments().front();
size_t bufSize;
if(sinkFilterOnly_) {
if(segment->getLength() > 0 ) {
if(static_cast<uint64_t>(segment->getPosition()+segment->getLength()) <=
static_cast<uint64_t>(getFileEntry()->getLastOffset())) {
bufSize = std::min(segment->getLength()-segment->getWrittenLength(),
BUFSIZE);
} else {
bufSize =
std::min
(static_cast<size_t>
(getFileEntry()->getLastOffset()-segment->getPositionToWrite()),
BUFSIZE);
}
} else {
bufSize = BUFSIZE;
}
getSocket()->readData(buf_, bufSize);
streamFilter_->transform(diskAdaptor, segment, buf_, bufSize);
} else {
decoded = transferEncodingDecoder_->decode(buf_, bufSize);
bufFinal = reinterpret_cast<const unsigned char*>(decoded.c_str());
bufSizeFinal = decoded.size();
}
if(contentEncodingDecoder_.isNull()) {
diskAdaptor->writeData(bufFinal, bufSizeFinal,
segment->getPositionToWrite());
} else {
std::string out = contentEncodingDecoder_->decode(bufFinal, bufSizeFinal);
diskAdaptor->writeData(reinterpret_cast<const unsigned char*>(out.data()),
out.size(),
segment->getPositionToWrite());
bufSizeFinal = out.size();
}
#ifdef ENABLE_MESSAGE_DIGEST
if(pieceHashValidationEnabled_) {
segment->updateHash(segment->getWrittenLength(), bufFinal, bufSizeFinal);
}
#endif // ENABLE_MESSAGE_DIGEST
if(bufSizeFinal > 0) {
segment->updateWrittenLength(bufSizeFinal);
// It is possible that segment is completed but we have some bytes
// of stream to read. For example, chunked encoding has "0"+CRLF
// after data. After we read data(at this moment segment is
// completed), we need another 3bytes(or more if it has trailers).
bufSize = BUFSIZE;
getSocket()->peekData(buf_, bufSize);
streamFilter_->transform(diskAdaptor, segment, buf_, bufSize);
bufSize = streamFilter_->getBytesProcessed();
getSocket()->readData(buf_, bufSize);
}
peerStat_->updateDownloadLength(bufSize);
getSegmentMan()->updateDownloadSpeedFor(peerStat_);
bool segmentPartComplete = false;
// Note that GrowSegment::complete() always returns false.
if(transferEncodingDecoder_.isNull() && contentEncodingDecoder_.isNull()) {
if(sinkFilterOnly_) {
if(segment->complete() ||
segment->getPositionToWrite() == getFileEntry()->getLastOffset()) {
segmentPartComplete = true;
@ -203,23 +172,20 @@ bool DownloadCommand::executeInternal() {
}
} else {
off_t loff = getFileEntry()->gtoloff(segment->getPositionToWrite());
if(!transferEncodingDecoder_.isNull() &&
((loff == getRequestEndOffset() && transferEncodingDecoder_->finished())
if(getFileEntry()->getLength() > 0 && !sinkFilterOnly_ &&
((loff == getRequestEndOffset() && streamFilter_->finished())
|| loff < getRequestEndOffset()) &&
(segment->complete() ||
segment->getPositionToWrite() == getFileEntry()->getLastOffset())) {
// In this case, transferEncodingDecoder is used and
// Content-Length is known. We check
// transferEncodingDecoder_->finished() only if the requested
// end offset equals to written position in file local offset;
// in other words, data in the requested ranage is all received.
// If requested end offset is greater than this segment, then
// transferEncodingDecoder_ is not finished in this segment.
// In this case, StreamFilter other than *SinkStreamFilter is
// used and Content-Length is known. We check
// streamFilter_->finished() only if the requested end offset
// equals to written position in file local offset; in other
// words, data in the requested ranage is all received. If
// requested end offset is greater than this segment, then
// streamFilter_ is not finished in this segment.
segmentPartComplete = true;
} else if((transferEncodingDecoder_.isNull() ||
transferEncodingDecoder_->finished()) &&
(contentEncodingDecoder_.isNull() ||
contentEncodingDecoder_->finished())) {
} else if(streamFilter_->finished()) {
segmentPartComplete = true;
}
}
@ -392,18 +358,18 @@ void DownloadCommand::validatePieceHash(const SharedHandle<Segment>& segment,
}
}
void DownloadCommand::installStreamFilter
(const SharedHandle<StreamFilter>& streamFilter)
{
if(streamFilter.isNull()) {
return;
}
streamFilter->installDelegate(streamFilter_);
streamFilter_ = streamFilter;
sinkFilterOnly_ =
util::endsWith(streamFilter_->getName(), SinkStreamFilter::NAME);
}
#endif // ENABLE_MESSAGE_DIGEST
void DownloadCommand::setTransferEncodingDecoder
(const SharedHandle<Decoder>& decoder)
{
this->transferEncodingDecoder_ = decoder;
}
void DownloadCommand::setContentEncodingDecoder
(const SharedHandle<Decoder>& decoder)
{
contentEncodingDecoder_ = decoder;
}
} // namespace aria2

19
src/DownloadCommand.h
View File

@ -39,8 +39,8 @@
namespace aria2 {
class Decoder;
class PeerStat;
class StreamFilter;
#ifdef ENABLE_MESSAGE_DIGEST
class MessageDigestContext;
#endif // ENABLE_MESSAGE_DIGEST
@ -67,9 +67,9 @@ private:
void checkLowestDownloadSpeed() const;
SharedHandle<Decoder> transferEncodingDecoder_;
SharedHandle<StreamFilter> streamFilter_;
SharedHandle<Decoder> contentEncodingDecoder_;
bool sinkFilterOnly_;
protected:
virtual bool executeInternal();
@ -86,19 +86,12 @@ public:
const SharedHandle<SocketCore>& s);
virtual ~DownloadCommand();
const SharedHandle<Decoder>& getTransferEncodingDecoder() const
const SharedHandle<StreamFilter>& getStreamFilter() const
{
return transferEncodingDecoder_;
return streamFilter_;
}
void setTransferEncodingDecoder(const SharedHandle<Decoder>& decoder);
const SharedHandle<Decoder>& getContentEncodingDecoder() const
{
return contentEncodingDecoder_;
}
void setContentEncodingDecoder(const SharedHandle<Decoder>& decoder);
void installStreamFilter(const SharedHandle<StreamFilter>& streamFilter);
void setStartupIdleTime(time_t startupIdleTime)
{

131
src/GZipDecodingStreamFilter.cc Normal file
View File

@ -0,0 +1,131 @@
/* <!-- copyright */
/*
* aria2 - The high speed download utility
*
* Copyright (C) 2010 Tatsuhiro Tsujikawa
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*
* In addition, as a special exception, the copyright holders give
* permission to link the code of portions of this program with the
* OpenSSL library under certain conditions as described in each
* individual source file, and distribute linked combinations
* including the two.
* You must obey the GNU General Public License in all respects
* for all of the code used other than OpenSSL. If you modify
* file(s) with this exception, you may extend this exception to your
* version of the file(s), but you are not obligated to do so. If you
* do not wish to do so, delete this exception statement from your
* version. If you delete this exception statement from all source
* files in the program, then also delete it here.
*/
/* copyright --> */
#include "GZipDecodingStreamFilter.h"
#include <cassert>
#include "StringFormat.h"
#include "DlAbortEx.h"
namespace aria2 {
const std::string GZipDecodingStreamFilter::NAME("GZipDecodingStreamFilter");
GZipDecodingStreamFilter::GZipDecodingStreamFilter
(const SharedHandle<StreamFilter>& delegate):
StreamFilter(delegate), strm_(0), finished_(false), bytesProcessed_(0) {}
GZipDecodingStreamFilter::~GZipDecodingStreamFilter()
{
release();
}
void GZipDecodingStreamFilter::init()
{
finished_ = false;
release();
strm_ = new z_stream();
strm_->zalloc = Z_NULL;
strm_->zfree = Z_NULL;
strm_->opaque = Z_NULL;
strm_->avail_in = 0;
strm_->next_in = Z_NULL;
// initalize z_stream with gzip/zlib format auto detection enabled.
if(Z_OK != inflateInit2(strm_, 47)) {
throw DL_ABORT_EX("Initializing z_stream failed.");
}
}
void GZipDecodingStreamFilter::release()
{
if(strm_) {
inflateEnd(strm_);
delete strm_;
strm_ = 0;
}
}
ssize_t GZipDecodingStreamFilter::transform
(const SharedHandle<BinaryStream>& out,
const SharedHandle<Segment>& segment,
const unsigned char* inbuf, size_t inlen)
{
bytesProcessed_ = 0;
ssize_t outlen = 0;
if(inlen == 0) {
return outlen;
}
strm_->avail_in = inlen;
strm_->next_in = const_cast<unsigned char*>(inbuf);
unsigned char outbuf[OUTBUF_LENGTH];
while(1) {
strm_->avail_out = OUTBUF_LENGTH;
strm_->next_out = outbuf;
int ret = ::inflate(strm_, Z_NO_FLUSH);
if(ret == Z_STREAM_END) {
finished_ = true;
} else if(ret != Z_OK) {
throw DL_ABORT_EX(StringFormat("libz::inflate() failed. cause:%s",
strm_->msg).str());
}
size_t produced = OUTBUF_LENGTH-strm_->avail_out;
outlen += getDelegate()->transform(out, segment, outbuf, produced);
if(strm_->avail_out > 0) {
break;
}
}
assert(inlen >= strm_->avail_in);
bytesProcessed_ = inlen-strm_->avail_in;
return outlen;
}
bool GZipDecodingStreamFilter::finished()
{
return finished_ && getDelegate()->finished();
}
const std::string& GZipDecodingStreamFilter::getName() const
{
return NAME;
}
} // namespace aria2

81
src/GZipDecodingStreamFilter.h Normal file
View File

@ -0,0 +1,81 @@
/* <!-- copyright */
/*
* aria2 - The high speed download utility
*
* Copyright (C) 2010 Tatsuhiro Tsujikawa
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*
* In addition, as a special exception, the copyright holders give
* permission to link the code of portions of this program with the
* OpenSSL library under certain conditions as described in each
* individual source file, and distribute linked combinations
* including the two.
* You must obey the GNU General Public License in all respects
* for all of the code used other than OpenSSL. If you modify
* file(s) with this exception, you may extend this exception to your
* version of the file(s), but you are not obligated to do so. If you
* do not wish to do so, delete this exception statement from your
* version. If you delete this exception statement from all source
* files in the program, then also delete it here.
*/
/* copyright --> */
#ifndef D_GZIP_STREAM_FILTER_H
#define D_GZIP_STREAM_FILTER_H
#include "StreamFilter.h"
#include <zlib.h>
namespace aria2 {
// GZipDecodingStreamFilter can decode both gzip and deflate format.
class GZipDecodingStreamFilter : public StreamFilter {
private:
z_stream* strm_;
bool finished_;
size_t bytesProcessed_;
static const size_t OUTBUF_LENGTH = 16*1024;
public:
GZipDecodingStreamFilter
(const SharedHandle<StreamFilter>& delegate = SharedHandle<StreamFilter>());
virtual ~GZipDecodingStreamFilter();
virtual void init();
virtual ssize_t transform(const SharedHandle<BinaryStream>& out,
const SharedHandle<Segment>& segment,
const unsigned char* inbuf, size_t inlen);
virtual bool finished();
virtual void release();
virtual const std::string& getName() const;
virtual size_t getBytesProcessed() const
{
return bytesProcessed_;
}
static const std::string NAME;
};
} // namespace aria2
#endif // D_GZIP_STREAM_FILTER_H

2
src/HttpConnection.cc
View File

@ -134,7 +134,7 @@ SharedHandle<HttpResponse> HttpConnection::receiveResponse()
if(socket_->wantRead() || socket_->wantWrite()) {
return SharedHandle<HttpResponse>();
} else {
throw DL_RETRY_EX(EX_INVALID_RESPONSE);
throw DL_RETRY_EX(EX_GOT_EOF);
}
}
proc->update(buf, size);

24
src/HttpDownloadCommand.cc
View File

@ -47,12 +47,15 @@
#include "HttpHeader.h"
#include "Range.h"
#include "DownloadContext.h"
#include "Decoder.h"
#include "RequestGroupMan.h"
#include "FileAllocationEntry.h"
#include "CheckIntegrityEntry.h"
#include "ServerStatMan.h"
#include "Logger.h"
#include "StreamFilter.h"
#include "SinkStreamFilter.h"
#include "util.h"
namespace aria2 {
HttpDownloadCommand::HttpDownloadCommand
@ -88,18 +91,16 @@ bool HttpDownloadCommand::prepareForNextSegment() {
if(getRequest()->isPipeliningEnabled() ||
(getRequest()->isKeepAliveEnabled() &&
(
// Make sure that all decoders are finished to pool socket
((!getTransferEncodingDecoder().isNull() &&
getTransferEncodingDecoder()->finished()) ||
(getTransferEncodingDecoder().isNull() &&
!getContentEncodingDecoder().isNull() &&
getContentEncodingDecoder()->finished())) ||
// Make sure that all filters are finished to pool socket
(!util::endsWith(getStreamFilter()->getName(),
SinkStreamFilter::NAME) &&
getStreamFilter()->finished()) ||
getRequestEndOffset() ==
getFileEntry()->gtoloff(getSegments().front()->getPositionToWrite())
)
)
) {
// TODO What if server sends EOF when _contentEncodingDecoder is
// TODO What if server sends EOF when non-SinkStreamFilter is
// used and server didn't send Connection: close? We end up to
// pool terminated socket. In HTTP/1.1, keep-alive is default,
// so closing connection without Connection: close header means
@ -133,7 +134,12 @@ bool HttpDownloadCommand::prepareForNextSegment() {
off_t HttpDownloadCommand::getRequestEndOffset() const
{
return httpResponse_->getHttpHeader()->getRange()->getEndByte()+1;
off_t endByte = httpResponse_->getHttpHeader()->getRange()->getEndByte();
if(endByte > 0) {
return endByte+1;
} else {
return endByte;
}
}
} // namespace aria2

25
src/HttpResponse.cc
View File

@ -46,14 +46,13 @@
#include "DlRetryEx.h"
#include "StringFormat.h"
#include "A2STR.h"
#include "Decoder.h"
#include "ChunkedDecoder.h"
#ifdef HAVE_LIBZ
# include "GZipDecoder.h"
#endif // HAVE_LIBZ
#include "CookieStorage.h"
#include "AuthConfigFactory.h"
#include "AuthConfig.h"
#include "ChunkedDecodingStreamFilter.h"
#ifdef HAVE_LIBZ
# include "GZipDecodingStreamFilter.h"
#endif // HAVE_LIBZ
namespace aria2 {
@ -177,20 +176,21 @@ bool HttpResponse::isTransferEncodingSpecified() const
std::string HttpResponse::getTransferEncoding() const
{
// TODO See TODO in getTransferEncodingDecoder()
// TODO See TODO in getTransferEncodingStreamFilter()
return httpHeader_->getFirst(HttpHeader::TRANSFER_ENCODING);
}
SharedHandle<Decoder> HttpResponse::getTransferEncodingDecoder() const
SharedHandle<StreamFilter> HttpResponse::getTransferEncodingStreamFilter() const
{
SharedHandle<StreamFilter> filter;
// TODO Transfer-Encoding header field can contains multiple tokens. We should
// parse the field and retrieve each token.
if(isTransferEncodingSpecified()) {
if(getTransferEncoding() == HttpHeader::CHUNKED) {
return SharedHandle<Decoder>(new ChunkedDecoder());
filter.reset(new ChunkedDecodingStreamFilter());
}
}
return SharedHandle<Decoder>();
return filter;
}
bool HttpResponse::isContentEncodingSpecified() const
@ -203,15 +203,16 @@ const std::string& HttpResponse::getContentEncoding() const
return httpHeader_->getFirst(HttpHeader::CONTENT_ENCODING);
}
SharedHandle<Decoder> HttpResponse::getContentEncodingDecoder() const
SharedHandle<StreamFilter> HttpResponse::getContentEncodingStreamFilter() const
{
SharedHandle<StreamFilter> filter;
#ifdef HAVE_LIBZ
if(getContentEncoding() == HttpHeader::GZIP ||
getContentEncoding() == HttpHeader::DEFLATE) {
return SharedHandle<Decoder>(new GZipDecoder());
filter.reset(new GZipDecodingStreamFilter());
}
#endif // HAVE_LIBZ
return SharedHandle<Decoder>();
return filter;
}
uint64_t HttpResponse::getContentLength() const

6
src/HttpResponse.h
View File

@ -48,7 +48,7 @@ namespace aria2 {
class HttpRequest;
class HttpHeader;
class Logger;
class Decoder;
class StreamFilter;
class HttpResponse {
private:
@ -86,13 +86,13 @@ public:
std::string getTransferEncoding() const;
SharedHandle<Decoder> getTransferEncodingDecoder() const;
SharedHandle<StreamFilter> getTransferEncodingStreamFilter() const;
bool isContentEncodingSpecified() const;
const std::string& getContentEncoding() const;
SharedHandle<Decoder> getContentEncodingDecoder() const;
SharedHandle<StreamFilter> getContentEncodingStreamFilter() const;
uint64_t getContentLength() const;

120
src/HttpResponseCommand.cc
View File

@ -69,14 +69,20 @@
#include "ServerStatMan.h"
#include "FileAllocationEntry.h"
#include "CheckIntegrityEntry.h"
#include "StreamFilter.h"
#include "SinkStreamFilter.h"
#include "ChunkedDecodingStreamFilter.h"
#include "GZipDecodingStreamFilter.h"
namespace aria2 {
static SharedHandle<Decoder> getTransferEncodingDecoder
(const SharedHandle<HttpResponse>& httpResponse);
static SharedHandle<StreamFilter> getTransferEncodingStreamFilter
(const SharedHandle<HttpResponse>& httpResponse,
const SharedHandle<StreamFilter>& delegate = SharedHandle<StreamFilter>());
static SharedHandle<Decoder> getContentEncodingDecoder
(const SharedHandle<HttpResponse>& httpResponse);
static SharedHandle<StreamFilter> getContentEncodingStreamFilter
(const SharedHandle<HttpResponse>& httpResponse,
const SharedHandle<StreamFilter>& delegate = SharedHandle<StreamFilter>());
HttpResponseCommand::HttpResponseCommand
(cuid_t cuid,
@ -198,12 +204,16 @@ bool HttpResponseCommand::executeInternal()
// anyway.
getPieceStorage()->getDiskAdaptor()->truncate(0);
getDownloadEngine()->addCommand
(createHttpDownloadCommand(httpResponse,
getTransferEncodingDecoder(httpResponse),
getContentEncodingDecoder(httpResponse)));
(createHttpDownloadCommand
(httpResponse,
getTransferEncodingStreamFilter
(httpResponse,
getContentEncodingStreamFilter(httpResponse))));
} else {
getDownloadEngine()->addCommand(createHttpDownloadCommand
(httpResponse, getTransferEncodingDecoder(httpResponse)));
getDownloadEngine()->addCommand
(createHttpDownloadCommand
(httpResponse,
getTransferEncodingStreamFilter(httpResponse)));
}
return true;
}
@ -275,7 +285,8 @@ bool HttpResponseCommand::handleDefaultEncoding
!segment.isNull() && segment->getPositionToWrite() == 0 &&
!getRequest()->isPipeliningEnabled()) {
command = createHttpDownloadCommand
(httpResponse, getTransferEncodingDecoder(httpResponse));
(httpResponse,
getTransferEncodingStreamFilter(httpResponse));
} else {
getSegmentMan()->cancelSegment(getCuid());
getFileEntry()->poolRequest(getRequest());
@ -291,39 +302,49 @@ bool HttpResponseCommand::handleDefaultEncoding
return true;
}
static SharedHandle<Decoder> getTransferEncodingDecoder
(const SharedHandle<HttpResponse>& httpResponse)
static SharedHandle<StreamFilter> getTransferEncodingStreamFilter
(const SharedHandle<HttpResponse>& httpResponse,
const SharedHandle<StreamFilter>& delegate)
{
SharedHandle<Decoder> decoder;
SharedHandle<StreamFilter> filter;
if(httpResponse->isTransferEncodingSpecified()) {
decoder = httpResponse->getTransferEncodingDecoder();
if(decoder.isNull()) {
filter = httpResponse->getTransferEncodingStreamFilter();
if(filter.isNull()) {
throw DL_ABORT_EX
(StringFormat(EX_TRANSFER_ENCODING_NOT_SUPPORTED,
httpResponse->getTransferEncoding().c_str()).str());
}
decoder->init();
filter->init();
filter->installDelegate(delegate);
}
return decoder;
if(filter.isNull()) {
filter = delegate;
}
return filter;
}
static SharedHandle<Decoder> getContentEncodingDecoder
(const SharedHandle<HttpResponse>& httpResponse)
static SharedHandle<StreamFilter> getContentEncodingStreamFilter
(const SharedHandle<HttpResponse>& httpResponse,
const SharedHandle<StreamFilter>& delegate)
{
SharedHandle<Decoder> decoder;
SharedHandle<StreamFilter> filter;
if(httpResponse->isContentEncodingSpecified()) {
decoder = httpResponse->getContentEncodingDecoder();
if(decoder.isNull()) {
filter = httpResponse->getContentEncodingStreamFilter();
if(filter.isNull()) {
LogFactory::getInstance()->info
("Content-Encoding %s is specified, but the current implementation"
"doesn't support it. The decoding process is skipped and the"
"downloaded content will be still encoded.",
httpResponse->getContentEncoding().c_str());
} else {
decoder->init();
filter->init();
filter->installDelegate(delegate);
}
}
return decoder;
if(filter.isNull()) {
filter = delegate;
}
return filter;
}
bool HttpResponseCommand::handleOtherEncoding
@ -357,9 +378,20 @@ bool HttpResponseCommand::handleOtherEncoding
getRequestGroup()->shouldCancelDownloadForSafety();
getRequestGroup()->initPieceStorage();
getPieceStorage()->getDiskAdaptor()->initAndOpenFile();
SharedHandle<StreamFilter> streamFilter =
getTransferEncodingStreamFilter
(httpResponse,
getContentEncodingStreamFilter(httpResponse));
// In this context, knowsTotalLength() is true only when the file is
// really zero-length.
if(getDownloadContext()->knowsTotalLength()) {
if(getDownloadContext()->knowsTotalLength() &&
(streamFilter.isNull() ||
streamFilter->getName() != ChunkedDecodingStreamFilter::NAME)) {
// If chunked transfer-encoding is specified, we have to read end
// of chunk markers(0\r\n\r\n, for example), so cannot pool
// connection here.
poolConnection();
return true;
}
@ -369,16 +401,15 @@ bool HttpResponseCommand::handleOtherEncoding
getSegmentMan()->getSegmentWithIndex(getCuid(), 0);
getDownloadEngine()->addCommand
(createHttpDownloadCommand(httpResponse,
getTransferEncodingDecoder(httpResponse),
getContentEncodingDecoder(httpResponse)));
(createHttpDownloadCommand(httpResponse, streamFilter));
return true;
}
bool HttpResponseCommand::skipResponseBody
(const SharedHandle<HttpResponse>& httpResponse)
{
SharedHandle<Decoder> decoder = getTransferEncodingDecoder(httpResponse);
SharedHandle<StreamFilter> filter =
getTransferEncodingStreamFilter(httpResponse);
// We don't use Content-Encoding here because this response body is just
// thrown away.
@ -386,7 +417,7 @@ bool HttpResponseCommand::skipResponseBody
(getCuid(), getRequest(), getFileEntry(), getRequestGroup(),
httpConnection_, httpResponse,
getDownloadEngine(), getSocket());
command->setTransferEncodingDecoder(decoder);
command->installStreamFilter(filter);
// If request method is HEAD or the response body is zero-length,
// set command's status to real time so that avoid read check blocking
@ -403,10 +434,23 @@ bool HttpResponseCommand::skipResponseBody
return true;
}
static bool decideFileAllocation
(const SharedHandle<StreamFilter>& filter)
{
for(SharedHandle<StreamFilter> f = filter; !f.isNull(); f = f->getDelegate()){
// Since the compressed file's length are returned in the response header
// and the decompressed file size is unknown at this point, disable file
// allocation here.
if(f->getName() == GZipDecodingStreamFilter::NAME) {
return false;
}
}
return true;
}
HttpDownloadCommand* HttpResponseCommand::createHttpDownloadCommand
(const SharedHandle<HttpResponse>& httpResponse,
const SharedHandle<Decoder>& transferEncodingDecoder,
const SharedHandle<Decoder>& contentEncodingDecoder)
const SharedHandle<StreamFilter>& filter)
{
HttpDownloadCommand* command =
@ -417,16 +461,8 @@ HttpDownloadCommand* HttpResponseCommand::createHttpDownloadCommand
command->setStartupIdleTime(getOption()->getAsInt(PREF_STARTUP_IDLE_TIME));
command->setLowestDownloadSpeedLimit
(getOption()->getAsInt(PREF_LOWEST_SPEED_LIMIT));
command->setTransferEncodingDecoder(transferEncodingDecoder);
if(!contentEncodingDecoder.isNull()) {
command->setContentEncodingDecoder(contentEncodingDecoder);
// Since the compressed file's length are returned in the response header
// and the decompressed file size is unknown at this point, disable file
// allocation here.
getRequestGroup()->setFileAllocationEnabled(false);
}
command->installStreamFilter(filter);
getRequestGroup()->setFileAllocationEnabled(decideFileAllocation(filter));
getRequestGroup()->getURISelector()->tuneDownloadCommand
(getFileEntry()->getRemainingUris(), command);

10
src/HttpResponseCommand.h
View File

@ -36,7 +36,6 @@
#define _D_HTTP_RESPONSE_COMMAND_H_
#include "AbstractCommand.h"
#include "Decoder.h"
#include "TimeA2.h"
namespace aria2 {
@ -45,6 +44,7 @@ class HttpConnection;
class HttpDownloadCommand;
class HttpResponse;
class SocketCore;
class StreamFilter;
class HttpResponseCommand : public AbstractCommand {
private:
@ -55,11 +55,9 @@ private:
bool skipResponseBody(const SharedHandle<HttpResponse>& httpResponse);
HttpDownloadCommand*
createHttpDownloadCommand(const SharedHandle<HttpResponse>& httpResponse,
const SharedHandle<Decoder>& transferEncodingDecoder
= SharedHandle<Decoder>(),
const SharedHandle<Decoder>& contentEncodingDecoder
= SharedHandle<Decoder>());
createHttpDownloadCommand
(const SharedHandle<HttpResponse>& httpResponse,
const SharedHandle<StreamFilter>& streamFilter);
void updateLastModifiedTime(const Time& lastModified);

45
src/HttpSkipResponseCommand.cc
View File

@ -37,7 +37,6 @@
#include "HttpResponse.h"
#include "message.h"
#include "SocketCore.h"
#include "Decoder.h"
#include "DlRetryEx.h"
#include "Request.h"
#include "DownloadEngine.h"
@ -58,6 +57,10 @@
#include "FileAllocationEntry.h"
#include "CheckIntegrityEntry.h"
#include "ServerStatMan.h"
#include "StreamFilter.h"
#include "BinaryStream.h"
#include "NullSinkStreamFilter.h"
#include "SinkStreamFilter.h"
namespace aria2 {
@ -73,22 +76,30 @@ HttpSkipResponseCommand::HttpSkipResponseCommand
AbstractCommand(cuid, req, fileEntry, requestGroup, e, s),
httpConnection_(httpConnection),
httpResponse_(httpResponse),
streamFilter_(new NullSinkStreamFilter()),
sinkFilterOnly_(true),
totalLength_(httpResponse_->getEntityLength()),
receivedBytes_(0)
{}
HttpSkipResponseCommand::~HttpSkipResponseCommand() {}
void HttpSkipResponseCommand::setTransferEncodingDecoder
(const SharedHandle<Decoder>& decoder)
void HttpSkipResponseCommand::installStreamFilter
(const SharedHandle<StreamFilter>& streamFilter)
{
transferEncodingDecoder_ = decoder;
if(streamFilter.isNull()) {
return;
}
streamFilter->installDelegate(streamFilter_);
streamFilter_ = streamFilter;
sinkFilterOnly_ =
util::endsWith(streamFilter_->getName(), SinkStreamFilter::NAME);
}
bool HttpSkipResponseCommand::executeInternal()
{
if(getRequest()->getMethod() == Request::METHOD_HEAD ||
(totalLength_ == 0 && transferEncodingDecoder_.isNull())) {
(totalLength_ == 0 && sinkFilterOnly_)) {
// If request method is HEAD or content-length header is present and
// it's value is 0, then pool socket for reuse.
// If content-length header is not present, then EOF is expected in the end.
@ -101,17 +112,25 @@ bool HttpSkipResponseCommand::executeInternal()
}
const size_t BUFSIZE = 16*1024;
unsigned char buf[BUFSIZE];
size_t bufSize = BUFSIZE;
size_t bufSize;
if(sinkFilterOnly_ && totalLength_ > 0) {
bufSize = totalLength_-receivedBytes_;
} else {
bufSize = BUFSIZE;
}
try {
getSocket()->readData(buf, bufSize);
if(transferEncodingDecoder_.isNull()) {
if(sinkFilterOnly_) {
getSocket()->readData(buf, bufSize);
receivedBytes_ += bufSize;
} else {
getSocket()->peekData(buf, bufSize);
// receivedBytes_ is not updated if transferEncoding is set.
// The return value is safely ignored here.
transferEncodingDecoder_->decode(buf, bufSize);
streamFilter_->transform(SharedHandle<BinaryStream>(),
SharedHandle<Segment>(),
buf, bufSize);
bufSize = streamFilter_->getBytesProcessed();
getSocket()->readData(buf, bufSize);
}
if(totalLength_ != 0 && bufSize == 0 &&
!getSocket()->wantRead() && !getSocket()->wantWrite()) {
@ -125,7 +144,7 @@ bool HttpSkipResponseCommand::executeInternal()
}
bool finished = false;
if(transferEncodingDecoder_.isNull()) {
if(sinkFilterOnly_) {
if(bufSize == 0) {
if(!getSocket()->wantRead() && !getSocket()->wantWrite()) {
return processResponse();
@ -134,7 +153,7 @@ bool HttpSkipResponseCommand::executeInternal()
finished = (totalLength_ == receivedBytes_);
}
} else {
finished = transferEncodingDecoder_->finished();
finished = streamFilter_->finished();
}
if(finished) {
poolConnection();

8
src/HttpSkipResponseCommand.h
View File

@ -41,7 +41,7 @@ namespace aria2 {
class HttpConnection;
class HttpResponse;
class Decoder;
class StreamFilter;
class HttpSkipResponseCommand : public AbstractCommand {
private:
@ -49,7 +49,9 @@ private:
SharedHandle<HttpResponse> httpResponse_;
SharedHandle<Decoder> transferEncodingDecoder_;
SharedHandle<StreamFilter> streamFilter_;
bool sinkFilterOnly_;
uint64_t totalLength_;
@ -72,7 +74,7 @@ public:
virtual ~HttpSkipResponseCommand();
void setTransferEncodingDecoder(const SharedHandle<Decoder>& decoder);
void installStreamFilter(const SharedHandle<StreamFilter>& streamFilter);
void disableSocketCheck();
};

7
src/Makefile.am
View File

@ -206,7 +206,12 @@ SRCS = Socket.h\
ValueBase.cc ValueBase.h\
ContextAttribute.h\
TorrentAttribute.h\
AdaptiveFileAllocationIterator.cc AdaptiveFileAllocationIterator.h
AdaptiveFileAllocationIterator.cc AdaptiveFileAllocationIterator.h\
StreamFilter.cc StreamFilter.h\
SinkStreamFilter.cc SinkStreamFilter.h\
GZipDecodingStreamFilter.cc GZipDecodingStreamFilter.h\
ChunkedDecodingStreamFilter.cc ChunkedDecodingStreamFilter.h\
NullSinkStreamFilter.cc NullSinkStreamFilter.h
if ENABLE_XML_RPC
SRCS += XmlRpcRequestParserController.cc XmlRpcRequestParserController.h\

42
src/Makefile.in
View File

@ -442,7 +442,11 @@ am__libaria2c_a_SOURCES_DIST = Socket.h SocketCore.cc SocketCore.h \
MetadataInfo.h SessionSerializer.cc SessionSerializer.h \
Event.h timespec.h ValueBase.cc ValueBase.h ContextAttribute.h \
TorrentAttribute.h AdaptiveFileAllocationIterator.cc \
AdaptiveFileAllocationIterator.h \
AdaptiveFileAllocationIterator.h StreamFilter.cc \
StreamFilter.h SinkStreamFilter.cc SinkStreamFilter.h \
GZipDecodingStreamFilter.cc GZipDecodingStreamFilter.h \
ChunkedDecodingStreamFilter.cc ChunkedDecodingStreamFilter.h \
NullSinkStreamFilter.cc NullSinkStreamFilter.h \
XmlRpcRequestParserController.cc \
XmlRpcRequestParserController.h \
XmlRpcRequestParserStateMachine.cc \
@ -883,17 +887,20 @@ am__objects_32 = SocketCore.$(OBJEXT) Command.$(OBJEXT) \
CreateRequestCommand.$(OBJEXT) download_helper.$(OBJEXT) \
MetadataInfo.$(OBJEXT) SessionSerializer.$(OBJEXT) \
ValueBase.$(OBJEXT) AdaptiveFileAllocationIterator.$(OBJEXT) \
$(am__objects_1) $(am__objects_2) $(am__objects_3) \
$(am__objects_4) $(am__objects_5) $(am__objects_6) \
$(am__objects_7) $(am__objects_8) $(am__objects_9) \
$(am__objects_10) $(am__objects_11) $(am__objects_12) \
$(am__objects_13) $(am__objects_14) $(am__objects_15) \
$(am__objects_16) $(am__objects_17) $(am__objects_18) \
$(am__objects_19) $(am__objects_20) $(am__objects_21) \
$(am__objects_22) $(am__objects_23) $(am__objects_24) \
$(am__objects_25) $(am__objects_26) $(am__objects_27) \
$(am__objects_28) $(am__objects_29) $(am__objects_30) \
$(am__objects_31)
StreamFilter.$(OBJEXT) SinkStreamFilter.$(OBJEXT) \
GZipDecodingStreamFilter.$(OBJEXT) \
ChunkedDecodingStreamFilter.$(OBJEXT) \
NullSinkStreamFilter.$(OBJEXT) $(am__objects_1) \
$(am__objects_2) $(am__objects_3) $(am__objects_4) \
$(am__objects_5) $(am__objects_6) $(am__objects_7) \
$(am__objects_8) $(am__objects_9) $(am__objects_10) \
$(am__objects_11) $(am__objects_12) $(am__objects_13) \
$(am__objects_14) $(am__objects_15) $(am__objects_16) \
$(am__objects_17) $(am__objects_18) $(am__objects_19) \
$(am__objects_20) $(am__objects_21) $(am__objects_22) \
$(am__objects_23) $(am__objects_24) $(am__objects_25) \
$(am__objects_26) $(am__objects_27) $(am__objects_28) \
$(am__objects_29) $(am__objects_30) $(am__objects_31)
am_libaria2c_a_OBJECTS = $(am__objects_32)
libaria2c_a_OBJECTS = $(am_libaria2c_a_OBJECTS)
am__installdirs = "$(DESTDIR)$(bindir)"
@ -1227,7 +1234,11 @@ SRCS = Socket.h SocketCore.cc SocketCore.h BinaryStream.h Command.cc \
MetadataInfo.h SessionSerializer.cc SessionSerializer.h \
Event.h timespec.h ValueBase.cc ValueBase.h ContextAttribute.h \
TorrentAttribute.h AdaptiveFileAllocationIterator.cc \
AdaptiveFileAllocationIterator.h $(am__append_1) \
AdaptiveFileAllocationIterator.h StreamFilter.cc \
StreamFilter.h SinkStreamFilter.cc SinkStreamFilter.h \
GZipDecodingStreamFilter.cc GZipDecodingStreamFilter.h \
ChunkedDecodingStreamFilter.cc ChunkedDecodingStreamFilter.h \
NullSinkStreamFilter.cc NullSinkStreamFilter.h $(am__append_1) \
$(am__append_2) $(am__append_3) $(am__append_4) \
$(am__append_5) $(am__append_6) $(am__append_7) \
$(am__append_8) $(am__append_9) $(am__append_10) \
@ -1396,6 +1407,7 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/CheckIntegrityEntry.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ChecksumCheckIntegrityEntry.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ChunkedDecoder.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ChunkedDecodingStreamFilter.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Command.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ConsoleStatCalc.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ContentTypeRequestGroupCriteria.Po@am__quote@
@ -1498,6 +1510,7 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FtpTunnelRequestCommand.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FtpTunnelResponseCommand.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/GZipDecoder.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/GZipDecodingStreamFilter.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/GZipEncoder.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/GrowSegment.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/HandshakeExtensionMessage.Po@am__quote@
@ -1559,6 +1572,7 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Netrc.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/NetrcAuthResolver.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/NsCookieParser.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/NullSinkStreamFilter.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Option.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/OptionHandler.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/OptionHandlerException.Po@am__quote@
@ -1606,6 +1620,7 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/SimpleLogFormatter.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/SimpleRandomizer.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/SingleFileAllocationIterator.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/SinkStreamFilter.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/SleepCommand.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/SocketBuffer.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/SocketCore.Po@am__quote@
@ -1614,6 +1629,7 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Sqlite3CookieParserImpl.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/StreamCheckIntegrityEntry.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/StreamFileAllocationEntry.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/StreamFilter.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/StringFormat.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/TimeA2.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/TimeBasedCommand.Po@am__quote@

41
src/NullSinkStreamFilter.cc Normal file
View File

@ -0,0 +1,41 @@
/* <!-- copyright */
/*
* aria2 - The high speed download utility
*
* Copyright (C) 2010 Tatsuhiro Tsujikawa
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*
* In addition, as a special exception, the copyright holders give
* permission to link the code of portions of this program with the
* OpenSSL library under certain conditions as described in each
* individual source file, and distribute linked combinations
* including the two.
* You must obey the GNU General Public License in all respects
* for all of the code used other than OpenSSL. If you modify
* file(s) with this exception, you may extend this exception to your
* version of the file(s), but you are not obligated to do so. If you
* do not wish to do so, delete this exception statement from your
* version. If you delete this exception statement from all source
* files in the program, then also delete it here.
*/
/* copyright --> */
#include "NullSinkStreamFilter.h"
namespace aria2 {
const std::string NullSinkStreamFilter::NAME("NullSinkStreamFilter");
} // namespace aria2

86
src/NullSinkStreamFilter.h Normal file
View File

@ -0,0 +1,86 @@
/* <!-- copyright */
/*
* aria2 - The high speed download utility
*
* Copyright (C) 2010 Tatsuhiro Tsujikawa
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*
* In addition, as a special exception, the copyright holders give
* permission to link the code of portions of this program with the
* OpenSSL library under certain conditions as described in each
* individual source file, and distribute linked combinations
* including the two.
* You must obey the GNU General Public License in all respects
* for all of the code used other than OpenSSL. If you modify
* file(s) with this exception, you may extend this exception to your
* version of the file(s), but you are not obligated to do so. If you
* do not wish to do so, delete this exception statement from your
* version. If you delete this exception statement from all source
* files in the program, then also delete it here.
*/
/* copyright --> */
#ifndef D_NULL_SINK_STREAM_FILTER_H
#define D_NULL_SINK_STREAM_FILTER_H
#include "StreamFilter.h"
namespace aria2 {
class NullSinkStreamFilter:public StreamFilter {
private:
size_t bytesProcessed_;
public:
NullSinkStreamFilter():bytesProcessed_(0) {}
virtual void init() {}
virtual ssize_t transform
(const SharedHandle<BinaryStream>& out,
const SharedHandle<Segment>& segment,
const unsigned char* inbuf, size_t inlen)
{
bytesProcessed_ = inlen;
return bytesProcessed_;
}
virtual bool finished()
{
return true;
}
virtual void release() {}
virtual const std::string& getName() const
{
return NAME;
}
static const std::string NAME;
virtual size_t getBytesProcessed() const
{
return bytesProcessed_;
}
virtual bool installDelegate(const SharedHandle<StreamFilter>& filter)
{
return false;
}
};
} // namespace aria2
#endif // D_NULL_SINK_STREAM_FILTER_H

4
src/RequestGroup.cc
View File

@ -525,7 +525,9 @@ void RequestGroup::processCheckIntegrityEntry
void RequestGroup::initPieceStorage()
{
SharedHandle<PieceStorage> tempPieceStorage;
if(downloadContext_->knowsTotalLength()) {
if(downloadContext_->knowsTotalLength() &&
(downloadContext_->getTotalLength() > 0 ||
downloadContext_->hasAttribute(bittorrent::BITTORRENT))) {
#ifdef ENABLE_BITTORRENT
SharedHandle<DefaultPieceStorage> ps
(new DefaultPieceStorage(downloadContext_, option_.get()));

65
src/SinkStreamFilter.cc Normal file
View File

@ -0,0 +1,65 @@
/* <!-- copyright */
/*
* aria2 - The high speed download utility
*
* Copyright (C) 2010 Tatsuhiro Tsujikawa
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*
* In addition, as a special exception, the copyright holders give
* permission to link the code of portions of this program with the
* OpenSSL library under certain conditions as described in each
* individual source file, and distribute linked combinations
* including the two.
* You must obey the GNU General Public License in all respects
* for all of the code used other than OpenSSL. If you modify
* file(s) with this exception, you may extend this exception to your
* version of the file(s), but you are not obligated to do so. If you
* do not wish to do so, delete this exception statement from your
* version. If you delete this exception statement from all source
* files in the program, then also delete it here.
*/
/* copyright --> */
#include "SinkStreamFilter.h"
#include "BinaryStream.h"
#include "Segment.h"
namespace aria2 {
const std::string SinkStreamFilter::NAME("SinkStreamFilter");
SinkStreamFilter::SinkStreamFilter(bool hashUpdate):
hashUpdate_(hashUpdate),
bytesProcessed_(0) {}
ssize_t SinkStreamFilter::transform
(const SharedHandle<BinaryStream>& out,
const SharedHandle<Segment>& segment,
const unsigned char* inbuf, size_t inlen)
{
if(inlen > 0) {
out->writeData(inbuf, inlen, segment->getPositionToWrite());
#ifdef ENABLE_MESSAGE_DIGEST
if(hashUpdate_) {
segment->updateHash(segment->getWrittenLength(), inbuf, inlen);
}
#endif // ENABLE_MESSAGE_DIGEST
segment->updateWrittenLength(inlen);
}
bytesProcessed_ = inlen;
return bytesProcessed_;
}
} // namespace aria2

84
src/SinkStreamFilter.h Normal file
View File

@ -0,0 +1,84 @@
/* <!-- copyright */
/*
* aria2 - The high speed download utility
*
* Copyright (C) 2010 Tatsuhiro Tsujikawa
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*
* In addition, as a special exception, the copyright holders give
* permission to link the code of portions of this program with the
* OpenSSL library under certain conditions as described in each
* individual source file, and distribute linked combinations
* including the two.
* You must obey the GNU General Public License in all respects
* for all of the code used other than OpenSSL. If you modify
* file(s) with this exception, you may extend this exception to your
* version of the file(s), but you are not obligated to do so. If you
* do not wish to do so, delete this exception statement from your
* version. If you delete this exception statement from all source
* files in the program, then also delete it here.
*/
/* copyright --> */
#ifndef D_SINK_STREAM_FILTER_H
#define D_SINK_STREAM_FILTER_H
#include "StreamFilter.h"
namespace aria2 {
class SinkStreamFilter:public StreamFilter {
private:
bool hashUpdate_;
size_t bytesProcessed_;
public:
SinkStreamFilter(bool hashUpdate = false);
virtual void init() {}
virtual ssize_t transform
(const SharedHandle<BinaryStream>& out,
const SharedHandle<Segment>& segment,
const unsigned char* inbuf, size_t inlen);
virtual bool finished()
{
return true;
}
virtual void release() {}
virtual const std::string& getName() const
{
return NAME;
}
static const std::string NAME;
virtual size_t getBytesProcessed() const
{
return bytesProcessed_;
}
virtual bool installDelegate(const SharedHandle<StreamFilter>& filter)
{
return false;
}
};
} // namespace aria2
#endif // D_SINK_STREAM_FILTER_H

53
src/StreamFilter.cc Normal file
View File

@ -0,0 +1,53 @@
/* <!-- copyright */
/*
* aria2 - The high speed download utility
*
* Copyright (C) 2010 Tatsuhiro Tsujikawa
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*
* In addition, as a special exception, the copyright holders give
* permission to link the code of portions of this program with the
* OpenSSL library under certain conditions as described in each
* individual source file, and distribute linked combinations
* including the two.
* You must obey the GNU General Public License in all respects
* for all of the code used other than OpenSSL. If you modify
* file(s) with this exception, you may extend this exception to your
* version of the file(s), but you are not obligated to do so. If you
* do not wish to do so, delete this exception statement from your
* version. If you delete this exception statement from all source
* files in the program, then also delete it here.
*/
/* copyright --> */
#include "StreamFilter.h"
namespace aria2 {
StreamFilter::StreamFilter
(const SharedHandle<StreamFilter>& delegate):
delegate_(delegate) {}
bool StreamFilter::installDelegate(const SharedHandle<StreamFilter>& filter)
{
if(delegate_.isNull()) {
delegate_ = filter;
return true;
} else {
return delegate_->installDelegate(filter);
}
}
} // namespace aria2

84
src/StreamFilter.h Normal file
View File

@ -0,0 +1,84 @@
/* <!-- copyright */
/*
* aria2 - The high speed download utility
*
* Copyright (C) 2010 Tatsuhiro Tsujikawa
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*
* In addition, as a special exception, the copyright holders give
* permission to link the code of portions of this program with the
* OpenSSL library under certain conditions as described in each
* individual source file, and distribute linked combinations
* including the two.
* You must obey the GNU General Public License in all respects
* for all of the code used other than OpenSSL. If you modify
* file(s) with this exception, you may extend this exception to your
* version of the file(s), but you are not obligated to do so. If you
* do not wish to do so, delete this exception statement from your
* version. If you delete this exception statement from all source
* files in the program, then also delete it here.
*/
/* copyright --> */
#ifndef D_STREAM_FILTER_H
#define D_STREAM_FILTER_H
#include "common.h"
#include <string>
#include "SharedHandle.h"
namespace aria2 {
class BinaryStream;
class Segment;
// Interface for basic decoding functionality.
class StreamFilter {
private:
SharedHandle<StreamFilter> delegate_;
public:
StreamFilter
(const SharedHandle<StreamFilter>& delegate = SharedHandle<StreamFilter>());
virtual ~StreamFilter() {}
// init() must be called before calling decode().
virtual void init() = 0;
virtual ssize_t transform(const SharedHandle<BinaryStream>& out,
const SharedHandle<Segment>& segment,
const unsigned char* inbuf, size_t inlen) = 0;
virtual bool finished() = 0;
// The call of release() will free allocated resources.
// After calling release(), the object can be reused by calling init().
virtual void release() = 0;
virtual const std::string& getName() const = 0;
virtual size_t getBytesProcessed() const = 0;
virtual bool installDelegate(const SharedHandle<StreamFilter>& filter);
SharedHandle<StreamFilter> getDelegate() const
{
return delegate_;
}
};
} // namespace aria2
#endif // D_STREAM_FILTER_H

240
test/ChunkedDecodingStreamFilterTest.cc Normal file
View File

@ -0,0 +1,240 @@
#include "ChunkedDecodingStreamFilter.h"
#include <cstdlib>
#include <iostream>
#include <cppunit/extensions/HelperMacros.h>
#include "DlAbortEx.h"
#include "Segment.h"
#include "ByteArrayDiskWriter.h"
#include "SinkStreamFilter.h"
#include "MockSegment.h"
namespace aria2 {
class ChunkedDecodingStreamFilterTest:public CppUnit::TestFixture {
CPPUNIT_TEST_SUITE(ChunkedDecodingStreamFilterTest);
CPPUNIT_TEST(testTransform);
CPPUNIT_TEST(testTransform_withoutTrailer);
CPPUNIT_TEST(testTransform_with2Trailers);
CPPUNIT_TEST(testTransform_largeChunkSize);
CPPUNIT_TEST(testTransform_tooLargeChunkSize);
CPPUNIT_TEST(testTransform_chunkSizeMismatch);
CPPUNIT_TEST(testGetName);
CPPUNIT_TEST_SUITE_END();
SharedHandle<ChunkedDecodingStreamFilter> filter_;
SharedHandle<SinkStreamFilter> sinkFilter_;
SharedHandle<ByteArrayDiskWriter> writer_;
SharedHandle<Segment> segment_;
void clearWriter()
{
writer_->setString("");
}
public:
void setUp()
{
writer_.reset(new ByteArrayDiskWriter());
sinkFilter_.reset(new SinkStreamFilter());
filter_.reset(new ChunkedDecodingStreamFilter(sinkFilter_));
sinkFilter_->init();
filter_->init();
segment_.reset(new MockSegment());
}
void testTransform();
void testTransform_withoutTrailer();
void testTransform_with2Trailers();
void testTransform_largeChunkSize();
void testTransform_tooLargeChunkSize();
void testTransform_chunkSizeMismatch();
void testGetName();
};
CPPUNIT_TEST_SUITE_REGISTRATION( ChunkedDecodingStreamFilterTest );
void ChunkedDecodingStreamFilterTest::testTransform()
{
try {
{
std::basic_string<unsigned char> msg =
reinterpret_cast<const unsigned char*>("a\r\n1234567890\r\n");
ssize_t r = filter_->transform(writer_, segment_, msg.data(), msg.size());
CPPUNIT_ASSERT_EQUAL((ssize_t)10, r);
CPPUNIT_ASSERT_EQUAL(std::string("1234567890"), writer_->getString());
}
clearWriter();
// Feed extension; see it is ignored.
{
std::basic_string<unsigned char> msg =
reinterpret_cast<const unsigned char*>
("3;extensionIgnored\r\n123\r\n");
ssize_t r = filter_->transform(writer_, segment_, msg.data(), msg.size());
CPPUNIT_ASSERT_EQUAL((ssize_t)3, r);
CPPUNIT_ASSERT_EQUAL(std::string("123"), writer_->getString());
}
clearWriter();
// Feed 2extensions; see it is ignored.
{
std::basic_string<unsigned char> msg =
reinterpret_cast<const unsigned char*>
("3;extension1;extension2;\r\n123\r\n");
ssize_t r = filter_->transform(writer_, segment_, msg.data(), msg.size());
CPPUNIT_ASSERT_EQUAL((ssize_t)3, r);
CPPUNIT_ASSERT_EQUAL(std::string("123"), writer_->getString());
}
clearWriter();
// Not all chunk size is available
{
std::basic_string<unsigned char> msg =
reinterpret_cast<const unsigned char*>("1");
ssize_t r = filter_->transform(writer_, segment_, msg.data(), msg.size());
CPPUNIT_ASSERT_EQUAL((ssize_t)0, r);
}
clearWriter();
{
std::basic_string<unsigned char> msg =
reinterpret_cast<const unsigned char*>("0\r\n1234567890123456\r\n");
ssize_t r = filter_->transform(writer_, segment_, msg.data(), msg.size());
CPPUNIT_ASSERT_EQUAL((ssize_t)16, r);
CPPUNIT_ASSERT_EQUAL(std::string("1234567890123456"),
writer_->getString());
}
clearWriter();
// Not all chunk data is available
{
std::basic_string<unsigned char> msg =
reinterpret_cast<const unsigned char*>("10\r\n1234567890");
ssize_t r = filter_->transform(writer_, segment_, msg.data(), msg.size());
CPPUNIT_ASSERT_EQUAL((ssize_t)10, r);
CPPUNIT_ASSERT_EQUAL(std::string("1234567890"), writer_->getString());
}
clearWriter();
{
std::basic_string<unsigned char> msg =
reinterpret_cast<const unsigned char*>("123456\r\n");
ssize_t r = filter_->transform(writer_, segment_, msg.data(), msg.size());
CPPUNIT_ASSERT_EQUAL((ssize_t)6, r);
CPPUNIT_ASSERT_EQUAL(std::string("123456"), writer_->getString());
}
clearWriter();
// no trailing CR LF.
{
std::basic_string<unsigned char> msg =
reinterpret_cast<const unsigned char*>("10\r\n1234567890123456");
ssize_t r = filter_->transform(writer_, segment_, msg.data(), msg.size());
CPPUNIT_ASSERT_EQUAL((ssize_t)16, r);
CPPUNIT_ASSERT_EQUAL(std::string("1234567890123456"),
writer_->getString());
}
clearWriter();
// feed only CR
{
std::basic_string<unsigned char> msg =
reinterpret_cast<const unsigned char*>("\r");
ssize_t r = filter_->transform(writer_, segment_, msg.data(), msg.size());
CPPUNIT_ASSERT_EQUAL((ssize_t)0, r);
}
// feed next LF
{
std::basic_string<unsigned char> msg =
reinterpret_cast<const unsigned char*>("\n");
ssize_t r = filter_->transform(writer_, segment_, msg.data(), msg.size());
CPPUNIT_ASSERT_EQUAL((ssize_t)0, r);
CPPUNIT_ASSERT_EQUAL(std::string(""), writer_->getString());
}
// feed 0 CR LF.
{
std::basic_string<unsigned char> msg =
reinterpret_cast<const unsigned char*>("0\r\n");
ssize_t r = filter_->transform(writer_, segment_, msg.data(), msg.size());
CPPUNIT_ASSERT_EQUAL((ssize_t)0, r);
}
// feed trailer
{
std::basic_string<unsigned char> msg =
reinterpret_cast<const unsigned char*>("trailer\r\n");
ssize_t r = filter_->transform(writer_, segment_, msg.data(), msg.size());
CPPUNIT_ASSERT_EQUAL((ssize_t)0, r);
}
// feed final CRLF
{
std::basic_string<unsigned char> msg =
reinterpret_cast<const unsigned char*>("\r\n");
ssize_t r = filter_->transform(writer_, segment_, msg.data(), msg.size());
CPPUNIT_ASSERT_EQUAL((ssize_t)0, r);
}
// input is over
CPPUNIT_ASSERT(filter_->finished());
} catch(DlAbortEx& e) {
CPPUNIT_FAIL(e.stackTrace());
}
}
void ChunkedDecodingStreamFilterTest::testTransform_withoutTrailer()
{
CPPUNIT_ASSERT_EQUAL
((ssize_t)0,
filter_->transform
(writer_, segment_,
reinterpret_cast<const unsigned char*>("0\r\n\r\n"), 5));
CPPUNIT_ASSERT(filter_->finished());
}
void ChunkedDecodingStreamFilterTest::testTransform_with2Trailers()
{
CPPUNIT_ASSERT_EQUAL
((ssize_t)0,
filter_->transform
(writer_, segment_,
reinterpret_cast<const unsigned char*>("0\r\nt1\r\nt2\r\n\r\n"), 13));
CPPUNIT_ASSERT(filter_->finished());
}
void ChunkedDecodingStreamFilterTest::testTransform_largeChunkSize()
{
// chunkSize should be under 2^64-1
{
std::basic_string<unsigned char> msg =
reinterpret_cast<const unsigned char*>("ffffffffffffffff\r\n");
filter_->transform(writer_, segment_, msg.data(), msg.size());
}
}
void ChunkedDecodingStreamFilterTest::testTransform_tooLargeChunkSize()
{
// chunkSize 2^64 causes error
{
std::basic_string<unsigned char> msg =
reinterpret_cast<const unsigned char*>("10000000000000000\r\n");
try {
filter_->transform(writer_, segment_, msg.data(), msg.size());
CPPUNIT_FAIL("exception must be thrown.");
} catch(DlAbortEx& e) {
// success
}
}
}
void ChunkedDecodingStreamFilterTest::testTransform_chunkSizeMismatch()
{
std::basic_string<unsigned char> msg =
reinterpret_cast<const unsigned char*>("3\r\n1234\r\n");
try {
filter_->transform(writer_, segment_, msg.data(), msg.size());
CPPUNIT_FAIL("exception must be thrown.");
} catch(DlAbortEx& e) {
// success
}
}
void ChunkedDecodingStreamFilterTest::testGetName()
{
CPPUNIT_ASSERT_EQUAL
(std::string("ChunkedDecodingStreamFilter"), filter_->getName());
}
} // namespace aria2

81
test/GZipDecodingStreamFilterTest.cc Normal file
View File

@ -0,0 +1,81 @@
#include "GZipDecodingStreamFilter.h"
#include <cassert>
#include <iostream>
#include <fstream>
#include <cppunit/extensions/HelperMacros.h>
#include "Exception.h"
#include "util.h"
#include "Segment.h"
#include "ByteArrayDiskWriter.h"
#include "SinkStreamFilter.h"
#include "MockSegment.h"
#ifdef ENABLE_MESSAGE_DIGEST
# include "MessageDigestHelper.h"
#endif // ENABLE_MESSAGE_DIGEST
namespace aria2 {
class GZipDecodingStreamFilterTest:public CppUnit::TestFixture {
CPPUNIT_TEST_SUITE(GZipDecodingStreamFilterTest);
CPPUNIT_TEST(testTransform);
CPPUNIT_TEST_SUITE_END();
class MockSegment2:public MockSegment {
private:
off_t positionToWrite_;
public:
MockSegment2():positionToWrite_(0) {}
virtual void updateWrittenLength(size_t bytes)
{
positionToWrite_ += bytes;
}
virtual off_t getPositionToWrite() const
{
return positionToWrite_;
}
};
SharedHandle<GZipDecodingStreamFilter> filter_;
SharedHandle<SinkStreamFilter> sinkFilter_;
SharedHandle<ByteArrayDiskWriter> writer_;
SharedHandle<MockSegment2> segment_;
public:
void setUp()
{
writer_.reset(new ByteArrayDiskWriter());
sinkFilter_.reset(new SinkStreamFilter());
filter_.reset(new GZipDecodingStreamFilter(sinkFilter_));
sinkFilter_->init();
filter_->init();
segment_.reset(new MockSegment2());
}
void testTransform();
};
CPPUNIT_TEST_SUITE_REGISTRATION(GZipDecodingStreamFilterTest);
void GZipDecodingStreamFilterTest::testTransform()
{
unsigned char buf[4096];
std::ifstream in("gzip_decode_test.gz", std::ios::binary);
while(in) {
in.read(reinterpret_cast<char*>(buf), sizeof(buf));
filter_->transform(writer_, segment_, buf, in.gcount());
}
CPPUNIT_ASSERT(filter_->finished());
#ifdef ENABLE_MESSAGE_DIGEST
CPPUNIT_ASSERT_EQUAL(std::string("8b577b33c0411b2be9d4fa74c7402d54a8d21f96"),
MessageDigestHelper::digestString
(MessageDigestContext::SHA1, writer_->getString()));
#endif // ENABLE_MESSAGE_DIGEST
}
} // namespace aria2

41
test/HttpResponseTest.cc
View File

@ -12,11 +12,11 @@
#include "HttpRequest.h"
#include "Exception.h"
#include "A2STR.h"
#include "Decoder.h"
#include "DlRetryEx.h"
#include "CookieStorage.h"
#include "AuthConfigFactory.h"
#include "AuthConfig.h"
#include "StreamFilter.h"
namespace aria2 {
@ -36,10 +36,10 @@ class HttpResponseTest : public CppUnit::TestFixture {
CPPUNIT_TEST(testIsRedirect);
CPPUNIT_TEST(testIsTransferEncodingSpecified);
CPPUNIT_TEST(testGetTransferEncoding);
CPPUNIT_TEST(testGetTransferEncodingDecoder);
CPPUNIT_TEST(testGetTransferEncodingStreamFilter);
CPPUNIT_TEST(testIsContentEncodingSpecified);
CPPUNIT_TEST(testGetContentEncoding);
CPPUNIT_TEST(testGetContentEncodingDecoder);
CPPUNIT_TEST(testGetContentEncodingStreamFilter);
CPPUNIT_TEST(testValidateResponse);
CPPUNIT_TEST(testValidateResponse_good_range);
CPPUNIT_TEST(testValidateResponse_bad_range);
@ -68,10 +68,10 @@ public:
void testIsRedirect();
void testIsTransferEncodingSpecified();
void testGetTransferEncoding();
void testGetTransferEncodingDecoder();
void testGetTransferEncodingStreamFilter();
void testIsContentEncodingSpecified();
void testGetContentEncoding();
void testGetContentEncodingDecoder();
void testGetContentEncodingStreamFilter();
void testValidateResponse();
void testValidateResponse_good_range();
void testValidateResponse_bad_range();
@ -253,18 +253,18 @@ void HttpResponseTest::testGetTransferEncoding()
httpResponse.getTransferEncoding());
}
void HttpResponseTest::testGetTransferEncodingDecoder()
void HttpResponseTest::testGetTransferEncodingStreamFilter()
{
HttpResponse httpResponse;
SharedHandle<HttpHeader> httpHeader(new HttpHeader());
httpResponse.setHttpHeader(httpHeader);
CPPUNIT_ASSERT(httpResponse.getTransferEncodingDecoder().isNull());
CPPUNIT_ASSERT(httpResponse.getTransferEncodingStreamFilter().isNull());
httpHeader->put("Transfer-Encoding", "chunked");
CPPUNIT_ASSERT(!httpResponse.getTransferEncodingDecoder().isNull());
CPPUNIT_ASSERT(!httpResponse.getTransferEncodingStreamFilter().isNull());
}
void HttpResponseTest::testIsContentEncodingSpecified()
@ -295,37 +295,42 @@ void HttpResponseTest::testGetContentEncoding()
CPPUNIT_ASSERT_EQUAL(std::string("gzip"), httpResponse.getContentEncoding());
}
void HttpResponseTest::testGetContentEncodingDecoder()
void HttpResponseTest::testGetContentEncodingStreamFilter()
{
HttpResponse httpResponse;
SharedHandle<HttpHeader> httpHeader(new HttpHeader());
httpResponse.setHttpHeader(httpHeader);
CPPUNIT_ASSERT(httpResponse.getContentEncodingDecoder().isNull());
CPPUNIT_ASSERT(httpResponse.getContentEncodingStreamFilter().isNull());
#ifdef HAVE_LIBZ
httpHeader->put("Content-Encoding", "gzip");
{
SharedHandle<Decoder> decoder = httpResponse.getContentEncodingDecoder();
CPPUNIT_ASSERT(!decoder.isNull());
CPPUNIT_ASSERT_EQUAL(std::string("GZipDecoder"), decoder->getName());
SharedHandle<StreamFilter> filter =
httpResponse.getContentEncodingStreamFilter();
CPPUNIT_ASSERT(!filter.isNull());
CPPUNIT_ASSERT_EQUAL(std::string("GZipDecodingStreamFilter"),
filter->getName());
}
httpHeader.reset(new HttpHeader());
httpResponse.setHttpHeader(httpHeader);
httpHeader->put("Content-Encoding", "deflate");
{
SharedHandle<Decoder> decoder = httpResponse.getContentEncodingDecoder();
CPPUNIT_ASSERT(!decoder.isNull());
CPPUNIT_ASSERT_EQUAL(std::string("GZipDecoder"), decoder->getName());
SharedHandle<StreamFilter> filter =
httpResponse.getContentEncodingStreamFilter();
CPPUNIT_ASSERT(!filter.isNull());
CPPUNIT_ASSERT_EQUAL(std::string("GZipDecodingStreamFilter"),
filter->getName());
}
#endif // HAVE_LIBZ
httpHeader.reset(new HttpHeader());
httpResponse.setHttpHeader(httpHeader);
httpHeader->put("Content-Encoding", "bzip2");
{
SharedHandle<Decoder> decoder = httpResponse.getContentEncodingDecoder();
CPPUNIT_ASSERT(decoder.isNull());
SharedHandle<StreamFilter> filter =
httpResponse.getContentEncodingStreamFilter();
CPPUNIT_ASSERT(filter.isNull());
}
}

4
test/Makefile.am
View File

@ -72,7 +72,9 @@ aria2c_SOURCES = AllTest.cc\
bitfieldTest.cc\
DownloadContextTest.cc\
SessionSerializerTest.cc\
ValueBaseTest.cc
ValueBaseTest.cc\
ChunkedDecodingStreamFilterTest.cc\
GZipDecodingStreamFilterTest.cc
if ENABLE_XML_RPC
aria2c_SOURCES += XmlRpcRequestParserControllerTest.cc\

16
test/Makefile.in
View File

@ -213,6 +213,8 @@ am__aria2c_SOURCES_DIST = AllTest.cc TestUtil.cc TestUtil.h \
InOrderPieceSelector.h LongestSequencePieceSelectorTest.cc \
a2algoTest.cc bitfieldTest.cc DownloadContextTest.cc \
SessionSerializerTest.cc ValueBaseTest.cc \
ChunkedDecodingStreamFilterTest.cc \
GZipDecodingStreamFilterTest.cc \
XmlRpcRequestParserControllerTest.cc \
XmlRpcRequestProcessorTest.cc XmlRpcMethodTest.cc \
FallocFileAllocationIteratorTest.cc GZipDecoderTest.cc \
@ -404,9 +406,11 @@ am_aria2c_OBJECTS = AllTest.$(OBJEXT) TestUtil.$(OBJEXT) \
LongestSequencePieceSelectorTest.$(OBJEXT) \
a2algoTest.$(OBJEXT) bitfieldTest.$(OBJEXT) \
DownloadContextTest.$(OBJEXT) SessionSerializerTest.$(OBJEXT) \
ValueBaseTest.$(OBJEXT) $(am__objects_1) $(am__objects_2) \
$(am__objects_3) $(am__objects_4) $(am__objects_5) \
$(am__objects_6) $(am__objects_7)
ValueBaseTest.$(OBJEXT) \
ChunkedDecodingStreamFilterTest.$(OBJEXT) \
GZipDecodingStreamFilterTest.$(OBJEXT) $(am__objects_1) \
$(am__objects_2) $(am__objects_3) $(am__objects_4) \
$(am__objects_5) $(am__objects_6) $(am__objects_7)
aria2c_OBJECTS = $(am_aria2c_OBJECTS)
am__DEPENDENCIES_1 =
aria2c_DEPENDENCIES = ../src/libaria2c.a $(am__DEPENDENCIES_1)
@ -639,7 +643,9 @@ aria2c_SOURCES = AllTest.cc TestUtil.cc TestUtil.h SocketCoreTest.cc \
RarestPieceSelectorTest.cc PieceStatManTest.cc \
InOrderPieceSelector.h LongestSequencePieceSelectorTest.cc \
a2algoTest.cc bitfieldTest.cc DownloadContextTest.cc \
SessionSerializerTest.cc ValueBaseTest.cc $(am__append_1) \
SessionSerializerTest.cc ValueBaseTest.cc \
ChunkedDecodingStreamFilterTest.cc \
GZipDecodingStreamFilterTest.cc $(am__append_1) \
$(am__append_2) $(am__append_3) $(am__append_4) \
$(am__append_5) $(am__append_6) $(am__append_7)
@ -773,6 +779,7 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/BtUnchokeMessageTest.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ByteArrayDiskWriterTest.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ChunkedDecoderTest.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ChunkedDecodingStreamFilterTest.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/CookieParserTest.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/CookieStorageTest.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/CookieTest.Po@am__quote@
@ -822,6 +829,7 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FileTest.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FtpConnectionTest.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/GZipDecoderTest.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/GZipDecodingStreamFilterTest.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/GZipEncoderTest.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/GrowSegmentTest.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/HandshakeExtensionMessageTest.Po@am__quote@

80
test/MockSegment.h Normal file
View File

@ -0,0 +1,80 @@
#ifndef D_MOCK_SEGMENT_H
#define D_MOCK_SEGMENT_H
#include "Segment.h"
#include "Piece.h"
#include "A2STR.h"
namespace aria2 {
class MockSegment:public Segment {
public:
virtual bool complete() const
{
return false;
}
virtual size_t getIndex() const
{
return 0;
}
virtual off_t getPosition() const
{
return 0;
}
virtual off_t getPositionToWrite() const
{
return 0;
}
virtual size_t getLength() const
{
return 0;
}
virtual size_t getSegmentLength() const
{
return 0;
}
virtual size_t getWrittenLength() const
{
return 0;
}
virtual void updateWrittenLength(size_t bytes) {}
#ifdef ENABLE_MESSAGE_DIGEST
// `begin' is a offset inside this segment.
virtual bool updateHash
(uint32_t begin, const unsigned char* data, size_t dataLength)
{
return false;
}
virtual bool isHashCalculated() const
{
return false;
}
virtual std::string getHashString()
{
return A2STR::NIL;
}
#endif // ENABLE_MESSAGE_DIGEST
virtual void clear() {}
virtual SharedHandle<Piece> getPiece() const
{
return SharedHandle<Piece>();
}
};
} // namespace aria2
#endif // D_MOCK_SEGMENT_H