diff --git a/ChangeLog b/ChangeLog index ab70bf3a..b47d8428 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,7 @@ +2007-05-20 Tatsuhiro Tsujikawa + + * Added the simultaneous download feature. + 2007-04-06 Tatsuhiro Tsujikawa * src/PeerAbstractCommand.cc diff --git a/TODO b/TODO index 711cf693..cc222e7f 100644 --- a/TODO +++ b/TODO @@ -21,3 +21,19 @@ * Fix DefaultBtProgressInfoFile.cc: save(), load() * remove blockIndex * Add seed mode. + +* Add ChecksumCommand +* Add fancy console read out. like this: +100K/300M(10%)(3cn)(3more) 100KB/s [FileAlloc:35MB/40MB(90%)][Checksum:10MB/20MB(50%)] +* exit status: all downloads have been successful-> EXIT_SUCCESS, + some of downloads have been failed -> EXIT_FAILURE +* consider life cycle of requestGroup and segmentMan +* use hintFilename and hintTotalLength if these are provided. + -> test against ftp downloads +* make sure that the same file name is not used at the same time. +* Add a option like "line-feed-readout" which causes readout to be printed with proper line feed. +* Rewrite ByteArrayDiskWriter, TrackerUpdateCommand with stringstream +* Make trakcerwatchercommand and trackerUploadCommand poses requestGroup +* Do not use ufilename in multi-simultaneous download mode. +* Merge umask patch. +https://sourceforge.net/tracker/index.php?func=detail&aid=1718641&group_id=159897&atid=813673 diff --git a/src/AbstractCommand.cc b/src/AbstractCommand.cc index cfbdee81..51051ac8 100644 --- a/src/AbstractCommand.cc +++ b/src/AbstractCommand.cc @@ -41,31 +41,35 @@ #include "SleepCommand.h" #include "prefs.h" #include "DNSCache.h" +#include "FatalException.h" AbstractCommand::AbstractCommand(int cuid, const RequestHandle& req, + RequestGroup* requestGroup, DownloadEngine* e, const SocketHandle& s): - Command(cuid), req(req), e(e), socket(s), + Command(cuid), req(req), _requestGroup(requestGroup), e(e), socket(s), checkSocketIsReadable(false), checkSocketIsWritable(false), nameResolverCheck(false) { setReadCheckSocket(socket); timeout = this->e->option->getAsInt(PREF_TIMEOUT); + ++_requestGroup->numConnection; } AbstractCommand::~AbstractCommand() { disableReadCheckSocket(); disableWriteCheckSocket(); + --_requestGroup->numConnection; } bool AbstractCommand::execute() { try { - if(e->segmentMan->finished()) { + if(_requestGroup->getSegmentMan()->finished()) { logger->debug("CUID#%d - finished.", cuid); return true; } - PeerStatHandle peerStat = e->segmentMan->getPeerStat(cuid); + PeerStatHandle peerStat = _requestGroup->getSegmentMan()->getPeerStat(cuid); if(peerStat.get()) { if(peerStat->getStatus() == PeerStat::REQUEST_IDLE) { logger->info("CUID#%d - Request idle.", cuid); @@ -82,10 +86,10 @@ bool AbstractCommand::execute() { #endif // ENABLE_ASYNC_DNS !checkSocketIsReadable && !checkSocketIsWritable && !nameResolverCheck) { checkPoint.reset(); - if(e->segmentMan->downloadStarted) { + if(_requestGroup->getSegmentMan()->downloadStarted) { // TODO Segment::isNull(), Change method name, it is very confusing. if(segment->isNull()) { - segment = e->segmentMan->getSegment(cuid); + segment = _requestGroup->getSegmentMan()->getSegment(cuid); if(segment.isNull()) { logger->info(MSG_NO_SEGMENT_AVAILABLE, cuid); return prepareForRetry(1); @@ -101,12 +105,19 @@ bool AbstractCommand::execute() { e->commands.push_back(this); return false; } + } catch(FatalException* err) { + logger->error(MSG_DOWNLOAD_ABORTED, err, cuid); + onAbort(err); + delete(err); + req->resetUrl(); + _requestGroup->getSegmentMan()->errors++; + return true; } catch(DlAbortEx* err) { logger->error(MSG_DOWNLOAD_ABORTED, err, cuid); onAbort(err); delete(err); req->resetUrl(); - e->segmentMan->errors++; + _requestGroup->getSegmentMan()->errors++; tryReserved(); return true; } catch(DlRetryEx* err) { @@ -120,7 +131,7 @@ bool AbstractCommand::execute() { delete(err); if(isAbort) { logger->error(MSG_MAX_TRY, cuid, req->getTryCount()); - e->segmentMan->errors++; + _requestGroup->getSegmentMan()->errors++; tryReserved(); return true; } else { @@ -130,17 +141,13 @@ bool AbstractCommand::execute() { } void AbstractCommand::tryReserved() { - if(!e->segmentMan->reserved.empty()) { - RequestHandle req = e->segmentMan->reserved.front(); - e->segmentMan->reserved.pop_front(); - Command* command = InitiateConnectionCommandFactory::createInitiateConnectionCommand(cuid, req, e); - e->commands.push_back(command); - } + Commands commands = _requestGroup->getNextCommand(e, 1); + e->addCommand(commands); } bool AbstractCommand::prepareForRetry(int wait) { - e->segmentMan->cancelSegment(cuid); - Command* command = InitiateConnectionCommandFactory::createInitiateConnectionCommand(cuid, req, e); + _requestGroup->getSegmentMan()->cancelSegment(cuid); + Command* command = InitiateConnectionCommandFactory::createInitiateConnectionCommand(cuid, req, _requestGroup, e); if(wait == 0) { e->commands.push_back(command); } else { @@ -150,10 +157,10 @@ bool AbstractCommand::prepareForRetry(int wait) { return true; } -void AbstractCommand::onAbort(RecoverableException* ex) { +void AbstractCommand::onAbort(Exception* ex) { logger->debug(MSG_UNREGISTER_CUID, cuid); - //e->segmentMan->unregisterId(cuid); - e->segmentMan->cancelSegment(cuid); + //_segmentMan->unregisterId(cuid); + _requestGroup->getSegmentMan()->cancelSegment(cuid); } void AbstractCommand::disableReadCheckSocket() { diff --git a/src/AbstractCommand.h b/src/AbstractCommand.h index bda7f9f5..7e6bcd7a 100644 --- a/src/AbstractCommand.h +++ b/src/AbstractCommand.h @@ -41,6 +41,7 @@ #include "SegmentMan.h" #include "TimeA2.h" #include "RecoverableException.h" +#include "RequestGroup.h" class AbstractCommand : public Command { private: @@ -48,13 +49,14 @@ private: int timeout; protected: RequestHandle req; + RequestGroup* _requestGroup; DownloadEngine* e; SocketHandle socket; SegmentHandle segment; void tryReserved(); virtual bool prepareForRetry(int wait); - virtual void onAbort(RecoverableException* ex); + virtual void onAbort(Exception* ex); virtual bool executeInternal() = 0; void setReadCheckSocket(const SocketHandle& socket); @@ -75,7 +77,7 @@ private: SocketHandle writeCheckTarget; bool nameResolverCheck; public: - AbstractCommand(int cuid, const RequestHandle& req, DownloadEngine* e, const SocketHandle& s = SocketHandle()); + AbstractCommand(int cuid, const RequestHandle& req, RequestGroup* requestGroup, DownloadEngine* e, const SocketHandle& s = SocketHandle()); virtual ~AbstractCommand(); bool execute(); }; diff --git a/src/AbstractDiskWriter.cc b/src/AbstractDiskWriter.cc index 1e9bcaed..a901f42b 100644 --- a/src/AbstractDiskWriter.cc +++ b/src/AbstractDiskWriter.cc @@ -45,7 +45,7 @@ #include AbstractDiskWriter::AbstractDiskWriter(): - fd(0), + fd(-1), fileAllocator(0), glowFileAllocator(0), logger(LogFactory::getInstance()) @@ -164,3 +164,16 @@ int32_t AbstractDiskWriter::readData(char* data, int32_t len, int64_t offset) { return ret; } +void AbstractDiskWriter::truncate(int64_t length) +{ + ftruncate(fd, length); +} + +int64_t AbstractDiskWriter::size() const +{ + struct stat fileStat; + if(fstat(fd, &fileStat) < 0) { + return 0; + } + return fileStat.st_size; +} diff --git a/src/AbstractDiskWriter.h b/src/AbstractDiskWriter.h index 2ba973df..316ab3fc 100644 --- a/src/AbstractDiskWriter.h +++ b/src/AbstractDiskWriter.h @@ -83,6 +83,10 @@ public: { this->glowFileAllocator = fileAllocator; } + + virtual void truncate(int64_t length); + + virtual int64_t size() const; }; #endif // _D_ABSTRACT_DISK_WRITER_H_ diff --git a/src/AbstractProxyRequestCommand.cc b/src/AbstractProxyRequestCommand.cc index d4142045..b0257a89 100644 --- a/src/AbstractProxyRequestCommand.cc +++ b/src/AbstractProxyRequestCommand.cc @@ -38,9 +38,10 @@ AbstractProxyRequestCommand::AbstractProxyRequestCommand(int cuid, const RequestHandle& req, + RequestGroup* requestGroup, DownloadEngine* e, const SocketHandle& s) - :AbstractCommand(cuid, req, e, s), httpConnection(0) { + :AbstractCommand(cuid, req, requestGroup, e, s), httpConnection(0) { disableReadCheckSocket(); setWriteCheckSocket(socket); } diff --git a/src/AbstractProxyRequestCommand.h b/src/AbstractProxyRequestCommand.h index 7b2c6feb..184ccd92 100644 --- a/src/AbstractProxyRequestCommand.h +++ b/src/AbstractProxyRequestCommand.h @@ -46,6 +46,7 @@ protected: public: AbstractProxyRequestCommand(int cuid, const RequestHandle& req, + RequestGroup* requestGroup, DownloadEngine* e, const SocketHandle& s); virtual ~AbstractProxyRequestCommand(); diff --git a/src/AbstractProxyResponseCommand.cc b/src/AbstractProxyResponseCommand.cc index f35fd706..eaf89762 100644 --- a/src/AbstractProxyResponseCommand.cc +++ b/src/AbstractProxyResponseCommand.cc @@ -39,10 +39,11 @@ AbstractProxyResponseCommand::AbstractProxyResponseCommand(int cuid, const RequestHandle& req, + RequestGroup* requestGroup, const HttpConnectionHandle& httpConnection, DownloadEngine* e, const SocketHandle& s) - :AbstractCommand(cuid, req, e, s), + :AbstractCommand(cuid, req, requestGroup, e, s), httpConnection(httpConnection) {} AbstractProxyResponseCommand::~AbstractProxyResponseCommand() {} diff --git a/src/AbstractProxyResponseCommand.h b/src/AbstractProxyResponseCommand.h index 99ef02d2..a5770903 100644 --- a/src/AbstractProxyResponseCommand.h +++ b/src/AbstractProxyResponseCommand.h @@ -46,6 +46,7 @@ protected: public: AbstractProxyResponseCommand(int cuid, const RequestHandle& req, + RequestGroup* requestGroup, const HttpConnectionHandle& httpConnection, DownloadEngine* e, const SocketHandle& s); diff --git a/src/ByteArrayDiskWriter.h b/src/ByteArrayDiskWriter.h index 7e1b0b02..c5390507 100644 --- a/src/ByteArrayDiskWriter.h +++ b/src/ByteArrayDiskWriter.h @@ -61,6 +61,14 @@ public: virtual void writeData(const char* data, int32_t len, int64_t position = 0); virtual int readData(char* data, int32_t len, int64_t position); + // Not implemented yet + virtual void truncate(int64_t length) {} + + virtual int64_t size() const + { + return bufLength; + } + // not implemented yet #ifdef ENABLE_MESSAGE_DIGEST virtual string messageDigest(int64_t offset, int64_t length, @@ -72,6 +80,8 @@ public: const char* getByteArray() const { return buf; } + + // can be deleted. Use size() instead. int getByteArrayLength() const { return bufLength; } diff --git a/src/ChunkChecksumValidator.cc b/src/ChunkChecksumValidator.cc index eed5e85e..cb9590fb 100644 --- a/src/ChunkChecksumValidator.cc +++ b/src/ChunkChecksumValidator.cc @@ -48,9 +48,9 @@ void ChunkChecksumValidator::validateSameLengthChecksum(BitfieldMan* bitfieldMan int64_t offset = ((int64_t)index)*checksumLength; string actualChecksum = diskWriter->messageDigest(offset, dataLength, algo); if(actualChecksum != expectedChecksum) { - logger->error(EX_INVALID_CHUNK_CHECKSUM, - index, offset, dataLength, - expectedChecksum.c_str(), actualChecksum.c_str()); + logger->info(EX_INVALID_CHUNK_CHECKSUM, + index, offset, dataLength, + expectedChecksum.c_str(), actualChecksum.c_str()); bitfieldMan->unsetBit(index); } } @@ -70,9 +70,9 @@ void ChunkChecksumValidator::validateDifferentLengthChecksum(BitfieldMan* bitfie string actualChecksum = diskWriter->messageDigest(offset, dataLength, algo); if(expectedChecksum != actualChecksum) { // wrong checksum - logger->error(EX_INVALID_CHUNK_CHECKSUM, - index, offset, dataLength, - expectedChecksum.c_str(), actualChecksum.c_str()); + logger->info(EX_INVALID_CHUNK_CHECKSUM, + index, offset, dataLength, + expectedChecksum.c_str(), actualChecksum.c_str()); bitfieldMan->unsetBitRange(startIndex, endIndex); } } diff --git a/src/ChunkChecksumValidator.h b/src/ChunkChecksumValidator.h index 89f7b22c..717151a8 100644 --- a/src/ChunkChecksumValidator.h +++ b/src/ChunkChecksumValidator.h @@ -47,7 +47,7 @@ class ChunkChecksumValidator { #ifdef ENABLE_MESSAGE_DIGEST -private: +protected: DiskWriterHandle diskWriter; MessageDigestContext::DigestAlgo algo; diff --git a/src/Command.cc b/src/Command.cc index de53c59a..e9b7f25f 100644 --- a/src/Command.cc +++ b/src/Command.cc @@ -38,11 +38,15 @@ int Command::uuidGen = 0; bool Command::statusMatch(Command::STATUS statusFilter) const { - if(statusFilter == STATUS_ALL) { - return true; - } else if(statusFilter == status) { - return true; - } else { - return false; + return statusFilter <= status; +} + +void Command::transitStatus() +{ + switch(status) { + case STATUS_REALTIME: + break; + default: + status = STATUS_INACTIVE; } } diff --git a/src/Command.h b/src/Command.h index f9fcb222..87cdd825 100644 --- a/src/Command.h +++ b/src/Command.h @@ -44,8 +44,9 @@ class Command { public: enum STATUS { STATUS_ALL, + STATUS_INACTIVE, STATUS_ACTIVE, - STATUS_INACTIVE + STATUS_REALTIME }; private: CommandUuid uuid; @@ -68,7 +69,13 @@ public: void setStatusInactive() { this->status = STATUS_INACTIVE; } + void setStatusRealtime() { this->status = STATUS_REALTIME; } + + void transitStatus(); + bool statusMatch(Command::STATUS statusFilter) const; }; +typedef deque Commands; + #endif // _D_COMMAND_H_ diff --git a/src/ConsoleDownloadEngine.cc b/src/ConsoleDownloadEngine.cc index 94669e64..e7f82680 100644 --- a/src/ConsoleDownloadEngine.cc +++ b/src/ConsoleDownloadEngine.cc @@ -43,6 +43,7 @@ ConsoleDownloadEngine::ConsoleDownloadEngine() {} ConsoleDownloadEngine::~ConsoleDownloadEngine() {} void ConsoleDownloadEngine::sendStatistics(long long int currentSize, long long int totalSize) { + /* printf("\r "); printf("\r"); printf("%s/%s Bytes %d%% %s %.2f KB/s %d connections", @@ -53,6 +54,45 @@ void ConsoleDownloadEngine::sendStatistics(long long int currentSize, long long speed/1024.0, commands.size()); fflush(stdout); + */ + + cout << "\r "; + cout << "\r"; + if(_requestGroupMan->countRequestGroup() > 0) { + RequestGroupHandle firstRequestGroup = _requestGroupMan->getRequestGroup(0); + cout << "["; + cout << Util::llitos(firstRequestGroup->getDownloadLength(), true) + << "/" + << Util::llitos(firstRequestGroup->getTotalLength(), true); + if(firstRequestGroup->getTotalLength() > 0) { + cout << "(" + << 100*firstRequestGroup->getDownloadLength()/firstRequestGroup->getTotalLength() + << "%)"; + } + cout << "(" + << firstRequestGroup->numConnection + << "cn)"; + if(_requestGroupMan->countRequestGroup() > 1) { + cout << "(" + << _requestGroupMan->countRequestGroup()-1 + << "more...)"; + } + cout << "]"; + } + cout << "[" << speed/1024.0 << "KB/s" << "]"; + + FileAllocationEntryHandle entry = _fileAllocationMan->getCurrentFileAllocationEntry(); + if(!entry.isNull()) { + cout << "[FileAlloc:" + << entry->getOffset() + << "/" + << entry->getRequestGroup()->getTotalLength() + << "(" + << entry->getOffset()/(entry->getRequestGroup()->getTotalLength()/100) + << "%)" + << "]"; + } + cout << flush; } void ConsoleDownloadEngine::initStatistics() { @@ -67,7 +107,7 @@ void ConsoleDownloadEngine::initStatistics() { } void ConsoleDownloadEngine::calculateStatistics() { - long long int dlSize = segmentMan->getDownloadLength(); + long long int dlSize = _requestGroupMan->getDownloadLength(); if(!isStartupLengthSet && dlSize > 0) { startupLength = dlSize; psize = dlSize; @@ -87,32 +127,38 @@ void ConsoleDownloadEngine::calculateStatistics() { if(elapsedFromStartup > 0) { avgSpeed = (int)((dlSize-startupLength)/elapsedFromStartup); } + int64_t totalLength = _requestGroupMan->getTotalLength(); if(avgSpeed < 0) { avgSpeed = 0; - } else if(avgSpeed != 0 && segmentMan->totalSize > 0) { - eta = (segmentMan->totalSize-dlSize)/avgSpeed; + } else if(avgSpeed != 0 && totalLength > 0) { + eta = (totalLength-dlSize)/avgSpeed; } - sendStatistics(dlSize, segmentMan->totalSize); + sendStatistics(dlSize, totalLength); } } void ConsoleDownloadEngine::onEndOfRun() { - segmentMan->diskWriter->closeFile(); - if(segmentMan->finished()) { - segmentMan->remove(); - } else { - segmentMan->save(); - } + _requestGroupMan->closeFile(); +// if(segmentMan->finished()) { +// segmentMan->remove(); +// } else { +// segmentMan->save(); +// } } void ConsoleDownloadEngine::afterEachIteration() { if(haltRequested) { printf(_("\nstopping application...\n")); fflush(stdout); - segmentMan->save(); - segmentMan->diskWriter->closeFile(); + _requestGroupMan->save(); + _requestGroupMan->closeFile(); printf(_("done\n")); exit(EXIT_SUCCESS); } } + +void ConsoleDownloadEngine::fillCommand() +{ + addCommand(_requestGroupMan->getInitialCommands(this)); +} diff --git a/src/ConsoleDownloadEngine.h b/src/ConsoleDownloadEngine.h index 625a35a3..0a0fae80 100644 --- a/src/ConsoleDownloadEngine.h +++ b/src/ConsoleDownloadEngine.h @@ -61,6 +61,8 @@ protected: public: ConsoleDownloadEngine(); ~ConsoleDownloadEngine(); + + void fillCommand(); }; #endif // _D_CONSOLE_DOWNLOAD_ENGINE_H_ diff --git a/src/DiskAdaptorWriter.h b/src/DiskAdaptorWriter.h index ea67d2f0..3c400ddc 100644 --- a/src/DiskAdaptorWriter.h +++ b/src/DiskAdaptorWriter.h @@ -37,6 +37,7 @@ #include "DiskWriter.h" #include "DiskAdaptor.h" +#include "FatalException.h" class DiskAdaptorWriter : public DiskWriter { private: @@ -82,6 +83,16 @@ public: { return diskAdaptor->messageDigest(offset, length, algo); } + + virtual void truncate(int64_t length) + { + throw new FatalException("DiskAdaptorWriter::truncate() is not implemented yet."); + } + + virtual int64_t size() const + { + throw new FatalException("DiskAdaptorWriter::size() is not implemented yet."); + } }; #endif // _D_DISK_ADAPTOR_WRITER_H_ diff --git a/src/DiskWriter.h b/src/DiskWriter.h index dc4c57a3..785c20f7 100644 --- a/src/DiskWriter.h +++ b/src/DiskWriter.h @@ -95,6 +95,11 @@ public: virtual string messageDigest(int64_t offset, int64_t length, const MessageDigestContext::DigestAlgo& algo) = 0; #endif // ENABLE_MESSAGE_DIGEST + + virtual void truncate(int64_t length) = 0; + + // Returns file length + virtual int64_t size() const = 0; }; typedef SharedHandle DiskWriterHandle; diff --git a/src/DownloadCommand.cc b/src/DownloadCommand.cc index 8b0cd6df..f28cc4c6 100644 --- a/src/DownloadCommand.cc +++ b/src/DownloadCommand.cc @@ -44,16 +44,17 @@ DownloadCommand::DownloadCommand(int cuid, const RequestHandle req, + RequestGroup* requestGroup, DownloadEngine* e, const SocketHandle& s): - AbstractCommand(cuid, req, e, s), + AbstractCommand(cuid, req, requestGroup, e, s), peerStat(0), transferDecoder(0) { - peerStat = this->e->segmentMan->getPeerStat(cuid); + peerStat = _requestGroup->getSegmentMan()->getPeerStat(cuid); if(!peerStat.get()) { peerStat = new PeerStat(cuid); - this->e->segmentMan->registerPeerStat(peerStat); + _requestGroup->getSegmentMan()->registerPeerStat(peerStat); } peerStat->downloadStart(); } @@ -64,8 +65,9 @@ DownloadCommand::~DownloadCommand() { } bool DownloadCommand::executeInternal() { + // TODO we need to specify the sum of all segmentMan's download speed here. if(maxDownloadSpeedLimit > 0 && - maxDownloadSpeedLimit < e->segmentMan->calculateDownloadSpeed()) { + maxDownloadSpeedLimit < _requestGroup->getSegmentMan()->calculateDownloadSpeed()) { usleep(1); e->commands.push_back(this); return false; @@ -75,16 +77,16 @@ bool DownloadCommand::executeInternal() { socket->readData(buf, bufSize); if(transferDecoder.isNull()) { - e->segmentMan->diskWriter->writeData(buf, bufSize, - segment->getPositionToWrite()); + _requestGroup->getSegmentMan()->diskWriter->writeData(buf, bufSize, + segment->getPositionToWrite()); segment->writtenLength += bufSize; peerStat->updateDownloadLength(bufSize); } else { int32_t infbufSize = 16*1024; char infbuf[infbufSize]; transferDecoder->inflate(infbuf, infbufSize, buf, bufSize); - e->segmentMan->diskWriter->writeData(infbuf, infbufSize, - segment->getPositionToWrite()); + _requestGroup->getSegmentMan()->diskWriter->writeData(infbuf, infbufSize, + segment->getPositionToWrite()); segment->writtenLength += infbufSize; peerStat->updateDownloadLength(infbufSize); } @@ -98,7 +100,7 @@ bool DownloadCommand::executeInternal() { lowestDownloadSpeedLimit); } } - if(e->segmentMan->totalSize != 0 && bufSize == 0) { + if(_requestGroup->getSegmentMan()->totalSize != 0 && bufSize == 0) { throw new DlRetryEx(EX_GOT_EOF); } if(!transferDecoder.isNull() && transferDecoder->finished() @@ -106,10 +108,10 @@ bool DownloadCommand::executeInternal() { || bufSize == 0) { if(!transferDecoder.isNull()) transferDecoder->end(); logger->info(MSG_DOWNLOAD_COMPLETED, cuid); - e->segmentMan->completeSegment(cuid, segment); + _requestGroup->getSegmentMan()->completeSegment(cuid, segment); #ifdef ENABLE_MESSAGE_DIGEST if(e->option->get(PREF_REALTIME_CHUNK_CHECKSUM) == V_TRUE) { - e->segmentMan->tryChunkChecksumValidation(segment); + _requestGroup->getSegmentMan()->tryChunkChecksumValidation(segment); } #endif // ENABLE_MESSAGE_DIGEST // this unit is going to download another segment. @@ -122,23 +124,25 @@ bool DownloadCommand::executeInternal() { } bool DownloadCommand::prepareForNextSegment() { - if(e->segmentMan->finished()) { + if(_requestGroup->getSegmentMan()->finished()) { return true; } else { // Merge segment with next segment, if segment.index+1 == nextSegment.index SegmentHandle tempSegment = segment; while(1) { - SegmentHandle nextSegment = e->segmentMan->getSegment(cuid, - tempSegment->index+1); + SegmentHandle nextSegment = + _requestGroup->getSegmentMan()->getSegment(cuid, + tempSegment->index+1); if(nextSegment.isNull()) { break; } else { if(nextSegment->writtenLength > 0) { return prepareForRetry(0); } - nextSegment->writtenLength = tempSegment->writtenLength-tempSegment->length; + nextSegment->writtenLength = + tempSegment->writtenLength-tempSegment->length; if(nextSegment->complete()) { - e->segmentMan->completeSegment(cuid, nextSegment); + _requestGroup->getSegmentMan()->completeSegment(cuid, nextSegment); tempSegment = nextSegment; } else { segment = nextSegment; diff --git a/src/DownloadCommand.h b/src/DownloadCommand.h index 7654e579..c8176dcb 100644 --- a/src/DownloadCommand.h +++ b/src/DownloadCommand.h @@ -55,7 +55,10 @@ protected: virtual bool prepareForNextSegment(); public: - DownloadCommand(int cuid, const RequestHandle req, DownloadEngine* e, + DownloadCommand(int cuid, + const RequestHandle req, + RequestGroup* requestGroup, + DownloadEngine* e, const SocketHandle& s); virtual ~DownloadCommand(); diff --git a/src/DownloadEngine.cc b/src/DownloadEngine.cc index 478291ec..5d414462 100644 --- a/src/DownloadEngine.cc +++ b/src/DownloadEngine.cc @@ -45,13 +45,13 @@ using namespace std; -DownloadEngine::DownloadEngine():noWait(false), segmentMan(0) { - logger = LogFactory::getInstance(); -} +DownloadEngine::DownloadEngine():logger(LogFactory::getInstance()), + noWait(false), + _requestGroupMan(0), + _fileAllocationMan(0) {} DownloadEngine::~DownloadEngine() { cleanQueue(); - delete segmentMan; } void DownloadEngine::cleanQueue() { @@ -69,7 +69,7 @@ void DownloadEngine::executeCommand(Command::STATUS statusFilter) if(com->execute()) { delete com; } else { - com->setStatusInactive(); + com->transitStatus(); } } else { commands.push_back(com); @@ -90,7 +90,7 @@ void DownloadEngine::run() { executeCommand(Command::STATUS_ACTIVE); } afterEachIteration(); - if(!noWait && !commands.empty()) { + if(!commands.empty()) { waitData(); } noWait = false; @@ -117,7 +117,7 @@ void DownloadEngine::waitData() { memcpy(&rfds, &rfdset, sizeof(fd_set)); memcpy(&wfds, &wfdset, sizeof(fd_set)); - tv.tv_sec = 1; + tv.tv_sec = noWait ? 0 : 1; tv.tv_usec = 0; retval = select(fdmax+1, &rfds, &wfds, NULL, &tv); if(retval > 0) { diff --git a/src/DownloadEngine.h b/src/DownloadEngine.h index fecba8d2..5610abdd 100644 --- a/src/DownloadEngine.h +++ b/src/DownloadEngine.h @@ -42,9 +42,10 @@ #include "Logger.h" #include "Option.h" #include "NameResolver.h" +#include "RequestGroupMan.h" +#include "FileAllocationMan.h" typedef deque Sockets; -typedef deque Commands; class SocketEntry { public: @@ -117,7 +118,8 @@ protected: public: bool noWait; Commands commands; - SegmentMan* segmentMan; + RequestGroupManHandle _requestGroupMan; + FileAllocationManHandle _fileAllocationMan; const Option* option; DownloadEngine(); @@ -143,6 +145,11 @@ public: bool deleteNameResolverCheck(const NameResolverHandle& resolver, Command* command); #endif // ENABLE_ASYNC_DNS + + void addCommand(const Commands& commands) + { + this->commands.insert(this->commands.end(), commands.begin(), commands.end()); + } }; #endif // _D_DOWNLOAD_ENGINE_H_ diff --git a/src/DownloadEngineFactory.cc b/src/DownloadEngineFactory.cc index 0921eaac..d447dc90 100644 --- a/src/DownloadEngineFactory.cc +++ b/src/DownloadEngineFactory.cc @@ -36,10 +36,13 @@ #include "prefs.h" #include "DefaultDiskWriter.h" #include "InitiateConnectionCommandFactory.h" -#include "ByteArrayDiskWriter.h" #include "Util.h" #include "FileAllocator.h" #include "FileAllocationMonitor.h" +#include "FillRequestGroupCommand.h" +#include "CUIDCounter.h" +#include "FileAllocationDispatcherCommand.h" +#include "FileAllocationMan.h" #ifdef ENABLE_BITTORRENT # include "PeerListenCommand.h" # include "TrackerWatcherCommand.h" @@ -58,6 +61,33 @@ # include "DefaultBtProgressInfoFile.h" #endif // ENABLE_BITTORRENT +ConsoleDownloadEngine* +DownloadEngineFactory::newConsoleEngine(const Option* op, + const RequestGroups& requestGroups) +{ + RequestGroups workingSet; + RequestGroups reservedSet; + if(op->getAsInt(PREF_MAX_SIMULTANEOUS_DOWNLOADS) < (int32_t)requestGroups.size()) { + copy(requestGroups.begin(), requestGroups.begin()+op->getAsInt(PREF_MAX_SIMULTANEOUS_DOWNLOADS), back_inserter(workingSet)); + copy(requestGroups.begin()+op->getAsInt(PREF_MAX_SIMULTANEOUS_DOWNLOADS), + requestGroups.end(), back_inserter(reservedSet)); + } else { + workingSet = requestGroups; + } + + ConsoleDownloadEngine* e = new ConsoleDownloadEngine(); + e->option = op; + RequestGroupManHandle requestGroupMan = new RequestGroupMan(workingSet, + op->getAsInt(PREF_MAX_SIMULTANEOUS_DOWNLOADS)); + requestGroupMan->addReservedGroup(reservedSet); + e->_requestGroupMan = requestGroupMan; + e->_fileAllocationMan = new FileAllocationMan(); + + e->commands.push_back(new FillRequestGroupCommand(CUIDCounterSingletonHolder::instance()->newID(), e, 1)); + e->commands.push_back(new FileAllocationDispatcherCommand(CUIDCounterSingletonHolder::instance()->newID(), e)); + return e; +} + ConsoleDownloadEngine* DownloadEngineFactory::newConsoleEngine(const Option* op, const Requests& requests, @@ -65,19 +95,19 @@ DownloadEngineFactory::newConsoleEngine(const Option* op, { ConsoleDownloadEngine* e = new ConsoleDownloadEngine(); e->option = op; - e->segmentMan = new SegmentMan(); - e->segmentMan->diskWriter = DefaultDiskWriter::createNewDiskWriter(op); - e->segmentMan->dir = op->get(PREF_DIR); - e->segmentMan->ufilename = op->get(PREF_OUT); - e->segmentMan->option = op; - e->segmentMan->reserved = reserved; +// e->segmentMan = new SegmentMan(); +// e->segmentMan->diskWriter = DefaultDiskWriter::createNewDiskWriter(op); +// e->segmentMan->dir = op->get(PREF_DIR); +// e->segmentMan->ufilename = op->get(PREF_OUT); +// e->segmentMan->option = op; +// e->segmentMan->reserved = reserved; - int cuidCounter = 1; - for(Requests::const_iterator itr = requests.begin(); - itr != requests.end(); - itr++, cuidCounter++) { - e->commands.push_back(InitiateConnectionCommandFactory::createInitiateConnectionCommand(cuidCounter, *itr, e)); - } +// int cuidCounter = 1; +// for(Requests::const_iterator itr = requests.begin(); +// itr != requests.end(); +// itr++, cuidCounter++) { +// e->commands.push_back(InitiateConnectionCommandFactory::createInitiateConnectionCommand(cuidCounter, *itr, e)); +// } return e; } @@ -89,10 +119,12 @@ DownloadEngineFactory::newTorrentConsoleEngine(const BtContextHandle& btContext, { TorrentConsoleDownloadEngine* te = new TorrentConsoleDownloadEngine(); te->option = op; - ByteArrayDiskWriter* byteArrayDiskWriter = new ByteArrayDiskWriter(); - te->segmentMan = new SegmentMan(); - te->segmentMan->diskWriter = byteArrayDiskWriter; - te->segmentMan->option = op; + RequestGroupManHandle requestGroupMan = new RequestGroupMan(); + te->_requestGroupMan = requestGroupMan; + // ByteArrayDiskWriter* byteArrayDiskWriter = new ByteArrayDiskWriter(); +// te->segmentMan = new SegmentMan(); +// te->segmentMan->diskWriter = byteArrayDiskWriter; +// te->segmentMan->option = op; BtRuntimeHandle btRuntime(new BtRuntime()); BtRegistry::registerBtRuntime(btContext->getInfoHashAsString(), btRuntime); diff --git a/src/DownloadEngineFactory.h b/src/DownloadEngineFactory.h index f5f801c3..f9d11e5d 100644 --- a/src/DownloadEngineFactory.h +++ b/src/DownloadEngineFactory.h @@ -43,6 +43,10 @@ class DownloadEngineFactory { public: + static ConsoleDownloadEngine* + newConsoleEngine(const Option* op, + const RequestGroups& requestGroups); + static ConsoleDownloadEngine* newConsoleEngine(const Option* option, const Requests& requests, diff --git a/src/FtpDownloadCommand.cc b/src/FtpDownloadCommand.cc index c13abeb1..e1c6ef8e 100644 --- a/src/FtpDownloadCommand.cc +++ b/src/FtpDownloadCommand.cc @@ -36,10 +36,11 @@ FtpDownloadCommand::FtpDownloadCommand(int cuid, const RequestHandle req, + RequestGroup* requestGroup, DownloadEngine* e, const SocketHandle& dataSocket, const SocketHandle& ctrlSocket) - :DownloadCommand(cuid, req, e, dataSocket), + :DownloadCommand(cuid, req, requestGroup, e, dataSocket), ctrlSocket(ctrlSocket) {} FtpDownloadCommand::~FtpDownloadCommand() {} diff --git a/src/FtpDownloadCommand.h b/src/FtpDownloadCommand.h index fdf67c91..1b799d46 100644 --- a/src/FtpDownloadCommand.h +++ b/src/FtpDownloadCommand.h @@ -41,7 +41,10 @@ class FtpDownloadCommand : public DownloadCommand { private: SocketHandle ctrlSocket; public: - FtpDownloadCommand(int cuid, const RequestHandle req, DownloadEngine* e, + FtpDownloadCommand(int cuid, + const RequestHandle req, + RequestGroup* requestGroup, + DownloadEngine* e, const SocketHandle& dataSocket, const SocketHandle& ctrlSocket); virtual ~FtpDownloadCommand(); diff --git a/src/FtpInitiateConnectionCommand.cc b/src/FtpInitiateConnectionCommand.cc index 5dfcc317..4837769e 100644 --- a/src/FtpInitiateConnectionCommand.cc +++ b/src/FtpInitiateConnectionCommand.cc @@ -43,10 +43,12 @@ FtpInitiateConnectionCommand::FtpInitiateConnectionCommand(int cuid, const RequestHandle& req, + RequestGroup* requestGroup, DownloadEngine* e) - :AbstractCommand(cuid, req, e) + :AbstractCommand(cuid, req, requestGroup, e) { setTimeout(e->option->getAsInt(PREF_DNS_TIMEOUT)); + setStatusActive(); disableReadCheckSocket(); disableWriteCheckSocket(); } @@ -81,9 +83,9 @@ bool FtpInitiateConnectionCommand::executeInternal() { e->option->getAsInt(PREF_HTTP_PROXY_PORT)); if(useHttpProxyGet()) { - command = new HttpRequestCommand(cuid, req, e, socket); + command = new HttpRequestCommand(cuid, req, _requestGroup, e, socket); } else if(useHttpProxyConnect()) { - command = new FtpTunnelRequestCommand(cuid, req, e, socket); + command = new FtpTunnelRequestCommand(cuid, req, _requestGroup, e, socket); } else { // TODO throw new DlAbortEx("ERROR"); @@ -92,7 +94,7 @@ bool FtpInitiateConnectionCommand::executeInternal() { logger->info(MSG_CONNECTING_TO_SERVER, cuid, req->getHost().c_str(), req->getPort()); socket->establishConnection(hostname, req->getPort()); - command = new FtpNegotiationCommand(cuid, req, e, socket); + command = new FtpNegotiationCommand(cuid, req, _requestGroup, e, socket); } e->commands.push_back(command); return true; diff --git a/src/FtpInitiateConnectionCommand.h b/src/FtpInitiateConnectionCommand.h index d5d94624..acda5af0 100644 --- a/src/FtpInitiateConnectionCommand.h +++ b/src/FtpInitiateConnectionCommand.h @@ -52,7 +52,7 @@ private: protected: virtual bool executeInternal(); public: - FtpInitiateConnectionCommand(int cuid, const RequestHandle& req, DownloadEngine* e); + FtpInitiateConnectionCommand(int cuid, const RequestHandle& req, RequestGroup* requestGroup, DownloadEngine* e); virtual ~FtpInitiateConnectionCommand(); }; diff --git a/src/FtpNegotiationCommand.cc b/src/FtpNegotiationCommand.cc index e0b4926f..d9f8d741 100644 --- a/src/FtpNegotiationCommand.cc +++ b/src/FtpNegotiationCommand.cc @@ -43,9 +43,10 @@ FtpNegotiationCommand::FtpNegotiationCommand(int cuid, const RequestHandle& req, + RequestGroup* requestGroup, DownloadEngine* e, const SocketHandle& s): - AbstractCommand(cuid, req, e, s), sequence(SEQ_RECV_GREETING) + AbstractCommand(cuid, req, requestGroup, e, s), sequence(SEQ_RECV_GREETING) { ftp = new FtpConnection(cuid, socket, req, e->option); disableReadCheckSocket(); @@ -62,13 +63,13 @@ bool FtpNegotiationCommand::executeInternal() { return prepareForRetry(0); } else if(sequence == SEQ_NEGOTIATION_COMPLETED) { FtpDownloadCommand* command = - new FtpDownloadCommand(cuid, req, e, dataSocket, socket); + new FtpDownloadCommand(cuid, req, _requestGroup, e, dataSocket, socket); command->setMaxDownloadSpeedLimit(e->option->getAsInt(PREF_MAX_DOWNLOAD_LIMIT)); command->setStartupIdleTime(e->option->getAsInt(PREF_STARTUP_IDLE_TIME)); command->setLowestDownloadSpeedLimit(e->option->getAsInt(PREF_LOWEST_SPEED_LIMIT)); e->commands.push_back(command); return true; - } else if(sequence == SEQ_HEAD_OK) { + } else if(sequence == SEQ_HEAD_OK || sequence == SEQ_DOWNLOAD_ALREADY_COMPLETED || sequence == SEQ_FILE_PREPARATION) { return true; } else { e->commands.push_back(this); @@ -188,26 +189,50 @@ bool FtpNegotiationCommand::recvSize() { if(size == INT64_MAX || size < 0) { throw new DlAbortEx(EX_TOO_LARGE_FILE, size); } - if(!e->segmentMan->downloadStarted) { - e->segmentMan->downloadStarted = true; - e->segmentMan->totalSize = size; - e->segmentMan->initBitfield(e->option->getAsInt(PREF_SEGMENT_SIZE), - e->segmentMan->totalSize); - e->segmentMan->filename = Util::urldecode(req->getFile()); + if(!_requestGroup->getSegmentMan()->downloadStarted) { + _requestGroup->getSegmentMan()->downloadStarted = true; + _requestGroup->getSegmentMan()->totalSize = size; + _requestGroup->getSegmentMan()->filename = Util::urldecode(req->getFile()); + + // TODO validate filename and totalsize against hintFilename and hintTotalSize if these are provided. + _requestGroup->validateTotalLengthByHint(size); + if(req->getMethod() == Request::METHOD_HEAD) { - e->segmentMan->isSplittable = false; // TODO because we don't want segment file to be saved. + _requestGroup->getSegmentMan()->isSplittable = false; // TODO because we don't want segment file to be saved. sequence = SEQ_HEAD_OK; return false; } - bool segFileExists = e->segmentMan->segmentFileExists(); - if(segFileExists) { - e->segmentMan->load(); - e->segmentMan->diskWriter->openExistingFile(e->segmentMan->getFilePath()); - } else { - e->segmentMan->diskWriter->initAndOpenFile(e->segmentMan->getFilePath(), size); + + if(e->option->get(PREF_CHECK_INTEGRITY) != V_TRUE) { + if(_requestGroup->downloadFinishedByFileLength()) { + logger->notice(MSG_DOWNLOAD_ALREADY_COMPLETED, cuid, _requestGroup->getFilePath().c_str()); + sequence = SEQ_DOWNLOAD_ALREADY_COMPLETED; + return false; + } } - } else if(e->segmentMan->totalSize != size) { - throw new DlAbortEx(EX_SIZE_MISMATCH, e->segmentMan->totalSize, size); + _requestGroup->loadAndOpenFile(); + _requestGroup->prepareForNextAction(cuid, req, e); + + sequence = SEQ_FILE_PREPARATION; + e->noWait = true; + return false; + + /* + _requestGroup->getSegmentMan()->initBitfield(e->option->getAsInt(PREF_SEGMENT_SIZE), + _requestGroup->getSegmentMan()->totalSize); + bool segFileExists = _requestGroup->getSegmentMan()->segmentFileExists(); + if(segFileExists) { + _requestGroup->getSegmentMan()->load(); + _requestGroup->getSegmentMan()->diskWriter->openExistingFile(_requestGroup->getSegmentMan()->getFilePath()); + } else { + _requestGroup->getSegmentMan()->diskWriter->initAndOpenFile(_requestGroup->getSegmentMan()->getFilePath(), size); + } + */ + + } else { + _requestGroup->validateTotalLength(size); + //if(_requestGroup->getSegmentMan()->totalSize != size) { + //throw new DlAbortEx(EX_SIZE_MISMATCH, _requestGroup->getSegmentMan()->totalSize, size); } if(e->option->get(PREF_FTP_PASV) == V_TRUE) { sequence = SEQ_SEND_PASV; @@ -281,7 +306,7 @@ bool FtpNegotiationCommand::recvRest() { if(status == 0) { return false; } - // TODO if we recieve negative response, then we set e->segmentMan->splittable = false, and continue. + // TODO if we recieve negative response, then we set _requestGroup->getSegmentMan()->splittable = false, and continue. if(status != 350) { throw new DlRetryEx(EX_BAD_STATUS, status); } diff --git a/src/FtpNegotiationCommand.h b/src/FtpNegotiationCommand.h index 971faba6..2413f586 100644 --- a/src/FtpNegotiationCommand.h +++ b/src/FtpNegotiationCommand.h @@ -63,7 +63,9 @@ private: SEQ_RECV_RETR, SEQ_NEGOTIATION_COMPLETED, SEQ_RETRY, - SEQ_HEAD_OK + SEQ_HEAD_OK, + SEQ_DOWNLOAD_ALREADY_COMPLETED, + SEQ_FILE_PREPARATION }; bool recvGreeting(); bool sendUser(); @@ -96,6 +98,7 @@ protected: public: FtpNegotiationCommand(int cuid, const RequestHandle& req, + RequestGroup* requestGroup, DownloadEngine* e, const SocketHandle& s); virtual ~FtpNegotiationCommand(); diff --git a/src/FtpTunnelRequestCommand.cc b/src/FtpTunnelRequestCommand.cc index 88e0a85f..b1ee1d3d 100644 --- a/src/FtpTunnelRequestCommand.cc +++ b/src/FtpTunnelRequestCommand.cc @@ -37,13 +37,14 @@ FtpTunnelRequestCommand::FtpTunnelRequestCommand(int cuid, const RequestHandle& req, + RequestGroup* requestGroup, DownloadEngine* e, const SocketHandle& s) - :AbstractProxyRequestCommand(cuid, req, e, s) {} + :AbstractProxyRequestCommand(cuid, req, requestGroup, e, s) {} FtpTunnelRequestCommand::~FtpTunnelRequestCommand() {} Command* FtpTunnelRequestCommand::getNextCommand() { - return new FtpTunnelResponseCommand(cuid, req, httpConnection, e, socket); + return new FtpTunnelResponseCommand(cuid, req, _requestGroup, httpConnection, e, socket); } diff --git a/src/FtpTunnelRequestCommand.h b/src/FtpTunnelRequestCommand.h index a1c42a11..3ce63e2f 100644 --- a/src/FtpTunnelRequestCommand.h +++ b/src/FtpTunnelRequestCommand.h @@ -41,6 +41,7 @@ class FtpTunnelRequestCommand : public AbstractProxyRequestCommand { public: FtpTunnelRequestCommand(int cuid, const RequestHandle& req, + RequestGroup* requestGroup, DownloadEngine* e, const SocketHandle& s); virtual ~FtpTunnelRequestCommand(); diff --git a/src/FtpTunnelResponseCommand.cc b/src/FtpTunnelResponseCommand.cc index 8411e452..9167e6d6 100644 --- a/src/FtpTunnelResponseCommand.cc +++ b/src/FtpTunnelResponseCommand.cc @@ -37,14 +37,15 @@ FtpTunnelResponseCommand::FtpTunnelResponseCommand(int cuid, const RequestHandle& req, + RequestGroup* requestGroup, const HttpConnectionHandle& httpConnection, DownloadEngine* e, const SocketHandle& s) - :AbstractProxyResponseCommand(cuid, req, httpConnection,e, s) {} + :AbstractProxyResponseCommand(cuid, req, requestGroup, httpConnection,e, s) {} FtpTunnelResponseCommand::~FtpTunnelResponseCommand() {} Command* FtpTunnelResponseCommand::getNextCommand() { - return new FtpNegotiationCommand(cuid, req, e, socket); + return new FtpNegotiationCommand(cuid, req, _requestGroup, e, socket); } diff --git a/src/FtpTunnelResponseCommand.h b/src/FtpTunnelResponseCommand.h index d33e97b8..79b2ec1a 100644 --- a/src/FtpTunnelResponseCommand.h +++ b/src/FtpTunnelResponseCommand.h @@ -41,6 +41,7 @@ class FtpTunnelResponseCommand : public AbstractProxyResponseCommand { public: FtpTunnelResponseCommand(int cuid, const RequestHandle& req, + RequestGroup* requestGroup, const HttpConnectionHandle& httpConnection, DownloadEngine* e, const SocketHandle& s); diff --git a/src/HttpDownloadCommand.cc b/src/HttpDownloadCommand.cc index 969addf9..c23142ee 100644 --- a/src/HttpDownloadCommand.cc +++ b/src/HttpDownloadCommand.cc @@ -39,18 +39,19 @@ HttpDownloadCommand::HttpDownloadCommand(int cuid, const RequestHandle req, + RequestGroup* requestGroup, DownloadEngine* e, const SocketHandle& socket) - :DownloadCommand(cuid, req, e, socket) {} + :DownloadCommand(cuid, req, requestGroup, e, socket) {} HttpDownloadCommand::~HttpDownloadCommand() {} bool HttpDownloadCommand::prepareForNextSegment() { - if(e->segmentMan->finished()) { + if(_requestGroup->getSegmentMan()->finished()) { return true; } else { if(req->isKeepAlive()) { - Command* command = new HttpRequestCommand(cuid, req, e, socket); + Command* command = new HttpRequestCommand(cuid, req, _requestGroup, e, socket); e->commands.push_back(command); return true; } else { diff --git a/src/HttpDownloadCommand.h b/src/HttpDownloadCommand.h index 0b6edb63..88953831 100644 --- a/src/HttpDownloadCommand.h +++ b/src/HttpDownloadCommand.h @@ -41,7 +41,10 @@ class HttpDownloadCommand : public DownloadCommand { protected: virtual bool prepareForNextSegment(); public: - HttpDownloadCommand(int cuid, const RequestHandle req, DownloadEngine* e, + HttpDownloadCommand(int cuid, + const RequestHandle req, + RequestGroup* requestGroup, + DownloadEngine* e, const SocketHandle& s); virtual ~HttpDownloadCommand(); }; diff --git a/src/HttpInitiateConnectionCommand.cc b/src/HttpInitiateConnectionCommand.cc index 93e2df4d..707ff73e 100644 --- a/src/HttpInitiateConnectionCommand.cc +++ b/src/HttpInitiateConnectionCommand.cc @@ -43,10 +43,12 @@ HttpInitiateConnectionCommand::HttpInitiateConnectionCommand(int cuid, const RequestHandle& req, + RequestGroup* requestGroup, DownloadEngine* e): - AbstractCommand(cuid, req, e) + AbstractCommand(cuid, req, requestGroup, e) { setTimeout(e->option->getAsInt(PREF_DNS_TIMEOUT)); + setStatusActive(); disableReadCheckSocket(); disableWriteCheckSocket(); } @@ -80,9 +82,9 @@ bool HttpInitiateConnectionCommand::executeInternal() { socket->establishConnection(hostname, e->option->getAsInt(PREF_HTTP_PROXY_PORT)); if(useProxyTunnel()) { - command = new HttpProxyRequestCommand(cuid, req, e, socket); + command = new HttpProxyRequestCommand(cuid, req, _requestGroup, e, socket); } else if(useProxyGet()) { - command = new HttpRequestCommand(cuid, req, e, socket); + command = new HttpRequestCommand(cuid, req, _requestGroup, e, socket); } else { // TODO throw new DlAbortEx("ERROR"); @@ -91,7 +93,7 @@ bool HttpInitiateConnectionCommand::executeInternal() { logger->info(MSG_CONNECTING_TO_SERVER, cuid, req->getHost().c_str(), req->getPort()); socket->establishConnection(hostname, req->getPort()); - command = new HttpRequestCommand(cuid, req, e, socket); + command = new HttpRequestCommand(cuid, req, _requestGroup, e, socket); } e->commands.push_back(command); return true; diff --git a/src/HttpInitiateConnectionCommand.h b/src/HttpInitiateConnectionCommand.h index 84201445..b77ede4e 100644 --- a/src/HttpInitiateConnectionCommand.h +++ b/src/HttpInitiateConnectionCommand.h @@ -60,7 +60,7 @@ protected: } #endif // ENABLE_ASYNC_DNS public: - HttpInitiateConnectionCommand(int cuid, const RequestHandle& req, DownloadEngine* e); + HttpInitiateConnectionCommand(int cuid, const RequestHandle& req, RequestGroup* requestGroup, DownloadEngine* e); virtual ~HttpInitiateConnectionCommand(); }; diff --git a/src/HttpProxyRequestCommand.cc b/src/HttpProxyRequestCommand.cc index 34ac623c..65a68bf7 100644 --- a/src/HttpProxyRequestCommand.cc +++ b/src/HttpProxyRequestCommand.cc @@ -37,13 +37,14 @@ HttpProxyRequestCommand::HttpProxyRequestCommand(int cuid, const RequestHandle& req, + RequestGroup* requestGroup, DownloadEngine* e, const SocketHandle& s) - :AbstractProxyRequestCommand(cuid, req, e, s) {} + :AbstractProxyRequestCommand(cuid, req, requestGroup, e, s) {} HttpProxyRequestCommand::~HttpProxyRequestCommand() {} Command* HttpProxyRequestCommand::getNextCommand() { - return new HttpProxyResponseCommand(cuid, req, httpConnection, e, socket); + return new HttpProxyResponseCommand(cuid, req, _requestGroup, httpConnection, e, socket); } diff --git a/src/HttpProxyRequestCommand.h b/src/HttpProxyRequestCommand.h index e2aa052a..975e2574 100644 --- a/src/HttpProxyRequestCommand.h +++ b/src/HttpProxyRequestCommand.h @@ -41,6 +41,7 @@ class HttpProxyRequestCommand : public AbstractProxyRequestCommand { public: HttpProxyRequestCommand(int cuid, const RequestHandle& req, + RequestGroup* requestGroup, DownloadEngine* e, const SocketHandle& s); virtual ~HttpProxyRequestCommand(); diff --git a/src/HttpProxyResponseCommand.cc b/src/HttpProxyResponseCommand.cc index 23c1fe47..19edcb13 100644 --- a/src/HttpProxyResponseCommand.cc +++ b/src/HttpProxyResponseCommand.cc @@ -37,14 +37,15 @@ HttpProxyResponseCommand::HttpProxyResponseCommand(int cuid, const RequestHandle& req, + RequestGroup* requestGroup, const HttpConnectionHandle& httpConnection, DownloadEngine* e, const SocketHandle& s) - :AbstractProxyResponseCommand(cuid, req, httpConnection, e, s) {} + :AbstractProxyResponseCommand(cuid, req, requestGroup, httpConnection, e, s) {} HttpProxyResponseCommand::~HttpProxyResponseCommand() {} Command* HttpProxyResponseCommand::getNextCommand() { - return new HttpRequestCommand(cuid, req, e, socket); + return new HttpRequestCommand(cuid, req, _requestGroup, e, socket); } diff --git a/src/HttpProxyResponseCommand.h b/src/HttpProxyResponseCommand.h index 4cd6471a..3cdd5fa3 100644 --- a/src/HttpProxyResponseCommand.h +++ b/src/HttpProxyResponseCommand.h @@ -41,6 +41,7 @@ class HttpProxyResponseCommand : public AbstractProxyResponseCommand { public: HttpProxyResponseCommand(int cuid, const RequestHandle& req, + RequestGroup* requestGroup, const HttpConnectionHandle& httpConnection, DownloadEngine* e, const SocketHandle& s); diff --git a/src/HttpRequestCommand.cc b/src/HttpRequestCommand.cc index 13cf35af..b708dfd0 100644 --- a/src/HttpRequestCommand.cc +++ b/src/HttpRequestCommand.cc @@ -39,9 +39,10 @@ HttpRequestCommand::HttpRequestCommand(int cuid, const RequestHandle& req, + RequestGroup* requestGroup, DownloadEngine* e, const SocketHandle& s) - :AbstractCommand(cuid, req, e, s) { + :AbstractCommand(cuid, req, requestGroup, e, s) { disableReadCheckSocket(); setWriteCheckSocket(socket); } @@ -60,14 +61,14 @@ bool HttpRequestCommand::executeInternal() { httpRequest->setUserAgent(e->option->get(PREF_USER_AGENT)); httpRequest->setRequest(req); httpRequest->setSegment(segment); - httpRequest->setEntityLength(e->segmentMan->totalSize); + httpRequest->setEntityLength(_requestGroup->getSegmentMan()->totalSize); httpRequest->configure(e->option); HttpConnectionHandle httpConnection = new HttpConnection(cuid, socket, e->option); httpConnection->sendRequest(httpRequest); - Command* command = new HttpResponseCommand(cuid, req, httpConnection, e, socket); + Command* command = new HttpResponseCommand(cuid, req, _requestGroup, httpConnection, e, socket); e->commands.push_back(command); return true; } diff --git a/src/HttpRequestCommand.h b/src/HttpRequestCommand.h index 1063b255..3d2d3d2f 100644 --- a/src/HttpRequestCommand.h +++ b/src/HttpRequestCommand.h @@ -41,7 +41,10 @@ class HttpRequestCommand:public AbstractCommand { protected: virtual bool executeInternal(); public: - HttpRequestCommand(int cuid, const RequestHandle& req, DownloadEngine* e, + HttpRequestCommand(int cuid, + const RequestHandle& req, + RequestGroup* requestGroup, + DownloadEngine* e, const SocketHandle& s); virtual ~HttpRequestCommand(); }; diff --git a/src/HttpResponse.cc b/src/HttpResponse.cc index 2886f30f..72b28916 100644 --- a/src/HttpResponse.cc +++ b/src/HttpResponse.cc @@ -145,16 +145,3 @@ int64_t HttpResponse::getEntityLength() const return httpHeader->getRange()->getEntityLength(); } } - -void HttpResponse::validateFilename(const string& expectedFilename) const -{ - if(expectedFilename.size() == 0) { - return; - } - string actualFilename = determinFilename(); - if(expectedFilename != actualFilename) { - throw new DlAbortEx(EX_FILENAME_MISMATCH, - actualFilename.c_str(), - expectedFilename.c_str()); - } -} diff --git a/src/HttpResponse.h b/src/HttpResponse.h index 6b5dcfe7..ac7afc9f 100644 --- a/src/HttpResponse.h +++ b/src/HttpResponse.h @@ -89,12 +89,6 @@ public: int64_t getEntityLength() const; - /** - * Compares actual filename with specified expectedFilename. - * The actual filename is the string returned from determinFilename(). - */ - void validateFilename(const string& expectedFilename) const; - void setHttpHeader(const HttpHeaderHandle& httpHeader) { this->httpHeader = httpHeader; diff --git a/src/HttpResponseCommand.cc b/src/HttpResponseCommand.cc index cc92e0a5..1ba9b680 100644 --- a/src/HttpResponseCommand.cc +++ b/src/HttpResponseCommand.cc @@ -40,15 +40,17 @@ #include "Util.h" #include "prefs.h" #include "File.h" +#include "InitiateConnectionCommandFactory.h" #include #include HttpResponseCommand::HttpResponseCommand(int32_t cuid, const RequestHandle& req, + RequestGroup* requestGroup, const HttpConnectionHandle& httpConnection, DownloadEngine* e, const SocketHandle& s) - :AbstractCommand(cuid, req, e, s), + :AbstractCommand(cuid, req, requestGroup, e, s), httpConnection(httpConnection) {} HttpResponseCommand::~HttpResponseCommand() {} @@ -84,11 +86,18 @@ bool HttpResponseCommand::executeInternal() e->noWait = true; return prepareForRetry(0); } - httpResponse->validateFilename(e->segmentMan->filename); - if(e->segmentMan->downloadStarted) { + if(_requestGroup->getSegmentMan()->downloadStarted) { + // TODO validate totalsize + _requestGroup->validateFilename(httpResponse->determinFilename()); + _requestGroup->validateTotalLength(httpResponse->getEntityLength()); + createHttpDownloadCommand(httpResponse); return true; } else { + // TODO validate totalsize against hintTotalSize if it is provided. + _requestGroup->validateFilenameByHint(httpResponse->determinFilename()); + _requestGroup->validateTotalLengthByHint(httpResponse->getEntityLength()); + if(httpResponse->isTransferEncodingSpecified()) { return handleOtherEncoding(httpResponse); } else { @@ -101,54 +110,53 @@ bool HttpResponseCommand::handleDefaultEncoding(const HttpResponseHandle& httpRe { HttpRequestHandle httpRequest = httpResponse->getHttpRequest(); // TODO quick and dirty way - if(httpRequest->getRequest()->isTorrent) { + if(_requestGroup->isTorrent) { return doTorrentStuff(httpResponse); } int64_t size = httpResponse->getEntityLength(); if(size == INT64_MAX || size < 0) { throw new DlAbortEx(EX_TOO_LARGE_FILE, size); } - e->segmentMan->isSplittable = !(size == 0); - e->segmentMan->filename = httpResponse->determinFilename(); - e->segmentMan->downloadStarted = true; - e->segmentMan->totalSize = size; + _requestGroup->getSegmentMan()->isSplittable = !(size == 0); + _requestGroup->getSegmentMan()->filename = httpResponse->determinFilename(); + _requestGroup->getSegmentMan()->downloadStarted = true; + _requestGroup->getSegmentMan()->totalSize = size; // quick hack for method 'head' if(httpRequest->getMethod() == Request::METHOD_HEAD) { // TODO because we don't want segment file to be saved. - e->segmentMan->isSplittable = false; + _requestGroup->getSegmentMan()->isSplittable = false; return true; } - bool segFileExists = e->segmentMan->segmentFileExists(); - if(segFileExists) { - e->segmentMan->load(); - e->segmentMan->diskWriter->openExistingFile(e->segmentMan->getFilePath()); - // send request again to the server with Range header - return prepareForRetry(0); - } else { - e->segmentMan->initBitfield(e->option->getAsInt(PREF_SEGMENT_SIZE), - e->segmentMan->totalSize); - e->segmentMan->diskWriter->initAndOpenFile(e->segmentMan->getFilePath(), - size); - return prepareForRetry(0); + + if(e->option->get(PREF_CHECK_INTEGRITY) != V_TRUE) { + if(_requestGroup->downloadFinishedByFileLength()) { + logger->notice(MSG_DOWNLOAD_ALREADY_COMPLETED, cuid, _requestGroup->getFilePath().c_str()); + return true; + } } + + _requestGroup->loadAndOpenFile(); + _requestGroup->prepareForNextAction(cuid, req, e); + e->noWait = true; + return true; } bool HttpResponseCommand::handleOtherEncoding(const HttpResponseHandle& httpResponse) { HttpRequestHandle httpRequest = httpResponse->getHttpRequest(); // we ignore content-length when transfer-encoding is set - e->segmentMan->downloadStarted = true; - e->segmentMan->isSplittable = false; - e->segmentMan->filename = httpResponse->determinFilename(); - e->segmentMan->totalSize = 0; + _requestGroup->getSegmentMan()->downloadStarted = true; + _requestGroup->getSegmentMan()->isSplittable = false; + _requestGroup->getSegmentMan()->filename = httpResponse->determinFilename(); + _requestGroup->getSegmentMan()->totalSize = 0; // quick hack for method 'head' if(httpRequest->getMethod() == Request::METHOD_HEAD) { return true; } // disable keep-alive req->setKeepAlive(false); - segment = e->segmentMan->getSegment(cuid); - e->segmentMan->diskWriter->initAndOpenFile(e->segmentMan->getFilePath()); + segment = _requestGroup->getSegmentMan()->getSegment(cuid); + _requestGroup->getSegmentMan()->diskWriter->initAndOpenFile(_requestGroup->getSegmentMan()->getFilePath()); createHttpDownloadCommand(httpResponse); return true; } @@ -164,7 +172,8 @@ void HttpResponseCommand::createHttpDownloadCommand(const HttpResponseHandle& ht } enc->init(); } - HttpDownloadCommand* command = new HttpDownloadCommand(cuid, req, e, socket); + HttpDownloadCommand* command = + new HttpDownloadCommand(cuid, req, _requestGroup, e, socket); command->setMaxDownloadSpeedLimit(e->option->getAsInt(PREF_MAX_DOWNLOAD_LIMIT)); command->setStartupIdleTime(e->option->getAsInt(PREF_STARTUP_IDLE_TIME)); command->setLowestDownloadSpeedLimit(e->option->getAsInt(PREF_LOWEST_SPEED_LIMIT)); @@ -176,16 +185,16 @@ void HttpResponseCommand::createHttpDownloadCommand(const HttpResponseHandle& ht bool HttpResponseCommand::doTorrentStuff(const HttpResponseHandle& httpResponse) { int64_t size = httpResponse->getEntityLength(); - e->segmentMan->totalSize = size; + _requestGroup->getSegmentMan()->totalSize = size; if(size > 0) { - e->segmentMan->initBitfield(e->option->getAsInt(PREF_SEGMENT_SIZE), - e->segmentMan->totalSize); + _requestGroup->getSegmentMan()->initBitfield(e->option->getAsInt(PREF_SEGMENT_SIZE), + _requestGroup->getSegmentMan()->totalSize); } // disable keep-alive httpResponse->getHttpRequest()->getRequest()->setKeepAlive(false); - e->segmentMan->isSplittable = false; - e->segmentMan->downloadStarted = true; - e->segmentMan->diskWriter->initAndOpenFile("/tmp/aria2"+Util::itos((int32_t)getpid())); + _requestGroup->getSegmentMan()->isSplittable = false; + _requestGroup->getSegmentMan()->downloadStarted = true; + _requestGroup->getSegmentMan()->diskWriter->initAndOpenFile("/tmp/aria2"+Util::itos((int32_t)getpid())); createHttpDownloadCommand(httpResponse); return true; } diff --git a/src/HttpResponseCommand.h b/src/HttpResponseCommand.h index 00ec83c0..8c1c6576 100644 --- a/src/HttpResponseCommand.h +++ b/src/HttpResponseCommand.h @@ -51,6 +51,7 @@ protected: public: HttpResponseCommand(int32_t cuid, const RequestHandle& req, + RequestGroup* requestGroup, const HttpConnectionHandle& httpConnection, DownloadEngine* e, const SocketHandle& s); diff --git a/src/InitiateConnectionCommandFactory.cc b/src/InitiateConnectionCommandFactory.cc index 536cea0b..2ae70386 100644 --- a/src/InitiateConnectionCommandFactory.cc +++ b/src/InitiateConnectionCommandFactory.cc @@ -37,16 +37,16 @@ #include "FtpInitiateConnectionCommand.h" #include "DlAbortEx.h" -Command* InitiateConnectionCommandFactory::createInitiateConnectionCommand(int cuid, const RequestHandle req, DownloadEngine* e) { +Command* InitiateConnectionCommandFactory::createInitiateConnectionCommand(int cuid, const RequestHandle req, RequestGroup* requestGroup, DownloadEngine* e) { if(req->getProtocol() == "http" #ifdef ENABLE_SSL // for SSL || req->getProtocol() == "https" #endif // ENABLE_SSL ) { - return new HttpInitiateConnectionCommand(cuid, req, e); + return new HttpInitiateConnectionCommand(cuid, req, requestGroup, e); } else if(req->getProtocol() == "ftp") { - return new FtpInitiateConnectionCommand(cuid, req, e); + return new FtpInitiateConnectionCommand(cuid, req, requestGroup, e); } else { // these protocols are not supported yet throw new DlAbortEx("%s is not supported yet.", req->getProtocol().c_str()); diff --git a/src/InitiateConnectionCommandFactory.h b/src/InitiateConnectionCommandFactory.h index 820d38bf..afe5c8c3 100644 --- a/src/InitiateConnectionCommandFactory.h +++ b/src/InitiateConnectionCommandFactory.h @@ -35,14 +35,16 @@ #ifndef _D_INITIATE_CONNECTION_COMMAND_FACTORY_H_ #define _D_INITIATE_CONNECTION_COMMAND_FACTORY_H_ -#include "Request.h" -#include "DownloadEngine.h" #include "common.h" +#include "Request.h" +#include "RequestGroup.h" +#include "DownloadEngine.h" class InitiateConnectionCommandFactory { public: static Command* createInitiateConnectionCommand(int cuid, - const RequestHandle req, + const RequestHandle req, + RequestGroup* requestGroup, DownloadEngine* e); }; diff --git a/src/LogFactory.cc b/src/LogFactory.cc index c48a4601..a0b07293 100644 --- a/src/LogFactory.cc +++ b/src/LogFactory.cc @@ -43,6 +43,8 @@ Logger* LogFactory::getInstance() { SimpleLogger* slogger = new SimpleLogger(); slogger->openFile(filename); slogger->setStdout(Logger::NOTICE, true); + slogger->setStdout(Logger::WARN, true); + slogger->setStdout(Logger::ERROR, true); logger = slogger; } return logger; diff --git a/src/Makefile.am b/src/Makefile.am index aeff28a1..2da5d272 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -81,7 +81,21 @@ SRCS = Socket.h\ GlowFileAllocator.cc GlowFileAllocator.h\ OptionParser.cc OptionParser.h\ OptionHandlerFactory.cc OptionHandlerFactory.h\ - NameResolver.cc NameResolver.h + NameResolver.cc NameResolver.h\ + RequestGroup.cc RequestGroup.h\ + RequestGroupMan.cc RequestGroupMan.h\ + FileAllocationCommand.cc FileAllocationCommand.h\ + FillRequestGroupCommand.cc FillRequestGroupCommand.h\ + FileAllocationDispatcherCommand.cc FileAllocationDispatcherCommand.h\ + FileAllocationEntry.cc FileAllocationEntry.h\ + IteratableChunkChecksumValidator.cc IteratableChunkChecksumValidator.h\ + CheckIntegrityCommand.cc CheckIntegrityCommand.h\ + SingleUrlRequestInfo.cc SingleUrlRequestInfo.h\ + MultiUrlRequestInfo.cc MultiUrlRequestInfo.h\ + UriFileListParser.cc UriFileListParser.h\ + SegmentManFactory.h\ + AbstractSegmentManFactory.h\ + DefaultSegmentManFactory.cc DefaultSegmentManFactory.h # debug_new.cpp if ENABLE_BITTORRENT @@ -187,7 +201,8 @@ SRCS += MetaEntry.h\ BtInteractive.h\ DefaultBtInteractive.cc DefaultBtInteractive.h\ PeerObject.h\ - ActivePeerConnectionCommand.cc ActivePeerConnectionCommand.h + ActivePeerConnectionCommand.cc ActivePeerConnectionCommand.h\ + TrackerSegmentManFactory.cc TrackerSegmentManFactory.h endif # ENABLE_BITTORRENT if ENABLE_METALINK diff --git a/src/Makefile.in b/src/Makefile.in index 64535cfd..98e61e0a 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -141,7 +141,8 @@ bin_PROGRAMS = aria2c$(EXEEXT) @ENABLE_BITTORRENT_TRUE@ BtInteractive.h\ @ENABLE_BITTORRENT_TRUE@ DefaultBtInteractive.cc DefaultBtInteractive.h\ @ENABLE_BITTORRENT_TRUE@ PeerObject.h\ -@ENABLE_BITTORRENT_TRUE@ ActivePeerConnectionCommand.cc ActivePeerConnectionCommand.h +@ENABLE_BITTORRENT_TRUE@ ActivePeerConnectionCommand.cc ActivePeerConnectionCommand.h\ +@ENABLE_BITTORRENT_TRUE@ TrackerSegmentManFactory.cc TrackerSegmentManFactory.h @ENABLE_METALINK_TRUE@am__append_2 = Metalinker.cc Metalinker.h\ @ENABLE_METALINK_TRUE@ MetalinkEntry.cc MetalinkEntry.h\ @@ -224,12 +225,23 @@ am__libaria2c_a_SOURCES_DIST = Socket.h SocketCore.cc SocketCore.h \ DefaultFileAllocator.h GlowFileAllocator.cc \ GlowFileAllocator.h OptionParser.cc OptionParser.h \ OptionHandlerFactory.cc OptionHandlerFactory.h NameResolver.cc \ - NameResolver.h MetaEntry.h Data.cc Data.h Dictionary.cc \ - Dictionary.h List.cc List.h MetaFileUtil.cc MetaFileUtil.h \ - MetaEntryVisitor.h ShaVisitor.cc ShaVisitor.h \ - PeerConnection.cc PeerConnection.h PeerMessageUtil.cc \ - PeerMessageUtil.h PeerAbstractCommand.cc PeerAbstractCommand.h \ - PeerInitiateConnectionCommand.cc \ + NameResolver.h RequestGroup.cc RequestGroup.h \ + RequestGroupMan.cc RequestGroupMan.h FileAllocationCommand.cc \ + FileAllocationCommand.h FillRequestGroupCommand.cc \ + FillRequestGroupCommand.h FileAllocationDispatcherCommand.cc \ + FileAllocationDispatcherCommand.h FileAllocationEntry.cc \ + FileAllocationEntry.h IteratableChunkChecksumValidator.cc \ + IteratableChunkChecksumValidator.h CheckIntegrityCommand.cc \ + CheckIntegrityCommand.h SingleUrlRequestInfo.cc \ + SingleUrlRequestInfo.h MultiUrlRequestInfo.cc \ + MultiUrlRequestInfo.h UriFileListParser.cc UriFileListParser.h \ + SegmentManFactory.h AbstractSegmentManFactory.h \ + DefaultSegmentManFactory.cc DefaultSegmentManFactory.h \ + MetaEntry.h Data.cc Data.h Dictionary.cc Dictionary.h List.cc \ + List.h MetaFileUtil.cc MetaFileUtil.h MetaEntryVisitor.h \ + ShaVisitor.cc ShaVisitor.h PeerConnection.cc PeerConnection.h \ + PeerMessageUtil.cc PeerMessageUtil.h PeerAbstractCommand.cc \ + PeerAbstractCommand.h PeerInitiateConnectionCommand.cc \ PeerInitiateConnectionCommand.h PeerInteractionCommand.cc \ PeerInteractionCommand.h Peer.cc Peer.h \ TorrentDownloadEngine.cc TorrentDownloadEngine.h \ @@ -290,6 +302,7 @@ am__libaria2c_a_SOURCES_DIST = Socket.h SocketCore.cc SocketCore.h \ BtChokedEvent.h BtChokingEvent.h BtInteractive.h \ DefaultBtInteractive.cc DefaultBtInteractive.h PeerObject.h \ ActivePeerConnectionCommand.cc ActivePeerConnectionCommand.h \ + TrackerSegmentManFactory.cc TrackerSegmentManFactory.h \ Metalinker.cc Metalinker.h MetalinkEntry.cc MetalinkEntry.h \ MetalinkResource.cc MetalinkResource.h MetalinkProcessor.h \ Xml2MetalinkProcessor.cc Xml2MetalinkProcessor.h \ @@ -358,7 +371,8 @@ am__libaria2c_a_SOURCES_DIST = Socket.h SocketCore.cc SocketCore.h \ @ENABLE_BITTORRENT_TRUE@ DefaultBtMessageReceiver.$(OBJEXT) \ @ENABLE_BITTORRENT_TRUE@ DefaultBtRequestFactory.$(OBJEXT) \ @ENABLE_BITTORRENT_TRUE@ DefaultBtInteractive.$(OBJEXT) \ -@ENABLE_BITTORRENT_TRUE@ ActivePeerConnectionCommand.$(OBJEXT) +@ENABLE_BITTORRENT_TRUE@ ActivePeerConnectionCommand.$(OBJEXT) \ +@ENABLE_BITTORRENT_TRUE@ TrackerSegmentManFactory.$(OBJEXT) @ENABLE_METALINK_TRUE@am__objects_2 = Metalinker.$(OBJEXT) \ @ENABLE_METALINK_TRUE@ MetalinkEntry.$(OBJEXT) \ @ENABLE_METALINK_TRUE@ MetalinkResource.$(OBJEXT) \ @@ -397,7 +411,16 @@ am__objects_3 = SocketCore.$(OBJEXT) Command.$(OBJEXT) \ NetrcAuthResolver.$(OBJEXT) RequestFactory.$(OBJEXT) \ DefaultFileAllocator.$(OBJEXT) GlowFileAllocator.$(OBJEXT) \ OptionParser.$(OBJEXT) OptionHandlerFactory.$(OBJEXT) \ - NameResolver.$(OBJEXT) $(am__objects_1) $(am__objects_2) + NameResolver.$(OBJEXT) RequestGroup.$(OBJEXT) \ + RequestGroupMan.$(OBJEXT) FileAllocationCommand.$(OBJEXT) \ + FillRequestGroupCommand.$(OBJEXT) \ + FileAllocationDispatcherCommand.$(OBJEXT) \ + FileAllocationEntry.$(OBJEXT) \ + IteratableChunkChecksumValidator.$(OBJEXT) \ + CheckIntegrityCommand.$(OBJEXT) SingleUrlRequestInfo.$(OBJEXT) \ + MultiUrlRequestInfo.$(OBJEXT) UriFileListParser.$(OBJEXT) \ + DefaultSegmentManFactory.$(OBJEXT) $(am__objects_1) \ + $(am__objects_2) am_libaria2c_a_OBJECTS = $(am__objects_3) libaria2c_a_OBJECTS = $(am_libaria2c_a_OBJECTS) am__installdirs = "$(DESTDIR)$(bindir)" @@ -614,7 +637,19 @@ SRCS = Socket.h SocketCore.cc SocketCore.h Command.cc Command.h \ DefaultFileAllocator.h GlowFileAllocator.cc \ GlowFileAllocator.h OptionParser.cc OptionParser.h \ OptionHandlerFactory.cc OptionHandlerFactory.h NameResolver.cc \ - NameResolver.h $(am__append_1) $(am__append_2) + NameResolver.h RequestGroup.cc RequestGroup.h \ + RequestGroupMan.cc RequestGroupMan.h FileAllocationCommand.cc \ + FileAllocationCommand.h FillRequestGroupCommand.cc \ + FillRequestGroupCommand.h FileAllocationDispatcherCommand.cc \ + FileAllocationDispatcherCommand.h FileAllocationEntry.cc \ + FileAllocationEntry.h IteratableChunkChecksumValidator.cc \ + IteratableChunkChecksumValidator.h CheckIntegrityCommand.cc \ + CheckIntegrityCommand.h SingleUrlRequestInfo.cc \ + SingleUrlRequestInfo.h MultiUrlRequestInfo.cc \ + MultiUrlRequestInfo.h UriFileListParser.cc UriFileListParser.h \ + SegmentManFactory.h AbstractSegmentManFactory.h \ + DefaultSegmentManFactory.cc DefaultSegmentManFactory.h \ + $(am__append_1) $(am__append_2) noinst_LIBRARIES = libaria2c.a libaria2c_a_SOURCES = $(SRCS) aria2c_LDADD = libaria2c.a @LIBINTL@ @ALLOCA@ @LIBGNUTLS_LIBS@\ @@ -733,6 +768,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/BtSuggestPieceMessage.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/BtUnchokeMessage.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ByteArrayDiskWriter.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/CheckIntegrityCommand.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ChunkChecksumValidator.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ChunkedEncoding.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Command.Po@am__quote@ @@ -756,6 +792,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DefaultPeerListProcessor.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DefaultPeerStorage.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DefaultPieceStorage.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DefaultSegmentManFactory.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DelegatingPeerListProcessor.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Dictionary.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DirectDiskAdaptor.Po@am__quote@ @@ -766,8 +803,12 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DownloadEngineFactory.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FeatureConfig.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/File.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FileAllocationCommand.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FileAllocationDispatcherCommand.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FileAllocationEntry.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FileAllocationMonitor.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FileEntry.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FillRequestGroupCommand.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FtpConnection.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FtpDownloadCommand.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FtpInitiateConnectionCommand.Po@am__quote@ @@ -787,6 +828,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/HttpResponse.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/HttpResponseCommand.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/InitiateConnectionCommandFactory.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/IteratableChunkChecksumValidator.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/List.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/LogFactory.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/MetaFileUtil.Po@am__quote@ @@ -795,6 +837,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/MetalinkResource.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Metalinker.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/MultiDiskAdaptor.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/MultiUrlRequestInfo.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/NameResolver.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Netrc.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/NetrcAuthResolver.Po@am__quote@ @@ -812,6 +855,8 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Piece.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Request.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/RequestFactory.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/RequestGroup.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/RequestGroupMan.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/RequestSlot.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/SeedCheckCommand.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Segment.Po@am__quote@ @@ -820,6 +865,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/SimpleBtMessage.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/SimpleLogger.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/SimpleRandomizer.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/SingleUrlRequestInfo.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/SleepCommand.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/SocketCore.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/SpeedCalc.Po@am__quote@ @@ -828,8 +874,10 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/TorrentConsoleDownloadEngine.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/TorrentDownloadEngine.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/TorrentRequestInfo.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/TrackerSegmentManFactory.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/TrackerUpdateCommand.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/TrackerWatcherCommand.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/UriFileListParser.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/UrlRequestInfo.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Util.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Xml2MetalinkProcessor.Po@am__quote@ diff --git a/src/MetalinkRequestInfo.cc b/src/MetalinkRequestInfo.cc index b7fbb037..495a5c3e 100644 --- a/src/MetalinkRequestInfo.cc +++ b/src/MetalinkRequestInfo.cc @@ -36,7 +36,7 @@ #include "Xml2MetalinkProcessor.h" #include "prefs.h" #include "DlAbortEx.h" -#include "UrlRequestInfo.h" +#include "MultiUrlRequestInfo.h" class AccumulateNonP2PUrl { private: @@ -87,6 +87,7 @@ RequestInfos MetalinkRequestInfo::execute() { printf("No file matched with your preference.\n"); throw new DlAbortEx("No file matched with your preference."); } + RequestGroups groups; for(MetalinkEntries::iterator itr = entries.begin(); itr != entries.end(); itr++) { MetalinkEntryHandle& entry = *itr; @@ -121,19 +122,25 @@ RequestInfos MetalinkRequestInfo::execute() { // BitTorrent downloading urls.push_back((*itr)->url); } - UrlRequestInfoHandle reqInfo = new UrlRequestInfo(urls, maxConnection, op); - reqInfo->setFilename(entry->filename); - reqInfo->setTotalLength(entry->size); + RequestGroupHandle rg = new RequestGroup(urls, op); + rg->setHintFilename(entry->filename); + rg->setHintTotalLength(entry->size); + #ifdef ENABLE_MESSAGE_DIGEST - reqInfo->setChecksum(checksum); - if(!entry->chunkChecksum.isNull()) { - reqInfo->setDigestAlgo(entry->chunkChecksum->digestAlgo); - reqInfo->setChunkChecksumLength(entry->chunkChecksum->pieceLength); - reqInfo->setChunkChecksums(entry->chunkChecksum->pieceHashes); + if(entry->chunkChecksum.isNull()) { + rg->setChecksum(checksum); + } else { + ChunkChecksumHandle cc = + new ChunkChecksum(entry->chunkChecksum->digestAlgo, + entry->chunkChecksum->pieceHashes, + entry->chunkChecksum->pieceLength); + rg->setChunkChecksum(cc); } #endif // ENABLE_MESSAGE_DIGEST - nextReqInfos.push_front(reqInfo); + groups.push_front(rg); } + MultiUrlRequestInfoHandle reqInfo = new MultiUrlRequestInfo(groups, op); + nextReqInfos.push_back(reqInfo); } catch(RecoverableException* ex) { logger->error("Exception caught", ex); delete ex; diff --git a/src/OptionHandlerFactory.cc b/src/OptionHandlerFactory.cc index a3c90eb1..22064c5e 100644 --- a/src/OptionHandlerFactory.cc +++ b/src/OptionHandlerFactory.cc @@ -90,6 +90,7 @@ OptionHandlers OptionHandlerFactory::createOptionHandlers() handlers.push_back(new BooleanOptionHandler(PREF_CONTINUE)); handlers.push_back(new DefaultOptionHandler(PREF_USER_AGENT)); handlers.push_back(new BooleanOptionHandler(PREF_NO_NETRC)); + handlers.push_back(new DefaultOptionHandler(PREF_INPUT_FILE)); return handlers; } diff --git a/src/PeerAbstractCommand.cc b/src/PeerAbstractCommand.cc index 3611530e..f6e9f01f 100644 --- a/src/PeerAbstractCommand.cc +++ b/src/PeerAbstractCommand.cc @@ -96,7 +96,7 @@ bool PeerAbstractCommand::prepareForRetry(int wait) { return true; } -void PeerAbstractCommand::onAbort(RecoverableException* ex) { +void PeerAbstractCommand::onAbort(Exception* ex) { peerStorage->returnPeer(peer); } diff --git a/src/PeerAbstractCommand.h b/src/PeerAbstractCommand.h index 61ad3801..0aed6824 100644 --- a/src/PeerAbstractCommand.h +++ b/src/PeerAbstractCommand.h @@ -39,7 +39,7 @@ #include "Request.h" #include "TorrentDownloadEngine.h" #include "TimeA2.h" -#include "RecoverableException.h" +#include "Exception.h" class PeerAbstractCommand : public BtContextAwareCommand { private: @@ -53,7 +53,7 @@ protected: void setTimeout(int timeout) { this->timeout = timeout; } virtual bool prepareForNextPeer(int wait); virtual bool prepareForRetry(int wait); - virtual void onAbort(RecoverableException* ex); + virtual void onAbort(Exception* ex); virtual bool executeInternal() = 0; void setReadCheckSocket(const SocketHandle& socket); void setWriteCheckSocket(const SocketHandle& socket); diff --git a/src/PeerInteractionCommand.cc b/src/PeerInteractionCommand.cc index ce1cbcb9..87a3bf52 100644 --- a/src/PeerInteractionCommand.cc +++ b/src/PeerInteractionCommand.cc @@ -218,7 +218,7 @@ bool PeerInteractionCommand::prepareForRetry(int wait) { return false; } -void PeerInteractionCommand::onAbort(RecoverableException* ex) { +void PeerInteractionCommand::onAbort(Exception* ex) { btInteractive->cancelAllPiece(); PeerAbstractCommand::onAbort(ex); } diff --git a/src/PeerInteractionCommand.h b/src/PeerInteractionCommand.h index 41fc358d..ebb62359 100644 --- a/src/PeerInteractionCommand.h +++ b/src/PeerInteractionCommand.h @@ -47,7 +47,7 @@ protected: virtual bool executeInternal(); virtual bool prepareForRetry(int wait); virtual bool prepareForNextPeer(int wait); - virtual void onAbort(RecoverableException* ex); + virtual void onAbort(Exception* ex); public: PeerInteractionCommand(int cuid, const PeerHandle& peer, diff --git a/src/PeerListenCommand.cc b/src/PeerListenCommand.cc index eebd457c..c2610455 100644 --- a/src/PeerListenCommand.cc +++ b/src/PeerListenCommand.cc @@ -34,6 +34,7 @@ /* copyright --> */ #include "PeerListenCommand.h" #include "PeerInteractionCommand.h" +#include "RecoverableException.h" PeerListenCommand::PeerListenCommand(int cuid, TorrentDownloadEngine* e, diff --git a/src/Request.cc b/src/Request.cc index c289ed68..bfe2ab81 100644 --- a/src/Request.cc +++ b/src/Request.cc @@ -43,8 +43,7 @@ const string Request::METHOD_HEAD = "head"; Request::Request():port(0), tryCount(0), keepAlive(true), method(METHOD_GET), _httpAuthResolver(0), _httpProxyAuthResolver(0), - _ftpAuthResolver(0), - isTorrent(false) + _ftpAuthResolver(0) { cookieBox = new CookieBox(); } diff --git a/src/Request.h b/src/Request.h index 6f64c617..8afa1ea9 100644 --- a/src/Request.h +++ b/src/Request.h @@ -81,7 +81,6 @@ private: bool parseUrl(const string& url); public: CookieBox* cookieBox; - bool isTorrent; public: Request(); virtual ~Request(); diff --git a/src/RequestInfo.h b/src/RequestInfo.h index 481cfc0e..99e43beb 100644 --- a/src/RequestInfo.h +++ b/src/RequestInfo.h @@ -107,7 +107,7 @@ protected: } void printDownloadAbortMessage() { - printf(_("\nThe download was not complete because of errors." + printf(_("\nSome downloads were not complete because of errors." " Check the log.\n" "aria2 will resume download if the transfer is restarted.")); } diff --git a/src/SegmentMan.cc b/src/SegmentMan.cc index 6969ded0..15db29ee 100644 --- a/src/SegmentMan.cc +++ b/src/SegmentMan.cc @@ -101,7 +101,7 @@ void SegmentMan::load() { } void SegmentMan::save() const { - if(!isSplittable || totalSize == 0) { + if(!isSplittable || totalSize == 0 || !bitfield) { return; } string segFilename = getSegmentFilePath(); @@ -434,11 +434,11 @@ int32_t SegmentMan::calculateDownloadSpeed() const { return speed; } -bool SegmentMan::fileExists() { +bool SegmentMan::fileExists() const { return File(getFilePath()).exists(); } -bool SegmentMan::shouldCancelDownloadForSafety() { +bool SegmentMan::shouldCancelDownloadForSafety() const { return fileExists() && !segmentFileExists() && option->get(PREF_ALLOW_OVERWRITE) != V_TRUE; } @@ -456,6 +456,7 @@ void SegmentMan::markPieceDone(int64_t length) if(length == bitfield->getTotalLength()) { bitfield->setAllBit(); } else { + bitfield->clearAllBit(); int32_t numSegment = length/bitfield->getBlockLength(); int32_t remainingLength = length%bitfield->getBlockLength(); bitfield->setBitRange(0, numSegment-1); @@ -535,10 +536,10 @@ void SegmentMan::tryChunkChecksumValidation(const SegmentHandle& segment) if(expectedChecksum == actualChecksum) { logger->info("Good chunk checksum."); } else { - logger->error(EX_INVALID_CHUNK_CHECKSUM, - index, offset, dataLength, - expectedChecksum.c_str(), actualChecksum.c_str()); - logger->info("Unset bit from %d to %d(inclusive)", startIndex, endIndex); + logger->info(EX_INVALID_CHUNK_CHECKSUM, + index, offset, + expectedChecksum.c_str(), actualChecksum.c_str()); + logger->debug("Unset bit from %d to %d(inclusive)", startIndex, endIndex); bitfield->unsetBitRange(startIndex, endIndex); break; } diff --git a/src/SegmentMan.h b/src/SegmentMan.h index ac72994f..eca5b925 100644 --- a/src/SegmentMan.h +++ b/src/SegmentMan.h @@ -245,6 +245,10 @@ public: * Initializes bitfield with the provided length parameters. */ void initBitfield(int32_t segmentLength, int64_t totalLength); + BitfieldMan* getBitfield() const + { + return bitfield; + } /** * Returns true if the segment whose index is index has been downloaded. */ @@ -279,9 +283,9 @@ public: */ int32_t calculateDownloadSpeed() const; - bool fileExists(); + bool fileExists() const; - bool shouldCancelDownloadForSafety(); + bool shouldCancelDownloadForSafety() const; void markAllPiecesDone(); @@ -296,4 +300,5 @@ public: #endif // ENABLE_MESSAGE_DIGEST }; +typedef SharedHandle SegmentManHandle; #endif // _D_SEGMENT_MAN_H_ diff --git a/src/TrackerUpdateCommand.cc b/src/TrackerUpdateCommand.cc index d619a754..53796e76 100644 --- a/src/TrackerUpdateCommand.cc +++ b/src/TrackerUpdateCommand.cc @@ -62,7 +62,7 @@ char* TrackerUpdateCommand::getTrackerResponse(size_t& trackerResponseLength) { char data[2048]; try { while(1) { - int dataLength = e->segmentMan->diskWriter->readData(data, sizeof(data), bufLength); + int dataLength = e->_requestGroupMan->getRequestGroup(0)->getSegmentMan()->diskWriter->readData(data, sizeof(data), bufLength); if(bufLength+dataLength >= maxBufLength) { maxBufLength = Util::expandBuffer(&buf, bufLength, bufLength+dataLength); } @@ -84,7 +84,8 @@ bool TrackerUpdateCommand::execute() { if(btAnnounce->noMoreAnnounce()) { return true; } - if(!e->segmentMan->finished()) { + if(e->_requestGroupMan->countRequestGroup() == 0 || + !e->_requestGroupMan->downloadFinished()) { return prepareForRetry(); } char* trackerResponse = 0; @@ -112,10 +113,10 @@ bool TrackerUpdateCommand::execute() { } btAnnounce->announceSuccess(); btAnnounce->resetAnnounce(); - e->segmentMan->init(); + e->_requestGroupMan->removeStoppedGroup(); } catch(RecoverableException* err) { logger->error("CUID#%d - Error occurred while processing tracker response.", cuid, err); - e->segmentMan->errors++; + e->_requestGroupMan->getRequestGroup(0)->getSegmentMan()->errors++; delete err; } if(trackerResponse) { diff --git a/src/TrackerWatcherCommand.cc b/src/TrackerWatcherCommand.cc index 5e22e2d5..4c5eec74 100644 --- a/src/TrackerWatcherCommand.cc +++ b/src/TrackerWatcherCommand.cc @@ -38,6 +38,7 @@ #include "SleepCommand.h" #include "prefs.h" #include "RequestFactory.h" +#include "TrackerSegmentManFactory.h" TrackerWatcherCommand::TrackerWatcherCommand(int cuid, TorrentDownloadEngine* e, @@ -66,9 +67,9 @@ Command* TrackerWatcherCommand::createCommand() { if(btAnnounce->isAnnounceReady()) { command = createRequestCommand(btAnnounce->getAnnounceUrl()); btAnnounce->announceStart(); // inside it, trackers++. - } else if(e->segmentMan->errors > 0) { + } else if(e->_requestGroupMan->getErrors() > 0) { btAnnounce->announceFailure(); // inside it, trackers = 0. - e->segmentMan->init(); + e->_requestGroupMan->removeStoppedGroup(); if(btAnnounce->isAllAnnounceFailed()) { btAnnounce->resetAnnounce(); // sleep a few seconds. @@ -83,12 +84,18 @@ Command* TrackerWatcherCommand::createCommand() { Command* TrackerWatcherCommand::createRequestCommand(const string& url) { - RequestHandle req = RequestFactorySingletonHolder::instance()->createRequest(); - req->setUrl(url); - req->isTorrent = true; - Command* command = - InitiateConnectionCommandFactory::createInitiateConnectionCommand(btRuntime->getNewCuid(), req, e); + RequestGroupHandle rg = new RequestGroup(url, e->option); + rg->isTorrent = true; + rg->setSegmentManFactory(new TrackerSegmentManFactory(e->option)); + e->_requestGroupMan->addRequestGroup(rg); + Commands commands = e->_requestGroupMan->getInitialCommands(e); + + if(commands.empty()) { + logger->error("CUID#%d - Cannot create tracker request."); + return 0; + } + logger->info("CUID#%d - Creating new tracker request command #%d", cuid, - command->getCuid()); - return command; + commands.front()->getCuid()); + return commands.front(); } diff --git a/src/UrlRequestInfo.cc b/src/UrlRequestInfo.cc index 7be0106e..40fe7c5e 100644 --- a/src/UrlRequestInfo.cc +++ b/src/UrlRequestInfo.cc @@ -88,7 +88,7 @@ RequestInfo* UrlRequestInfo::createNextRequestInfo() const void handler(int signal) { haltRequested = true; } - +/* class CreateRequest { private: Requests* requestsPtr; @@ -120,13 +120,13 @@ public: } } }; - +*/ void UrlRequestInfo::printUrls(const Strings& urls) const { for(Strings::const_iterator itr = urls.begin(); itr != urls.end(); itr++) { logger->info("Adding URL: %s", itr->c_str()); } } - +/* HeadResultHandle UrlRequestInfo::getHeadResult() { Requests requests; for_each(urls.begin(), urls.end(), @@ -151,7 +151,7 @@ HeadResultHandle UrlRequestInfo::getHeadResult() { return hr; } - +*/ RequestInfos UrlRequestInfo::execute() { { @@ -159,136 +159,54 @@ RequestInfos UrlRequestInfo::execute() { DNSCacheSingletonHolder::instance(dnsCache); } - Requests requests; - Requests reserved; - printUrls(urls); - RequestInfo* next = 0; - try { - HeadResultHandle hr(0); - if(_totalLength == 0 || _filename.length() == 0) { - hr = getHeadResult(); - - if(hr.isNull()) { - logger->notice(MSG_NO_URL_TO_DOWNLOAD); - return RequestInfos(); - } - } else { - hr = new HeadResult(); - hr->filename = _filename; - hr->totalLength = _totalLength; - } - logger->info("Head result: filename=%s, total length=%s", - hr->filename.c_str(), Util::ullitos(hr->totalLength, true).c_str()); - - for_each(urls.begin(), urls.end(), - CreateRequest(&requests, - op->get(PREF_REFERER), - op->getAsInt(PREF_SPLIT))); - - if(requests.size() == 0) { - logger->notice(MSG_NO_URL_TO_DOWNLOAD); - return RequestInfos(); - } - adjustRequestSize(requests, reserved, maxConnections); + try { + RequestGroups requestGroups; + + Strings urls; + urls.push_back("http://localhost/~tujikawa/linux-2.6.6.tar.bz2"); + RequestGroupHandle rg1 = new RequestGroup(urls, op); + + Strings urls2; + urls2.push_back("http://localhost/~tujikawa/linux-2.6.19.1.tar.bz2"); + RequestGroupHandle rg2 = new RequestGroup(urls2, op); + + requestGroups.push_back(rg1); + requestGroups.push_back(rg2); - SharedHandle e(DownloadEngineFactory::newConsoleEngine(op, requests, reserved)); + SharedHandle e(DownloadEngineFactory::newConsoleEngine(op, requestGroups)); + + + Strings reservedUrls1; + reservedUrls1.push_back("http://localhost/~tujikawa/linux-2.6.1.tar.bz2"); + + RequestGroupHandle rrg1 = new RequestGroup(reservedUrls1, op); + + e->_requestGroupMan->addReservedGroup(rrg1); - if(hr->totalLength > 0) { - e->segmentMan->filename = hr->filename; - e->segmentMan->totalSize = hr->totalLength; - e->segmentMan->downloadStarted = true; - } - -#ifdef ENABLE_MESSAGE_DIGEST - if(chunkChecksumLength > 0) { - e->segmentMan->digestAlgo = digestAlgo; - e->segmentMan->chunkHashLength = chunkChecksumLength; - e->segmentMan->pieceHashes = chunkChecksums; - } -#endif // ENABLE_MESSAGE_DIGEST - - if(op->get(PREF_CONTINUE) == V_TRUE && e->segmentMan->fileExists()) { - if(e->segmentMan->totalSize == 0) { - logger->notice("Cannot get file length. Download aborted."); - return RequestInfos(); - } - File existingFile(e->segmentMan->getFilePath()); - if(e->segmentMan->totalSize < existingFile.size()) { - logger->notice("The local file length is larger than the remote file size. Download aborted."); - return RequestInfos(); - } - e->segmentMan->initBitfield(e->option->getAsInt(PREF_SEGMENT_SIZE), - e->segmentMan->totalSize); - - e->segmentMan->diskWriter->openExistingFile(e->segmentMan->getFilePath(), - e->segmentMan->totalSize); - if(e->option->get(PREF_CHECK_INTEGRITY) == V_TRUE) { -#ifdef ENABLE_MESSAGE_DIGEST - if(!e->segmentMan->isChunkChecksumValidationReady()) { - throw new DlAbortEx("Chunk checksums are not provided."); - } - e->segmentMan->markAllPiecesDone(); - e->segmentMan->checkIntegrity(); -#endif // ENABLE_MESSAGE_DIGEST - } else { - e->segmentMan->markPieceDone(existingFile.size()); - } - } else if(e->segmentMan->segmentFileExists()) { - e->segmentMan->load(); - e->segmentMan->diskWriter->openExistingFile(e->segmentMan->getFilePath(), - e->segmentMan->totalSize); -#ifdef ENABLE_MESSAGE_DIGEST - if(op->get(PREF_CHECK_INTEGRITY) == V_TRUE) { - if(!e->segmentMan->isChunkChecksumValidationReady()) { - throw new DlAbortEx("Chunk checksums are not provided."); - } - e->segmentMan->checkIntegrity(); - } -#endif // ENABLE_MESSAGE_DIGEST - } else { - if(e->segmentMan->shouldCancelDownloadForSafety()) { - throw new FatalException(EX_FILE_ALREADY_EXISTS, - e->segmentMan->getFilePath().c_str(), - e->segmentMan->getSegmentFilePath().c_str()); - } - if(e->segmentMan->totalSize > 0) { - e->segmentMan->initBitfield(e->option->getAsInt(PREF_SEGMENT_SIZE), - e->segmentMan->totalSize); - if(e->segmentMan->fileExists() && e->option->get(PREF_CHECK_INTEGRITY) == V_TRUE) { -#ifdef ENABLE_MESSAGE_DIGEST - if(!e->segmentMan->isChunkChecksumValidationReady()) { - throw new DlAbortEx("Chunk checksums are not provided."); - } -#endif // ENABLE_MESSAGE_DIGEST - e->segmentMan->diskWriter->openExistingFile(e->segmentMan->getFilePath(), - e->segmentMan->totalSize); -#ifdef ENABLE_MESSAGE_DIGEST - e->segmentMan->markAllPiecesDone(); - e->segmentMan->checkIntegrity(); -#endif // ENABLE_MESSAGE_DIGEST - } else { - e->segmentMan->diskWriter->initAndOpenFile(e->segmentMan->getFilePath(), - e->segmentMan->totalSize); - } - } - } + e->fillCommand(); + + + + + // The number of simultaneous download is specified by PREF_MAX_SIMULTANEOUS_DOWNLOADS. + // The remaining urls are queued into FillRequestGroupCommand. + // It observes the number of simultaneous downloads and if it is under + // the limit, it adds RequestGroup object from its queue to DownloadEngine. + // This is done every 1 second. At the same time, it removes finished/error + // RequestGroup from DownloadEngine. + Util::setGlobalSignalHandler(SIGINT, handler, 0); Util::setGlobalSignalHandler(SIGTERM, handler, 0); e->run(); - if(e->segmentMan->finished()) { - printDownloadCompeleteMessage(e->segmentMan->getFilePath()); - fileInfo.filename = e->segmentMan->getFilePath(); - fileInfo.length = e->segmentMan->totalSize; - fileInfo.checksum = checksum; - + if(e->_requestGroupMan->downloadFinished()) { next = createNextRequestInfo(); } else { - e->segmentMan->save(); - e->segmentMan->diskWriter->closeFile(); + e->_requestGroupMan->save(); + e->_requestGroupMan->closeFile(); printDownloadAbortMessage(); } } catch(RecoverableException *ex) { diff --git a/src/main.cc b/src/main.cc index bab2777b..3ca62f78 100644 --- a/src/main.cc +++ b/src/main.cc @@ -38,6 +38,7 @@ #include "prefs.h" #include "FeatureConfig.h" #include "UrlRequestInfo.h" +#include "MultiUrlRequestInfo.h" #include "TorrentRequestInfo.h" #include "BitfieldManFactory.h" #include "SimpleRandomizer.h" @@ -48,6 +49,8 @@ #include "OptionHandlerFactory.h" #include "FatalException.h" #include "File.h" +#include "CUIDCounter.h" +#include "UriFileListParser.h" #include #include #include @@ -354,6 +357,7 @@ int main(int argc, char* argv[]) { op->put(PREF_CONTINUE, V_FALSE); op->put(PREF_USER_AGENT, "aria2"); op->put(PREF_NO_NETRC, V_FALSE); + op->put(PREF_MAX_SIMULTANEOUS_DOWNLOADS, "5"); while(1) { int optIndex = 0; int lopt; @@ -389,6 +393,7 @@ int main(int argc, char* argv[]) { { "continue", no_argument, 0, 'c' }, { "user-agent", required_argument, 0, 'U' }, { "no-netrc", no_argument, 0, 'n' }, + { "input-file", required_argument, 0, 'i' }, #ifdef ENABLE_BITTORRENT { "torrent-file", required_argument, NULL, 'T' }, { "listen-port", required_argument, &lopt, 15 }, @@ -416,7 +421,7 @@ int main(int argc, char* argv[]) { { "help", no_argument, NULL, 'h' }, { 0, 0, 0, 0 } }; - c = getopt_long(argc, argv, "Dd:o:l:s:pt:m:vhST:M:C:a:cU:n", longOpts, &optIndex); + c = getopt_long(argc, argv, "Dd:o:l:s:pt:m:vhST:M:C:a:cU:ni:", longOpts, &optIndex); if(c == -1) { break; } @@ -570,6 +575,9 @@ int main(int argc, char* argv[]) { case 'n': cmdstream << PREF_NO_NETRC << "=" << V_TRUE << "\n"; break; + case 'i': + cmdstream << PREF_INPUT_FILE << "=" << optarg << "\n"; + break; case 'v': showVersion(); exit(EXIT_SUCCESS); @@ -664,6 +672,8 @@ int main(int argc, char* argv[]) { } } RequestFactorySingletonHolder::instance(requestFactory); + CUIDCounterHandle cuidCounter = new CUIDCounter(); + CUIDCounterSingletonHolder::instance(cuidCounter); Util::setGlobalSignalHandler(SIGPIPE, SIG_IGN, 0); @@ -683,12 +693,26 @@ int main(int argc, char* argv[]) { #ifdef ENABLE_METALINK if(op->defined(PREF_METALINK_FILE)) { firstReqInfo = new MetalinkRequestInfo(op->get(PREF_METALINK_FILE), - op); - } else + op); + } + else #endif // ENABLE_METALINK - { - firstReqInfo = new UrlRequestInfo(args, 0, op); + if(op->defined(PREF_INPUT_FILE)) { + UriFileListParser flparser(op->get(PREF_INPUT_FILE)); + RequestGroups groups; + while(flparser.hasNext()) { + Strings uris = flparser.next(); + if(!uris.empty()) { + RequestGroupHandle rg = new RequestGroup(uris, op); + groups.push_back(rg); + } + } + firstReqInfo = new MultiUrlRequestInfo(groups, op); } + else + { + firstReqInfo = new MultiUrlRequestInfo(args, op); + } RequestInfos reqInfos; if(firstReqInfo) { @@ -699,6 +723,7 @@ int main(int argc, char* argv[]) { reqInfos.pop_front(); RequestInfos nextReqInfos = reqInfo->execute(); copy(nextReqInfos.begin(), nextReqInfos.end(), front_inserter(reqInfos)); + /* if(reqInfo->isFail()) { exitStatus = EXIT_FAILURE; } else if(reqInfo->getFileInfo().checkReady()) { @@ -713,6 +738,7 @@ int main(int argc, char* argv[]) { exitStatus = EXIT_FAILURE; } } + */ } } catch(Exception* ex) { cerr << ex->getMsg() << endl; diff --git a/src/message.h b/src/message.h index 26d42139..901118d2 100644 --- a/src/message.h +++ b/src/message.h @@ -55,9 +55,10 @@ #define MSG_RECEIVE_PEER_MESSAGE "CUID#%d - From: %s:%d %s" #define MSG_GOT_NEW_PIECE _("CUID#%d - we got new piece. index=%d") #define MSG_GOT_WRONG_PIECE _("CUID#%d - we got wrong piece. index=%d") +#define MSG_DOWNLOAD_NOT_COMPLETE _("CUID#%d - Download not complete: %s") +#define MSG_DOWNLOAD_ALREADY_COMPLETED _("CUID#%d - Download already completed: %s") #define MSG_TRACKER_WARNING_MESSAGE _("Tracker returned warning message: %s") - #define MSG_SEGMENT_FILE_EXISTS _("The segment file %s exists.") #define MSG_SEGMENT_FILE_DOES_NOT_EXIST _("The segment file %s does not exist.") #define MSG_SAVING_SEGMENT_FILE _("Saving the segment file %s") @@ -65,6 +66,7 @@ #define MSG_LOADING_SEGMENT_FILE _("Loading the segment file %s.") #define MSG_LOADED_SEGMENT_FILE _("The segment file was loaded successfully.") #define MSG_NO_URL_TO_DOWNLOAD _("No URI to download. Download aborted.") +#define MSG_FILE_ALREADY_EXISTS _("File %s exists, but %s does not exist. The download was canceled in order to prevent your file from being truncated to 0. If you are sure to download file all over again, then delete it or add --allow-overwrite=true option and restart aria2.") #define EX_TIME_OUT _("Timeout.") #define EX_INVALID_CHUNK_SIZE _("Invalid chunk size.") @@ -75,12 +77,12 @@ #define EX_NO_STATUS_HEADER _("No status header.") #define EX_PROXY_CONNECTION_FAILED _("Proxy connection failed.") #define EX_CONNECTION_FAILED _("Connection failed.") -#define EX_FILENAME_MISMATCH _("The requested filename and the previously registered one are not same. %s != %s") +#define EX_FILENAME_MISMATCH _("The requested filename and the previously registered one are not same. Expected:%s Actual:%s") #define EX_BAD_STATUS _("The response status is not successful. status=%d") #define EX_TOO_LARGE_FILE _("Too large file size. size=%lld") #define EX_TRANSFER_ENCODING_NOT_SUPPORTED _("Transfer encoding %s is not supported.") #define EX_SSL_INIT_FAILURE _("SSL initialization failed.") -#define EX_SIZE_MISMATCH _("Size mismatch %lld != %lld") +#define EX_SIZE_MISMATCH _("Size mismatch Expected:%lld Actual:%lld") #define EX_AUTH_FAILED _("Authorization failed.") #define EX_GOT_EOF _("Got EOF from the server.") #define EX_EOF_FROM_PEER _("Got EOF from peer.") @@ -112,8 +114,8 @@ #define EX_SOCKET_SEND _("Failed to send data, cause: %s") #define EX_SOCKET_RECV _("Failed to receive data, cause: %s") #define EX_SOCKET_PEEK _("Failed to peek data, cause: %s") -#define EX_FILE_ALREADY_EXISTS _("File %s exists, but %s does not exist. The download was canceled in order to prevent your file from being truncated to 0. If you are sure to download file all over again, then delete it or add --allow-overwrite=true option and restart aria2.") +#define EX_FILE_ALREADY_EXISTS _("File %s exists, but %s does not exist.") #define EX_INVALID_PAYLOAD_SIZE _("Invalid payload size for %s, size=%d. It should be %d.") #define EX_INVALID_BT_MESSAGE_ID _("Invalid ID=%d for %s. It should be %d.") -#define EX_INVALID_CHUNK_CHECKSUM _("Chunk checksum validation failed. checksumIndex=%d, offset=%lld, length=%d, expectedHash=%s, actualHash=%s") +#define EX_INVALID_CHUNK_CHECKSUM _("Chunk checksum validation failed. checksumIndex=%d, offset=%lld, expectedHash=%s, actualHash=%s") #endif // _D_MESSAGE_H_ diff --git a/src/prefs.h b/src/prefs.h index 49eea67c..d4cf95fa 100644 --- a/src/prefs.h +++ b/src/prefs.h @@ -96,6 +96,12 @@ #define PREF_CONTINUE "continue" // value: #define PREF_NO_NETRC "no-netrc" +// value: 1*digit +#define PREF_MAX_DOWNLOADS "max-downloads" +// value: string that your file system recognizes as a file name. +#define PREF_INPUT_FILE "input-file" +// value: 1*digit +#define PREF_MAX_SIMULTANEOUS_DOWNLOADS "max-simultaneous-downloads" /** * FTP related preferences diff --git a/test/Makefile.am b/test/Makefile.am index 1723bb28..5d2dadeb 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -1,76 +1,78 @@ TESTS = aria2c check_PROGRAMS = $(TESTS) aria2c_SOURCES = AllTest.cc\ - PeerTest.cc\ - DefaultPeerStorageTest.cc\ - RequestFactoryTest.cc\ - NetrcAuthResolverTest.cc\ - DefaultAuthResolverTest.cc\ - RequestTest.cc\ - HttpRequestTest.cc\ - UtilTest.cc\ - OptionHandlerTest.cc\ - SegmentManTest.cc\ - BitfieldManTest.cc\ - GlowFileAllocatorTest.cc\ - NetrcTest.cc\ - SingletonHolderTest.cc\ - HttpHeaderTest.cc\ - HttpResponseTest.cc\ - SharedHandleTest.cc\ - ChunkedEncodingTest.cc\ - FileTest.cc\ - OptionTest.cc\ - Base64Test.cc\ - CookieBoxTest.cc\ - DataTest.cc\ - DictionaryTest.cc\ - ListTest.cc\ - MetaFileUtilTest.cc\ - ShaVisitorTest.cc\ - PeerMessageUtilTest.cc\ - DefaultDiskWriterTest.cc\ - MultiDiskAdaptorTest.cc\ - Xml2MetalinkProcessorTest.cc\ - MetalinkerTest.cc\ - MetalinkEntryTest.cc\ - FeatureConfigTest.cc\ - ShareRatioSeedCriteriaTest.cc\ - TimeSeedCriteriaTest.cc\ - SpeedCalcTest.cc\ - DefaultPeerListProcessorTest.cc\ - AnnounceListTest.cc\ - TrackerWatcherCommandTest.cc\ - DefaultBtContextTest.cc\ - DefaultPieceStorageTest.cc\ - DefaultBtAnnounceTest.cc\ - BtRegistryTest.cc\ - DefaultBtMessageDispatcherTest.cc\ - MockPeerStorage.h\ - DefaultBtRequestFactoryTest.cc\ - BtAllowedFastMessageTest.cc\ - BtBitfieldMessageTest.cc\ - BtCancelMessageTest.cc\ - BtChokeMessageTest.cc\ - BtHaveAllMessageTest.cc\ - BtHaveMessageTest.cc\ - BtHaveNoneMessageTest.cc\ - BtInterestedMessageTest.cc\ - BtKeepAliveMessageTest.cc\ - BtNotInterestedMessageTest.cc\ - BtPieceMessageTest.cc\ - BtPortMessageTest.cc\ - BtRejectMessageTest.cc\ - BtRequestMessageTest.cc\ - BtSuggestPieceMessageTest.cc\ - BtUnchokeMessageTest.cc\ - BtHandshakeMessageTest.cc\ - MockBtMessageDispatcher.h\ - FixedNumberRandomizer.h\ - MockBtMessageFactory.h\ - MockBtMessage.h\ - ConsoleFileAllocationMonitorTest.cc\ - ChunkChecksumValidatorTest.cc + IteratableChunkChecksumValidatorTest.cc +# UriFileListParserTest.cc +# PeerTest.cc\ +# DefaultPeerStorageTest.cc\ +# RequestFactoryTest.cc\ +# NetrcAuthResolverTest.cc\ +# DefaultAuthResolverTest.cc\ +# RequestTest.cc\ +# HttpRequestTest.cc\ +# UtilTest.cc\ +# OptionHandlerTest.cc\ +# SegmentManTest.cc\ +# BitfieldManTest.cc\ +# GlowFileAllocatorTest.cc\ +# NetrcTest.cc\ +# SingletonHolderTest.cc\ +# HttpHeaderTest.cc\ +# HttpResponseTest.cc\ +# SharedHandleTest.cc\ +# ChunkedEncodingTest.cc\ +# FileTest.cc\ +# OptionTest.cc\ +# Base64Test.cc\ +# CookieBoxTest.cc\ +# DataTest.cc\ +# DictionaryTest.cc\ +# ListTest.cc\ +# MetaFileUtilTest.cc\ +# ShaVisitorTest.cc\ +# PeerMessageUtilTest.cc\ +# DefaultDiskWriterTest.cc\ +# MultiDiskAdaptorTest.cc\ +# Xml2MetalinkProcessorTest.cc\ +# MetalinkerTest.cc\ +# MetalinkEntryTest.cc\ +# FeatureConfigTest.cc\ +# ShareRatioSeedCriteriaTest.cc\ +# TimeSeedCriteriaTest.cc\ +# SpeedCalcTest.cc\ +# DefaultPeerListProcessorTest.cc\ +# AnnounceListTest.cc\ +# TrackerWatcherCommandTest.cc\ +# DefaultBtContextTest.cc\ +# DefaultPieceStorageTest.cc\ +# DefaultBtAnnounceTest.cc\ +# BtRegistryTest.cc\ +# DefaultBtMessageDispatcherTest.cc\ +# MockPeerStorage.h\ +# DefaultBtRequestFactoryTest.cc\ +# BtAllowedFastMessageTest.cc\ +# BtBitfieldMessageTest.cc\ +# BtCancelMessageTest.cc\ +# BtChokeMessageTest.cc\ +# BtHaveAllMessageTest.cc\ +# BtHaveMessageTest.cc\ +# BtHaveNoneMessageTest.cc\ +# BtInterestedMessageTest.cc\ +# BtKeepAliveMessageTest.cc\ +# BtNotInterestedMessageTest.cc\ +# BtPieceMessageTest.cc\ +# BtPortMessageTest.cc\ +# BtRejectMessageTest.cc\ +# BtRequestMessageTest.cc\ +# BtSuggestPieceMessageTest.cc\ +# BtUnchokeMessageTest.cc\ +# BtHandshakeMessageTest.cc\ +# MockBtMessageDispatcher.h\ +# FixedNumberRandomizer.h\ +# MockBtMessageFactory.h\ +# MockBtMessage.h\ +# ConsoleFileAllocationMonitorTest.cc\ +# ChunkChecksumValidatorTest.cc #aria2c_CXXFLAGS = ${CPPUNIT_CFLAGS} -I../src -I../lib -Wall -D_FILE_OFFSET_BITS=64 #aria2c_LDFLAGS = ${CPPUNIT_LIBS} diff --git a/test/Makefile.in b/test/Makefile.in index e33b52e3..51b1fdd1 100644 --- a/test/Makefile.in +++ b/test/Makefile.in @@ -57,47 +57,8 @@ mkinstalldirs = $(SHELL) $(top_srcdir)/mkinstalldirs CONFIG_HEADER = $(top_builddir)/config.h CONFIG_CLEAN_FILES = am__EXEEXT_1 = aria2c$(EXEEXT) -am_aria2c_OBJECTS = AllTest.$(OBJEXT) PeerTest.$(OBJEXT) \ - DefaultPeerStorageTest.$(OBJEXT) RequestFactoryTest.$(OBJEXT) \ - NetrcAuthResolverTest.$(OBJEXT) \ - DefaultAuthResolverTest.$(OBJEXT) RequestTest.$(OBJEXT) \ - HttpRequestTest.$(OBJEXT) UtilTest.$(OBJEXT) \ - OptionHandlerTest.$(OBJEXT) SegmentManTest.$(OBJEXT) \ - BitfieldManTest.$(OBJEXT) GlowFileAllocatorTest.$(OBJEXT) \ - NetrcTest.$(OBJEXT) SingletonHolderTest.$(OBJEXT) \ - HttpHeaderTest.$(OBJEXT) HttpResponseTest.$(OBJEXT) \ - SharedHandleTest.$(OBJEXT) ChunkedEncodingTest.$(OBJEXT) \ - FileTest.$(OBJEXT) OptionTest.$(OBJEXT) Base64Test.$(OBJEXT) \ - CookieBoxTest.$(OBJEXT) DataTest.$(OBJEXT) \ - DictionaryTest.$(OBJEXT) ListTest.$(OBJEXT) \ - MetaFileUtilTest.$(OBJEXT) ShaVisitorTest.$(OBJEXT) \ - PeerMessageUtilTest.$(OBJEXT) DefaultDiskWriterTest.$(OBJEXT) \ - MultiDiskAdaptorTest.$(OBJEXT) \ - Xml2MetalinkProcessorTest.$(OBJEXT) MetalinkerTest.$(OBJEXT) \ - MetalinkEntryTest.$(OBJEXT) FeatureConfigTest.$(OBJEXT) \ - ShareRatioSeedCriteriaTest.$(OBJEXT) \ - TimeSeedCriteriaTest.$(OBJEXT) SpeedCalcTest.$(OBJEXT) \ - DefaultPeerListProcessorTest.$(OBJEXT) \ - AnnounceListTest.$(OBJEXT) TrackerWatcherCommandTest.$(OBJEXT) \ - DefaultBtContextTest.$(OBJEXT) \ - DefaultPieceStorageTest.$(OBJEXT) \ - DefaultBtAnnounceTest.$(OBJEXT) BtRegistryTest.$(OBJEXT) \ - DefaultBtMessageDispatcherTest.$(OBJEXT) \ - DefaultBtRequestFactoryTest.$(OBJEXT) \ - BtAllowedFastMessageTest.$(OBJEXT) \ - BtBitfieldMessageTest.$(OBJEXT) BtCancelMessageTest.$(OBJEXT) \ - BtChokeMessageTest.$(OBJEXT) BtHaveAllMessageTest.$(OBJEXT) \ - BtHaveMessageTest.$(OBJEXT) BtHaveNoneMessageTest.$(OBJEXT) \ - BtInterestedMessageTest.$(OBJEXT) \ - BtKeepAliveMessageTest.$(OBJEXT) \ - BtNotInterestedMessageTest.$(OBJEXT) \ - BtPieceMessageTest.$(OBJEXT) BtPortMessageTest.$(OBJEXT) \ - BtRejectMessageTest.$(OBJEXT) BtRequestMessageTest.$(OBJEXT) \ - BtSuggestPieceMessageTest.$(OBJEXT) \ - BtUnchokeMessageTest.$(OBJEXT) \ - BtHandshakeMessageTest.$(OBJEXT) \ - ConsoleFileAllocationMonitorTest.$(OBJEXT) \ - ChunkChecksumValidatorTest.$(OBJEXT) +am_aria2c_OBJECTS = AllTest.$(OBJEXT) \ + IteratableChunkChecksumValidatorTest.$(OBJEXT) aria2c_OBJECTS = $(am_aria2c_OBJECTS) am__DEPENDENCIES_1 = aria2c_DEPENDENCIES = ../src/libaria2c.a $(am__DEPENDENCIES_1) @@ -109,10 +70,6 @@ CXXCOMPILE = $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \ CXXLD = $(CXX) CXXLINK = $(CXXLD) $(AM_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) $(LDFLAGS) \ -o $@ -COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \ - $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -CCLD = $(CC) -LINK = $(CCLD) $(AM_CFLAGS) $(CFLAGS) $(AM_LDFLAGS) $(LDFLAGS) -o $@ SOURCES = $(aria2c_SOURCES) DIST_SOURCES = $(aria2c_SOURCES) ETAGS = etags @@ -262,77 +219,79 @@ sysconfdir = @sysconfdir@ target_alias = @target_alias@ TESTS = aria2c aria2c_SOURCES = AllTest.cc\ - PeerTest.cc\ - DefaultPeerStorageTest.cc\ - RequestFactoryTest.cc\ - NetrcAuthResolverTest.cc\ - DefaultAuthResolverTest.cc\ - RequestTest.cc\ - HttpRequestTest.cc\ - UtilTest.cc\ - OptionHandlerTest.cc\ - SegmentManTest.cc\ - BitfieldManTest.cc\ - GlowFileAllocatorTest.cc\ - NetrcTest.cc\ - SingletonHolderTest.cc\ - HttpHeaderTest.cc\ - HttpResponseTest.cc\ - SharedHandleTest.cc\ - ChunkedEncodingTest.cc\ - FileTest.cc\ - OptionTest.cc\ - Base64Test.cc\ - CookieBoxTest.cc\ - DataTest.cc\ - DictionaryTest.cc\ - ListTest.cc\ - MetaFileUtilTest.cc\ - ShaVisitorTest.cc\ - PeerMessageUtilTest.cc\ - DefaultDiskWriterTest.cc\ - MultiDiskAdaptorTest.cc\ - Xml2MetalinkProcessorTest.cc\ - MetalinkerTest.cc\ - MetalinkEntryTest.cc\ - FeatureConfigTest.cc\ - ShareRatioSeedCriteriaTest.cc\ - TimeSeedCriteriaTest.cc\ - SpeedCalcTest.cc\ - DefaultPeerListProcessorTest.cc\ - AnnounceListTest.cc\ - TrackerWatcherCommandTest.cc\ - DefaultBtContextTest.cc\ - DefaultPieceStorageTest.cc\ - DefaultBtAnnounceTest.cc\ - BtRegistryTest.cc\ - DefaultBtMessageDispatcherTest.cc\ - MockPeerStorage.h\ - DefaultBtRequestFactoryTest.cc\ - BtAllowedFastMessageTest.cc\ - BtBitfieldMessageTest.cc\ - BtCancelMessageTest.cc\ - BtChokeMessageTest.cc\ - BtHaveAllMessageTest.cc\ - BtHaveMessageTest.cc\ - BtHaveNoneMessageTest.cc\ - BtInterestedMessageTest.cc\ - BtKeepAliveMessageTest.cc\ - BtNotInterestedMessageTest.cc\ - BtPieceMessageTest.cc\ - BtPortMessageTest.cc\ - BtRejectMessageTest.cc\ - BtRequestMessageTest.cc\ - BtSuggestPieceMessageTest.cc\ - BtUnchokeMessageTest.cc\ - BtHandshakeMessageTest.cc\ - MockBtMessageDispatcher.h\ - FixedNumberRandomizer.h\ - MockBtMessageFactory.h\ - MockBtMessage.h\ - ConsoleFileAllocationMonitorTest.cc\ - ChunkChecksumValidatorTest.cc + IteratableChunkChecksumValidatorTest.cc +# UriFileListParserTest.cc +# PeerTest.cc\ +# DefaultPeerStorageTest.cc\ +# RequestFactoryTest.cc\ +# NetrcAuthResolverTest.cc\ +# DefaultAuthResolverTest.cc\ +# RequestTest.cc\ +# HttpRequestTest.cc\ +# UtilTest.cc\ +# OptionHandlerTest.cc\ +# SegmentManTest.cc\ +# BitfieldManTest.cc\ +# GlowFileAllocatorTest.cc\ +# NetrcTest.cc\ +# SingletonHolderTest.cc\ +# HttpHeaderTest.cc\ +# HttpResponseTest.cc\ +# SharedHandleTest.cc\ +# ChunkedEncodingTest.cc\ +# FileTest.cc\ +# OptionTest.cc\ +# Base64Test.cc\ +# CookieBoxTest.cc\ +# DataTest.cc\ +# DictionaryTest.cc\ +# ListTest.cc\ +# MetaFileUtilTest.cc\ +# ShaVisitorTest.cc\ +# PeerMessageUtilTest.cc\ +# DefaultDiskWriterTest.cc\ +# MultiDiskAdaptorTest.cc\ +# Xml2MetalinkProcessorTest.cc\ +# MetalinkerTest.cc\ +# MetalinkEntryTest.cc\ +# FeatureConfigTest.cc\ +# ShareRatioSeedCriteriaTest.cc\ +# TimeSeedCriteriaTest.cc\ +# SpeedCalcTest.cc\ +# DefaultPeerListProcessorTest.cc\ +# AnnounceListTest.cc\ +# TrackerWatcherCommandTest.cc\ +# DefaultBtContextTest.cc\ +# DefaultPieceStorageTest.cc\ +# DefaultBtAnnounceTest.cc\ +# BtRegistryTest.cc\ +# DefaultBtMessageDispatcherTest.cc\ +# MockPeerStorage.h\ +# DefaultBtRequestFactoryTest.cc\ +# BtAllowedFastMessageTest.cc\ +# BtBitfieldMessageTest.cc\ +# BtCancelMessageTest.cc\ +# BtChokeMessageTest.cc\ +# BtHaveAllMessageTest.cc\ +# BtHaveMessageTest.cc\ +# BtHaveNoneMessageTest.cc\ +# BtInterestedMessageTest.cc\ +# BtKeepAliveMessageTest.cc\ +# BtNotInterestedMessageTest.cc\ +# BtPieceMessageTest.cc\ +# BtPortMessageTest.cc\ +# BtRejectMessageTest.cc\ +# BtRequestMessageTest.cc\ +# BtSuggestPieceMessageTest.cc\ +# BtUnchokeMessageTest.cc\ +# BtHandshakeMessageTest.cc\ +# MockBtMessageDispatcher.h\ +# FixedNumberRandomizer.h\ +# MockBtMessageFactory.h\ +# MockBtMessage.h\ +# ConsoleFileAllocationMonitorTest.cc\ +# ChunkChecksumValidatorTest.cc #aria2c_CXXFLAGS = ${CPPUNIT_CFLAGS} -I../src -I../lib -Wall -D_FILE_OFFSET_BITS=64 #aria2c_LDFLAGS = ${CPPUNIT_LIBS} aria2c_LDADD = ../src/libaria2c.a\ @@ -395,71 +354,7 @@ distclean-compile: -rm -f *.tab.c @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/AllTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/AnnounceListTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Base64Test.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/BitfieldManTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/BtAllowedFastMessageTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/BtBitfieldMessageTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/BtCancelMessageTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/BtChokeMessageTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/BtHandshakeMessageTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/BtHaveAllMessageTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/BtHaveMessageTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/BtHaveNoneMessageTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/BtInterestedMessageTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/BtKeepAliveMessageTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/BtNotInterestedMessageTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/BtPieceMessageTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/BtPortMessageTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/BtRegistryTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/BtRejectMessageTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/BtRequestMessageTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/BtSuggestPieceMessageTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/BtUnchokeMessageTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ChunkChecksumValidatorTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ChunkedEncodingTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ConsoleFileAllocationMonitorTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/CookieBoxTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DataTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DefaultAuthResolverTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DefaultBtAnnounceTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DefaultBtContextTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DefaultBtMessageDispatcherTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DefaultBtRequestFactoryTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DefaultDiskWriterTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DefaultPeerListProcessorTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DefaultPeerStorageTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DefaultPieceStorageTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DictionaryTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FeatureConfigTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FileTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/GlowFileAllocatorTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/HttpHeaderTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/HttpRequestTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/HttpResponseTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ListTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/MetaFileUtilTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/MetalinkEntryTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/MetalinkerTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/MultiDiskAdaptorTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/NetrcAuthResolverTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/NetrcTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/OptionHandlerTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/OptionTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/PeerMessageUtilTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/PeerTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/RequestFactoryTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/RequestTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/SegmentManTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ShaVisitorTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ShareRatioSeedCriteriaTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/SharedHandleTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/SingletonHolderTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/SpeedCalcTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/TimeSeedCriteriaTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/TrackerWatcherCommandTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/UtilTest.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Xml2MetalinkProcessorTest.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/IteratableChunkChecksumValidatorTest.Po@am__quote@ .cc.o: @am__fastdepCXX_TRUE@ if $(CXXCOMPILE) -MT $@ -MD -MP -MF "$(DEPDIR)/$*.Tpo" -c -o $@ $<; \