diff --git a/ChangeLog b/ChangeLog index e36dfbb0..9e59f582 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,24 @@ +2009-06-30 Tatsuhiro Tsujikawa + + Removed ServerHost. Same functionality is implemented using + FileEntry's in-flight Request objects. + * src/AbstractCommand.cc + * src/BtDependency.cc + * src/CreateRequestCommand.cc + * src/FileEntry.cc + * src/FileEntry.h + * src/FtpNegotiationCommand.cc + * src/HttpResponseCommand.cc + * src/Makefile.am + * src/Metalink2RequestGroup.cc + * src/RequestGroup.cc + * src/RequestGroup.h + * src/ServerHost.cc: Removed + * src/ServerHost.h: Removed + * test/BtDependencyTest.cc + * test/FileEntryTest.cc + * test/RequestGroupTest.cc + 2009-06-30 Tatsuhiro Tsujikawa Rewritten PeerStat handling. In the previous implementation, diff --git a/src/AbstractCommand.cc b/src/AbstractCommand.cc index 540f8db8..54e96e67 100644 --- a/src/AbstractCommand.cc +++ b/src/AbstractCommand.cc @@ -140,10 +140,6 @@ bool AbstractCommand::execute() { Command* command = InitiateConnectionCommandFactory::createInitiateConnectionCommand (cuid, fasterRequest, _fileEntry, _requestGroup, e); - // TODO1.5 Here is ServerHost stuff - //ServerHostHandle sv(new ServerHost(command->getCuid(), req->getHost())); - //registerServerHost(sv); - e->setNoWait(true); e->commands.push_back(command); return true; @@ -248,7 +244,6 @@ bool AbstractCommand::execute() { } void AbstractCommand::tryReserved() { - _requestGroup->removeServerHost(cuid); if(_requestGroup->getDownloadContext()->getFileEntries().size() == 1) { const SharedHandle& entry = _requestGroup->getDownloadContext()->getFirstFileEntry(); diff --git a/src/BtDependency.cc b/src/BtDependency.cc index d8f5c0d3..97d39778 100644 --- a/src/BtDependency.cc +++ b/src/BtDependency.cc @@ -80,12 +80,17 @@ bool BtDependency::resolve() } // Copy file path in _dependant's FileEntries to newly created // context's FileEntries to endorse the path structure of - // _dependant. + // _dependant. URIs and singleHostMultiConnection are also copied. for(std::vector >::const_iterator s = _dependant->getDownloadContext()->getFileEntries().begin(), d = context->getFileEntries().begin(); d != context->getFileEntries().end(); ++s, ++d) { (*d)->setPath((*s)->getPath()); + (*d)->addUris((*s)->getRemainingUris().begin(), + (*s)->getRemainingUris().end()); + if(!(*s)->isSingleHostMultiConnectionEnabled()) { + (*d)->disableSingleHostMultiConnection(); + } } } catch(RecoverableException& e) { _logger->error(EX_EXCEPTION_CAUGHT, e); diff --git a/src/CreateRequestCommand.cc b/src/CreateRequestCommand.cc index 824d26c4..9dac078a 100644 --- a/src/CreateRequestCommand.cc +++ b/src/CreateRequestCommand.cc @@ -75,8 +75,6 @@ bool CreateRequestCommand::executeInternal() Command* command = InitiateConnectionCommandFactory::createInitiateConnectionCommand (cuid, req, _fileEntry, _requestGroup, e); - //ServerHostHandle sv(new ServerHost(command->getCuid(), req->getHost())); - //registerServerHost(sv); e->setNoWait(true); e->commands.push_back(command); return true; diff --git a/src/FileEntry.cc b/src/FileEntry.cc index e2e67da5..28427faf 100644 --- a/src/FileEntry.cc +++ b/src/FileEntry.cc @@ -49,10 +49,12 @@ FileEntry::FileEntry(const std::string& path, const std::deque& uris): path(path), _uris(uris), length(length), offset(offset), extracted(false), requested(true), + _singleHostMultiConnection(true), _logger(LogFactory::getInstance()) {} FileEntry::FileEntry(): length(0), offset(0), extracted(false), requested(false), + _singleHostMultiConnection(true), _logger(LogFactory::getInstance()) {} FileEntry::~FileEntry() {} @@ -101,11 +103,26 @@ std::string FileEntry::selectUri(const SharedHandle& uriSelector) return uriSelector->select(this); } +template +static bool inFlightHost(InputIterator first, InputIterator last, + const std::string& hostname) +{ + // TODO1.5 redirection should be considered here. We need to parse + // original URI to get hostname. + for(; first != last; ++first) { + if((*first)->getHost() == hostname) { + return true; + } + } + return false; +} + SharedHandle FileEntry::getRequest(const SharedHandle& selector) { SharedHandle req; if(_requestPool.empty()) { + std::deque pending; while(1) { std::string uri = selector->select(this); if(uri.empty()) { @@ -113,19 +130,28 @@ FileEntry::getRequest(const SharedHandle& selector) } req.reset(new Request()); if(req->setUrl(uri)) { + if(!_singleHostMultiConnection) { + if(inFlightHost(_inFlightRequests.begin(), _inFlightRequests.end(), + req->getHost())) { + pending.push_back(uri); + req.reset(); + continue; + } + } _spentUris.push_back(uri); _inFlightRequests.push_back(req); - return req; + break; } else { req.reset(); } } + _uris.insert(_uris.begin(), pending.begin(), pending.end()); } else { req = _requestPool.front(); _requestPool.pop_front(); _inFlightRequests.push_back(req); - return req; } + return req; } SharedHandle diff --git a/src/FileEntry.h b/src/FileEntry.h index 93137fb7..f0b55700 100644 --- a/src/FileEntry.h +++ b/src/FileEntry.h @@ -68,7 +68,7 @@ private: // URIResult is stored in the ascending order of the time when its result is // available. std::deque _uriResults; - + bool _singleHostMultiConnection; Logger* _logger; void storePool(const SharedHandle& request); @@ -142,6 +142,12 @@ public: _uris = uris; } + template + void addUris(InputIterator first, InputIterator last) + { + _uris.insert(_uris.end(), first, last); + } + // Inserts _uris and _spentUris into uris. void getUris(std::deque& uris) const; @@ -201,6 +207,16 @@ public: // The extracted URIResults are removed from _uriResults. void extractURIResult (std::deque& res, downloadresultcode::RESULT r); + + void disableSingleHostMultiConnection() + { + _singleHostMultiConnection = false; + } + + bool isSingleHostMultiConnectionEnabled() const + { + return _singleHostMultiConnection; + } }; typedef SharedHandle FileEntryHandle; diff --git a/src/FtpNegotiationCommand.cc b/src/FtpNegotiationCommand.cc index 9abdaa34..95cda465 100644 --- a/src/FtpNegotiationCommand.cc +++ b/src/FtpNegotiationCommand.cc @@ -57,7 +57,6 @@ #include "DefaultBtProgressInfoFile.h" #include "RequestGroupMan.h" #include "DownloadFailureException.h" -#include "ServerHost.h" #include "Socket.h" #include "StringFormat.h" #include "DiskAdaptor.h" @@ -103,12 +102,8 @@ bool FtpNegotiationCommand::executeInternal() { (cuid, req, _fileEntry, _requestGroup, ftp, e, dataSocket, socket); command->setStartupIdleTime(getOption()->getAsInt(PREF_STARTUP_IDLE_TIME)); command->setLowestDownloadSpeedLimit(getOption()->getAsInt(PREF_LOWEST_SPEED_LIMIT)); - if(!_requestGroup->isSingleHostMultiConnectionEnabled()) { - SharedHandle sv = - _requestGroup->searchServerHost(req->getHost()); - if(!sv.isNull()) { - _fileEntry->removeURIWhoseHostnameIs(sv->getHostname()); - } + if(!_fileEntry->isSingleHostMultiConnectionEnabled()) { + _fileEntry->removeURIWhoseHostnameIs(req->getHost()); } _requestGroup->getURISelector()->tuneDownloadCommand (_fileEntry->getRemainingUris(), command); diff --git a/src/HttpResponseCommand.cc b/src/HttpResponseCommand.cc index 4b4d8080..5bc6229b 100644 --- a/src/HttpResponseCommand.cc +++ b/src/HttpResponseCommand.cc @@ -37,7 +37,6 @@ #include "DownloadContext.h" #include "FileEntry.h" #include "RequestGroup.h" -#include "ServerHost.h" #include "RequestGroupMan.h" #include "Request.h" #include "HttpRequest.h" @@ -122,18 +121,10 @@ bool HttpResponseCommand::executeInternal() } return skipResponseBody(httpResponse); } - if(!_requestGroup->isSingleHostMultiConnectionEnabled()) { - // Query by hostname. Searching by CUID may returns NULL. In case - // when resuming download, ServerHost is registered with CUID A. - // Then if requested range is not equal to saved one, - // StreamFileAllocationEntry is created with _nextCommand NULL. - // This results creating new command CUID, say B and same URI. So - // searching ServerHost by CUID B fails. - SharedHandle sv = - _requestGroup->searchServerHost(req->getHost()); - if(!sv.isNull()) { - _fileEntry->removeURIWhoseHostnameIs(sv->getHostname()); - } + if(!_fileEntry->isSingleHostMultiConnectionEnabled()) { + // TODO1.5 redirection should be considered here. We need to parse + // original URI to get hostname. + _fileEntry->removeURIWhoseHostnameIs(req->getHost()); } if(_requestGroup->getPieceStorage().isNull()) { uint64_t totalLength = httpResponse->getEntityLength(); diff --git a/src/Makefile.am b/src/Makefile.am index 043b29e4..a0ad4423 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -150,7 +150,6 @@ SRCS = Socket.h\ PeerConnection.cc PeerConnection.h\ ByteArrayDiskWriter.cc ByteArrayDiskWriter.h\ ByteArrayDiskWriterFactory.cc ByteArrayDiskWriterFactory.h\ - ServerHost.cc ServerHost.h\ DownloadContext.cc DownloadContext.h\ TimedHaltCommand.cc TimedHaltCommand.h\ CUIDCounter.h\ diff --git a/src/Makefile.in b/src/Makefile.in index 348a3088..60f7cb27 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -398,27 +398,26 @@ am__libaria2c_a_SOURCES_DIST = Socket.h SocketCore.cc SocketCore.h \ MultiFileAllocationIterator.cc MultiFileAllocationIterator.h \ PeerConnection.cc PeerConnection.h ByteArrayDiskWriter.cc \ ByteArrayDiskWriter.h ByteArrayDiskWriterFactory.cc \ - ByteArrayDiskWriterFactory.h ServerHost.cc ServerHost.h \ - DownloadContext.cc DownloadContext.h TimedHaltCommand.cc \ - TimedHaltCommand.h CUIDCounter.h DNSCache.h DownloadResult.h \ - Sequence.h IntSequence.h PostDownloadHandler.h \ - PreDownloadHandler.h SingletonHolder.h \ - TrueRequestGroupCriteria.h a2algo.h a2functional.h a2io.h \ - a2netcompat.h a2time.h array_fun.h help_tags.h prefs.cc \ - prefs.h usage_text.h ProtocolDetector.cc ProtocolDetector.h \ - NullStatCalc.h StringFormat.cc StringFormat.h \ - HttpSkipResponseCommand.cc HttpSkipResponseCommand.h \ - InitiateConnectionCommand.cc InitiateConnectionCommand.h \ - FtpFinishDownloadCommand.cc FtpFinishDownloadCommand.h \ - A2STR.cc A2STR.h RarestPieceSelector.cc RarestPieceSelector.h \ - Decoder.h ChunkedDecoder.cc ChunkedDecoder.h Signature.cc \ - Signature.h ServerStat.cc ServerStat.h ServerStatMan.cc \ - ServerStatMan.h URISelector.h AdaptiveURISelector.cc \ - AdaptiveURISelector.h InOrderURISelector.cc \ - InOrderURISelector.h FeedbackURISelector.cc \ - FeedbackURISelector.h NsCookieParser.cc NsCookieParser.h \ - CookieStorage.cc CookieStorage.h SocketBuffer.cc \ - SocketBuffer.h OptionHandlerException.cc \ + ByteArrayDiskWriterFactory.h DownloadContext.cc \ + DownloadContext.h TimedHaltCommand.cc TimedHaltCommand.h \ + CUIDCounter.h DNSCache.h DownloadResult.h Sequence.h \ + IntSequence.h PostDownloadHandler.h PreDownloadHandler.h \ + SingletonHolder.h TrueRequestGroupCriteria.h a2algo.h \ + a2functional.h a2io.h a2netcompat.h a2time.h array_fun.h \ + help_tags.h prefs.cc prefs.h usage_text.h ProtocolDetector.cc \ + ProtocolDetector.h NullStatCalc.h StringFormat.cc \ + StringFormat.h HttpSkipResponseCommand.cc \ + HttpSkipResponseCommand.h InitiateConnectionCommand.cc \ + InitiateConnectionCommand.h FtpFinishDownloadCommand.cc \ + FtpFinishDownloadCommand.h A2STR.cc A2STR.h \ + RarestPieceSelector.cc RarestPieceSelector.h Decoder.h \ + ChunkedDecoder.cc ChunkedDecoder.h Signature.cc Signature.h \ + ServerStat.cc ServerStat.h ServerStatMan.cc ServerStatMan.h \ + URISelector.h AdaptiveURISelector.cc AdaptiveURISelector.h \ + InOrderURISelector.cc InOrderURISelector.h \ + FeedbackURISelector.cc FeedbackURISelector.h NsCookieParser.cc \ + NsCookieParser.h CookieStorage.cc CookieStorage.h \ + SocketBuffer.cc SocketBuffer.h OptionHandlerException.cc \ OptionHandlerException.h URIResult.cc URIResult.h EventPoll.h \ SelectEventPoll.cc SelectEventPoll.h SequentialPicker.h \ SequentialDispatcherCommand.h PieceSelector.h \ @@ -830,10 +829,10 @@ am__objects_26 = SocketCore.$(OBJEXT) Command.$(OBJEXT) \ PeerSessionResource.$(OBJEXT) BtRegistry.$(OBJEXT) \ MultiFileAllocationIterator.$(OBJEXT) PeerConnection.$(OBJEXT) \ ByteArrayDiskWriter.$(OBJEXT) \ - ByteArrayDiskWriterFactory.$(OBJEXT) ServerHost.$(OBJEXT) \ - DownloadContext.$(OBJEXT) TimedHaltCommand.$(OBJEXT) \ - prefs.$(OBJEXT) ProtocolDetector.$(OBJEXT) \ - StringFormat.$(OBJEXT) HttpSkipResponseCommand.$(OBJEXT) \ + ByteArrayDiskWriterFactory.$(OBJEXT) DownloadContext.$(OBJEXT) \ + TimedHaltCommand.$(OBJEXT) prefs.$(OBJEXT) \ + ProtocolDetector.$(OBJEXT) StringFormat.$(OBJEXT) \ + HttpSkipResponseCommand.$(OBJEXT) \ InitiateConnectionCommand.$(OBJEXT) \ FtpFinishDownloadCommand.$(OBJEXT) A2STR.$(OBJEXT) \ RarestPieceSelector.$(OBJEXT) ChunkedDecoder.$(OBJEXT) \ @@ -1153,27 +1152,26 @@ SRCS = Socket.h SocketCore.cc SocketCore.h BinaryStream.h Command.cc \ MultiFileAllocationIterator.cc MultiFileAllocationIterator.h \ PeerConnection.cc PeerConnection.h ByteArrayDiskWriter.cc \ ByteArrayDiskWriter.h ByteArrayDiskWriterFactory.cc \ - ByteArrayDiskWriterFactory.h ServerHost.cc ServerHost.h \ - DownloadContext.cc DownloadContext.h TimedHaltCommand.cc \ - TimedHaltCommand.h CUIDCounter.h DNSCache.h DownloadResult.h \ - Sequence.h IntSequence.h PostDownloadHandler.h \ - PreDownloadHandler.h SingletonHolder.h \ - TrueRequestGroupCriteria.h a2algo.h a2functional.h a2io.h \ - a2netcompat.h a2time.h array_fun.h help_tags.h prefs.cc \ - prefs.h usage_text.h ProtocolDetector.cc ProtocolDetector.h \ - NullStatCalc.h StringFormat.cc StringFormat.h \ - HttpSkipResponseCommand.cc HttpSkipResponseCommand.h \ - InitiateConnectionCommand.cc InitiateConnectionCommand.h \ - FtpFinishDownloadCommand.cc FtpFinishDownloadCommand.h \ - A2STR.cc A2STR.h RarestPieceSelector.cc RarestPieceSelector.h \ - Decoder.h ChunkedDecoder.cc ChunkedDecoder.h Signature.cc \ - Signature.h ServerStat.cc ServerStat.h ServerStatMan.cc \ - ServerStatMan.h URISelector.h AdaptiveURISelector.cc \ - AdaptiveURISelector.h InOrderURISelector.cc \ - InOrderURISelector.h FeedbackURISelector.cc \ - FeedbackURISelector.h NsCookieParser.cc NsCookieParser.h \ - CookieStorage.cc CookieStorage.h SocketBuffer.cc \ - SocketBuffer.h OptionHandlerException.cc \ + ByteArrayDiskWriterFactory.h DownloadContext.cc \ + DownloadContext.h TimedHaltCommand.cc TimedHaltCommand.h \ + CUIDCounter.h DNSCache.h DownloadResult.h Sequence.h \ + IntSequence.h PostDownloadHandler.h PreDownloadHandler.h \ + SingletonHolder.h TrueRequestGroupCriteria.h a2algo.h \ + a2functional.h a2io.h a2netcompat.h a2time.h array_fun.h \ + help_tags.h prefs.cc prefs.h usage_text.h ProtocolDetector.cc \ + ProtocolDetector.h NullStatCalc.h StringFormat.cc \ + StringFormat.h HttpSkipResponseCommand.cc \ + HttpSkipResponseCommand.h InitiateConnectionCommand.cc \ + InitiateConnectionCommand.h FtpFinishDownloadCommand.cc \ + FtpFinishDownloadCommand.h A2STR.cc A2STR.h \ + RarestPieceSelector.cc RarestPieceSelector.h Decoder.h \ + ChunkedDecoder.cc ChunkedDecoder.h Signature.cc Signature.h \ + ServerStat.cc ServerStat.h ServerStatMan.cc ServerStatMan.h \ + URISelector.h AdaptiveURISelector.cc AdaptiveURISelector.h \ + InOrderURISelector.cc InOrderURISelector.h \ + FeedbackURISelector.cc FeedbackURISelector.h NsCookieParser.cc \ + NsCookieParser.h CookieStorage.cc CookieStorage.h \ + SocketBuffer.cc SocketBuffer.h OptionHandlerException.cc \ OptionHandlerException.h URIResult.cc URIResult.h EventPoll.h \ SelectEventPoll.cc SelectEventPoll.h SequentialPicker.h \ SequentialDispatcherCommand.h PieceSelector.h \ @@ -1527,7 +1525,6 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/SeedCheckCommand.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/SegmentMan.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/SelectEventPoll.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ServerHost.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ServerStat.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ServerStatMan.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Signature.Po@am__quote@ diff --git a/src/Metalink2RequestGroup.cc b/src/Metalink2RequestGroup.cc index dcad7912..9ca59766 100644 --- a/src/Metalink2RequestGroup.cc +++ b/src/Metalink2RequestGroup.cc @@ -217,6 +217,9 @@ Metalink2RequestGroup::createRequestGroup strconcat(option->get(PREF_DIR), "/", entry->file->getPath()))); dctx->setDir(option->get(PREF_DIR)); dctx->getFirstFileEntry()->setUris(uris); + if(option->getAsBool(PREF_METALINK_ENABLE_UNIQUE_PROTOCOL)) { + dctx->getFirstFileEntry()->disableSingleHostMultiConnection(); + } #ifdef ENABLE_MESSAGE_DIGEST if(entry->chunkChecksum.isNull()) { if(!entry->checksum.isNull()) { @@ -236,8 +239,6 @@ Metalink2RequestGroup::createRequestGroup option->getAsInt(PREF_METALINK_SERVERS) : std::min(option->getAsInt(PREF_METALINK_SERVERS), static_cast(entry->maxConnections))); - // In metalink, multi connection to a single host is not allowed by default. - rg->setSingleHostMultiConnectionEnabled(!option->getAsBool(PREF_METALINK_ENABLE_UNIQUE_PROTOCOL)); // remove "metalink" from Accept Type list to avoid loop in tranparent // metalink rg->removeAcceptType(RequestGroup::ACCEPT_METALINK); diff --git a/src/RequestGroup.cc b/src/RequestGroup.cc index 3dfa75f1..dab3a3c2 100644 --- a/src/RequestGroup.cc +++ b/src/RequestGroup.cc @@ -65,7 +65,6 @@ #include "DownloadHandlerFactory.h" #include "MemoryBufferPreDownloadHandler.h" #include "DownloadHandlerConstants.h" -#include "ServerHost.h" #include "Option.h" #include "FileEntry.h" #include "Request.h" @@ -124,7 +123,6 @@ RequestGroup::RequestGroup(const SharedHandle