diff --git a/ChangeLog b/ChangeLog index e294e945..d89c0989 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,38 @@ +2010-03-28 Tatsuhiro Tsujikawa + + Added --always-resume and --max-resume-failure-tries option. If + --always-resume=false is given, when all given URIs does not + support resume or aria2 encounters N URIs which does not support + resume + (N is the value specified using --max-resume-failure-tries + option), aria2 download file from scratch. The default behavior + is --always-resume=true, which means if all URIs do not support + resume, download fails. I think this is OK because user normally + don't like to see that partially downloaded file is + overwritten(this is particularly true if file size is big). This + option is useful when aria2 is used as download backend and + graceful falling back to overwritten behavior is preferable. + Added exit status value 8, which means download failed because + server did not support resume. + * src/AbstractCommand.cc + * src/DefaultPieceStorage.cc + * src/DownloadCommand.cc + * src/DownloadResultCode.h + * src/FileEntry.h + * src/FtpNegotiationCommand.cc + * src/HttpResponse.cc + * src/HttpResponseCommand.cc + * src/OptionHandlerFactory.cc + * src/RequestGroup.cc + * src/RequestGroup.h + * src/SegmentMan.cc + * src/SegmentMan.h + * src/prefs.cc + * src/prefs.h + * src/usage_text.h + * test/DefaultPieceStorageTest.cc + * test/SegmentManTest.cc + 2010-03-25 Tatsuhiro Tsujikawa Added --remove-control-file option to -i list options. diff --git a/src/AbstractCommand.cc b/src/AbstractCommand.cc index 084b10c2..85b55475 100644 --- a/src/AbstractCommand.cc +++ b/src/AbstractCommand.cc @@ -148,6 +148,15 @@ bool AbstractCommand::execute() { if(!_requestGroup->getPieceStorage().isNull()) { _segments.clear(); _requestGroup->getSegmentMan()->getInFlightSegment(_segments, cuid); + if(!req.isNull() && _segments.empty()) { + // This command previously has assigned segments, but it is + // canceled. So discard current request chain. + if(logger->debug()) { + logger->debug("CUID#%s - It seems previously assigned segments are" + "canceled. Restart.", util::itos(cuid).c_str()); + } + return prepareForRetry(0); + } if(req.isNull() || req->getMaxPipelinedRequest() == 1 || _requestGroup->getDownloadContext()->getFileEntries().size() == 1) { if(_segments.empty()) { @@ -215,6 +224,9 @@ bool AbstractCommand::execute() { util::itos(cuid).c_str(), req->getUri().c_str()); _fileEntry->addURIResult(req->getUri(), err.getCode()); _requestGroup->setLastUriResult(req->getUri(), err.getCode()); + if(err.getCode() == downloadresultcode::CANNOT_RESUME) { + _requestGroup->increaseResumeFailureCount(); + } } onAbort(); tryReserved(); @@ -233,7 +245,6 @@ bool AbstractCommand::execute() { const unsigned int maxTries = getOption()->getAsInt(PREF_MAX_TRIES); bool isAbort = maxTries != 0 && req->getTryCount() >= maxTries; if(isAbort) { - onAbort(); if(logger->info()) { logger->info(MSG_MAX_TRY, util::itos(cuid).c_str(), req->getTryCount()); } @@ -241,6 +252,10 @@ bool AbstractCommand::execute() { req->getUri().c_str()); _fileEntry->addURIResult(req->getUri(), err.getCode()); _requestGroup->setLastUriResult(req->getUri(), err.getCode()); + if(err.getCode() == downloadresultcode::CANNOT_RESUME) { + _requestGroup->increaseResumeFailureCount(); + } + onAbort(); tryReserved(); return true; } else { @@ -323,7 +338,52 @@ void AbstractCommand::onAbort() { logger->debug("CUID#%s - Aborting download", util::itos(cuid).c_str()); } if(!_requestGroup->getPieceStorage().isNull()) { - _requestGroup->getSegmentMan()->cancelSegment(cuid); + SharedHandle segmentMan = _requestGroup->getSegmentMan(); + segmentMan->cancelSegment(cuid); + // Don't do following process if BitTorrent is involved or files + // in DownloadContext is more than 1. The latter condition is + // limitation of current implementation. + if(!getOption()->getAsBool(PREF_ALWAYS_RESUME) && + !_fileEntry.isNull() && + segmentMan->calculateSessionDownloadLength() == 0 && + !_requestGroup->p2pInvolved() && + _requestGroup->getDownloadContext()->getFileEntries().size() == 1) { + const int maxTries = getOption()->getAsInt(PREF_MAX_RESUME_FAILURE_TRIES); + if((maxTries > 0 && _requestGroup->getResumeFailureCount() >= maxTries)|| + _fileEntry->emptyRequestUri()) { + // Local file exists, but given servers(or at least contacted + // ones) doesn't support resume. Let's restart download from + // scratch. + logger->notice("CUID#%s - Failed to resume download." + " Download from scratch.", + util::itos(cuid).c_str()); + if(logger->debug()) { + logger->debug("CUID#%s - Gathering URIs that has CANNOT_RESUME error", + util::itos(cuid).c_str()); + } + // Set PREF_ALWAYS_RESUME to V_TRUE to avoid repeating this + // process. + getOption()->put(PREF_ALWAYS_RESUME, V_TRUE); + std::deque res; + _fileEntry->extractURIResult(res, downloadresultcode::CANNOT_RESUME); + if(!res.empty()) { + segmentMan->cancelAllSegments(); + segmentMan->eraseSegmentWrittenLengthMemo(); + _requestGroup->getPieceStorage()->markPiecesDone(0); + std::vector uris; + uris.reserve(res.size()); + std::transform(res.begin(), res.end(), std::back_inserter(uris), + std::mem_fun_ref(&URIResult::getURI)); + if(logger->debug()) { + logger->debug("CUID#%s - %lu URIs found.", + util::itos(cuid).c_str(), + static_cast(uris.size())); + } + _fileEntry->addUris(uris.begin(), uris.end()); + segmentMan->recognizeSegmentFor(_fileEntry); + } + } + } } } diff --git a/src/DefaultPieceStorage.cc b/src/DefaultPieceStorage.cc index dc39d621..dd963201 100644 --- a/src/DefaultPieceStorage.cc +++ b/src/DefaultPieceStorage.cc @@ -596,6 +596,10 @@ void DefaultPieceStorage::markPiecesDone(uint64_t length) { if(length == bitfieldMan->getTotalLength()) { bitfieldMan->setAllBit(); + } else if(length == 0) { + // TODO this would go to markAllPiecesUndone() + bitfieldMan->clearAllBit(); + usedPieces.clear(); } else { size_t numPiece = length/bitfieldMan->getBlockLength(); if(numPiece > 0) { diff --git a/src/DownloadCommand.cc b/src/DownloadCommand.cc index e2bcf335..b01e2c40 100644 --- a/src/DownloadCommand.cc +++ b/src/DownloadCommand.cc @@ -304,14 +304,15 @@ bool DownloadCommand::prepareForNextSegment() { if(!tempSegment->complete()) { return prepareForRetry(0); } + SharedHandle segmentMan = _requestGroup->getSegmentMan(); SharedHandle nextSegment = - _requestGroup->getSegmentMan()->getSegment(cuid, - tempSegment->getIndex()+1); - if(!nextSegment.isNull() && nextSegment->getWrittenLength() == 0) { + segmentMan->getCleanSegmentIfOwnerIsIdle + (cuid, tempSegment->getIndex()+1); + if(nextSegment.isNull()) { + return prepareForRetry(0); + } else { e->commands.push_back(this); return false; - } else { - return prepareForRetry(0); } } else { return prepareForRetry(0); diff --git a/src/DownloadResultCode.h b/src/DownloadResultCode.h index a0fb3f41..f32821c0 100644 --- a/src/DownloadResultCode.h +++ b/src/DownloadResultCode.h @@ -50,6 +50,7 @@ enum RESULT { TOO_SLOW_DOWNLOAD_SPEED = 5, NETWORK_PROBLEM = 6, IN_PROGRESS = 7, + CANNOT_RESUME = 8, }; } // namespace downloadresultcode diff --git a/src/FileEntry.h b/src/FileEntry.h index 93b93802..a049b0e5 100644 --- a/src/FileEntry.h +++ b/src/FileEntry.h @@ -269,6 +269,11 @@ public: } bool removeUri(const std::string& uri); + + bool emptyRequestUri() const + { + return _uris.empty() && _inFlightRequests.empty() && _requestPool.empty(); + } }; // Returns the first FileEntry which isRequested() method returns diff --git a/src/FtpNegotiationCommand.cc b/src/FtpNegotiationCommand.cc index 16540fd7..4fa26fcd 100644 --- a/src/FtpNegotiationCommand.cc +++ b/src/FtpNegotiationCommand.cc @@ -394,7 +394,10 @@ bool FtpNegotiationCommand::onFileSizeDetermined(uint64_t totalLength) poolConnection(); return false; } - + // We have to make sure that command that has Request object must + // have segment after PieceStorage is initialized. See + // AbstractCommand::execute() + _requestGroup->getSegmentMan()->getSegment(cuid, 0); return true; } else { _requestGroup->adjustFilename @@ -424,6 +427,10 @@ bool FtpNegotiationCommand::onFileSizeDetermined(uint64_t totalLength) return false; } _requestGroup->loadAndOpenFile(infoFile); + // We have to make sure that command that has Request object must + // have segment after PieceStorage is initialized. See + // AbstractCommand::execute() + _requestGroup->getSegmentMan()->getSegment(cuid, 0); prepareForNextAction(this); @@ -574,7 +581,8 @@ bool FtpNegotiationCommand::recvRest(const SharedHandle& segment) { // then throw exception here. if(status != 350) { if(!segment.isNull() && segment->getPositionToWrite() != 0) { - throw DL_ABORT_EX("FTP server doesn't support resuming."); + throw DL_ABORT_EX2("FTP server doesn't support resuming.", + downloadresultcode::CANNOT_RESUME); } } sequence = SEQ_SEND_RETR; diff --git a/src/HttpResponse.cc b/src/HttpResponse.cc index f4ad9862..6957fe92 100644 --- a/src/HttpResponse.cc +++ b/src/HttpResponse.cc @@ -79,7 +79,7 @@ void HttpResponse::validateResponse() const // compare the received range against the requested range RangeHandle responseRange = httpHeader->getRange(); if(!httpRequest->isRangeSatisfied(responseRange)) { - throw DL_ABORT_EX + throw DL_ABORT_EX2 (StringFormat (EX_INVALID_RANGE_HEADER, util::itos(httpRequest->getStartByte(), true).c_str(), @@ -87,7 +87,8 @@ void HttpResponse::validateResponse() const util::uitos(httpRequest->getEntityLength(), true).c_str(), util::itos(responseRange->getStartByte(), true).c_str(), util::itos(responseRange->getEndByte(), true).c_str(), - util::uitos(responseRange->getEntityLength(), true).c_str()).str()); + util::uitos(responseRange->getEntityLength(), true).c_str()).str(), + downloadresultcode::CANNOT_RESUME); } } } diff --git a/src/HttpResponseCommand.cc b/src/HttpResponseCommand.cc index 05a50e36..f410adfe 100644 --- a/src/HttpResponseCommand.cc +++ b/src/HttpResponseCommand.cc @@ -245,6 +245,9 @@ bool HttpResponseCommand::handleDefaultEncoding _requestGroup->loadAndOpenFile(infoFile); File file(_requestGroup->getFirstFilePath()); + // We have to make sure that command that has Request object must + // have segment after PieceStorage is initialized. See + // AbstractCommand::execute() SharedHandle segment = _requestGroup->getSegmentMan()->getSegment(cuid, 0); // pipelining requires implicit range specified. But the request for @@ -348,6 +351,11 @@ bool HttpResponseCommand::handleOtherEncoding poolConnection(); return true; } + // We have to make sure that command that has Request object must + // have segment after PieceStorage is initialized. See + // AbstractCommand::execute() + _requestGroup->getSegmentMan()->getSegment(cuid, 0); + e->commands.push_back (createHttpDownloadCommand(httpResponse, getTransferEncodingDecoder(httpResponse), diff --git a/src/OptionHandlerFactory.cc b/src/OptionHandlerFactory.cc index ae2418c6..0df5c502 100644 --- a/src/OptionHandlerFactory.cc +++ b/src/OptionHandlerFactory.cc @@ -65,7 +65,18 @@ OptionHandlers OptionHandlerFactory::createOptionHandlers() V_FALSE)); op->addTag(TAG_ADVANCED); handlers.push_back(op); - } + } + { + SharedHandle op(new BooleanOptionHandler + (PREF_ALWAYS_RESUME, + TEXT_ALWAYS_RESUME, + V_TRUE, + OptionHandler::OPT_ARG)); + op->addTag(TAG_ADVANCED); + op->addTag(TAG_FTP); + op->addTag(TAG_HTTP); + handlers.push_back(op); + } #ifdef ENABLE_ASYNC_DNS { SharedHandle op(new BooleanOptionHandler @@ -322,6 +333,17 @@ OptionHandlers OptionHandlerFactory::createOptionHandlers() op->addTag(TAG_HTTP); handlers.push_back(op); } + { + SharedHandle op(new NumberOptionHandler + (PREF_MAX_RESUME_FAILURE_TRIES, + TEXT_MAX_RESUME_FAILURE_TRIES, + "0", + 0)); + op->addTag(TAG_ADVANCED); + op->addTag(TAG_FTP); + op->addTag(TAG_HTTP); + handlers.push_back(op); + } { SharedHandle op(new BooleanOptionHandler (PREF_NO_CONF, diff --git a/src/RequestGroup.cc b/src/RequestGroup.cc index e6fee4e0..fef4f88c 100644 --- a/src/RequestGroup.cc +++ b/src/RequestGroup.cc @@ -131,6 +131,7 @@ RequestGroup::RequestGroup(const SharedHandle