From 91de1de749f142a32ceb28c4e9d17f1c6085e379 Mon Sep 17 00:00:00 2001 From: JJenkx Date: Tue, 21 Jun 2022 23:44:43 -0400 Subject: [PATCH] Applied some patches --- configure.ac | 18 ++++++++ src/AbstractProxyRequestCommand.cc | 1 + src/ByteArrayDiskWriter.h | 2 +- src/DownloadCommand.cc | 2 +- src/DownloadEngineFactory.cc | 36 ++++++++++++---- src/HttpListenCommand.cc | 27 +++++++----- src/HttpListenCommand.h | 8 +++- src/HttpRequest.cc | 4 +- src/HttpRequestCommand.cc | 1 + src/HttpResponse.cc | 26 +++++++++--- src/HttpSkipResponseCommand.cc | 42 +++++++++++++++--- src/Makefile.am | 2 + src/OptionHandlerFactory.cc | 68 ++++++++++++++++++++++++++---- src/Piece.cc | 9 +++- src/SinkStreamFilter.cc | 10 ++++- src/SocketBuffer.cc | 3 +- src/SocketCore.cc | 14 +++++- src/SocketCore.h | 5 +++ src/message.h | 5 ++- src/prefs.cc | 10 +++++ src/prefs.h | 10 +++++ src/usage_text.h | 24 +++++++++++ 22 files changed, 275 insertions(+), 52 deletions(-) diff --git a/configure.ac b/configure.ac index 10acaba0..e67d0b80 100644 --- a/configure.ac +++ b/configure.ac @@ -67,6 +67,7 @@ ARIA2_ARG_WITHOUT([libz]) ARIA2_ARG_WITH([tcmalloc]) ARIA2_ARG_WITH([jemalloc]) ARIA2_ARG_WITHOUT([libssh2]) +ARIA2_ARG_WITHOUT([systemd]) ARIA2_ARG_DISABLE([ssl]) ARIA2_ARG_DISABLE([bittorrent]) @@ -497,6 +498,19 @@ if test "x$with_libssh2" = "xyes"; then fi fi +have_systemd=no +if test "x$with_systemd" = "xyes"; then + PKG_CHECK_MODULES([SYSTEMD], [libsystemd], [have_systemd=yes], [have_systemd=no]) + if test "x$have_systemd" = "xyes"; then + AC_DEFINE([HAVE_SYSTEMD], [1], [Define to 1 if you have systemd.]) + else + AC_MSG_WARN([$SYSTEMD_PKG_ERRORS]) + if test "x$with_systemd_requested" = "xyes"; then + ARIA2_DEP_NOT_MET([systemd]) + fi + fi +fi + have_libcares=no if test "x$with_libcares" = "xyes"; then PKG_CHECK_MODULES([LIBCARES], [libcares >= 1.7.0], [have_libcares=yes], @@ -648,6 +662,9 @@ AM_CONDITIONAL([HAVE_SQLITE3], [test "x$have_sqlite3" = "xyes"]) # Set conditional for libssh2 AM_CONDITIONAL([HAVE_LIBSSH2], [test "x$have_libssh2" = "xyes"]) +# Set conditional for systemd +AM_CONDITIONAL([HAVE_SYSTEMD], [test "x$have_systemd" = "xyes"]) + case "$host" in *solaris*) save_LIBS=$LIBS @@ -1117,6 +1134,7 @@ LibExpat: $have_libexpat (CFLAGS='$EXPAT_CFLAGS' LIBS='$EXPAT_LIBS') LibCares: $have_libcares (CFLAGS='$LIBCARES_CFLAGS' LIBS='$LIBCARES_LIBS') Zlib: $have_zlib (CFLAGS='$ZLIB_CFLAGS' LIBS='$ZLIB_LIBS') Libssh2: $have_libssh2 (CFLAGS='$LIBSSH2_CFLAGS' LIBS='$LIBSSH2_LIBS') +Systemd: $have_systemd (CFLAGS='$SYSTEMD_CFLAGS' LIBS='$SYSTEMD_LIBS') Tcmalloc: $have_tcmalloc (CFLAGS='$TCMALLOC_CFLAGS' LIBS='$TCMALLOC_LIBS') Jemalloc: $have_jemalloc (CFLAGS='$JEMALLOC_CFLAGS' LIBS='$JEMALLOC_LIBS') Epoll: $have_epoll diff --git a/src/AbstractProxyRequestCommand.cc b/src/AbstractProxyRequestCommand.cc index bd2bcb37..1feed07e 100644 --- a/src/AbstractProxyRequestCommand.cc +++ b/src/AbstractProxyRequestCommand.cc @@ -72,6 +72,7 @@ bool AbstractProxyRequestCommand::executeInternal() if (httpConnection_->sendBufferIsEmpty()) { auto httpRequest = make_unique(); httpRequest->setUserAgent(getOption()->get(PREF_USER_AGENT)); + httpRequest->setNoWantDigest(!getOption()->getAsBool(PREF_HTTP_WANT_DIGEST)); httpRequest->setRequest(getRequest()); httpRequest->setProxyRequest(proxyRequest_); diff --git a/src/ByteArrayDiskWriter.h b/src/ByteArrayDiskWriter.h index a52cb8c8..0e8efb45 100644 --- a/src/ByteArrayDiskWriter.h +++ b/src/ByteArrayDiskWriter.h @@ -49,7 +49,7 @@ private: void clear(); public: - ByteArrayDiskWriter(size_t maxLength = 5_m); + ByteArrayDiskWriter(size_t maxLength = 15_m); virtual ~ByteArrayDiskWriter(); virtual void initAndOpenFile(int64_t totalLength = 0) CXX11_OVERRIDE; diff --git a/src/DownloadCommand.cc b/src/DownloadCommand.cc index 2db41e46..f49eb807 100644 --- a/src/DownloadCommand.cc +++ b/src/DownloadCommand.cc @@ -306,7 +306,7 @@ void DownloadCommand::checkLowestDownloadSpeed() const startupIdleTime_) { int nowSpeed = peerStat_->calculateDownloadSpeed(); if (nowSpeed <= lowestDownloadSpeedLimit_) { - throw DL_ABORT_EX2(fmt(EX_TOO_SLOW_DOWNLOAD_SPEED, nowSpeed, + throw DL_RETRY_EX2(fmt(EX_TOO_SLOW_DOWNLOAD_SPEED, nowSpeed, lowestDownloadSpeedLimit_, getRequest()->getHost().c_str()), error_code::TOO_SLOW_DOWNLOAD_SPEED); diff --git a/src/DownloadEngineFactory.cc b/src/DownloadEngineFactory.cc index 64c5c76b..086165bb 100644 --- a/src/DownloadEngineFactory.cc +++ b/src/DownloadEngineFactory.cc @@ -35,6 +35,9 @@ #include "DownloadEngineFactory.h" #include +#ifdef HAVE_SYSTEMD +#include +#endif // HAVE_SYSTEMD #include "Option.h" #include "RequestGroup.h" @@ -205,14 +208,31 @@ std::unique_ptr DownloadEngineFactory::newDownloadEngine( if (secure) { A2_LOG_NOTICE("RPC transport will be encrypted."); } - static int families[] = {AF_INET, AF_INET6}; - size_t familiesLength = op->getAsBool(PREF_DISABLE_IPV6) ? 1 : 2; - for (size_t i = 0; i < familiesLength; ++i) { - auto httpListenCommand = make_unique( - e->newCUID(), e.get(), families[i], secure); - if (httpListenCommand->bindPort(op->getAsInt(PREF_RPC_LISTEN_PORT))) { - e->addCommand(std::move(httpListenCommand)); - ok = true; +#ifdef HAVE_SYSTEMD + int fds = sd_listen_fds(1); + if (0 < fds) { + for (int fd = SD_LISTEN_FDS_START; fd < SD_LISTEN_FDS_START + fds; fd++) { + auto httpListenCommand = make_unique( + e->newCUID(), e.get(), fd, secure); + if (httpListenCommand->listen()) { + e->addCommand(std::move(httpListenCommand)); + ok = true; + } + } + } else +#endif // HAVE_SYSTEMD + { + static int families[] = {AF_INET, AF_INET6}; + size_t familiesLength = op->getAsBool(PREF_DISABLE_IPV6) ? 1 : 2; + for (size_t i = 0; i < familiesLength; ++i) { + auto httpListenCommand = make_unique( + e->newCUID(), e.get(), families[i], + op->getAsInt(PREF_RPC_LISTEN_PORT), + secure); + if (httpListenCommand->listen()) { + e->addCommand(std::move(httpListenCommand)); + ok = true; + } } } if (!ok) { diff --git a/src/HttpListenCommand.cc b/src/HttpListenCommand.cc index 7c7794aa..9b632ba6 100644 --- a/src/HttpListenCommand.cc +++ b/src/HttpListenCommand.cc @@ -52,7 +52,7 @@ namespace aria2 { HttpListenCommand::HttpListenCommand(cuid_t cuid, DownloadEngine* e, int family, bool secure) - : Command(cuid), e_(e), family_(family), secure_(secure) + : Command(cuid), e_(e), fd_(fd), family_(AF_INET), port_(0), secure_(secure) { } @@ -89,27 +89,34 @@ bool HttpListenCommand::execute() return false; } -bool HttpListenCommand::bindPort(uint16_t port) +bool HttpListenCommand::listen() { if (serverSocket_) { e_->deleteSocketForReadCheck(serverSocket_, this); } serverSocket_ = std::make_shared(); - const int ipv = (family_ == AF_INET) ? 4 : 6; + std::string sListen; try { - int flags = 0; - if (e_->getOption()->getAsBool(PREF_RPC_LISTEN_ALL)) { - flags = AI_PASSIVE; + if (fd_ < 0) { + int flags = 0; + const int ipv = (family_ == AF_INET) ? 4 : 6; + sListen = fmt("TCP/IPv%d port %u", ipv, port_); + if (e_->getOption()->getAsBool(PREF_RPC_LISTEN_ALL)) { + flags = AI_PASSIVE; + } + serverSocket_->bind(nullptr, port_, family_, flags); + } else { + sListen = fmt("file descriptor fd=%d", fd_); + serverSocket_->bindExistingFd(fd_); } - serverSocket_->bind(nullptr, port, family_, flags); serverSocket_->beginListen(); - A2_LOG_INFO(fmt(MSG_LISTENING_PORT, getCuid(), port)); + A2_LOG_INFO(fmt(MSG_LISTENING_RPC, getCuid(), sListen.c_str())); e_->addSocketForReadCheck(serverSocket_, this); - A2_LOG_NOTICE(fmt(_("IPv%d RPC: listening on TCP port %u"), ipv, port)); + A2_LOG_NOTICE(fmt(_("RPC: listening on %s"), sListen.c_str())); return true; } catch (RecoverableException& e) { - A2_LOG_ERROR_EX(fmt("IPv%d RPC: failed to bind TCP port %u", ipv, port), e); + A2_LOG_ERROR_EX(fmt("RPC: failed to listen on %s", sListen.c_str()), e); serverSocket_->closeConnection(); } return false; diff --git a/src/HttpListenCommand.h b/src/HttpListenCommand.h index d63d5b3d..c023f2be 100644 --- a/src/HttpListenCommand.h +++ b/src/HttpListenCommand.h @@ -47,18 +47,22 @@ class SocketCore; class HttpListenCommand : public Command { private: DownloadEngine* e_; + int fd_; int family_; + uint16_t port_; std::shared_ptr serverSocket_; bool secure_; public: - HttpListenCommand(cuid_t cuid, DownloadEngine* e, int family, bool secure); + HttpListenCommand(cuid_t cuid, DownloadEngine* e, int family, uint16_t port, bool secure); + + HttpListenCommand(cuid_t cuid, DownloadEngine* e, int fd, bool secure); virtual ~HttpListenCommand(); virtual bool execute() CXX11_OVERRIDE; - bool bindPort(uint16_t port); + bool listen(); }; } // namespace aria2 diff --git a/src/HttpRequest.cc b/src/HttpRequest.cc index b5fb4f86..39672616 100644 --- a/src/HttpRequest.cc +++ b/src/HttpRequest.cc @@ -53,6 +53,7 @@ #include "Request.h" #include "DownloadHandlerConstants.h" #include "MessageDigest.h" +#include "LogFactory.h" namespace aria2 { @@ -124,7 +125,7 @@ bool HttpRequest::isRangeSatisfied(const Range& range) const return true; } return getStartByte() == range.startByte && - (getEndByte() == 0 || getEndByte() == range.endByte) && + (getEndByte() == 0 || getEndByte() <= range.endByte) && (fileEntry_->getLength() == 0 || fileEntry_->getLength() == range.entityLength); } @@ -262,7 +263,6 @@ std::string HttpRequest::createRequest() } if (!wantDigest.empty()) { wantDigest.erase(wantDigest.size() - 2); - builtinHds.emplace_back("Want-Digest:", wantDigest); } } for (const auto& builtinHd : builtinHds) { diff --git a/src/HttpRequestCommand.cc b/src/HttpRequestCommand.cc index 3ef3887c..b83a7849 100644 --- a/src/HttpRequestCommand.cc +++ b/src/HttpRequestCommand.cc @@ -90,6 +90,7 @@ createHttpRequest(const std::shared_ptr& req, { auto httpRequest = make_unique(); httpRequest->setUserAgent(option->get(PREF_USER_AGENT)); + httpRequest->setNoWantDigest(!option->getAsBool(PREF_HTTP_WANT_DIGEST)); httpRequest->setRequest(req); httpRequest->setFileEntry(fileEntry); httpRequest->setSegment(segment); diff --git a/src/HttpResponse.cc b/src/HttpResponse.cc index 500cd64e..37c0bd66 100644 --- a/src/HttpResponse.cc +++ b/src/HttpResponse.cc @@ -78,12 +78,26 @@ void HttpResponse::validateResponse() const // compare the received range against the requested range auto responseRange = httpHeader_->getRange(); if (!httpRequest_->isRangeSatisfied(responseRange)) { - throw DL_ABORT_EX2( - fmt(EX_INVALID_RANGE_HEADER, httpRequest_->getStartByte(), - httpRequest_->getEndByte(), httpRequest_->getEntityLength(), - responseRange.startByte, responseRange.endByte, - responseRange.entityLength), - error_code::CANNOT_RESUME); + if ( httpRequest_->getEndByte() > 0 && + httpRequest_->getEndByte() <= responseRange.endByte){ + // Some servers return full length of file as endByte + // regardless of what was requested. + // If server offers more, ignore for a while and hope for the best. + + A2_LOG_WARN( fmt(MSG_STRANGE_RANGE_HEADER, cuid_, + httpRequest_->getStartByte(), + httpRequest_->getEndByte(), httpRequest_->getEntityLength(), + responseRange.startByte, responseRange.endByte, + responseRange.entityLength)); + + } else { + throw DL_ABORT_EX2( + fmt(EX_INVALID_RANGE_HEADER, httpRequest_->getStartByte(), + httpRequest_->getEndByte(), httpRequest_->getEntityLength(), + responseRange.startByte, responseRange.endByte, + responseRange.entityLength), + error_code::CANNOT_RESUME); + } } } return; diff --git a/src/HttpSkipResponseCommand.cc b/src/HttpSkipResponseCommand.cc index a722d774..de4ad6ce 100644 --- a/src/HttpSkipResponseCommand.cc +++ b/src/HttpSkipResponseCommand.cc @@ -204,7 +204,7 @@ bool HttpSkipResponseCommand::processResponse() auto statusCode = httpResponse_->getStatusCode(); if (statusCode >= 400) { switch (statusCode) { - case 401: + case 401: // Unauthorized if (getOption()->getAsBool(PREF_HTTP_AUTH_CHALLENGE) && !httpResponse_->getHttpRequest()->authenticationUsed() && getDownloadEngine()->getAuthConfigFactory()->activateBasicCred( @@ -213,15 +213,41 @@ bool HttpSkipResponseCommand::processResponse() return prepareForRetry(0); } throw DL_ABORT_EX2(EX_AUTH_FAILED, error_code::HTTP_AUTH_FAILED); - case 404: + case 404: // Not Found if (getOption()->getAsInt(PREF_MAX_FILE_NOT_FOUND) == 0) { throw DL_ABORT_EX2(MSG_RESOURCE_NOT_FOUND, error_code::RESOURCE_NOT_FOUND); } throw DL_RETRY_EX2(MSG_RESOURCE_NOT_FOUND, error_code::RESOURCE_NOT_FOUND); - case 502: - case 503: + case 400: // Bad Request + if (getOption()->getAsBool(PREF_RETRY_ON_400) + && getOption()->getAsInt(PREF_RETRY_WAIT) > 0) { + throw DL_RETRY_EX2(fmt(EX_BAD_STATUS, statusCode), + error_code::HTTP_PROTOCOL_ERROR); + } + break; + case 403: // Forbidden + if (getOption()->getAsBool(PREF_RETRY_ON_403) + && getOption()->getAsInt(PREF_RETRY_WAIT) > 0) { + throw DL_RETRY_EX2(fmt(EX_BAD_STATUS, statusCode), + error_code::HTTP_PROTOCOL_ERROR); + } + break; + case 406: // Not Acceptable + if (getOption()->getAsBool(PREF_RETRY_ON_406) + && getOption()->getAsInt(PREF_RETRY_WAIT) > 0) { + throw DL_RETRY_EX2(fmt(EX_BAD_STATUS, statusCode), + error_code::HTTP_PROTOCOL_ERROR); + } + break; + case 408: // Request Timeout + case 429: // Too Many Requests + case 502: // Bad Gateway + case 503: // Service Unavailable + case 507: // Insufficient Storage + case 520: // https://github.com/aria2/aria2/issues/1229 + case 521: // https://github.com/aria2/aria2/issues/1229 // Only retry if pretry-wait > 0. Hammering 'busy' server is not // a good idea. if (getOption()->getAsInt(PREF_RETRY_WAIT) > 0) { @@ -230,12 +256,16 @@ bool HttpSkipResponseCommand::processResponse() } throw DL_ABORT_EX2(fmt(EX_BAD_STATUS, statusCode), error_code::HTTP_SERVICE_UNAVAILABLE); - case 504: + case 504: // Gateway Timeout // This is Gateway Timeout, so try again throw DL_RETRY_EX2(fmt(EX_BAD_STATUS, statusCode), error_code::HTTP_SERVICE_UNAVAILABLE); }; - + if (getOption()->getAsBool(PREF_RETRY_ON_UNKNOWN) + && getOption()->getAsInt(PREF_RETRY_WAIT) > 0) { + throw DL_RETRY_EX2(fmt(EX_BAD_STATUS, statusCode), + error_code::HTTP_PROTOCOL_ERROR); + } throw DL_ABORT_EX2(fmt(EX_BAD_STATUS, statusCode), error_code::HTTP_PROTOCOL_ERROR); } diff --git a/src/Makefile.am b/src/Makefile.am index cb6e3b7c..6fc656c2 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -712,6 +712,7 @@ AM_CPPFLAGS = \ @LIBGMP_CFLAGS@ \ @LIBGCRYPT_CFLAGS@ \ @LIBSSH2_CFLAGS@ \ + @SYSTEMD_CFLAGS@ \ @LIBCARES_CFLAGS@ \ @WSLAY_CFLAGS@ \ @TCMALLOC_CFLAGS@ \ @@ -735,6 +736,7 @@ EXTLDADD = @ALLOCA@ \ @LIBGMP_LIBS@ \ @LIBGCRYPT_LIBS@ \ @LIBSSH2_LIBS@ \ + @SYSTEMD_LIBS@ \ @LIBCARES_LIBS@ \ @WSLAY_LIBS@ \ @TCMALLOC_LIBS@ \ diff --git a/src/OptionHandlerFactory.cc b/src/OptionHandlerFactory.cc index 2cd2c43e..29c27902 100644 --- a/src/OptionHandlerFactory.cc +++ b/src/OptionHandlerFactory.cc @@ -158,7 +158,7 @@ std::vector OptionHandlerFactory::createOptionHandlers() } { OptionHandler* op(new BooleanOptionHandler( - PREF_CONTINUE, TEXT_CONTINUE, A2_V_FALSE, OptionHandler::OPT_ARG, 'c')); + PREF_CONTINUE, TEXT_CONTINUE, A2_V_TRUE, OptionHandler::OPT_ARG, 'c')); op->addTag(TAG_BASIC); op->addTag(TAG_FTP); op->addTag(TAG_HTTP); @@ -432,7 +432,7 @@ std::vector OptionHandlerFactory::createOptionHandlers() { OptionHandler* op(new NumberOptionHandler(PREF_MAX_CONCURRENT_DOWNLOADS, TEXT_MAX_CONCURRENT_DOWNLOADS, - "5", 1, -1, 'j')); + "8", 1, -1, 'j')); op->addTag(TAG_BASIC); op->setChangeGlobalOption(true); handlers.push_back(op); @@ -440,7 +440,7 @@ std::vector OptionHandlerFactory::createOptionHandlers() { OptionHandler* op(new NumberOptionHandler(PREF_MAX_CONNECTION_PER_SERVER, TEXT_MAX_CONNECTION_PER_SERVER, - "1", 1, 16, 'x')); + "1", 1, -1, 'x')); op->addTag(TAG_BASIC); op->addTag(TAG_FTP); op->addTag(TAG_HTTP); @@ -501,7 +501,7 @@ std::vector OptionHandlerFactory::createOptionHandlers() } { OptionHandler* op(new UnitNumberOptionHandler( - PREF_MIN_SPLIT_SIZE, TEXT_MIN_SPLIT_SIZE, "20M", 1_m, 1_g, 'k')); + PREF_MIN_SPLIT_SIZE, TEXT_MIN_SPLIT_SIZE, "1k", 1_k, 1_g, 'k')); op->addTag(TAG_BASIC); op->addTag(TAG_FTP); op->addTag(TAG_HTTP); @@ -834,7 +834,7 @@ std::vector OptionHandlerFactory::createOptionHandlers() } { OptionHandler* op(new NumberOptionHandler( - PREF_CONNECT_TIMEOUT, TEXT_CONNECT_TIMEOUT, "60", 1, 600)); + PREF_CONNECT_TIMEOUT, TEXT_CONNECT_TIMEOUT, "30", 1, 600)); op->addTag(TAG_FTP); op->addTag(TAG_HTTP); op->setInitialOption(true); @@ -905,7 +905,7 @@ std::vector OptionHandlerFactory::createOptionHandlers() } { OptionHandler* op(new UnitNumberOptionHandler( - PREF_PIECE_LENGTH, TEXT_PIECE_LENGTH, "1M", 1_m, 1_g)); + PREF_PIECE_LENGTH, TEXT_PIECE_LENGTH, "1k", 1_k, 1_g)); op->addTag(TAG_ADVANCED); op->addTag(TAG_FTP); op->addTag(TAG_HTTP); @@ -926,13 +926,53 @@ std::vector OptionHandlerFactory::createOptionHandlers() } { OptionHandler* op( - new NumberOptionHandler(PREF_RETRY_WAIT, TEXT_RETRY_WAIT, "0", 0, 600)); + new NumberOptionHandler(PREF_RETRY_WAIT, TEXT_RETRY_WAIT, "2", 0, 600)); op->addTag(TAG_FTP); op->addTag(TAG_HTTP); op->setInitialOption(true); op->setChangeGlobalOption(true); op->setChangeOptionForReserved(true); handlers.push_back(op); + } + { + OptionHandler* op(new BooleanOptionHandler( + PREF_RETRY_ON_400, TEXT_RETRY_ON_400, A2_V_FALSE, OptionHandler::OPT_ARG)); + op->addTag(TAG_ADVANCED); + op->addTag(TAG_HTTP); + op->setInitialOption(true); + op->setChangeGlobalOption(true); + op->setChangeOptionForReserved(true); + handlers.push_back(op); + } + { + OptionHandler* op(new BooleanOptionHandler( + PREF_RETRY_ON_403, TEXT_RETRY_ON_403, A2_V_FALSE, OptionHandler::OPT_ARG)); + op->addTag(TAG_ADVANCED); + op->addTag(TAG_HTTP); + op->setInitialOption(true); + op->setChangeGlobalOption(true); + op->setChangeOptionForReserved(true); + handlers.push_back(op); + } + { + OptionHandler* op(new BooleanOptionHandler( + PREF_RETRY_ON_406, TEXT_RETRY_ON_406, A2_V_FALSE, OptionHandler::OPT_ARG)); + op->addTag(TAG_ADVANCED); + op->addTag(TAG_HTTP); + op->setInitialOption(true); + op->setChangeGlobalOption(true); + op->setChangeOptionForReserved(true); + handlers.push_back(op); + } + { + OptionHandler* op(new BooleanOptionHandler( + PREF_RETRY_ON_UNKNOWN, TEXT_RETRY_ON_UNKNOWN, A2_V_FALSE, OptionHandler::OPT_ARG)); + op->addTag(TAG_ADVANCED); + op->addTag(TAG_HTTP); + op->setInitialOption(true); + op->setChangeGlobalOption(true); + op->setChangeOptionForReserved(true); + handlers.push_back(op); } { OptionHandler* op(new BooleanOptionHandler( @@ -971,7 +1011,7 @@ std::vector OptionHandlerFactory::createOptionHandlers() } { OptionHandler* op( - new NumberOptionHandler(PREF_SPLIT, TEXT_SPLIT, "5", 1, -1, 's')); + new NumberOptionHandler(PREF_SPLIT, TEXT_SPLIT, "8", 1, -1, 's')); op->addTag(TAG_BASIC); op->addTag(TAG_FTP); op->addTag(TAG_HTTP); @@ -1066,6 +1106,18 @@ std::vector OptionHandlerFactory::createOptionHandlers() op->setChangeOptionForReserved(true); handlers.push_back(op); } + { + OptionHandler* op( + new BooleanOptionHandler(PREF_HTTP_WANT_DIGEST, + TEXT_HTTP_WANT_DIGEST, + A2_V_TRUE, OptionHandler::OPT_ARG)); + op->addTag(TAG_ADVANCED); + op->addTag(TAG_HTTP); + op->setInitialOption(true); + op->setChangeGlobalOption(true); + op->setChangeOptionForReserved(true); + handlers.push_back(op); + } { OptionHandler* op(new BooleanOptionHandler( PREF_ENABLE_HTTP_KEEP_ALIVE, TEXT_ENABLE_HTTP_KEEP_ALIVE, A2_V_TRUE, diff --git a/src/Piece.cc b/src/Piece.cc index ccbb0f44..e1a53d91 100644 --- a/src/Piece.cc +++ b/src/Piece.cc @@ -347,8 +347,13 @@ void Piece::updateWrCache(WrDiskCache* diskCache, unsigned char* data, cell->len = len; cell->capacity = capacity; bool rv; - rv = wrCache_->cacheData(cell); - assert(rv); + try { + rv = wrCache_->cacheData(cell); + assert(rv); + } catch (RecoverableException& e) { + delete cell; + throw; + } rv = diskCache->update(wrCache_.get(), len); assert(rv); } diff --git a/src/SinkStreamFilter.cc b/src/SinkStreamFilter.cc index 47f529de..42d1d810 100644 --- a/src/SinkStreamFilter.cc +++ b/src/SinkStreamFilter.cc @@ -41,6 +41,7 @@ #include "Segment.h" #include "WrDiskCache.h" #include "Piece.h" +#include "DlAbortEx.h" namespace aria2 { @@ -81,8 +82,13 @@ ssize_t SinkStreamFilter::transform(const std::shared_ptr& out, size_t capacity = std::max(len, static_cast(4_k)); auto dataCopy = new unsigned char[capacity]; memcpy(dataCopy, inbuf + alen, len); - piece->updateWrCache(wrDiskCache_, dataCopy, 0, len, capacity, - segment->getPositionToWrite() + alen); + try { + piece->updateWrCache(wrDiskCache_, dataCopy, 0, len, capacity, + segment->getPositionToWrite() + alen); + } catch (RecoverableException& e) { + delete[] dataCopy; + throw; + } } } else { diff --git a/src/SocketBuffer.cc b/src/SocketBuffer.cc index 62862fff..19061732 100644 --- a/src/SocketBuffer.cc +++ b/src/SocketBuffer.cc @@ -39,6 +39,7 @@ #include "SocketCore.h" #include "DlAbortEx.h" +#include "DlRetryEx.h" #include "message.h" #include "fmt.h" #include "LogFactory.h" @@ -158,7 +159,7 @@ ssize_t SocketBuffer::send() } ssize_t slen = socket_->writeVector(iov, num); if (slen == 0 && !socket_->wantRead() && !socket_->wantWrite()) { - throw DL_ABORT_EX(fmt(EX_SOCKET_SEND, "Connection closed.")); + throw DL_RETRY_EX(fmt(EX_SOCKET_SEND, "Connection closed.")); } // A2_LOG_NOTICE(fmt("num=%zu, amount=%d, bufq.size()=%zu, SEND=%d", // num, amount, bufq_.size(), slen)); diff --git a/src/SocketCore.cc b/src/SocketCore.cc index e235f512..74ea2a4c 100644 --- a/src/SocketCore.cc +++ b/src/SocketCore.cc @@ -360,6 +360,18 @@ void SocketCore::bind(const struct sockaddr* addr, socklen_t addrlen) sockfd_ = fd; } +void SocketCore::bindExistingFd(int fd) +{ + std::string error; + if (fd < 0) { + error = fmt("Invalid file descriptor fd=%d", fd); + throw DL_ABORT_EX(fmt(EX_SOCKET_BIND, error.c_str())); + } + util::make_fd_cloexec(fd); + applySocketBufferSize(fd); + sockfd_ = fd; +} + void SocketCore::beginListen() { if (listen(sockfd_, 1024) == -1) { @@ -1015,7 +1027,7 @@ bool SocketCore::tlsHandshake(TLSContext* tlsctx, const std::string& hostname) if (rv == TLS_ERR_ERROR) { // Damn those error. - throw DL_ABORT_EX(fmt("SSL/TLS handshake failure: %s", + throw DL_RETRY_EX(fmt("SSL/TLS handshake failure: %s", handshakeError.empty() ? tlsSession_->getLastErrorString().c_str() : handshakeError.c_str())); diff --git a/src/SocketCore.h b/src/SocketCore.h index 60f33279..2cd7890a 100644 --- a/src/SocketCore.h +++ b/src/SocketCore.h @@ -162,6 +162,11 @@ public: void bind(const char* addrp, uint16_t port, int family, int flags = AI_PASSIVE); + /** + * Bind to an existing file descriptor. + */ + void bindExistingFd(int fd); + /** * Listens form connection on it. * Call bind(uint16_t) before calling this function. diff --git a/src/message.h b/src/message.h index 4705d7fb..9b89eb08 100644 --- a/src/message.h +++ b/src/message.h @@ -93,8 +93,8 @@ #define MSG_CONTENT_DISPOSITION_DETECTED \ "CUID#%" PRId64 " - Content-Disposition detected. Use %s as filename" #define MSG_PEER_BANNED "CUID#%" PRId64 " - Peer %s:%d banned." -#define MSG_LISTENING_PORT \ - "CUID#%" PRId64 " - Using port %d for accepting new connections" +#define MSG_LISTENING_RPC \ + "CUID#%" PRId64 " - Using %s for accepting new connections" #define MSG_BIND_FAILURE "CUID#%" PRId64 " - An error occurred while binding port=%d" #define MSG_INCOMING_PEER_CONNECTION \ "CUID#%" PRId64 " - Incoming connection, adding new command CUID#%" PRId64 "" @@ -211,6 +211,7 @@ #define MSG_REMOVING_UNSELECTED_FILE _("GID#%s - Removing unselected file.") #define MSG_FILE_REMOVED _("File %s removed.") #define MSG_FILE_COULD_NOT_REMOVED _("File %s could not be removed.") +#define MSG_STRANGE_RANGE_HEADER "CUID#%" PRId64 " Strange range header. Request: %" PRId64 "-%" PRId64 "/%" PRId64 ", Response: %" PRId64 "-%" PRId64 "/%" PRId64 "" #define EX_TIME_OUT _("Timeout.") #define EX_INVALID_CHUNK_SIZE _("Invalid chunk size.") diff --git a/src/prefs.cc b/src/prefs.cc index 2591b9f0..102b8784 100644 --- a/src/prefs.cc +++ b/src/prefs.cc @@ -326,6 +326,14 @@ PrefPtr PREF_ENABLE_ASYNC_DNS6 = makePref("enable-async-dns6"); PrefPtr PREF_MAX_DOWNLOAD_RESULT = makePref("max-download-result"); // value: 1*digit PrefPtr PREF_RETRY_WAIT = makePref("retry-wait"); +// value: true | false +PrefPtr PREF_RETRY_ON_400 = makePref("retry-on-400"); +// value: true | false +PrefPtr PREF_RETRY_ON_403 = makePref("retry-on-403"); +// value: true | false +PrefPtr PREF_RETRY_ON_406 = makePref("retry-on-406"); +// value: true | false +PrefPtr PREF_RETRY_ON_UNKNOWN = makePref("retry-on-unknown"); // value: string PrefPtr PREF_ASYNC_DNS_SERVER = makePref("async-dns-server"); // value: true | false @@ -429,6 +437,8 @@ PrefPtr PREF_HTTP_ACCEPT_GZIP = makePref("http-accept-gzip"); // value: true | false PrefPtr PREF_CONTENT_DISPOSITION_DEFAULT_UTF8 = makePref("content-disposition-default-utf8"); +// values: true | false +PrefPtr PREF_HTTP_WANT_DIGEST = makePref("http-want-digest"); // value: true | false PrefPtr PREF_NO_WANT_DIGEST_HEADER = makePref("no-want-digest-header"); diff --git a/src/prefs.h b/src/prefs.h index 338fd6e6..52372bbd 100644 --- a/src/prefs.h +++ b/src/prefs.h @@ -279,6 +279,14 @@ extern PrefPtr PREF_ENABLE_ASYNC_DNS6; extern PrefPtr PREF_MAX_DOWNLOAD_RESULT; // value: 1*digit extern PrefPtr PREF_RETRY_WAIT; +// value: true | false +extern PrefPtr PREF_RETRY_ON_400; +// value: true | false +extern PrefPtr PREF_RETRY_ON_403; +// value: true | false +extern PrefPtr PREF_RETRY_ON_406; +// value: true | false +extern PrefPtr PREF_RETRY_ON_UNKNOWN; // value: string extern PrefPtr PREF_ASYNC_DNS_SERVER; // value: true | false @@ -382,6 +390,8 @@ extern PrefPtr PREF_HTTP_ACCEPT_GZIP; // value: true | false extern PrefPtr PREF_CONTENT_DISPOSITION_DEFAULT_UTF8; // value: true | false +extern PrefPtr PREF_HTTP_WANT_DIGEST; +// value: true | false extern PrefPtr PREF_NO_WANT_DIGEST_HEADER; /**; diff --git a/src/usage_text.h b/src/usage_text.h index 0052b798..44e50b05 100644 --- a/src/usage_text.h +++ b/src/usage_text.h @@ -64,6 +64,22 @@ _(" --retry-wait=SEC Set the seconds to wait between retries. \n" \ " With SEC > 0, aria2 will retry download when the\n" \ " HTTP server returns 503 response.") +#define TEXT_RETRY_ON_400 \ + _(" --retry-on-400[=true|false] Configure whether retry or not when\n" \ + " HTTP server returns 400 Bad Request.\n" \ + " Only effective if retry-wait > 0.") +#define TEXT_RETRY_ON_403 \ + _(" --retry-on-403[=true|false] Configure whether retry or not when\n" \ + " HTTP server returns 403 Forbidden.\n" \ + " Only effective if retry-wait > 0.") +#define TEXT_RETRY_ON_406 \ + _(" --retry-on-406[=true|false] Configure whether retry or not when\n" \ + " HTTP server returns 406 Not Acceptable.\n" \ + " Only effective if retry-wait > 0.") +#define TEXT_RETRY_ON_UNKNOWN \ + _(" --retry-on-unknown[=true|false] Configure whether retry or not when\n" \ + " HTTP server returns unknown status code.\n" \ + " Only effective if retry-wait > 0.") #define TEXT_TIMEOUT \ _(" -t, --timeout=SEC Set timeout in seconds.") #define TEXT_MAX_TRIES \ @@ -547,6 +563,14 @@ " Content-Disposition header as UTF-8 instead of\n" \ " ISO-8859-1, for example, the filename parameter,\n" \ " but not the extended version filename*.") +#define TEXT_HTTP_WANT_DIGEST \ + _(" --http-want-digest[=true|false] Send Want-Digest HTTP requser header\n" \ + " with only limited hash algorithms:\n" \ + " SHA-512, SHA-256, and SHA-1.\n" \ + " The Want-Digest HTTP header is primarily used\n" \ + " in a HTTP request, to ask the responder to\n" \ + " provide a digest of the requested resource\n" \ + " using the Digest response header") #define TEXT_EVENT_POLL \ _(" --event-poll=POLL Specify the method for polling events.") #define TEXT_BT_EXTERNAL_IP \