From 64538814dd961a414ef8532733d8213fd3feff75 Mon Sep 17 00:00:00 2001 From: sleepymac Date: Sat, 15 Aug 2020 09:58:47 -0700 Subject: [PATCH] Adds an option for in-order downloading in BitTorrent. A new --bt-piece-selector option is added. If left unspecified or set to "default", the existing rarest piece selection algorithm is used. If set to "inorder", it enables in-order downloading. This allows playing media files while they are downloading. --- doc/bash_completion/aria2c | 2 +- doc/manual-src/en/aria2c.rst | 10 ++++++ doc/xmlrpc/aria2rpc | 3 ++ src/DefaultPieceStorage.cc | 9 ++++- src/InorderPieceSelector.cc | 59 ++++++++++++++++++++++++++++++++ src/InorderPieceSelector.h | 55 +++++++++++++++++++++++++++++ src/Makefile.am | 1 + src/OptionHandlerFactory.cc | 10 ++++++ src/prefs.cc | 2 ++ src/prefs.h | 2 ++ src/usage_text.h | 11 ++++++ test/DefaultPieceStorageTest.cc | 12 +------ test/InorderPieceSelector.h | 26 -------------- test/InorderPieceSelectorTest.cc | 47 +++++++++++++++++++++++++ test/Makefile.am | 2 +- 15 files changed, 211 insertions(+), 40 deletions(-) create mode 100644 src/InorderPieceSelector.cc create mode 100644 src/InorderPieceSelector.h delete mode 100644 test/InorderPieceSelector.h create mode 100644 test/InorderPieceSelectorTest.cc diff --git a/doc/bash_completion/aria2c b/doc/bash_completion/aria2c index 9ab3ed02..8d94f8e9 100644 --- a/doc/bash_completion/aria2c +++ b/doc/bash_completion/aria2c @@ -78,7 +78,7 @@ _aria2c() esac case $cur in -*) - COMPREPLY=( $( compgen -W '--rpc-save-upload-metadata --rpc-save-upload-metadata=false --on-download-start --metalink-language --rpc-secret --torrent-file --enable-peer-exchange --enable-peer-exchange=false --http-proxy-passwd --bt-tracker-timeout --ftp-type --seed-time --keep-unfinished-download-result --keep-unfinished-download-result=false --bt-tracker-connect-timeout --bt-max-open-files --no-netrc --no-netrc=false --force-sequential --force-sequential=false --metalink-base-uri --private-key --ftp-passwd --allow-overwrite --allow-overwrite=false --rpc-allow-origin-all --rpc-allow-origin-all=false --bt-detach-seed-only --bt-detach-seed-only=false --dht-entry-point6 --summary-interval --lowest-speed-limit --bt-tracker-interval --proxy-method --metalink-preferred-protocol --enable-http-keep-alive --enable-http-keep-alive=false --metalink-version --stderr --stderr=false --bt-lpd-interface --force-save --force-save=false --rpc-secure --rpc-secure=false --listen-port --rpc-private-key --server-stat-of --server-stat-timeout --bt-load-saved-metadata --bt-load-saved-metadata=false --https-proxy-user --piece-length --dry-run --dry-run=false --truncate-console-readout --truncate-console-readout=false --save-not-found --save-not-found=false --async-dns-server --bt-max-peers --max-overall-upload-limit --rpc-user --optimize-concurrent-downloads --optimize-concurrent-downloads=true --optimize-concurrent-downloads=false --optimize-concurrent-downloads=A:B --dir --split --on-download-pause --auto-file-renaming --auto-file-renaming=false --http-proxy --save-session-interval --daemon --daemon=false --https-proxy --min-tls-version --save-cookies --out --rlimit-nofile --max-file-not-found --on-download-stop --certificate --bt-min-crypto-level --remove-control-file --remove-control-file=false --enable-dht --enable-dht=false --file-allocation --follow-metalink --on-bt-download-complete --ftp-proxy --show-files --show-files=false --timeout --bt-hash-check-seed --bt-hash-check-seed=false --ftp-pasv --ftp-pasv=false --check-certificate --check-certificate=false --always-resume --always-resume=false --load-cookies --bt-remove-unselected-file --bt-remove-unselected-file=false --bt-stop-timeout --version --max-concurrent-downloads --quiet --quiet=false --max-download-result --content-disposition-default-utf8 --content-disposition-default-utf8=false --max-resume-failure-tries --header --rpc-listen-all --rpc-listen-all=false --all-proxy-user --server-stat-if --dht-file-path6 --save-session --bt-external-ip --max-tries --conditional-get --conditional-get=false --ftp-reuse-connection --ftp-reuse-connection=false --gid --dscp --max-download-limit --bt-prioritize-piece --check-integrity --check-integrity=false --log-level --remote-time --remote-time=false --uri-selector --rpc-listen-port --index-out --bt-tracker --referer --ssh-host-key-md --console-log-level --connect-timeout --stream-piece-selector --dht-message-timeout --select-file --download-result --disable-ipv6 --disable-ipv6=false --rpc-max-request-size --rpc-passwd --stop-with-process --https-proxy-passwd --continue --continue=false --no-file-allocation-limit --netrc-path --ftp-proxy-user --enable-color --enable-color=false --metalink-location --allow-piece-length-change --allow-piece-length-change=false --max-connection-per-server --no-conf --no-conf=false --rpc-certificate --metalink-os --enable-http-pipelining --enable-http-pipelining=false --http-passwd --user-agent --enable-dht6 --enable-dht6=false --dht-file-path --http-auth-challenge --http-auth-challenge=false --bt-enable-hook-after-hash-check --bt-enable-hook-after-hash-check=false --peer-id-prefix --max-mmap-limit --enable-mmap --enable-mmap=false --use-head --use-head=false --bt-require-crypto --bt-require-crypto=false --show-console-readout --show-console-readout=false --conf-path --log --no-proxy --dht-entry-point --dht-listen-port --http-user --retry-wait --on-download-complete --help --help=#basic --help=#advanced --help=#http --help=#https --help=#ftp --help=#metalink --help=#bittorrent --help=#cookie --help=#hook --help=#file --help=#rpc --help=#checksum --help=#experimental --help=#deprecated --help=#help --help=#all --max-overall-download-limit --event-poll --http-accept-gzip --http-accept-gzip=false --metalink-file --all-proxy --disk-cache --hash-check-only --hash-check-only=false --dht-listen-addr6 --human-readable --human-readable=false --ftp-user --all-proxy-passwd --bt-exclude-tracker --pause-metadata --pause-metadata=false --http-proxy-user --deferred-input --deferred-input=false --metalink-enable-unique-protocol --metalink-enable-unique-protocol=false --stop --peer-agent --max-upload-limit --multiple-interface --realtime-chunk-checksum --realtime-chunk-checksum=false --http-no-cache --http-no-cache=false --ca-certificate --bt-force-encryption --bt-force-encryption=false --bt-save-metadata --bt-save-metadata=false --seed-ratio --follow-torrent --pause --pause=false --checksum --auto-save-interval --async-dns --async-dns=false --bt-enable-lpd --bt-enable-lpd=false --parameterized-uri --parameterized-uri=false --ftp-proxy-passwd --enable-rpc --enable-rpc=false --min-split-size --bt-seed-unverified --bt-seed-unverified=false --input-file --interface --enable-async-dns6 --enable-async-dns6=false --reuse-uri --reuse-uri=false --socket-recv-buffer-size --bt-request-peer-speed-limit --on-download-error --bt-metadata-only --bt-metadata-only=false ' -- "$cur" ) ) + COMPREPLY=( $( compgen -W '--version --help --help=#basic --help=#advanced --help=#http --help=#https --help=#ftp --help=#metalink --help=#bittorrent --help=#cookie --help=#hook --help=#file --help=#rpc --help=#checksum --help=#experimental --help=#deprecated --help=#help --help=#all --timeout --connect-timeout --max-tries --auto-save-interval --log --dir --out --split --daemon --daemon=false --referer --lowest-speed-limit --piece-length --max-overall-download-limit --max-download-limit --file-allocation --no-file-allocation-limit --allow-overwrite --allow-overwrite=false --realtime-chunk-checksum --realtime-chunk-checksum=false --check-integrity --check-integrity=false --netrc-path --continue --continue=false --no-netrc --no-netrc=false --input-file --deferred-input --deferred-input=false --max-concurrent-downloads --optimize-concurrent-downloads --optimize-concurrent-downloads=true --optimize-concurrent-downloads=false --optimize-concurrent-downloads=A:B --force-sequential --force-sequential=false --auto-file-renaming --auto-file-renaming=false --parameterized-uri --parameterized-uri=false --allow-piece-length-change --allow-piece-length-change=false --no-conf --no-conf=false --conf-path --stop --quiet --quiet=false --async-dns --async-dns=false --summary-interval --log-level --console-log-level --uri-selector --server-stat-timeout --server-stat-if --server-stat-of --remote-time --remote-time=false --max-file-not-found --event-poll --enable-rpc --enable-rpc=false --rpc-listen-port --rpc-user --rpc-passwd --rpc-max-request-size --rpc-listen-all --rpc-listen-all=false --rpc-allow-origin-all --rpc-allow-origin-all=false --rpc-certificate --rpc-private-key --rpc-secure --rpc-secure=false --rpc-save-upload-metadata --rpc-save-upload-metadata=false --dry-run --dry-run=false --reuse-uri --reuse-uri=false --on-download-start --on-download-pause --on-download-stop --on-download-complete --on-download-error --interface --multiple-interface --disable-ipv6 --disable-ipv6=false --human-readable --human-readable=false --remove-control-file --remove-control-file=false --always-resume --always-resume=false --max-resume-failure-tries --save-session --max-connection-per-server --min-split-size --conditional-get --conditional-get=false --enable-async-dns6 --enable-async-dns6=false --max-download-result --retry-wait --async-dns-server --show-console-readout --show-console-readout=false --stream-piece-selector --truncate-console-readout --truncate-console-readout=false --pause --pause=false --download-result --hash-check-only --hash-check-only=false --checksum --stop-with-process --enable-mmap --enable-mmap=false --force-save --force-save=false --save-not-found --save-not-found=false --disk-cache --gid --save-session-interval --enable-color --enable-color=false --rpc-secret --dscp --pause-metadata --pause-metadata=false --rlimit-nofile --min-tls-version --socket-recv-buffer-size --max-mmap-limit --stderr --stderr=false --keep-unfinished-download-result --keep-unfinished-download-result=false --ftp-user --ftp-passwd --ftp-type --ftp-pasv --ftp-pasv=false --ftp-reuse-connection --ftp-reuse-connection=false --ssh-host-key-md --http-user --http-passwd --user-agent --load-cookies --save-cookies --enable-http-keep-alive --enable-http-keep-alive=false --enable-http-pipelining --enable-http-pipelining=false --header --certificate --private-key --ca-certificate --check-certificate --check-certificate=false --use-head --use-head=false --http-auth-challenge --http-auth-challenge=false --http-no-cache --http-no-cache=false --http-accept-gzip --http-accept-gzip=false --content-disposition-default-utf8 --content-disposition-default-utf8=false --http-proxy --https-proxy --ftp-proxy --all-proxy --no-proxy --proxy-method --http-proxy-user --http-proxy-passwd --https-proxy-user --https-proxy-passwd --ftp-proxy-user --ftp-proxy-passwd --all-proxy-user --all-proxy-passwd --show-files --show-files=false --max-overall-upload-limit --max-upload-limit --torrent-file --listen-port --follow-torrent --select-file --seed-time --seed-ratio --peer-id-prefix --peer-agent --enable-peer-exchange --enable-peer-exchange=false --enable-dht --enable-dht=false --dht-listen-port --dht-entry-point --dht-file-path --enable-dht6 --enable-dht6=false --dht-listen-addr6 --dht-entry-point6 --dht-file-path6 --bt-min-crypto-level --bt-require-crypto --bt-require-crypto=false --bt-request-peer-speed-limit --bt-max-open-files --bt-seed-unverified --bt-seed-unverified=false --bt-hash-check-seed --bt-hash-check-seed=false --bt-max-peers --bt-external-ip --index-out --bt-tracker-interval --bt-stop-timeout --bt-piece-selector --bt-prioritize-piece --bt-save-metadata --bt-save-metadata=false --bt-metadata-only --bt-metadata-only=false --bt-enable-lpd --bt-enable-lpd=false --bt-lpd-interface --bt-tracker-timeout --bt-tracker-connect-timeout --dht-message-timeout --on-bt-download-complete --bt-tracker --bt-exclude-tracker --bt-remove-unselected-file --bt-remove-unselected-file=false --bt-detach-seed-only --bt-detach-seed-only=false --bt-force-encryption --bt-force-encryption=false --bt-enable-hook-after-hash-check --bt-enable-hook-after-hash-check=false --bt-load-saved-metadata --bt-load-saved-metadata=false --metalink-file --metalink-version --metalink-language --metalink-os --metalink-location --follow-metalink --metalink-preferred-protocol --metalink-enable-unique-protocol --metalink-enable-unique-protocol=false --metalink-base-uri ' -- "$cur" ) ) ;; *) _filedir '@(torrent|meta4|metalink|text|txt|list|lst)' diff --git a/doc/manual-src/en/aria2c.rst b/doc/manual-src/en/aria2c.rst index f093f733..b9c629be 100644 --- a/doc/manual-src/en/aria2c.rst +++ b/doc/manual-src/en/aria2c.rst @@ -753,6 +753,15 @@ BitTorrent Specific Options one which satisfies the given level. Default: ``plain`` +.. option:: --bt-piece-selector= + + Specify the piece selection algorithm used for BitTorrent downloads. + If unspecified or 'default' is given, aria2 prioritizes downloads of the + rarest pieces, that is the ones that are held by the least amount of peers. + This is the optimal behavior for the BitTorrent swarm. + If 'inorder' is given, aria2 tries to download pieces in order. This allows + playing media files while downloading them. + .. option:: --bt-prioritize-piece=head[=],tail[=] Try to download first and last pieces of each file first. This is @@ -2132,6 +2141,7 @@ of URIs. These optional lines must start with white space(s). * :option:`bt-max-peers <--bt-max-peers>` * :option:`bt-metadata-only <--bt-metadata-only>` * :option:`bt-min-crypto-level <--bt-min-crypto-level>` + * :option:`bt-piece-selector <--bt-piece-selector>` * :option:`bt-prioritize-piece <--bt-prioritize-piece>` * :option:`bt-remove-unselected-file <--bt-remove-unselected-file>` * :option:`bt-request-peer-speed-limit <--bt-request-peer-speed-limit>` diff --git a/doc/xmlrpc/aria2rpc b/doc/xmlrpc/aria2rpc index f52b7f6a..3fee3d84 100755 --- a/doc/xmlrpc/aria2rpc +++ b/doc/xmlrpc/aria2rpc @@ -116,6 +116,9 @@ OptionParser.new do |opt| opt.on("--bt-min-crypto-level LEVEL",["plain","arc4"]){|val| options["bt-min-crypto-level"]=val } + opt.on("--bt-piece-selector SELECTOR",["default","inorder"){|val| + options["bt-piece-selector"]=val + } opt.on("--bt-prioritize-piece RANGE") {|val| options["bt-prioritize-piece"]=val } diff --git a/src/DefaultPieceStorage.cc b/src/DefaultPieceStorage.cc index e08b1b18..ae27192e 100644 --- a/src/DefaultPieceStorage.cc +++ b/src/DefaultPieceStorage.cc @@ -56,6 +56,7 @@ #include "Option.h" #include "fmt.h" #include "RarestPieceSelector.h" +#include "InorderPieceSelector.h" #include "DefaultStreamPieceSelector.h" #include "InorderStreamPieceSelector.h" #include "RandomStreamPieceSelector.h" @@ -91,7 +92,6 @@ DefaultPieceStorage::DefaultPieceStorage( nextHaveIndex_(1), pieceStatMan_(std::make_shared( downloadContext->getNumPieces(), true)), - pieceSelector_(make_unique(pieceStatMan_)), wrDiskCache_(nullptr) { const std::string& pieceSelectorOpt = @@ -112,6 +112,13 @@ DefaultPieceStorage::DefaultPieceStorage( streamPieceSelector_ = make_unique(bitfieldMan_.get(), 1.5); } + + const std::string& btPieceSelectorOpt = option_->get(PREF_BT_PIECE_SELECTOR); + if (btPieceSelectorOpt.empty() || btPieceSelectorOpt == A2_V_DEFAULT) { + pieceSelector_ = make_unique(pieceStatMan_); + } else if (btPieceSelectorOpt == V_INORDER) { + pieceSelector_ = make_unique(); + } } DefaultPieceStorage::~DefaultPieceStorage() = default; diff --git a/src/InorderPieceSelector.cc b/src/InorderPieceSelector.cc new file mode 100644 index 00000000..0b8e7dca --- /dev/null +++ b/src/InorderPieceSelector.cc @@ -0,0 +1,59 @@ +/* */ +#include "InorderPieceSelector.h" + +#include "bitfield.h" + +namespace aria2 { + +InorderPieceSelector::InorderPieceSelector() +{ +} + +InorderPieceSelector::~InorderPieceSelector() = default; + +bool InorderPieceSelector::select(size_t& index, const unsigned char* bitfield, + size_t nbits) const +{ + for (size_t i = 0; i < nbits; ++i) { + if (bitfield::test(bitfield, nbits, i)) { + index = i; + return true; + } + } + return false; +} + +} // namespace aria2 diff --git a/src/InorderPieceSelector.h b/src/InorderPieceSelector.h new file mode 100644 index 00000000..95424318 --- /dev/null +++ b/src/InorderPieceSelector.h @@ -0,0 +1,55 @@ +/* */ +#ifndef D_INORDER_PIECE_SELECTOR_H +#define D_INORDER_PIECE_SELECTOR_H + +#include "PieceSelector.h" + +#include + +namespace aria2 { + +class InorderPieceSelector : public PieceSelector { +public: + InorderPieceSelector(); + virtual ~InorderPieceSelector(); + + virtual bool select(size_t& index, const unsigned char* bitfield, + size_t nbits) const CXX11_OVERRIDE; +}; + +} // namespace aria2 + +#endif // D_INORDER_PIECE_SELECTOR_H diff --git a/src/Makefile.am b/src/Makefile.am index cb6e3b7c..a52de372 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -139,6 +139,7 @@ SRCS = \ IndexedList.h\ InitiateConnectionCommand.cc InitiateConnectionCommand.h\ InitiateConnectionCommandFactory.cc InitiateConnectionCommandFactory.h\ + InorderPieceSelector.cc InorderPieceSelector.h\ InorderStreamPieceSelector.cc InorderStreamPieceSelector.h\ RandomStreamPieceSelector.cc RandomStreamPieceSelector.h\ InorderURISelector.cc InorderURISelector.h\ diff --git a/src/OptionHandlerFactory.cc b/src/OptionHandlerFactory.cc index 6bff4caa..79012963 100644 --- a/src/OptionHandlerFactory.cc +++ b/src/OptionHandlerFactory.cc @@ -1581,6 +1581,16 @@ std::vector OptionHandlerFactory::createOptionHandlers() op->setChangeOptionForReserved(true); handlers.push_back(op); } + { + OptionHandler* op(new ParameterOptionHandler( + PREF_BT_PIECE_SELECTOR, TEXT_BT_PIECE_SELECTOR, A2_V_DEFAULT, + {A2_V_DEFAULT, V_INORDER})); + op->addTag(TAG_BITTORRENT); + op->setInitialOption(true); + op->setChangeGlobalOption(true); + op->setChangeOptionForReserved(true); + handlers.push_back(op); + } { OptionHandler* op(new PrioritizePieceOptionHandler( PREF_BT_PRIORITIZE_PIECE, TEXT_BT_PRIORITIZE_PIECE)); diff --git a/src/prefs.cc b/src/prefs.cc index fe982685..e05f44aa 100644 --- a/src/prefs.cc +++ b/src/prefs.cc @@ -534,6 +534,8 @@ PrefPtr PREF_INDEX_OUT = makePref("index-out"); PrefPtr PREF_BT_TRACKER_INTERVAL = makePref("bt-tracker-interval"); // values: 1*digit PrefPtr PREF_BT_STOP_TIMEOUT = makePref("bt-stop-timeout"); +// values: default | inorder +PrefPtr PREF_BT_PIECE_SELECTOR = makePref("bt-piece-selector"); // values: head[=SIZE]|tail[=SIZE], ... PrefPtr PREF_BT_PRIORITIZE_PIECE = makePref("bt-prioritize-piece"); // values: true | false diff --git a/src/prefs.h b/src/prefs.h index 9719b3be..1ffb69c6 100644 --- a/src/prefs.h +++ b/src/prefs.h @@ -485,6 +485,8 @@ extern PrefPtr PREF_INDEX_OUT; extern PrefPtr PREF_BT_TRACKER_INTERVAL; // values: 1*digit extern PrefPtr PREF_BT_STOP_TIMEOUT; +// values: default | inorder +extern PrefPtr PREF_BT_PIECE_SELECTOR; // values: head[=SIZE]|tail[=SIZE], ... extern PrefPtr PREF_BT_PRIORITIZE_PIECE; // values: true | false diff --git a/src/usage_text.h b/src/usage_text.h index ebf67e3d..7986f200 100644 --- a/src/usage_text.h +++ b/src/usage_text.h @@ -618,6 +618,17 @@ _(" --bt-stop-timeout=SEC Stop BitTorrent download if download speed is 0 in\n" \ " consecutive SEC seconds. If 0 is given, this\n" \ " feature is disabled.") +#define TEXT_BT_PIECE_SELECTOR \ + _(" --bt-piece-selector=SELECTOR Specify the piece selection algorithm used for\n" \ + " BitTorrent downloads.\n" \ + " If unspecified or 'default' is given, aria2\n" \ + " prioritizes downloads of the rarest pieces, that\n" \ + " is the ones that are held by the least amount of\n" \ + " peers. This is the optimal behavior for the\n" \ + " BitTorrent swarm.\n" \ + " If 'inorder' is given, aria2 tries to download\n" \ + " pieces in order. This allows playing media files\n" \ + " while downloading them.") #define TEXT_BT_PRIORITIZE_PIECE \ _(" --bt-prioritize-piece=head[=SIZE],tail[=SIZE] Try to download first and last\n" \ " pieces of each file first. This is useful for\n" \ diff --git a/test/DefaultPieceStorageTest.cc b/test/DefaultPieceStorageTest.cc index cbb3b4bf..9e556c6b 100644 --- a/test/DefaultPieceStorageTest.cc +++ b/test/DefaultPieceStorageTest.cc @@ -8,8 +8,6 @@ #include "Peer.h" #include "Option.h" #include "FileEntry.h" -#include "RarestPieceSelector.h" -#include "InorderPieceSelector.h" #include "DownloadContext.h" #include "bittorrent_helper.h" #include "DiskAdaptor.h" @@ -46,19 +44,18 @@ private: std::shared_ptr dctx_; std::shared_ptr peer; std::shared_ptr