diff --git a/ChangeLog b/ChangeLog index 60f05de3..9a4caffb 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,191 @@ +2006-09-19 Tatsuhiro Tsujikawa + + To rewrite segment download mechanism for HTTP/FTP download. + Use BitfieldMan to manage segment download. + * src/HttpResponseCommand.h + (executeInternal): Pass the reference of segment. + * src/AbstractCommand.cc + (prepareForRetry): Call segmentMan->cancelSegment here. + (onAbort): Call segmentMan->cancelSegment here. + * src/HttpDownloadCommand.cc + (prepareForNextSegment): New function. + * src/DownloadEngineFactory.cc + (newConsoleEngine): Removed splitter. + (newTorrentConsoleEngine): Removed splitter. + * src/Request.h + (segment): Renamed from seg. + * src/FtpInitiateConnectionCommand.h + (executeInternal): Pass the reference of segment. + * src/AbstractCommand.h + (executeInternal): Pass the reference of segment. + * src/pref.h + (PREF_SEGMENT_SIZE): New definition. + * src/HttpProxyRequestCommand.h + (executeInternal): Pass the reference of segment. + * src/HttpResponseCommand.cc + (checkResponse): Allowed status 206 when a request range starts 0. + (handleDefaultEncoding): Rewritten the code related to Segment. + (handleOtherEncoding): Rewritten the code related to Segment. + * src/SegmentMan.h + (SegmentEntry): New class. + (SegmentEntries): New type definition. + (bitfield): New variable. + (usedSegmentEntries): New variable. + (onNullBitfield): New function. + (checkoutSegment): New function. + (segments): Removed. + (splitter): Removed. + (unregisterId): Removed. + (getSegment): New function(overload) + (getDownloadedSize): Removed. + (cancelSegment): New function. + (completeSegment): New function. + (initBitfield): New function. + (hasSegment): New function. + (getDownloadLength): New function. + * src/BitfieldMan.h + (getStartIndex): New function. + (getEndIndex): New function. + (getMissingUnusedIndex): New function(overload). + (getSparseMissingUnusedIndex): New function. + * src/BitfieldMan.cc + (getMissingIndexRandomly): Handle the last byte of bitfield properly. + (getMissingUnusedIndex): New function(overload). + (Range): New class. + (getStartIndex): New function. + (getEndIndex): New function. + (getSparseMissingUnusedIndex): New function. + (isBitSetInternal): Return false if the given index is less than 0. + * src/HttpInitiateConnectionCommand.h + (executeInternal): Pass the reference of segment. + * src/FtpNegotiateCommand.h + (executeInternal): Pass the reference of segment. + * src/FtpNegotiateCommand.cc + (recvSize): Initialize bitfield here. + * src/FtpTunnelResponseCommand.h + (executeInternal): Pass the reference of segment. + * src/HttpConnection.cc + (createRequest): Rewritten range header processing. + * src/DownloadCommand.h + (executeInternal): Pass the reference of segment. + (prepareForRetry): Removed. + (prepareForNextSegment): Added an argument segment. Made it a virtual + function. + * src/main.cc + (main): Set the initial value of PREF_SEGMENT_SIZE to 1MB. + * src/SegmentMan.cc + (SegmentMan): Added bitfield. Removed splitter. + (~SegmentMan): Added bitfield. Removed splitter. + (unregisterId): Removed. + (getSegment): Rewritten. + (updateSegment): Rewritten. + (save): Rewritten. + (read): Rewritten. + (finished): Rewritten. + (getDownloadedSize): Removed. + (initBitfield): New function. + (FindSegmentEntryByIndex): New function object. + (FindSegmentEntryByCuid): New function object. + (checkoutSegment): New function. + (onNullBitfield): New function. + (getSegment): New function(overload). + (CancelSegment): New function object. + (cancelSegment): New function. + (completeSegment): New function. + (hasSegment): New function. + (getDownloadLength): New function. + * src/FtpInitiateConnectionCommand.cc + (executeInternal): Load .aria2 file after hostname resolution finishes. + * src/Segment.h: Rewritten. + * src/Segment.cc (operator<<): New function. + * src/HttpDownloadCommand.h + (prepareForNextSegment): New function. + * src/Request.cc + (resetUrl): Made segment null. + * src/DownloadEngine.cc + (~DownloadEngine): Call cleanQueue before deleting segmentMan. + * src/HttpProxyRequestCommand.h + (executeInternal): Pass the reference of segment. + * src/DownloadCommand.cc + (executeInternal): Rewritten the code related to Segment. + (prepareForRetry): Removed. + (prepareForNextSegment): Rewritten. + * src/FtpTunnelResponseCommand.h + (executeInternal): Pass the reference of segment. + + To add HTTP 1.1 persistent connection support(experimental) + * src/HttpRequestCommand.cc + (executeInternal): Disable keep alive if it is disabled by + configuration. + * src/Request.h + (keepAlive): New variable. + (isKeepAlive): New function. + (setKeepAlive): New function. + * src/pref.h + (PREF_HTTP_KEEP_ALIVE): New definition. + * src/HttpResponseCommand.cc + (executeInternal): Check the remote server supports keep alive. + * src/HttpConnection.cc + (createRequest): Send "Connection: close" only if keep alive is + disabled. + * src/main.cc + (main): + Set the initial value(false) of PREF_KEEP_ALIVE to false. + + To add max download speed limit: + * src/pref.h + (PREF_MAX_SPEED_LIMIT): New definition. + * src/PeerInteractionCommand.cc + (executeInternal): Added max download speed limit. Not tested yet. + * src/SegmentMan.h + (PeerStats): New type definition. + (peerStats): New variable. + (registerPeerStat): New function. + (FindPeerStat): New function object. + (getPeerStat): New function. + (calculateDownloadSpeed): New function. + * src/SpeedCalc.h: New class. + * src/SpeedCalc.cc: New class. + * src/main.cc + (main): + Set the initial value of PREF_MAX_SPEED_LIMIT to 0(which means the + download speed is not restricted). + * src/PeerStat.h: New class. + * src/SegmentMan.cc + (registerPeerStat): New function. + (calculateDownloadSpeed): New function. + * src/DownloadCommand.cc + (STARTUP_IDLE_TIME): New definition. + (DownloadCommand): Register peerStat to segmentMan. Call peerStat-> + downloadStart. + (~DownloadCommand): Call peerStat->downloadStop. + (executeInternal): Added download speed limitter. Rewritten lowest + speed limitter. + + * src/HttpConnection.cc + (receiveResponse): Fixed: eohIndex[headerBuf] -> headerBuf[eohIndex]. + + * src/AbstractCommand.cc + (resolveHostname): Throw DlAbortEx if a name resolution failes. + Added hostname to the error message. + + * src/ConsoleDownloadEngine.cc + (calculateStatistics): Initialize psize with dlSize. + + * src/PieceMessage.cc + (receivedAction): Do not call peerInteraction->abortPiece here. + (onGotWrongPiece): Call peerInteraction->abortPiece here. + + * src/BitfieldMan.h + (clearAllUseBit): New function. + (setAllUseBit): New function. + * src/BitfieldMan.cc + (clearAllBit): Do not clear useBitfield here. + (clearAllUseBit): New function. + (setAllUseBit): New function. + * src/Piece.cc + (clearAllBlock): Call bitfield->clearAllUseBit(). + 2006-08-28 Tatsuhiro Tsujikawa To make filename URL-decoded: diff --git a/TODO b/TODO index 21a115f7..921f7cad 100644 --- a/TODO +++ b/TODO @@ -13,5 +13,11 @@ * List available os, version, etc for metalink * ipv6(RFC2428 for ftp) * Add silent mode. -* Add upload speed limit command-line option. * Save URLs and command-line arguments to .aria2 file. +* Add multi-file metalink support. +* Add a control port for GUI frontend + +0.8.0 + +* Add a statement for the permission to link with OpenSSL. +* Add upload speed limit command-line option(not tested for torrent yet). diff --git a/configure b/configure index 67a21420..47e8ea3b 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.59 for aria2c 0.7.3. +# Generated by GNU Autoconf 2.59 for aria2c 0.8.0. # # Report bugs to . # @@ -269,8 +269,8 @@ SHELL=${CONFIG_SHELL-/bin/sh} # Identity of this package. PACKAGE_NAME='aria2c' PACKAGE_TARNAME='aria2c' -PACKAGE_VERSION='0.7.3' -PACKAGE_STRING='aria2c 0.7.3' +PACKAGE_VERSION='0.8.0' +PACKAGE_STRING='aria2c 0.8.0' PACKAGE_BUGREPORT='tujikawa@rednoah.com' ac_unique_file="src/Socket.h" @@ -788,7 +788,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures aria2c 0.7.3 to adapt to many kinds of systems. +\`configure' configures aria2c 0.8.0 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -854,7 +854,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of aria2c 0.7.3:";; + short | recursive ) echo "Configuration of aria2c 0.8.0:";; esac cat <<\_ACEOF @@ -1004,7 +1004,7 @@ fi test -n "$ac_init_help" && exit 0 if $ac_init_version; then cat <<\_ACEOF -aria2c configure 0.7.3 +aria2c configure 0.8.0 generated by GNU Autoconf 2.59 Copyright (C) 2003 Free Software Foundation, Inc. @@ -1018,7 +1018,7 @@ cat >&5 <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by aria2c $as_me 0.7.3, which was +It was created by aria2c $as_me 0.8.0, which was generated by GNU Autoconf 2.59. Invocation command line was $ $0 $@ @@ -1661,7 +1661,7 @@ fi # Define the identity of the package. PACKAGE='aria2c' - VERSION='0.7.3' + VERSION='0.8.0' cat >>confdefs.h <<_ACEOF @@ -12227,7 +12227,7 @@ _ASBOX } >&5 cat >&5 <<_CSEOF -This file was extended by aria2c $as_me 0.7.3, which was +This file was extended by aria2c $as_me 0.8.0, which was generated by GNU Autoconf 2.59. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -12290,7 +12290,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF ac_cs_version="\\ -aria2c config.status 0.7.3 +aria2c config.status 0.8.0 configured by $0, generated by GNU Autoconf 2.59, with options \\"`echo "$ac_configure_args" | sed 's/[\\""\`\$]/\\\\&/g'`\\" diff --git a/configure.ac b/configure.ac index 6adf4dbe..b273cb1d 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. # AC_PREREQ(2.59) -AC_INIT(aria2c, 0.7.3, tujikawa@rednoah.com) +AC_INIT(aria2c, 0.8.0, tujikawa@rednoah.com) AM_INIT_AUTOMAKE() AM_PATH_CPPUNIT(1.10.2) AC_CONFIG_SRCDIR([src/Socket.h]) diff --git a/src/AbstractCommand.cc b/src/AbstractCommand.cc index 907c7ee9..6a471277 100644 --- a/src/AbstractCommand.cc +++ b/src/AbstractCommand.cc @@ -48,16 +48,18 @@ bool AbstractCommand::execute() { checkSocketIsWritable && writeCheckTarget->isWritable(0) || !checkSocketIsReadable && !checkSocketIsWritable) { checkPoint.reset(); - Segment seg = { 0, 0, 0, false }; + if(e->segmentMan->finished()) { + logger->debug("CUID#%d - finished.", cuid); + return true; + } + Segment segment; if(e->segmentMan->downloadStarted) { - // get segment information in order to set Range header. - if(!e->segmentMan->getSegment(seg, cuid)) { - // no segment available + if(!e->segmentMan->getSegment(segment, cuid)) { logger->info(MSG_NO_SEGMENT_AVAILABLE, cuid); return true; } } - return executeInternal(seg); + return executeInternal(segment); } else { if(checkPoint.elapsed(timeout)) { throw new DlRetryEx(EX_TIME_OUT); @@ -103,6 +105,7 @@ void AbstractCommand::tryReserved() { } bool AbstractCommand::prepareForRetry(int wait) { + e->segmentMan->cancelSegment(cuid); Command* command = InitiateConnectionCommandFactory::createInitiateConnectionCommand(cuid, req, e); if(wait == 0) { e->commands.push_back(command); @@ -115,7 +118,8 @@ bool AbstractCommand::prepareForRetry(int wait) { void AbstractCommand::onAbort(Exception* ex) { logger->debug(MSG_UNREGISTER_CUID, cuid); - e->segmentMan->unregisterId(cuid); + //e->segmentMan->unregisterId(cuid); + e->segmentMan->cancelSegment(cuid); } void AbstractCommand::disableReadCheckSocket() { @@ -193,7 +197,8 @@ bool AbstractCommand::resolveHostname(const string& hostname, return true; break; case NameResolver::STATUS_ERROR: - throw new DlRetryEx("CUID#%d - Name resolution failed:%s", cuid, + throw new DlAbortEx("CUID#%d - Name resolution for %s failed:%s", cuid, + hostname.c_str(), resolver->getError().c_str()); default: return false; diff --git a/src/AbstractCommand.h b/src/AbstractCommand.h index 3fbea81e..7ac5cd2c 100644 --- a/src/AbstractCommand.h +++ b/src/AbstractCommand.h @@ -40,7 +40,7 @@ protected: void tryReserved(); virtual bool prepareForRetry(int wait); virtual void onAbort(Exception* ex); - virtual bool executeInternal(Segment segment) = 0; + virtual bool executeInternal(Segment& segment) = 0; void setReadCheckSocket(const SocketHandle& socket); void setWriteCheckSocket(const SocketHandle& socket); diff --git a/src/BitfieldMan.cc b/src/BitfieldMan.cc index 54519c48..e7013e46 100644 --- a/src/BitfieldMan.cc +++ b/src/BitfieldMan.cc @@ -95,8 +95,20 @@ BitfieldMan::getMissingIndexRandomly(const unsigned char* bitfield, { int byte = (int)(((double)bitfieldLength)*random()/(RAND_MAX+1.0)); + unsigned char lastMask = 0; + int lastByteLength = totalLength%(blockLength*8); + int lastBlockCount = DIV_FLOOR(lastByteLength, blockLength); + for(int i = 0; i < lastBlockCount; i++) { + lastMask >>= 1; + lastMask |= 0x80; + } for(int i = 0; i < bitfieldLength; i++) { - unsigned char mask = 0xff; + unsigned char mask; + if(byte == bitfieldLength-1) { + mask = lastMask; + } else { + mask = 0xff; + } if(bitfield[byte]&mask) { int index = byte*8+getNthBitIndex(bitfield[byte], 1); return index; @@ -207,6 +219,82 @@ int BitfieldMan::getMissingIndex() const { return index; } +int BitfieldMan::getMissingUnusedIndex() const { + unsigned char* tempBitfield = new unsigned char[bitfieldLength]; + memset(tempBitfield, 0xff, bitfieldLength); + int index = getMissingUnusedIndex(tempBitfield, bitfieldLength); + delete [] tempBitfield; + return index; +} + +// [startIndex, endIndex) +class Range { +public: + int startIndex; + int endIndex; + Range(int startIndex = 0, int endIndex = 0):startIndex(startIndex), + endIndex(endIndex) {} + + int getSize() const { + return endIndex-startIndex; + } + + int getMidIndex() const { + return (endIndex-startIndex)/2+startIndex; + } + + bool operator<(const Range& range) const { + return getSize() < range.getSize(); + } +}; + +int BitfieldMan::getStartIndex(int index) const { + while(index < blocks && + (isUseBitSet(index) || isBitSet(index))) { + index++; + } + if(blocks <= index) { + return -1; + } else { + return index; + } +} + +int BitfieldMan::getEndIndex(int index) const { + while(index < blocks && + (!isUseBitSet(index) && !isBitSet(index))) { + index++; + } + return index; +} + +int BitfieldMan::getSparseMissingUnusedIndex() const { + Range maxRange; + int index = 0; + int blocks = countBlock(); + Range currentRange; + while(index < blocks) { + currentRange.startIndex = getStartIndex(index); + if(currentRange.startIndex == -1) { + break; + } + currentRange.endIndex = getEndIndex(currentRange.startIndex); + if(maxRange < currentRange) { + maxRange = currentRange; + } + index = currentRange.endIndex; + } + if(maxRange.getSize()) { + if(maxRange.startIndex == 0) { + return 0; + } else { + return maxRange.getMidIndex(); + } + } else { + return -1; + } +} + BlockIndexes BitfieldMan::getAllMissingIndexes() const { BlockIndexes missingIndexes; for(int i = 0; i < bitfieldLength; i++) { @@ -317,7 +405,7 @@ bool BitfieldMan::isAllBitSet() const { } bool BitfieldMan::isBitSetInternal(const unsigned char* bitfield, int index) const { - if(blocks <= index) { return false; } + if(index < 0 || blocks <= index) { return false; } unsigned char mask = 128 >> index%8; return (bitfield[index/8] & mask) != 0; } @@ -340,7 +428,6 @@ void BitfieldMan::setBitfield(const unsigned char* bitfield, int bitfieldLength) void BitfieldMan::clearAllBit() { memset(this->bitfield, 0, this->bitfieldLength); - memset(this->useBitfield, 0, this->bitfieldLength); } void BitfieldMan::setAllBit() { @@ -349,6 +436,16 @@ void BitfieldMan::setAllBit() { } } +void BitfieldMan::clearAllUseBit() { + memset(this->useBitfield, 0, this->bitfieldLength); +} + +void BitfieldMan::setAllUseBit() { + for(int i = 0; i < blocks; i++) { + setUseBit(i); + } +} + bool BitfieldMan::setFilterBit(int index) { return setBitInternal(filterBitfield, index, true); } diff --git a/src/BitfieldMan.h b/src/BitfieldMan.h index 369c3fd8..dfaf4020 100644 --- a/src/BitfieldMan.h +++ b/src/BitfieldMan.h @@ -43,6 +43,9 @@ private: bool isBitSetInternal(const unsigned char* bitfield, int index) const; bool setBitInternal(unsigned char* bitfield, int index, bool on); bool setFilterBit(int index); + + int getStartIndex(int index) const; + int getEndIndex(int index) const; public: BitfieldMan(int blockLength, long long int totalLength); BitfieldMan(const BitfieldMan& bitfieldMan); @@ -116,6 +119,14 @@ public: * affected by filter */ int getMissingUnusedIndex(const unsigned char* bitfield, int len) const; + /** + * affected by filter + */ + int getMissingUnusedIndex() const; + /** + * affected by filter + */ + int getSparseMissingUnusedIndex() const; /** * affected by filter */ @@ -155,6 +166,9 @@ public: void clearAllBit(); void setAllBit(); + void clearAllUseBit(); + void setAllUseBit(); + void addFilter(long long int offset, long long int length); /** * Clears filter and disables filter diff --git a/src/ConsoleDownloadEngine.cc b/src/ConsoleDownloadEngine.cc index fb2c1bc8..61c1919f 100644 --- a/src/ConsoleDownloadEngine.cc +++ b/src/ConsoleDownloadEngine.cc @@ -51,9 +51,10 @@ void ConsoleDownloadEngine::initStatistics() { } void ConsoleDownloadEngine::calculateStatistics() { - long long int dlSize = segmentMan->getDownloadedSize(); + long long int dlSize = segmentMan->getDownloadLength(); if(!isStartupLengthSet && dlSize > 0) { startupLength = dlSize; + psize = dlSize; isStartupLengthSet = true; } int elapsed = cp.difference(); diff --git a/src/DownloadCommand.cc b/src/DownloadCommand.cc index 4e948f53..5ae452dd 100644 --- a/src/DownloadCommand.cc +++ b/src/DownloadCommand.cc @@ -29,14 +29,30 @@ #include "prefs.h" #include +#define STARTUP_IDLE_TIME 10 + DownloadCommand::DownloadCommand(int cuid, Request* req, DownloadEngine* e, const SocketHandle& s): AbstractCommand(cuid, req, e, s), lastSize(0) { + PeerStatHandle peerStat = PeerStatHandle(new PeerStat(cuid)); + peerStat->downloadStart(); + this->e->segmentMan->registerPeerStat(peerStat); } -DownloadCommand::~DownloadCommand() {} +DownloadCommand::~DownloadCommand() { + PeerStatHandle peerStat = e->segmentMan->getPeerStat(cuid); + assert(peerStat.get()); + peerStat->downloadStop(); +} -bool DownloadCommand::executeInternal(Segment seg) { +bool DownloadCommand::executeInternal(Segment& segment) { + int maxSpeedLimit = e->option->getAsInt(PREF_MAX_SPEED_LIMIT); + if(maxSpeedLimit > 0 && + maxSpeedLimit < e->segmentMan->calculateDownloadSpeed()) { + usleep(1); + e->commands.push_back(this); + return false; + } TransferEncoding* te = NULL; if(transferEncoding.size()) { te = getTransferEncoding(transferEncoding); @@ -45,61 +61,78 @@ bool DownloadCommand::executeInternal(Segment seg) { int bufSize = 4096; char buf[bufSize]; socket->readData(buf, bufSize); + PeerStatHandle peerStat = e->segmentMan->getPeerStat(cuid); + assert(peerStat.get()); if(te != NULL) { int infbufSize = 4096; char infbuf[infbufSize]; te->inflate(infbuf, infbufSize, buf, bufSize); - e->segmentMan->diskWriter->writeData(infbuf, infbufSize, seg.sp+seg.ds); - seg.ds += infbufSize; + e->segmentMan->diskWriter->writeData(infbuf, infbufSize, + segment.getPosition()+segment.writtenLength); + segment.writtenLength += infbufSize; + peerStat->updateDownloadLength(infbufSize); } else { - e->segmentMan->diskWriter->writeData(buf, bufSize, seg.sp+seg.ds); - seg.ds += bufSize; + e->segmentMan->diskWriter->writeData(buf, bufSize, + segment.getPosition()+segment.writtenLength); + segment.writtenLength += bufSize; + peerStat->updateDownloadLength(bufSize); } // calculate downloading speed - int diff = sw.difference(); - if(diff >= 1) { - seg.speed = (int)((seg.ds-lastSize)/(diff*1.0)); - sw.reset(); - lastSize = seg.ds; + if(/*sw.elapsed(1) >= 1 && */peerStat->getDownloadStartTime().elapsed(STARTUP_IDLE_TIME)) { int lowestLimit = e->option->getAsInt(PREF_LOWEST_SPEED_LIMIT); - if(lowestLimit > 0 && seg.speed <= lowestLimit) { + int nowSpeed = peerStat->calculateDownloadSpeed(); + if(lowestLimit > 0 && nowSpeed <= lowestLimit) { throw new DlAbortEx("CUID#%d - Too slow Downloading speed: %d <= %d(B/s)", cuid, - seg.speed, + nowSpeed, lowestLimit); } + //sw.reset(); } if(e->segmentMan->totalSize != 0 && bufSize == 0) { throw new DlRetryEx(EX_GOT_EOF); } if(te != NULL && te->finished() - || te == NULL && seg.ds >= seg.ep-seg.sp+1 + || te == NULL && segment.complete() || bufSize == 0) { if(te != NULL) te->end(); logger->info(MSG_DOWNLOAD_COMPLETED, cuid); - seg.ds = seg.ep-seg.sp+1; - seg.finish = true; - e->segmentMan->updateSegment(seg); + e->segmentMan->completeSegment(cuid, segment); // this unit is going to download another segment. - return prepareForNextSegment(); + return prepareForNextSegment(segment); } else { - e->segmentMan->updateSegment(seg); + e->segmentMan->updateSegment(cuid, segment); e->commands.push_back(this); return false; } } -bool DownloadCommand::prepareForRetry(int wait) { - //req->resetUrl(); - return AbstractCommand::prepareForRetry(wait); -} - -bool DownloadCommand::prepareForNextSegment() { - req->resetUrl(); - if(!e->segmentMan->finished()) { - return AbstractCommand::prepareForRetry(0); - } else { +bool DownloadCommand::prepareForNextSegment(const Segment& currentSegment) { + if(e->segmentMan->finished()) { return true; + } else { + // Merge segment with next segment, if segment.index+1 == nextSegment.index + Segment tempSegment = currentSegment; + while(1) { + Segment nextSegment; + if(e->segmentMan->getSegment(nextSegment, cuid, tempSegment.index+1)) { + if(nextSegment.writtenLength > 0) { + return prepareForRetry(0); + } + nextSegment.writtenLength = tempSegment.writtenLength-tempSegment.length; + if(nextSegment.complete()) { + e->segmentMan->completeSegment(cuid, nextSegment); + tempSegment = nextSegment; + } else { + e->segmentMan->updateSegment(cuid, nextSegment); + e->commands.push_back(this); + return false; + } + } else { + break; + } + } + return prepareForRetry(0); } } diff --git a/src/DownloadCommand.h b/src/DownloadCommand.h index 60403adf..68c6b291 100644 --- a/src/DownloadCommand.h +++ b/src/DownloadCommand.h @@ -33,10 +33,9 @@ private: Time sw; long long int lastSize; protected: - bool executeInternal(Segment segment); + bool executeInternal(Segment& segment); - bool prepareForRetry(int wait); - bool prepareForNextSegment(); + virtual bool prepareForNextSegment(const Segment& currentSegment); public: DownloadCommand(int cuid, Request* req, DownloadEngine* e, const SocketHandle& s); diff --git a/src/DownloadEngine.cc b/src/DownloadEngine.cc index e1dafe5e..90fa8899 100644 --- a/src/DownloadEngine.cc +++ b/src/DownloadEngine.cc @@ -37,8 +37,8 @@ DownloadEngine::DownloadEngine():noWait(false), segmentMan(0) { } DownloadEngine::~DownloadEngine() { - delete segmentMan; cleanQueue(); + delete segmentMan; } void DownloadEngine::cleanQueue() { diff --git a/src/DownloadEngineFactory.cc b/src/DownloadEngineFactory.cc index 6c094592..5edd8eed 100644 --- a/src/DownloadEngineFactory.cc +++ b/src/DownloadEngineFactory.cc @@ -22,7 +22,6 @@ #include "DownloadEngineFactory.h" #include "prefs.h" #include "DefaultDiskWriter.h" -#include "SplitSlowestSegmentSplitter.h" #include "InitiateConnectionCommandFactory.h" #include "ByteArrayDiskWriter.h" #include "Util.h" @@ -51,8 +50,6 @@ DownloadEngineFactory::newConsoleEngine(const Option* op, e->segmentMan->dir = op->get(PREF_DIR); e->segmentMan->ufilename = op->get(PREF_OUT); e->segmentMan->option = op; - e->segmentMan->splitter = new SplitSlowestSegmentSplitter(); - e->segmentMan->splitter->setMinSegmentSize(op->getAsLLInt(PREF_MIN_SEGMENT_SIZE)); e->segmentMan->reserved = reserved; int cuidCounter = 1; @@ -79,8 +76,6 @@ DownloadEngineFactory::newTorrentConsoleEngine(const Option* op, te->segmentMan = new SegmentMan(); te->segmentMan->diskWriter = byteArrayDiskWriter; te->segmentMan->option = op; - te->segmentMan->splitter = new SplitSlowestSegmentSplitter(); - te->segmentMan->splitter->setMinSegmentSize(op->getAsLLInt(PREF_MIN_SEGMENT_SIZE)); te->torrentMan = new TorrentMan(); te->torrentMan->setStoreDir(op->get(PREF_DIR)); te->torrentMan->option = op; diff --git a/src/FtpConnection.cc b/src/FtpConnection.cc index ba8e2000..e425e0bf 100644 --- a/src/FtpConnection.cc +++ b/src/FtpConnection.cc @@ -97,7 +97,7 @@ SocketHandle FtpConnection::sendPort() const { } void FtpConnection::sendRest(const Segment& segment) const { - string request = "REST "+Util::llitos(segment.sp+segment.ds)+"\r\n"; + string request = "REST "+Util::llitos(segment.getPosition()+segment.writtenLength)+"\r\n"; logger->info(MSG_SENDING_REQUEST, cuid, request.c_str()); socket->writeData(request); } diff --git a/src/FtpDownloadCommand.cc b/src/FtpDownloadCommand.cc index 25368973..c6167d46 100644 --- a/src/FtpDownloadCommand.cc +++ b/src/FtpDownloadCommand.cc @@ -25,7 +25,8 @@ FtpDownloadCommand::FtpDownloadCommand(int cuid, Request* req, DownloadEngine* e, const SocketHandle& dataSocket, const SocketHandle& ctrlSocket) - :DownloadCommand(cuid, req, e, dataSocket), ctrlSocket(ctrlSocket) {} + :DownloadCommand(cuid, req, e, dataSocket), + ctrlSocket(ctrlSocket) {} FtpDownloadCommand::~FtpDownloadCommand() {} diff --git a/src/FtpInitiateConnectionCommand.cc b/src/FtpInitiateConnectionCommand.cc index 8a964870..32979488 100644 --- a/src/FtpInitiateConnectionCommand.cc +++ b/src/FtpInitiateConnectionCommand.cc @@ -43,18 +43,7 @@ FtpInitiateConnectionCommand::~FtpInitiateConnectionCommand() { #endif // ENABLE_ASYNC_DNS } -bool FtpInitiateConnectionCommand::executeInternal(Segment segment) { - if(!e->segmentMan->downloadStarted) { - e->segmentMan->filename = Util::urldecode(req->getFile()); - bool segFileExists = e->segmentMan->segmentFileExists(); - if(segFileExists) { - e->segmentMan->load(); - e->segmentMan->diskWriter->openExistingFile(e->segmentMan->getFilePath()); - e->segmentMan->downloadStarted = true; - } else { - e->segmentMan->diskWriter->initAndOpenFile(e->segmentMan->getFilePath()); - } - } +bool FtpInitiateConnectionCommand::executeInternal(Segment& segment) { string hostname; if(useHttpProxy()) { hostname = e->option->get(PREF_HTTP_PROXY_HOST); @@ -71,6 +60,17 @@ bool FtpInitiateConnectionCommand::executeInternal(Segment segment) { } } #endif // ENABLE_ASYNC_DNS + if(!e->segmentMan->downloadStarted) { + e->segmentMan->filename = Util::urldecode(req->getFile()); + bool segFileExists = e->segmentMan->segmentFileExists(); + if(segFileExists) { + e->segmentMan->load(); + e->segmentMan->diskWriter->openExistingFile(e->segmentMan->getFilePath()); + e->segmentMan->downloadStarted = true; + } else { + e->segmentMan->diskWriter->initAndOpenFile(e->segmentMan->getFilePath()); + } + } Command* command; if(useHttpProxy()) { logger->info(MSG_CONNECTING_TO_SERVER, cuid, diff --git a/src/FtpInitiateConnectionCommand.h b/src/FtpInitiateConnectionCommand.h index 95bd9df4..7fa82105 100644 --- a/src/FtpInitiateConnectionCommand.h +++ b/src/FtpInitiateConnectionCommand.h @@ -33,7 +33,7 @@ private: bool useHttpProxyGet() const; bool useHttpProxyConnect() const; protected: - bool executeInternal(Segment segment); + bool executeInternal(Segment& segment); public: FtpInitiateConnectionCommand(int cuid, Request* req, DownloadEngine* e); ~FtpInitiateConnectionCommand(); diff --git a/src/FtpNegotiationCommand.cc b/src/FtpNegotiationCommand.cc index 34641e27..e0248aca 100644 --- a/src/FtpNegotiationCommand.cc +++ b/src/FtpNegotiationCommand.cc @@ -40,7 +40,7 @@ FtpNegotiationCommand::~FtpNegotiationCommand() { delete ftp; } -bool FtpNegotiationCommand::executeInternal(Segment segment) { +bool FtpNegotiationCommand::executeInternal(Segment& segment) { while(processSequence(segment)); if(sequence == SEQ_RETRY) { return prepareForRetry(0); @@ -170,6 +170,8 @@ bool FtpNegotiationCommand::recvSize() { if(!e->segmentMan->downloadStarted) { e->segmentMan->downloadStarted = true; e->segmentMan->totalSize = size; + e->segmentMan->initBitfield(e->option->getAsInt(PREF_SEGMENT_SIZE), + e->segmentMan->totalSize); } else if(e->segmentMan->totalSize != size) { throw new DlAbortEx(EX_SIZE_MISMATCH, e->segmentMan->totalSize, size); } diff --git a/src/FtpNegotiationCommand.h b/src/FtpNegotiationCommand.h index e89317d6..8df16dc1 100644 --- a/src/FtpNegotiationCommand.h +++ b/src/FtpNegotiationCommand.h @@ -78,7 +78,7 @@ private: int sequence; FtpConnection* ftp; protected: - bool executeInternal(Segment segment); + bool executeInternal(Segment& segment); public: FtpNegotiationCommand(int cuid, Request* req, DownloadEngine* e, const SocketHandle& s); diff --git a/src/FtpTunnelRequestCommand.cc b/src/FtpTunnelRequestCommand.cc index 24a8049b..b3b56fdf 100644 --- a/src/FtpTunnelRequestCommand.cc +++ b/src/FtpTunnelRequestCommand.cc @@ -33,7 +33,7 @@ FtpTunnelRequestCommand::FtpTunnelRequestCommand(int cuid, Request* req, FtpTunnelRequestCommand::~FtpTunnelRequestCommand() {} -bool FtpTunnelRequestCommand::executeInternal(Segment segment) { +bool FtpTunnelRequestCommand::executeInternal(Segment& segment) { socket->setBlockingMode(); HttpConnection httpConnection(cuid, socket, req, e->option); httpConnection.sendProxyRequest(); diff --git a/src/FtpTunnelRequestCommand.h b/src/FtpTunnelRequestCommand.h index b02d1806..3a40aa13 100644 --- a/src/FtpTunnelRequestCommand.h +++ b/src/FtpTunnelRequestCommand.h @@ -26,7 +26,7 @@ class FtpTunnelRequestCommand : public AbstractCommand { protected: - bool executeInternal(Segment segment); + bool executeInternal(Segment& segment); public: FtpTunnelRequestCommand(int cuid, Request* req, DownloadEngine* e, const SocketHandle& s); diff --git a/src/FtpTunnelResponseCommand.cc b/src/FtpTunnelResponseCommand.cc index d0e63888..ec521f56 100644 --- a/src/FtpTunnelResponseCommand.cc +++ b/src/FtpTunnelResponseCommand.cc @@ -35,7 +35,7 @@ FtpTunnelResponseCommand::~FtpTunnelResponseCommand() { delete http; } -bool FtpTunnelResponseCommand::executeInternal(Segment segment) { +bool FtpTunnelResponseCommand::executeInternal(Segment& segment) { HttpHeader headers; int status = http->receiveResponse(headers); if(status == 0) { diff --git a/src/FtpTunnelResponseCommand.h b/src/FtpTunnelResponseCommand.h index 7d0e19ae..641096f2 100644 --- a/src/FtpTunnelResponseCommand.h +++ b/src/FtpTunnelResponseCommand.h @@ -29,7 +29,7 @@ class FtpTunnelResponseCommand : public AbstractCommand { private: HttpConnection* http; protected: - bool executeInternal(Segment segment); + bool executeInternal(Segment& segment); public: FtpTunnelResponseCommand(int cuid, Request* req, DownloadEngine* e, const SocketHandle& s); diff --git a/src/HttpConnection.cc b/src/HttpConnection.cc index e63000a2..47c0f6a0 100644 --- a/src/HttpConnection.cc +++ b/src/HttpConnection.cc @@ -71,14 +71,23 @@ string HttpConnection::createRequest(const Segment& segment) const { ((req->getDir() == "/" ? "/" : req->getDir()+"/")+req->getFile()))+ string(" HTTP/1.1\r\n")+ "User-Agent: "+USER_AGENT+"\r\n"+ - "Connection: close\r\n"+ + // use persistent connection + //"Connection: close\r\n"+ "Accept: */*\r\n"+ /* */ "Host: "+getHost(req->getHost(), req->getPort())+"\r\n"+ "Pragma: no-cache\r\n"+ "Cache-Control: no-cache\r\n"; - if(segment.sp+segment.ds > 0) { + if(!req->isKeepAlive()) { + request += "Connection: close\r\n"; + } + if(segment.length > 0) { request += "Range: bytes="+ - Util::llitos(segment.sp+segment.ds)+"-"+Util::llitos(segment.ep)+"\r\n"; + Util::llitos(segment.getPosition()+segment.writtenLength); + request += "-"; + if(req->isKeepAlive()) { + request += Util::llitos(segment.getPosition()+segment.length-1); + } + request += "\r\n"; } if(useProxy() && useProxyAuth() && useProxyGet()) { request += "Proxy-Connection: close\r\n"; @@ -144,7 +153,7 @@ int HttpConnection::receiveResponse(HttpHeader& headers) { socket->readData(headerBuf+headerBufLength, size); headerBufLength += size; } else { - if(eohIndex[headerBuf] == '\n') { + if(headerBuf[eohIndex] == '\n') { // for crapping non-standard HTTP server delimiterSwitch = 1; } else { diff --git a/src/HttpDownloadCommand.cc b/src/HttpDownloadCommand.cc index 3077ab16..143e1923 100644 --- a/src/HttpDownloadCommand.cc +++ b/src/HttpDownloadCommand.cc @@ -20,14 +20,15 @@ */ /* copyright --> */ #include "HttpDownloadCommand.h" -#include -#include -#include -#include #include "DlRetryEx.h" #include "HttpRequestCommand.h" #include "Util.h" #include "ChunkedEncoding.h" +#include "message.h" +#include +#include +#include +#include using namespace std; @@ -49,3 +50,17 @@ HttpDownloadCommand::~HttpDownloadCommand() { TransferEncoding* HttpDownloadCommand::getTransferEncoding(const string& name) { return transferEncodings[name]; } + +bool HttpDownloadCommand::prepareForNextSegment(const Segment& currentSegment) { + if(e->segmentMan->finished()) { + return true; + } else { + if(req->isKeepAlive()) { + Command* command = new HttpRequestCommand(cuid, req, e, socket); + e->commands.push_back(command); + return true; + } else { + return DownloadCommand::prepareForNextSegment(currentSegment); + } + } +} diff --git a/src/HttpDownloadCommand.h b/src/HttpDownloadCommand.h index 440fa9ec..8141d7ed 100644 --- a/src/HttpDownloadCommand.h +++ b/src/HttpDownloadCommand.h @@ -36,6 +36,8 @@ using namespace std; class HttpDownloadCommand:public DownloadCommand { private: map transferEncodings; +protected: + virtual bool prepareForNextSegment(const Segment& currentSegment); public: HttpDownloadCommand(int cuid, Request* req, DownloadEngine* e, const SocketHandle& s); diff --git a/src/HttpInitiateConnectionCommand.cc b/src/HttpInitiateConnectionCommand.cc index 57bca089..33fd74a9 100644 --- a/src/HttpInitiateConnectionCommand.cc +++ b/src/HttpInitiateConnectionCommand.cc @@ -43,7 +43,7 @@ HttpInitiateConnectionCommand::~HttpInitiateConnectionCommand() { #endif // ENABLE_ASYNC_DNS } -bool HttpInitiateConnectionCommand::executeInternal(Segment segment) { +bool HttpInitiateConnectionCommand::executeInternal(Segment& segment) { string hostname; if(useProxy()) { hostname = e->option->get(PREF_HTTP_PROXY_HOST); diff --git a/src/HttpInitiateConnectionCommand.h b/src/HttpInitiateConnectionCommand.h index 8a3839b4..fbaecfb5 100644 --- a/src/HttpInitiateConnectionCommand.h +++ b/src/HttpInitiateConnectionCommand.h @@ -41,7 +41,7 @@ protected: * Whether or not the connection is established successfully is * evaluated by RequestCommand. */ - bool executeInternal(Segment segment); + bool executeInternal(Segment& segment); public: HttpInitiateConnectionCommand(int cuid, Request* req, DownloadEngine* e); ~HttpInitiateConnectionCommand(); diff --git a/src/HttpProxyRequestCommand.cc b/src/HttpProxyRequestCommand.cc index 0bf66558..b9f660d8 100644 --- a/src/HttpProxyRequestCommand.cc +++ b/src/HttpProxyRequestCommand.cc @@ -33,7 +33,7 @@ HttpProxyRequestCommand::HttpProxyRequestCommand(int cuid, Request* req, HttpProxyRequestCommand::~HttpProxyRequestCommand() {} -bool HttpProxyRequestCommand::executeInternal(Segment segment) { +bool HttpProxyRequestCommand::executeInternal(Segment& segment) { socket->setBlockingMode(); HttpConnection httpConnection(cuid, socket, req, e->option); httpConnection.sendProxyRequest(); diff --git a/src/HttpProxyRequestCommand.h b/src/HttpProxyRequestCommand.h index 2b5a646a..0d873cc3 100644 --- a/src/HttpProxyRequestCommand.h +++ b/src/HttpProxyRequestCommand.h @@ -26,7 +26,7 @@ class HttpProxyRequestCommand : public AbstractCommand { protected: - bool executeInternal(Segment segment); + bool executeInternal(Segment& segment); public: HttpProxyRequestCommand(int cuid, Request* req, DownloadEngine* e, const SocketHandle& s); diff --git a/src/HttpProxyResponseCommand.cc b/src/HttpProxyResponseCommand.cc index 51cb6580..17d16097 100644 --- a/src/HttpProxyResponseCommand.cc +++ b/src/HttpProxyResponseCommand.cc @@ -35,7 +35,7 @@ HttpProxyResponseCommand::~HttpProxyResponseCommand() { delete http; } -bool HttpProxyResponseCommand::executeInternal(Segment segment) { +bool HttpProxyResponseCommand::executeInternal(Segment& segment) { HttpHeader headers; int status = http->receiveResponse(headers); if(status == 0) { diff --git a/src/HttpProxyResponseCommand.h b/src/HttpProxyResponseCommand.h index ad938cf0..5f800e84 100644 --- a/src/HttpProxyResponseCommand.h +++ b/src/HttpProxyResponseCommand.h @@ -29,7 +29,7 @@ class HttpProxyResponseCommand : public AbstractCommand { private: HttpConnection* http; protected: - bool executeInternal(Segment segment); + bool executeInternal(Segment& segment); public: HttpProxyResponseCommand(int cuid, Request* req, DownloadEngine* e, const SocketHandle& s); diff --git a/src/HttpRequestCommand.cc b/src/HttpRequestCommand.cc index 9f5ba45a..522c6a1b 100644 --- a/src/HttpRequestCommand.cc +++ b/src/HttpRequestCommand.cc @@ -22,6 +22,7 @@ #include "HttpRequestCommand.h" #include "HttpResponseCommand.h" #include "HttpConnection.h" +#include "prefs.h" HttpRequestCommand::HttpRequestCommand(int cuid, Request* req, DownloadEngine* e, @@ -33,15 +34,17 @@ HttpRequestCommand::HttpRequestCommand(int cuid, Request* req, HttpRequestCommand::~HttpRequestCommand() {} -bool HttpRequestCommand::executeInternal(Segment seg) { +bool HttpRequestCommand::executeInternal(Segment& segment) { socket->setBlockingMode(); if(req->getProtocol() == "https") { socket->initiateSecureConnection(); } + if(!e->option->getAsBool(PREF_HTTP_KEEP_ALIVE)) { + req->setKeepAlive(false); + } HttpConnection http(cuid, socket, req, e->option); - // set seg to request in order to remember the request range - req->seg = seg; - http.sendRequest(seg); + req->segment = segment; + http.sendRequest(segment); Command* command = getNextCommand(); e->commands.push_back(command); diff --git a/src/HttpRequestCommand.h b/src/HttpRequestCommand.h index d3fa923b..2834ebb0 100644 --- a/src/HttpRequestCommand.h +++ b/src/HttpRequestCommand.h @@ -26,7 +26,7 @@ class HttpRequestCommand:public AbstractCommand { protected: - bool executeInternal(Segment segment); + bool executeInternal(Segment& segment); Command* getNextCommand() const; public: HttpRequestCommand(int cuid, Request* req, DownloadEngine* e, diff --git a/src/HttpResponseCommand.cc b/src/HttpResponseCommand.cc index feea4a06..b95070e8 100644 --- a/src/HttpResponseCommand.cc +++ b/src/HttpResponseCommand.cc @@ -26,6 +26,7 @@ #include "HttpInitiateConnectionCommand.h" #include "message.h" #include "Util.h" +#include "prefs.h" #include #include @@ -40,8 +41,8 @@ HttpResponseCommand::~HttpResponseCommand() { delete http; } -bool HttpResponseCommand::executeInternal(Segment seg) { - if(req->seg.sp != seg.sp) { +bool HttpResponseCommand::executeInternal(Segment& segment) { + if(req->segment != segment) { logger->info(MSG_SEGMENT_CHANGED, cuid); return prepareForRetry(0); } @@ -53,8 +54,12 @@ bool HttpResponseCommand::executeInternal(Segment seg) { return false; } // check HTTP status number - checkResponse(status, seg); + checkResponse(status, segment); retrieveCookie(headers); + // check whether the server supports persistent connections. + if(Util::toLower(headers.getFirst("Connection")).find("close") != string::npos) { + req->setKeepAlive(false); + } // check whether Location header exists. If it does, update request object // with redirected URL. // then establish a connection to the new host and port @@ -64,7 +69,8 @@ bool HttpResponseCommand::executeInternal(Segment seg) { if(!e->segmentMan->downloadStarted) { string transferEncoding; if(headers.defined("Transfer-Encoding")) { - return handleOtherEncoding(headers.getFirst("Transfer-Encoding"), headers); + return handleOtherEncoding(headers.getFirst("Transfer-Encoding"), + headers); } else { return handleDefaultEncoding(headers); } @@ -82,8 +88,8 @@ void HttpResponseCommand::checkResponse(int status, const Segment& segment) { throw new DlAbortEx(EX_AUTH_FAILED); } if(!(300 <= status && status < 400 || - (segment.sp+segment.ds == 0 && status == 200) - || (segment.sp+segment.ds > 0 && status == 206))) { + (segment.getPosition()+segment.writtenLength == 0 && (status == 200 || status == 206)) || + (segment.getPosition()+segment.writtenLength > 0 && status == 206))) { throw new DlRetryEx(EX_BAD_STATUS, status); } } @@ -112,6 +118,8 @@ bool HttpResponseCommand::handleDefaultEncoding(const HttpHeader& headers) { if(req->isTorrent) { long long int size = headers.getFirstAsLLInt("Content-Length"); e->segmentMan->totalSize = size; + e->segmentMan->initBitfield(e->option->getAsInt(PREF_SEGMENT_SIZE), + e->segmentMan->totalSize); e->segmentMan->isSplittable = false; e->segmentMan->downloadStarted = true; e->segmentMan->diskWriter->initAndOpenFile("/tmp/aria2"+Util::itos(getpid())); @@ -134,11 +142,10 @@ bool HttpResponseCommand::handleDefaultEncoding(const HttpHeader& headers) { return prepareForRetry(0); } else { e->segmentMan->totalSize = size; - Segment seg; - e->segmentMan->getSegment(seg, cuid); + e->segmentMan->initBitfield(e->option->getAsInt(PREF_SEGMENT_SIZE), + e->segmentMan->totalSize); e->segmentMan->diskWriter->initAndOpenFile(e->segmentMan->getFilePath()); - createHttpDownloadCommand(); - return true; + return prepareForRetry(0); } } @@ -148,8 +155,10 @@ bool HttpResponseCommand::handleOtherEncoding(const string& transferEncoding, co e->segmentMan->isSplittable = false; e->segmentMan->filename = determinFilename(headers); e->segmentMan->totalSize = 0; - Segment seg; - e->segmentMan->getSegment(seg, cuid); + // disable keep-alive + req->setKeepAlive(false); + Segment segment; + e->segmentMan->getSegment(segment, cuid); e->segmentMan->diskWriter->initAndOpenFile(e->segmentMan->getFilePath()); createHttpDownloadCommand(transferEncoding); return true; diff --git a/src/HttpResponseCommand.h b/src/HttpResponseCommand.h index 664fe026..298085b7 100644 --- a/src/HttpResponseCommand.h +++ b/src/HttpResponseCommand.h @@ -39,7 +39,7 @@ private: string determinFilename(const HttpHeader& headers); HttpConnection* http; protected: - bool executeInternal(Segment segment); + bool executeInternal(Segment& segment); public: HttpResponseCommand(int cuid, Request* req, DownloadEngine* e, const SocketHandle& s); diff --git a/src/Makefile.am b/src/Makefile.am index 3ed2ba80..b4e85973 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -23,11 +23,8 @@ SRCS = Socket.h\ SleepCommand.cc SleepCommand.h\ DownloadEngine.cc DownloadEngine.h\ ConsoleDownloadEngine.cc ConsoleDownloadEngine.h\ - Segment.h\ + Segment.cc Segment.h\ SegmentMan.cc SegmentMan.h\ - SegmentSplitter.cc SegmentSplitter.h\ - SplitFirstSegmentSplitter.cc SplitFirstSegmentSplitter.h\ - SplitSlowestSegmentSplitter.cc SplitSlowestSegmentSplitter.h\ Util.cc Util.h\ Request.cc Request.h\ common.h\ @@ -55,7 +52,9 @@ SRCS = Socket.h\ FeatureConfig.cc FeatureConfig.h\ DownloadEngineFactory.cc DownloadEngineFactory.h\ RequestInfo.h\ - UrlRequestInfo.cc UrlRequestInfo.h + UrlRequestInfo.cc UrlRequestInfo.h\ + SpeedCalc.cc SpeedCalc.h\ + PeerStat.h if ENABLE_ASYNC_DNS SRCS += NameResolver.cc NameResolver.h diff --git a/src/Makefile.in b/src/Makefile.in index e1df1f2e..ec826441 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -151,26 +151,24 @@ am__libaria2c_a_SOURCES_DIST = Socket.h SocketCore.cc SocketCore.h \ FtpTunnelRequestCommand.h FtpTunnelResponseCommand.cc \ FtpTunnelResponseCommand.h SleepCommand.cc SleepCommand.h \ DownloadEngine.cc DownloadEngine.h ConsoleDownloadEngine.cc \ - ConsoleDownloadEngine.h Segment.h SegmentMan.cc SegmentMan.h \ - SegmentSplitter.cc SegmentSplitter.h \ - SplitFirstSegmentSplitter.cc SplitFirstSegmentSplitter.h \ - SplitSlowestSegmentSplitter.cc SplitSlowestSegmentSplitter.h \ - Util.cc Util.h Request.cc Request.h common.h message.h \ - Exception.h DlAbortEx.h DlRetryEx.h Logger.h SimpleLogger.cc \ - SimpleLogger.h TransferEncoding.h ChunkedEncoding.cc \ - ChunkedEncoding.h DiskWriter.h DefaultDiskWriter.cc \ - DefaultDiskWriter.h PreAllocationDiskWriter.cc \ - PreAllocationDiskWriter.h AbstractDiskWriter.cc \ - AbstractDiskWriter.h File.cc File.h Option.cc Option.h \ - Base64.cc Base64.h CookieBox.cc CookieBox.h messageDigest.h \ - LogFactory.cc LogFactory.h NullLogger.h TimeA2.cc TimeA2.h \ - SharedHandle.h FeatureConfig.cc FeatureConfig.h \ - DownloadEngineFactory.cc DownloadEngineFactory.h RequestInfo.h \ - UrlRequestInfo.cc UrlRequestInfo.h NameResolver.cc \ - NameResolver.h MetaEntry.h Data.cc Data.h Dictionary.cc \ - Dictionary.h List.cc List.h MetaFileUtil.cc MetaFileUtil.h \ - MetaEntryVisitor.h ShaVisitor.cc ShaVisitor.h TorrentMan.cc \ - TorrentMan.h PeerConnection.cc PeerConnection.h \ + ConsoleDownloadEngine.h Segment.cc Segment.h SegmentMan.cc \ + SegmentMan.h Util.cc Util.h Request.cc Request.h common.h \ + message.h Exception.h DlAbortEx.h DlRetryEx.h Logger.h \ + SimpleLogger.cc SimpleLogger.h TransferEncoding.h \ + ChunkedEncoding.cc ChunkedEncoding.h DiskWriter.h \ + DefaultDiskWriter.cc DefaultDiskWriter.h \ + PreAllocationDiskWriter.cc PreAllocationDiskWriter.h \ + AbstractDiskWriter.cc AbstractDiskWriter.h File.cc File.h \ + Option.cc Option.h Base64.cc Base64.h CookieBox.cc CookieBox.h \ + messageDigest.h LogFactory.cc LogFactory.h NullLogger.h \ + TimeA2.cc TimeA2.h SharedHandle.h FeatureConfig.cc \ + FeatureConfig.h DownloadEngineFactory.cc \ + DownloadEngineFactory.h RequestInfo.h UrlRequestInfo.cc \ + UrlRequestInfo.h SpeedCalc.cc SpeedCalc.h PeerStat.h \ + NameResolver.cc NameResolver.h MetaEntry.h Data.cc Data.h \ + Dictionary.cc Dictionary.h List.cc List.h MetaFileUtil.cc \ + MetaFileUtil.h MetaEntryVisitor.h ShaVisitor.cc ShaVisitor.h \ + TorrentMan.cc TorrentMan.h PeerConnection.cc PeerConnection.h \ PeerMessageUtil.cc PeerMessageUtil.h PeerAbstractCommand.cc \ PeerAbstractCommand.h PeerInitiateConnectionCommand.cc \ PeerInitiateConnectionCommand.h PeerInteractionCommand.cc \ @@ -280,17 +278,15 @@ am__objects_4 = SocketCore.$(OBJEXT) Command.$(OBJEXT) \ FtpTunnelRequestCommand.$(OBJEXT) \ FtpTunnelResponseCommand.$(OBJEXT) SleepCommand.$(OBJEXT) \ DownloadEngine.$(OBJEXT) ConsoleDownloadEngine.$(OBJEXT) \ - SegmentMan.$(OBJEXT) SegmentSplitter.$(OBJEXT) \ - SplitFirstSegmentSplitter.$(OBJEXT) \ - SplitSlowestSegmentSplitter.$(OBJEXT) Util.$(OBJEXT) \ + Segment.$(OBJEXT) SegmentMan.$(OBJEXT) Util.$(OBJEXT) \ Request.$(OBJEXT) SimpleLogger.$(OBJEXT) \ ChunkedEncoding.$(OBJEXT) DefaultDiskWriter.$(OBJEXT) \ PreAllocationDiskWriter.$(OBJEXT) AbstractDiskWriter.$(OBJEXT) \ File.$(OBJEXT) Option.$(OBJEXT) Base64.$(OBJEXT) \ CookieBox.$(OBJEXT) LogFactory.$(OBJEXT) TimeA2.$(OBJEXT) \ FeatureConfig.$(OBJEXT) DownloadEngineFactory.$(OBJEXT) \ - UrlRequestInfo.$(OBJEXT) $(am__objects_1) $(am__objects_2) \ - $(am__objects_3) + UrlRequestInfo.$(OBJEXT) SpeedCalc.$(OBJEXT) $(am__objects_1) \ + $(am__objects_2) $(am__objects_3) am_libaria2c_a_OBJECTS = $(am__objects_4) libaria2c_a_OBJECTS = $(am_libaria2c_a_OBJECTS) am__installdirs = "$(DESTDIR)$(bindir)" @@ -472,23 +468,21 @@ SRCS = Socket.h SocketCore.cc SocketCore.h Command.cc Command.h \ FtpTunnelRequestCommand.h FtpTunnelResponseCommand.cc \ FtpTunnelResponseCommand.h SleepCommand.cc SleepCommand.h \ DownloadEngine.cc DownloadEngine.h ConsoleDownloadEngine.cc \ - ConsoleDownloadEngine.h Segment.h SegmentMan.cc SegmentMan.h \ - SegmentSplitter.cc SegmentSplitter.h \ - SplitFirstSegmentSplitter.cc SplitFirstSegmentSplitter.h \ - SplitSlowestSegmentSplitter.cc SplitSlowestSegmentSplitter.h \ - Util.cc Util.h Request.cc Request.h common.h message.h \ - Exception.h DlAbortEx.h DlRetryEx.h Logger.h SimpleLogger.cc \ - SimpleLogger.h TransferEncoding.h ChunkedEncoding.cc \ - ChunkedEncoding.h DiskWriter.h DefaultDiskWriter.cc \ - DefaultDiskWriter.h PreAllocationDiskWriter.cc \ - PreAllocationDiskWriter.h AbstractDiskWriter.cc \ - AbstractDiskWriter.h File.cc File.h Option.cc Option.h \ - Base64.cc Base64.h CookieBox.cc CookieBox.h messageDigest.h \ - LogFactory.cc LogFactory.h NullLogger.h TimeA2.cc TimeA2.h \ - SharedHandle.h FeatureConfig.cc FeatureConfig.h \ - DownloadEngineFactory.cc DownloadEngineFactory.h RequestInfo.h \ - UrlRequestInfo.cc UrlRequestInfo.h $(am__append_1) \ - $(am__append_2) $(am__append_3) + ConsoleDownloadEngine.h Segment.cc Segment.h SegmentMan.cc \ + SegmentMan.h Util.cc Util.h Request.cc Request.h common.h \ + message.h Exception.h DlAbortEx.h DlRetryEx.h Logger.h \ + SimpleLogger.cc SimpleLogger.h TransferEncoding.h \ + ChunkedEncoding.cc ChunkedEncoding.h DiskWriter.h \ + DefaultDiskWriter.cc DefaultDiskWriter.h \ + PreAllocationDiskWriter.cc PreAllocationDiskWriter.h \ + AbstractDiskWriter.cc AbstractDiskWriter.h File.cc File.h \ + Option.cc Option.h Base64.cc Base64.h CookieBox.cc CookieBox.h \ + messageDigest.h LogFactory.cc LogFactory.h NullLogger.h \ + TimeA2.cc TimeA2.h SharedHandle.h FeatureConfig.cc \ + FeatureConfig.h DownloadEngineFactory.cc \ + DownloadEngineFactory.h RequestInfo.h UrlRequestInfo.cc \ + UrlRequestInfo.h SpeedCalc.cc SpeedCalc.h PeerStat.h \ + $(am__append_1) $(am__append_2) $(am__append_3) noinst_LIBRARIES = libaria2c.a libaria2c_a_SOURCES = $(SRCS) aria2c_LDADD = libaria2c.a @LIBINTL@ @ALLOCA@ @LIBGNUTLS_LIBS@\ @@ -655,15 +649,14 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/RequestMessage.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/RequestSlot.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/SeedCheckCommand.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Segment.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/SegmentMan.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/SegmentSplitter.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ShaVisitor.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/SimpleLogger.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/SimplePeerMessage.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/SleepCommand.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/SocketCore.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/SplitFirstSegmentSplitter.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/SplitSlowestSegmentSplitter.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/SpeedCalc.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/SuggestPieceMessage.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/TimeA2.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/TorrentAutoSaveCommand.Po@am__quote@ diff --git a/src/PeerInteractionCommand.cc b/src/PeerInteractionCommand.cc index 1b529264..6f0f1b89 100644 --- a/src/PeerInteractionCommand.cc +++ b/src/PeerInteractionCommand.cc @@ -127,7 +127,11 @@ bool PeerInteractionCommand::executeInternal() { } receiveMessages(); - peerInteraction->addRequests(); + int maxSpeedLimit = e->option->getAsInt(PREF_MAX_SPEED_LIMIT); + if(maxSpeedLimit == 0 || + maxSpeedLimit > 0 && maxSpeedLimit <= e->getDownloadSpeed()) { + peerInteraction->addRequests(); + } peerInteraction->sendMessages(e->getUploadSpeed()); break; } diff --git a/src/PeerStat.h b/src/PeerStat.h new file mode 100644 index 00000000..a0fe686d --- /dev/null +++ b/src/PeerStat.h @@ -0,0 +1,90 @@ +/* */ +#ifndef _D_PEER_STAT_H_ +#define _D_PEER_STAT_H_ + +#include "common.h" +#include "SpeedCalc.h" +#include "SharedHandle.h" + +class PeerStat { +public: + enum STATUS { + IDLE, + ACTIVE + }; +private: + int cuid; + SpeedCalc downloadSpeed; + Time downloadStartTime; + STATUS status; +public: + + PeerStat(int cuid):cuid(cuid), status(IDLE) {} + + ~PeerStat() {} + + /** + * Returns current download speed in byte per sec. + */ + int calculateDownloadSpeed() { + return downloadSpeed.calculateSpeed(); + } + + void updateDownloadLength(int bytes) { + downloadSpeed.update(bytes); + } + + int getMaxSpeed() const { + return downloadSpeed.getMaxSpeed(); + } + + void reset() { + downloadSpeed.reset(); + downloadStartTime.reset(); + } + + void downloadStart() { + reset(); + status = ACTIVE; + } + + void downloadStop() { + status = IDLE; + } + + const Time& getDownloadStartTime() const { + return downloadStartTime; + } + + STATUS getStatus() const { + return status; + } + + int getCuid() const { + return cuid; + } +}; + +typedef SharedHandle PeerStatHandle; + +#endif // _D_PEER_STAT_H_ diff --git a/src/Piece.cc b/src/Piece.cc index 9f906f58..424157bc 100644 --- a/src/Piece.cc +++ b/src/Piece.cc @@ -30,6 +30,7 @@ void Piece::completeBlock(int blockIndex) { void Piece::clearAllBlock() { bitfield->clearAllBit(); + bitfield->clearAllUseBit(); } void Piece::setAllBlock() { diff --git a/src/PieceMessage.cc b/src/PieceMessage.cc index 2c1422ed..8101595d 100644 --- a/src/PieceMessage.cc +++ b/src/PieceMessage.cc @@ -81,7 +81,6 @@ void PieceMessage::receivedAction() { onGotNewPiece(piece); } else { onGotWrongPiece(piece); - peerInteraction->abortPiece(piece); } } } @@ -212,6 +211,7 @@ void PieceMessage::onGotWrongPiece(Piece& piece) { erasePieceOnDisk(piece); piece.clearAllBlock(); torrentMan->updatePiece(piece); + peerInteraction->abortPiece(piece); } void PieceMessage::erasePieceOnDisk(const Piece& piece) { diff --git a/src/Request.cc b/src/Request.cc index e9e30b86..8dcabc16 100644 --- a/src/Request.cc +++ b/src/Request.cc @@ -23,11 +23,7 @@ #include "Util.h" #include "FeatureConfig.h" -Request::Request():port(0), tryCount(0), isTorrent(false) { - seg.sp = 0; - seg.ep = 0; - seg.ds = 0; - seg.finish = false; +Request::Request():port(0), tryCount(0), keepAlive(true), isTorrent(false) { cookieBox = new CookieBox(); } @@ -42,6 +38,7 @@ bool Request::setUrl(const string& url) { bool Request::resetUrl() { previousUrl = referer; + segment = Segment(); return setUrl(url); } diff --git a/src/Request.h b/src/Request.h index 5b5b15f1..313044e1 100644 --- a/src/Request.h +++ b/src/Request.h @@ -23,8 +23,8 @@ #define _D_REQUEST_H_ #include #include -#include #include "CookieBox.h" +#include "Segment.h" #include "common.h" using namespace std; @@ -59,9 +59,10 @@ private: string file; int tryCount; int trackerEvent; + bool keepAlive; bool parseUrl(const string& url); public: - Segment seg; + Segment segment; CookieBox* cookieBox; bool isTorrent; public: @@ -91,7 +92,8 @@ public: int getPort() const { return port; } string getDir() const { return dir; } string getFile() const { return file;} - + bool isKeepAlive() const { return keepAlive; } + void setKeepAlive(bool keepAlive) { this->keepAlive = keepAlive; } void setTrackerEvent(int event) { trackerEvent = event; } int getTrackerEvent() const { return trackerEvent; } diff --git a/src/Segment.cc b/src/Segment.cc new file mode 100644 index 00000000..c909f194 --- /dev/null +++ b/src/Segment.cc @@ -0,0 +1,31 @@ +/* */ + +#include "Segment.h" + +ostream& operator<<(ostream& o, const Segment& segment) { + o << "index = " << segment.index << ", "; + o << "length = " << segment.length << ", "; + o << "segmentLength = " << segment.segmentLength << ", "; + o << "writtenLength = " << segment.writtenLength; + return o; +} diff --git a/src/Segment.h b/src/Segment.h index dc2ea084..139a53b4 100644 --- a/src/Segment.h +++ b/src/Segment.h @@ -27,27 +27,46 @@ using namespace std; -/** - * Segment represents a download segment. - * sp, ep is a offset from a begining of a file. - * Therefore, if a file size is x, then 0 <= sp <= ep <= x-1. - * sp, ep is used in Http Range header. - * e.g. Range: bytes=sp-ep - * ds is downloaded bytes. - * If a download of this segement is complete, finish must be set to true. - */ -typedef struct { - int cuid; - long long int sp; - long long int ep; - long long int ds; - int speed; - bool finish; -} Segment; +class Segment { +public: + int index; + int length; + int segmentLength; + int writtenLength; -typedef deque Segments; + Segment(int index, int length, int segmentLength, int writtenLength = 0) + :index(index), length(length), segmentLength(segmentLength), + writtenLength(writtenLength) {} -#define SEGMENT_EQUAL(X, Y) (X.cuid == Y.cuid && X.sp == Y.sp && X.ep == Y.ep && X.ds == Y.ds && X.finish == Y.finish ? true : false) + Segment():index(-1), length(0), segmentLength(0), writtenLength(0) {} + + bool complete() const { + return length <= writtenLength; + } + + bool isNull() const { + return index == -1; + } + + long long int getPosition() const { + return ((long long int)index)*segmentLength; + } + + bool operator==(const Segment& segment) const { + return index == segment.index && + length == segment.length && + segmentLength == segment.segmentLength && + writtenLength == segment.writtenLength; + } + + bool operator!=(const Segment& segment) const { + return !(*this == segment); + } + + friend ostream& operator<<(ostream& o, const Segment& segment); +}; + +ostream& operator<<(ostream& o, const Segment& segment); #endif // _D_SEGMENT_H_ diff --git a/src/SegmentMan.cc b/src/SegmentMan.cc index 72929270..36602639 100644 --- a/src/SegmentMan.cc +++ b/src/SegmentMan.cc @@ -31,88 +31,21 @@ #include #include -SegmentMan::SegmentMan():totalSize(0), +SegmentMan::SegmentMan():bitfield(0), + totalSize(0), isSplittable(true), downloadStarted(false), dir("."), errors(0), - splitter(NULL), - diskWriter(NULL) { + diskWriter(0) { logger = LogFactory::getInstance(); } SegmentMan::~SegmentMan() { - if(splitter != NULL) { - delete splitter; - } - if(diskWriter != NULL) { - delete diskWriter; - } + delete bitfield; + delete diskWriter; } -void SegmentMan::unregisterId(int cuid) { - for(Segments::iterator itr = segments.begin(); itr != segments.end(); itr++) { - if((*itr).cuid == cuid) { - cuid = 0; - } - } -} - -bool SegmentMan::getSegment(Segment& seg, int cuid) { - //Segment s = { 0, 0, 0, false }; - - if(segments.empty()) { - logger->debug(string("assign new segment { sp = 0, ep = "+(totalSize == 0 ? "0" : Util::llitos(totalSize-1))+" } to cuid "+Util::llitos(cuid)).c_str()); - //seg = { cuid, 0, totalSize == 0 ? 0 : totalSize-1, 0, false }; - seg.cuid = cuid; - seg.sp = 0; - seg.ep = totalSize == 0 ? 0 : totalSize-1; - seg.ds = 0; - seg.speed = 0; - seg.finish = false; - segments.push_back(seg); - return true; - } - for(Segments::iterator itr = segments.begin(); itr != segments.end(); itr++) { - if((*itr).cuid == cuid && !(*itr).finish) { -// logger->debug("return an existing segment { " -// "sp = "+Util::ulitos((*itr).sp)+", "+ -// "ep = "+Util::ulitos((*itr).ep)+", " -// "ds = "+Util::ulitos((*itr).ds)+" } to "+ -// "cuid "+Util::ulitos((*itr).cuid)); - seg = *itr; - return true; - } - } - if(!isSplittable) { - return false; - } - for(Segments::iterator itr = segments.begin(); itr != segments.end(); itr++) { - Segment& s = *itr; - if(s.finish) { - continue; - } - if(s.cuid == 0) { - s.cuid = cuid; - seg = s; - return true; - } - } - return splitter->splitSegment(seg, cuid, segments); -} - -void SegmentMan::updateSegment(const Segment& segment) { - for(Segments::iterator itr = segments.begin(); itr != segments.end(); itr++) { - if((*itr).cuid == segment.cuid && - (*itr).sp == segment.sp && - (*itr).ep == segment.ep) { - *itr = segment; - break; - } - } -} - - bool SegmentMan::segmentFileExists() const { if(!isSplittable) { return false; @@ -144,10 +77,6 @@ void SegmentMan::load() { segFilename.c_str(), strerror(errno)); } logger->info(MSG_LOADED_SEGMENT_FILE); - for(Segments::iterator itr = segments.begin(); itr != segments.end(); - itr++) { - (*itr).cuid = 0; - } } void SegmentMan::save() const { @@ -161,8 +90,32 @@ void SegmentMan::save() const { if(fwrite(&totalSize, sizeof(totalSize), 1, segFile) < 1) { throw string("writeError"); } - for(Segments::const_iterator itr = segments.begin(); itr != segments.end(); itr++) { - if(fwrite(&*itr, sizeof(Segment), 1, segFile) < 1) { + int segmentLength = bitfield->getBlockLength(); + if(fwrite(&segmentLength, sizeof(segmentLength), 1, segFile) < 1) { + throw string("writeError"); + } + if(bitfield) { + int bitfieldLength = bitfield->getBitfieldLength(); + if(fwrite(&bitfieldLength, sizeof(bitfieldLength), 1, segFile) < 1) { + throw string("writeError"); + } + if(fwrite(bitfield->getBitfield(), bitfield->getBitfieldLength(), + 1, segFile) < 1) { + throw string("writeError"); + } + } else { + int i = 0; + if(fwrite(&i, sizeof(i), 1, segFile) < 1) { + throw string("writeError"); + } + } + int usedSegmentCount = usedSegmentEntries.size(); + if(fwrite(&usedSegmentCount, sizeof(usedSegmentCount), 1, segFile) < 1) { + throw string("writeError"); + } + for(SegmentEntries::const_iterator itr = usedSegmentEntries.begin(); + itr != usedSegmentEntries.end(); itr++) { + if(fwrite(&itr->segment, sizeof(Segment), 1, segFile) < 1) { throw string("writeError"); } } @@ -189,16 +142,35 @@ void SegmentMan::read(FILE* file) { if(fread(&totalSize, sizeof(totalSize), 1, file) < 1) { throw string("readError"); } - while(1) { + int segmentSize; + if(fread(&segmentSize, sizeof(segmentSize), 1, file) < 1) { + throw string("readError"); + } + int bitfieldLength; + if(fread(&bitfieldLength, sizeof(bitfieldLength), 1, file) < 1) { + throw string("readError"); + } + if(bitfieldLength > 0) { + initBitfield(segmentSize, totalSize); + unsigned char* savedBitfield = new unsigned char[bitfield->getBitfieldLength()]; + if(fread(savedBitfield, bitfield->getBitfieldLength(), 1, file) < 1) { + delete [] savedBitfield; + throw string("readError"); + } else { + bitfield->setBitfield(savedBitfield, bitfield->getBitfieldLength()); + delete [] savedBitfield; + } + } + int segmentCount; + if(fread(&segmentCount, sizeof(segmentCount), 1, file) < 1) { + throw string("readError"); + } + while(segmentCount--) { Segment seg; if(fread(&seg, sizeof(Segment), 1, file) < 1) { - if(ferror(file)) { - throw string("readError"); - } else if(feof(file)) { - break; - } + throw string("readError"); } - segments.push_back(seg); + usedSegmentEntries.push_back(SegmentEntry(0, seg)); } } @@ -213,15 +185,14 @@ void SegmentMan::remove() const { } bool SegmentMan::finished() const { - if(!downloadStarted || segments.size() == 0) { + if(!downloadStarted) { return false; } - for(Segments::const_iterator itr = segments.begin(); itr != segments.end(); itr++) { - if(!(*itr).finish) { - return false; - } + if(!bitfield) { + return false; } - return true; + assert(bitfield); + return bitfield->isAllBitSet(); } void SegmentMan::removeIfFinished() const { @@ -230,19 +201,219 @@ void SegmentMan::removeIfFinished() const { } } -long long int SegmentMan::getDownloadedSize() const { - long long int size = 0; - for(Segments::const_iterator itr = segments.begin(); itr != segments.end(); itr++) { - size += (*itr).ds; - } - return size; -} - void SegmentMan::init() { totalSize = 0; isSplittable = false; downloadStarted = false; errors = 0; - segments.clear(); + //segments.clear(); + usedSegmentEntries.clear(); + delete bitfield; + peerStats.clear(); diskWriter->closeFile(); + +} + +void SegmentMan::initBitfield(int segmentLength, long long int totalLength) { + this->bitfield = new BitfieldMan(segmentLength, totalLength); +} + +class FindSegmentEntryByIndex { +private: + int index; +public: + FindSegmentEntryByIndex(int index):index(index) {} + + bool operator()(const SegmentEntry& entry) { + return entry.segment.index == index; + } +}; + +class FindSegmentEntryByCuid { +private: + int cuid; +public: + FindSegmentEntryByCuid(int cuid):cuid(cuid) {} + + bool operator()(const SegmentEntry& entry) { + return entry.cuid == cuid; + } +}; + +Segment SegmentMan::checkoutSegment(int cuid, int index) { + logger->debug("Attach segment#%d to CUID#%d.", index, cuid); + bitfield->setUseBit(index); + + SegmentEntries::iterator itr = find_if(usedSegmentEntries.begin(), + usedSegmentEntries.end(), + FindSegmentEntryByIndex(index)); + Segment segment; + if(itr == usedSegmentEntries.end()) { + segment = Segment(index, bitfield->getBlockLength(index), + bitfield->getBlockLength()); + SegmentEntry entry(cuid, segment); + usedSegmentEntries.push_back(entry); + } else { + (*itr).cuid = cuid; + segment = (*itr).segment; + } + + logger->debug("index=%d, length=%d, segmentLength=%d, writtenLength=%d", + segment.index, segment.length, segment.segmentLength, + segment.writtenLength); + return segment; +} + +bool SegmentMan::onNullBitfield(Segment& segment, int cuid) { + if(usedSegmentEntries.size() == 0) { + segment = Segment(0, 0, 0); + usedSegmentEntries.push_back(SegmentEntry(cuid, segment)); + return true; + } else { + SegmentEntries::iterator uitr = find_if(usedSegmentEntries.begin(), + usedSegmentEntries.end(), + FindSegmentEntryByCuid(cuid)); + if(uitr == usedSegmentEntries.end()) { + return false; + } else { + segment = uitr->segment; + return true; + } + } +} + +bool SegmentMan::getSegment(Segment& segment, int cuid) { + if(!bitfield) { + return onNullBitfield(segment, cuid); + } + SegmentEntries::iterator uitr = find_if(usedSegmentEntries.begin(), + usedSegmentEntries.end(), + FindSegmentEntryByCuid(cuid)); + if(uitr != usedSegmentEntries.end()) { + segment = uitr->segment; + return true; + } + int index = bitfield->getSparseMissingUnusedIndex(); + if(index == -1) { + return false; + } else { + segment = checkoutSegment(cuid, index); + return true; + } +} + +bool SegmentMan::getSegment(Segment& segment, int cuid, int index) { + if(!bitfield) { + return onNullBitfield(segment, cuid); + } + if(index < 0 || bitfield->countBlock() <= index) { + return false; + } + if(bitfield->isBitSet(index) || bitfield->isUseBitSet(index)) { + return false; + } else { + segment = checkoutSegment(cuid, index); + return true; + } +} + +bool SegmentMan::updateSegment(int cuid, const Segment& segment) { + if(segment.isNull()) { + return false; + } + SegmentEntries::iterator itr = find_if(usedSegmentEntries.begin(), + usedSegmentEntries.end(), + FindSegmentEntryByCuid(cuid)); + if(itr == usedSegmentEntries.end()) { + return false; + } else { + (*itr).segment = segment; + return true; + } +} + +class CancelSegment { +private: + int cuid; + BitfieldMan* bitfield; +public: + CancelSegment(int cuid, BitfieldMan* bitfield):cuid(cuid), + bitfield(bitfield) {} + + void operator()(SegmentEntry& entry) { + if(entry.cuid == cuid) { + bitfield->unsetUseBit(entry.segment.index); + entry.cuid = 0; + } + } +}; + +void SegmentMan::cancelSegment(int cuid) { + if(bitfield) { + for_each(usedSegmentEntries.begin(), usedSegmentEntries.end(), + CancelSegment(cuid, bitfield)); + } else { + usedSegmentEntries.clear(); + } +} + +bool SegmentMan::completeSegment(int cuid, const Segment& segment) { + if(segment.isNull()) { + return false; + } + if(bitfield) { + bitfield->unsetUseBit(segment.index); + bitfield->setBit(segment.index); + } else { + initBitfield(option->getAsInt(PREF_SEGMENT_SIZE), segment.writtenLength); + bitfield->setAllBit(); + } + SegmentEntries::iterator itr = find_if(usedSegmentEntries.begin(), + usedSegmentEntries.end(), + FindSegmentEntryByCuid(cuid)); + if(itr == usedSegmentEntries.end()) { + return false; + } else { + usedSegmentEntries.erase(itr); + return true; + } +} + +bool SegmentMan::hasSegment(int index) const { + if(bitfield) { + return bitfield->isBitSet(index); + } else { + return false; + } +} + +long long int SegmentMan::getDownloadLength() const { + long long int dlLength = 0; + if(bitfield) { + dlLength += bitfield->getCompletedLength(); + } + for(SegmentEntries::const_iterator itr = usedSegmentEntries.begin(); + itr != usedSegmentEntries.end(); itr++) { + dlLength += itr->segment.writtenLength; + } + return dlLength; +} + +void SegmentMan::registerPeerStat(const PeerStatHandle& peerStat) { + PeerStatHandle temp = getPeerStat(peerStat->getCuid()); + if(!temp.get()) { + peerStats.push_back(peerStat); + } +} + +int SegmentMan::calculateDownloadSpeed() const { + int speed = 0; + for(PeerStats::const_iterator itr = peerStats.begin(); + itr != peerStats.end(); itr++) { + const PeerStatHandle& peerStat = *itr; + if(peerStat->getStatus() == PeerStat::ACTIVE) { + speed += peerStat->calculateDownloadSpeed(); + } + } + return speed; } diff --git a/src/SegmentMan.h b/src/SegmentMan.h index 53112d6d..6b36c670 100644 --- a/src/SegmentMan.h +++ b/src/SegmentMan.h @@ -26,23 +26,42 @@ #include "Logger.h" #include "Segment.h" #include "Option.h" -#include "SegmentSplitter.h" #include "DiskWriter.h" #include "Request.h" +#include "BitfieldMan.h" +#include "PeerStat.h" using namespace std; #define SEGMENT_FILE_EXTENSION ".aria2" +class SegmentEntry { +public: + int cuid; + Segment segment; +public: + SegmentEntry(int cuid, const Segment& segment) + :cuid(cuid), segment(segment) {} + ~SegmentEntry() {} +}; + +typedef deque SegmentEntries; +typedef deque PeerStats; + /** * This class holds the download progress of the one download entry. */ class SegmentMan { private: const Logger* logger; + BitfieldMan* bitfield; + SegmentEntries usedSegmentEntries; + PeerStats peerStats; void read(FILE* file); FILE* openSegFile(const string& segFilename, const string& mode) const; + bool onNullBitfield(Segment& segment, int cuid); + Segment checkoutSegment(int cuid, int index); public: /** * The total number of bytes to download. @@ -65,10 +84,6 @@ public: * The default value is false. */ bool downloadStarted; - /** - * Holds segments. - */ - Segments segments; /** * Respresents the file name of the downloaded file. * If the URL does not contain file name part(http://www.rednoah.com/, for @@ -91,7 +106,6 @@ public: int errors; const Option* option; - SegmentSplitter* splitter; DiskWriter* diskWriter; Requests reserved; @@ -116,31 +130,6 @@ public: return getFilePath()+SEGMENT_FILE_EXTENSION; } - /** - * Sets the cuid of the holded segments with specified cuid to 0. - */ - void unregisterId(int cuid); - /** - * There is a segment available for DownloadCommand specified by cuid, - * fills segment and returns true. - * There is no segment available, then returns false and segment is - * undefined in this case. - * - * @param segment segment to attach for cuid. - * @param cuid cuid of DownloadCommand. - * @returns true: there is a segment available, false: there is no segment - * available. - */ - bool getSegment(Segment& segment, int cuid); - /** - * Updates the ds value of the specified segment. - * Only a segment x is updated where x.sp == sgment.sp && x.ep == - * segment.ep && x.ds == segment.ds &&x.cuid == segment.cuid - * is true. - * - * @param segment segment to update - */ - void updateSegment(const Segment& segment); /** * Returns true only if the segment data file exists. * The file name of the segment data is filename appended by ".aria2". @@ -173,9 +162,84 @@ public: */ void removeIfFinished() const; /** - * Returns the total number of bytes to be downloaded. + * Returns a vacant segment. + * If there is no vacant segment, then returns a segment instance whose + * isNull call is true. */ - long long int getDownloadedSize() const; + bool getSegment(Segment& segment, int cuid); + /** + * Returns a segment whose index is index. + * If it has already assigned + * to another cuid or has been downloaded, then returns a segment instance + * whose isNull call is true. + */ + bool getSegment(Segment& segment, int cuid, int index); + /** + * Updates download status. + */ + bool updateSegment(int cuid, const Segment& segment); + /** + * Cancels all the segment which the command having given cuid + * uses. + */ + void cancelSegment(int cuid); + /** + * Tells SegmentMan that the segment has been downloaded successfully. + */ + bool completeSegment(int cuid, const Segment& segment); + /** + * Initializes bitfield with the provided length parameters. + */ + void initBitfield(int segmentLength, long long int totalLength); + /** + * Returns true if the segment whose index is index has been downloaded. + */ + bool hasSegment(int index) const; + /** + * Returns the length of bytes downloaded. + */ + long long int getDownloadLength() const; + + /** + * Registers given peerStat if it has not been registerd. + * Otherwise does nothing. + */ + void registerPeerStat(const PeerStatHandle& peerStat); + + class FindPeerStat { + private: + int cuid; + public: + FindPeerStat(int cuid):cuid(cuid) {} + + bool operator()(const PeerStatHandle& peerStat) { + if(peerStat->getCuid() == cuid) { + return true; + } else { + return false; + } + } + }; + + /** + * Returns peerStat whose cuid is given cuid. If it is not found, returns + * PeerStatHandle(0). + */ + PeerStatHandle getPeerStat(int cuid) const { + PeerStats::const_iterator itr = find_if(peerStats.begin(), peerStats.end(), + FindPeerStat(cuid)); + if(itr == peerStats.end()) { + // TODO + return PeerStatHandle(0); + } else { + return *itr; + } + } + + /** + * Returns current download speed in bytes per sec. + */ + int calculateDownloadSpeed() const; }; #endif // _D_SEGMENT_MAN_H_ diff --git a/src/SpeedCalc.cc b/src/SpeedCalc.cc new file mode 100644 index 00000000..10750cc2 --- /dev/null +++ b/src/SpeedCalc.cc @@ -0,0 +1,82 @@ +/* */ +#include "SpeedCalc.h" +#include +#include +#include + +#define CHANGE_INTERVAL_SEC 15 + +class Reset { +public: + void operator()(Time& tm) { + tm.reset(); + } +}; + +void SpeedCalc::reset() { + fill(&lengthArray[0], &lengthArray[2], 0); + for_each(&cpArray[0], &cpArray[2], Reset()); + sw = 0; + maxSpeed = 0; + prevSpeed = 0; +} + +int SpeedCalc::calculateSpeed() { + int milliElapsed = cpArray[sw].differenceInMillis(); + if(milliElapsed) { + int speed = lengthArray[sw]*1000/milliElapsed; + prevSpeed = speed; + maxSpeed = max(speed, maxSpeed); + return speed; + } else { + return prevSpeed; + } +} + +class Plus { +private: + int d; +public: + Plus(int d):d(d) {} + + void operator()(long long int& length) { + length += d; + } +}; + +void SpeedCalc::update(int bytes) { + for_each(&lengthArray[0], &lengthArray[2], Plus(bytes)); + if(isIntervalOver()) { + changeSw(); + } +} + +bool SpeedCalc::isIntervalOver() const { + return CHANGE_INTERVAL_SEC <= cpArray[sw].difference(); +} + +void SpeedCalc::changeSw() { + lengthArray[sw] = 0; + cpArray[sw].reset(); + sw ^= 0x01; +} diff --git a/src/SpeedCalc.h b/src/SpeedCalc.h new file mode 100644 index 00000000..e0ba7d28 --- /dev/null +++ b/src/SpeedCalc.h @@ -0,0 +1,59 @@ +/* */ +#ifndef _D_SPEED_CALC_H_ +#define _D_SPEED_CALC_H_ + +#include "common.h" +#include "TimeA2.h" + +class SpeedCalc { +private: + long long int lengthArray[2]; + int sw; + Time cpArray[2]; + int maxSpeed; + int prevSpeed; + + bool isIntervalOver() const; + void changeSw(); +public: + SpeedCalc() { + reset(); + } + + ~SpeedCalc() {} + + /** + * Returns download/upload speed in byte per sec + */ + int calculateSpeed(); + + int getMaxSpeed() const { + return maxSpeed; + } + + void update(int bytes); + + void reset(); +}; + +#endif // _D_SPEED_CALC_H_ diff --git a/src/main.cc b/src/main.cc index 3914d375..c3c4a7e3 100644 --- a/src/main.cc +++ b/src/main.cc @@ -22,7 +22,6 @@ #include "HttpInitiateConnectionCommand.h" #include "ConsoleDownloadEngine.h" #include "SegmentMan.h" -#include "SplitSlowestSegmentSplitter.h" #include "LogFactory.h" #include "common.h" #include "DefaultDiskWriter.h" @@ -272,6 +271,8 @@ int main(int argc, char* argv[]) { op->put(PREF_DIR, "."); op->put(PREF_SPLIT, "1"); op->put(PREF_DAEMON, V_FALSE); + op->put(PREF_SEGMENT_SIZE, Util::itos(1024*1024)); + op->put(PREF_HTTP_KEEP_ALIVE, V_FALSE); op->put(PREF_LISTEN_PORT, "-1"); op->put(PREF_METALINK_SERVERS, "15"); op->put(PREF_FOLLOW_TORRENT, @@ -303,6 +304,7 @@ int main(int argc, char* argv[]) { op->put(PREF_DIRECT_FILE_MAPPING, V_TRUE); op->put(PREF_UPLOAD_LIMIT, "0"); op->put(PREF_LOWEST_SPEED_LIMIT, "0"); + op->put(PREF_MAX_SPEED_LIMIT, "0"); while(1) { int optIndex = 0; int lopt; diff --git a/src/prefs.h b/src/prefs.h index 9c3e040a..70fc9207 100644 --- a/src/prefs.h +++ b/src/prefs.h @@ -57,8 +57,12 @@ #define PREF_DAEMON "daemon" // value: a string #define PREF_REFERER "referer" -// value' 1*digit +// value: 1*digit #define PREF_LOWEST_SPEED_LIMIT "lowest_speed_limit" +// value: 1*digit +#define PREF_SEGMENT_SIZE "segment_size" +// value: 1*digit +#define PREF_MAX_SPEED_LIMIT "max_speed_limit" /** * FTP related preferences @@ -86,6 +90,8 @@ # define V_BASIC "basic" // values: true | false #define PREF_HTTP_AUTH_ENABLED "http_auth_enabled" +// values: true | false +#define PREF_HTTP_KEEP_ALIVE "http_keep_alive" /** * HTTP proxy related preferences diff --git a/test/BitfieldManTest.cc b/test/BitfieldManTest.cc index e8dab885..257c4c2f 100644 --- a/test/BitfieldManTest.cc +++ b/test/BitfieldManTest.cc @@ -12,6 +12,7 @@ class BitfieldManTest:public CppUnit::TestFixture { CPPUNIT_TEST(testIsAllBitSet); CPPUNIT_TEST(testFilter); CPPUNIT_TEST(testGetMissingIndex); + CPPUNIT_TEST(testGetSparceMissingUnusedIndex); CPPUNIT_TEST_SUITE_END(); private: @@ -24,6 +25,7 @@ public: void testIsAllBitSet(); void testFilter(); void testGetMissingIndex(); + void testGetSparceMissingUnusedIndex(); }; @@ -209,3 +211,29 @@ void BitfieldManTest::testGetMissingIndex() { CPPUNIT_ASSERT_EQUAL(-1, bt1.getMissingIndex(bitArray5, 32)); } + +void BitfieldManTest::testGetSparceMissingUnusedIndex() { + BitfieldMan bitfield(1024*1024, 10*1024*1024); + + CPPUNIT_ASSERT_EQUAL(0, bitfield.getSparseMissingUnusedIndex()); + bitfield.setBit(0); + CPPUNIT_ASSERT_EQUAL(5, bitfield.getSparseMissingUnusedIndex()); + bitfield.setUseBit(5); + CPPUNIT_ASSERT_EQUAL(3, bitfield.getSparseMissingUnusedIndex()); + bitfield.setBit(3); + CPPUNIT_ASSERT_EQUAL(8, bitfield.getSparseMissingUnusedIndex()); + bitfield.setBit(8); + CPPUNIT_ASSERT_EQUAL(2, bitfield.getSparseMissingUnusedIndex()); + bitfield.setBit(2); + CPPUNIT_ASSERT_EQUAL(7, bitfield.getSparseMissingUnusedIndex()); + bitfield.setBit(7); + CPPUNIT_ASSERT_EQUAL(1, bitfield.getSparseMissingUnusedIndex()); + bitfield.setBit(1); + CPPUNIT_ASSERT_EQUAL(4, bitfield.getSparseMissingUnusedIndex()); + bitfield.setBit(4); + CPPUNIT_ASSERT_EQUAL(6, bitfield.getSparseMissingUnusedIndex()); + bitfield.setBit(6); + CPPUNIT_ASSERT_EQUAL(9, bitfield.getSparseMissingUnusedIndex()); + bitfield.setBit(9); + CPPUNIT_ASSERT_EQUAL(-1, bitfield.getSparseMissingUnusedIndex()); +} diff --git a/test/Makefile.am b/test/Makefile.am index 49465030..aae91f06 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -37,7 +37,9 @@ aria2c_SOURCES = AllTest.cc\ MetalinkEntryTest.cc\ FeatureConfigTest.cc\ ShareRatioSeedCriteriaTest.cc\ - TimeSeedCriteriaTest.cc + TimeSeedCriteriaTest.cc\ + SegmentManTest.cc\ + SpeedCalcTest.cc #aria2c_CXXFLAGS = ${CPPUNIT_CFLAGS} -I../src -I../lib -Wall -D_FILE_OFFSET_BITS=64 #aria2c_LDFLAGS = ${CPPUNIT_LIBS} diff --git a/test/Makefile.in b/test/Makefile.in index 52718918..bd5ef14b 100644 --- a/test/Makefile.in +++ b/test/Makefile.in @@ -76,7 +76,8 @@ am_aria2c_OBJECTS = AllTest.$(OBJEXT) RequestTest.$(OBJEXT) \ Xml2MetalinkProcessorTest.$(OBJEXT) MetalinkerTest.$(OBJEXT) \ MetalinkEntryTest.$(OBJEXT) FeatureConfigTest.$(OBJEXT) \ ShareRatioSeedCriteriaTest.$(OBJEXT) \ - TimeSeedCriteriaTest.$(OBJEXT) + TimeSeedCriteriaTest.$(OBJEXT) SegmentManTest.$(OBJEXT) \ + SpeedCalcTest.$(OBJEXT) aria2c_OBJECTS = $(am_aria2c_OBJECTS) am__DEPENDENCIES_1 = aria2c_DEPENDENCIES = ../src/libaria2c.a $(am__DEPENDENCIES_1) @@ -267,7 +268,9 @@ aria2c_SOURCES = AllTest.cc\ MetalinkEntryTest.cc\ FeatureConfigTest.cc\ ShareRatioSeedCriteriaTest.cc\ - TimeSeedCriteriaTest.cc + TimeSeedCriteriaTest.cc\ + SegmentManTest.cc\ + SpeedCalcTest.cc #aria2c_CXXFLAGS = ${CPPUNIT_CFLAGS} -I../src -I../lib -Wall -D_FILE_OFFSET_BITS=64 #aria2c_LDFLAGS = ${CPPUNIT_LIBS} @@ -358,8 +361,10 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/RejectMessageTest.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/RequestMessageTest.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/RequestTest.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/SegmentManTest.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ShaVisitorTest.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ShareRatioSeedCriteriaTest.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/SpeedCalcTest.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/SuggestPieceMessageTest.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/TimeSeedCriteriaTest.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/TorrentManTest.Po@am__quote@