github
/
aria2
mirror of https://github.com/aria2/aria2
1
0
Fork 0
Code Issues Releases Wiki Activity

2007-03-24 Tatsuhiro Tsujikawa <tujikawa at rednoah dot com> 

To add the ability to resume downloading a partially downloaded 
file
	which is downloaded from the beginning:
	* src/FileAllocator.h: Made abstract class. New 
DefaultFileAllocator
	takes this role.
	* src/main.cc: Added -c option.
	* src/BitfieldMan.h, src/BitfieldMan.cc (setBitRange): New 
function.
	* src/DiskWriter.h (openExistingFile): Added totalLength 
argument.
	* src/prefs.h (PREF_CONTINUE): New definition.
	* src/SegmentMan.h, src/SegmentMan.cc (markPieceDone): New 
function.
	* src/DefaultDiskWriter.cc
	(createNewDiskWriter): Add GlowFileAllocator to the new object.
	* src/AbstractDiskWriter.h
	(glowFileAllocator): New variable.
	* src/AbstractDiskWriter.cc
	(openExistingFile): Now preallocate file space from the end of 
the
	existing file if totalLength argument is specified and grater 
than 0.
	* src/UrlRequestInfo.cc: If -c option is specified and the file
	to download exists in local, continue the download of the file.
	--allow-overwrite=true is assumed in this context.
	* src/DefaultFileAllocator.h, src/DefaultFileAllocator.cc: New 
class.
	* src/GlowFileAllocator.h, src/GlowFileAllocator.cc: New class.

	Throw exception if --check-integrity=true is specified but chunk
	checksums are not provided:
	* src/UrlRequestInfo.cc
pull/1/head
Tatsuhiro Tsujikawa 2007-03-24 14:32:49 +00:00
parent 5649cf6cda
commit a37aaa9c0c
28 changed files with 498 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
ChangeLog
View File

@ -1,3 +1,31 @@
2007-03-24 Tatsuhiro Tsujikawa <tujikawa at rednoah dot com>
To add the ability to resume downloading a partially downloaded file
which is downloaded from the beginning:
* src/FileAllocator.h: Made abstract class. New DefaultFileAllocator
takes this role.
* src/main.cc: Added -c option.
* src/BitfieldMan.h, src/BitfieldMan.cc (setBitRange): New function.
* src/DiskWriter.h (openExistingFile): Added totalLength argument.
* src/prefs.h (PREF_CONTINUE): New definition.
* src/SegmentMan.h, src/SegmentMan.cc (markPieceDone): New function.
* src/DefaultDiskWriter.cc
(createNewDiskWriter): Add GlowFileAllocator to the new object.
* src/AbstractDiskWriter.h
(glowFileAllocator): New variable.
* src/AbstractDiskWriter.cc
(openExistingFile): Now preallocate file space from the end of the
existing file if totalLength argument is specified and grater than 0.
* src/UrlRequestInfo.cc: If -c option is specified and the file
to download exists in local, continue the download of the file.
--allow-overwrite=true is assumed in this context.
* src/DefaultFileAllocator.h, src/DefaultFileAllocator.cc: New class.
* src/GlowFileAllocator.h, src/GlowFileAllocator.cc: New class.
Throw exception if --check-integrity=true is specified but chunk
checksums are not provided:
* src/UrlRequestInfo.cc
2007-03-21 Tatsuhiro Tsujikawa <tujikawa at rednoah dot com>
* src/Request.h: Use AuthResolver to get authentication information.

3
TODO
View File

@ -28,4 +28,5 @@
* -c command line option to continue the download of existing file assuming
that it was downloaded from the beginning.
* Continue file allocation with existing file
* keep-alive
* Netrc, mode 600, enabled in ftp, http, all
* preallocate file in MultiDiskAdaptor

11
src/AbstractDiskWriter.cc
View File

@ -47,6 +47,7 @@
AbstractDiskWriter::AbstractDiskWriter():
fd(0),
fileAllocator(0),
glowFileAllocator(0),
logger(LogFactory::getInstance())
{}
@ -71,7 +72,7 @@ void AbstractDiskWriter::closeFile() {
}
}
void AbstractDiskWriter::openExistingFile(const string& filename) {
void AbstractDiskWriter::openExistingFile(const string& filename, int64_t totalLength) {
this->filename = filename;
File f(filename);
if(!f.isFile()) {
@ -81,6 +82,14 @@ void AbstractDiskWriter::openExistingFile(const string& filename) {
if((fd = open(filename.c_str(), O_RDWR, S_IRUSR|S_IWUSR)) < 0) {
throw new DlAbortEx(EX_FILE_OPEN, filename.c_str(), strerror(errno));
}
if(f.size() < totalLength) {
if(!fileAllocator.isNull()) {
logger->notice("Allocating file %s, %s bytes",
filename.c_str(),
Util::ullitos(totalLength).c_str());
glowFileAllocator->allocate(fd, totalLength);
}
}
}
void AbstractDiskWriter::createFile(const string& filename, int32_t addFlags) {

11
src/AbstractDiskWriter.h
View File

@ -44,6 +44,7 @@ protected:
string filename;
int32_t fd;
FileAllocatorHandle fileAllocator;
FileAllocatorHandle glowFileAllocator;
const Logger* logger;
void createFile(const string& filename, int32_t addFlags = 0);
@ -62,7 +63,7 @@ public:
virtual void closeFile();
virtual void openExistingFile(const string& filename);
virtual void openExistingFile(const string& filename, int64_t totalLength = 0);
#ifdef ENABLE_MESSAGE_DIGEST
virtual string messageDigest(int64_t offset, int64_t length,
@ -73,9 +74,15 @@ public:
virtual int32_t readData(char* data, int32_t len, int64_t offset);
void setFileAllocator(const FileAllocatorHandle& fileAllocator) {
void setFileAllocator(const FileAllocatorHandle& fileAllocator)
{
this->fileAllocator = fileAllocator;
}
void setGlowFileAllocator(const FileAllocatorHandle& fileAllocator)
{
this->glowFileAllocator = fileAllocator;
}
};
#endif // _D_ABSTRACT_DISK_WRITER_H_

8
src/BitfieldMan.cc
View File

@ -645,6 +645,14 @@ void BitfieldMan::unsetBitRange(int32_t startIndex, int32_t endIndex)
updateCache();
}
void BitfieldMan::setBitRange(int32_t startIndex, int32_t endIndex)
{
for(int32_t i = startIndex; i <= endIndex; ++i) {
setBit(i);
}
updateCache();
}
bool BitfieldMan::isBitSetOffsetRange(int64_t offset, int64_t length) const
{
if(length <= 0) {

2
src/BitfieldMan.h
View File

@ -255,6 +255,8 @@ public:
void unsetBitRange(int32_t startIndex, int32_t endIndex);
void setBitRange(int32_t startIndex, int32_t endIndex);
bool isBitSetOffsetRange(int64_t offset, int64_t length) const;
int64_t getMissingUnusedLength(int32_t startingIndex) const;

3
src/ByteArrayDiskWriter.cc
View File

@ -68,7 +68,8 @@ void ByteArrayDiskWriter::closeFile() {
clear();
}
void ByteArrayDiskWriter::openExistingFile(const string& filename) {
void ByteArrayDiskWriter::openExistingFile(const string& filename,
int64_t totalLength) {
openFile(filename);
}

2
src/ByteArrayDiskWriter.h
View File

@ -55,7 +55,7 @@ public:
virtual void closeFile();
virtual void openExistingFile(const string& filename);
virtual void openExistingFile(const string& filename, int64_t totalLength = 0);
// position is ignored
virtual void writeData(const char* data, int32_t len, int64_t position = 0);

10
src/DefaultDiskWriter.cc
View File

@ -35,7 +35,8 @@
#include "DefaultDiskWriter.h"
#include "DlAbortEx.h"
#include "message.h"
#include "FileAllocator.h"
#include "DefaultFileAllocator.h"
#include "GlowFileAllocator.h"
#include "prefs.h"
#include "Util.h"
#include <errno.h>
@ -69,11 +70,16 @@ DefaultDiskWriter* DefaultDiskWriter::createNewDiskWriter(const Option* option)
{
DefaultDiskWriter* diskWriter = new DefaultDiskWriter();
if(option->get(PREF_FILE_ALLOCATION) == V_PREALLOC) {
FileAllocatorHandle allocator = new FileAllocator();
DefaultFileAllocatorHandle allocator = new DefaultFileAllocator();
allocator->setFileAllocationMonitor(FileAllocationMonitorFactory::getFactory()->createNewMonitor());
diskWriter->setFileAllocator(allocator);
GlowFileAllocatorHandle glowAllocator = new GlowFileAllocator();
glowAllocator->setFileAllocationMonitor(FileAllocationMonitorFactory::getFactory()->createNewMonitor());
diskWriter->setGlowFileAllocator(glowAllocator);
} else {
diskWriter->setFileAllocator(0);
diskWriter->setGlowFileAllocator(0);
}
return diskWriter;
}

5
src/FileAllocator.cc → src/DefaultFileAllocator.cc
View File

@ -1,4 +1,3 @@
/* <!-- copyright */
/*
* aria2 - The high speed download utility
@ -33,14 +32,14 @@
* files in the program, then also delete it here.
*/
/* copyright --> */
#include "FileAllocator.h"
#include "DefaultFileAllocator.h"
#include "DlAbortEx.h"
#include "TimeA2.h"
#include <sys/types.h>
#include <unistd.h>
#include <errno.h>
void FileAllocator::allocate(int fd, int64_t totalLength)
void DefaultFileAllocator::allocate(int fd, int64_t totalLength)
{
if(0 != lseek(fd, 0, SEEK_SET)) {
throw new DlAbortEx("Seek failed: %s", strerror(errno));

51
src/DefaultFileAllocator.h Normal file
View File

@ -0,0 +1,51 @@
/* <!-- copyright */
/*
* aria2 - The high speed download utility
*
* Copyright (C) 2006 Tatsuhiro Tsujikawa
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*
* In addition, as a special exception, the copyright holders give
* permission to link the code of portions of this program with the
* OpenSSL library under certain conditions as described in each
* individual source file, and distribute linked combinations
* including the two.
* You must obey the GNU General Public License in all respects
* for all of the code used other than OpenSSL. If you modify
* file(s) with this exception, you may extend this exception to your
* version of the file(s), but you are not obligated to do so. If you
* do not wish to do so, delete this exception statement from your
* version. If you delete this exception statement from all source
* files in the program, then also delete it here.
*/
/* copyright --> */
#ifndef _D_DEFAULT_FILE_ALLOCATOR_H_
#define _D_DEFAULT_FILE_ALLOCATOR_H_
#include "FileAllocator.h"
class DefaultFileAllocator : public FileAllocator {
public:
DefaultFileAllocator() {}
virtual ~DefaultFileAllocator() {}
virtual void allocate(int fd, int64_t totalLength);
};
typedef SharedHandle<DefaultFileAllocator> DefaultFileAllocatorHandle;
#endif // _D_DEFAULT_FILE_ALLOCATOR_H_

2
src/DiskAdaptorWriter.h
View File

@ -62,7 +62,7 @@ public:
diskAdaptor->closeFile();
}
virtual void openExistingFile(const string& filename)
virtual void openExistingFile(const string& filename, int64_t totalLength = 0)
{
diskAdaptor->openExistingFile();
}

2
src/DiskWriter.h
View File

@ -70,7 +70,7 @@ public:
*
* @param filename the file name to be opened.
*/
virtual void openExistingFile(const string& filename) = 0;
virtual void openExistingFile(const string& filename, int64_t totalLength = 0) = 0;
/*
* Writes len bytes from data to this binary stream at offset position.

6
src/FileAllocator.h
View File

@ -40,14 +40,14 @@
#include "NullFileAllocationMonitor.h"
class FileAllocator {
private:
protected:
FileAllocationMonitorHandle fileAllocationMonitor;
public:
FileAllocator():fileAllocationMonitor(new NullFileAllocationMonitor()) {}
~FileAllocator() {}
virtual ~FileAllocator() {}
void allocate(int fd, int64_t totalLength);
virtual void allocate(int fd, int64_t totalLength) = 0;
void setFileAllocationMonitor(const FileAllocationMonitorHandle& monitor)
{

74
src/GlowFileAllocator.cc Normal file
View File

@ -0,0 +1,74 @@
/* <!-- copyright */
/*
* aria2 - The high speed download utility
*
* Copyright (C) 2006 Tatsuhiro Tsujikawa
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*
* In addition, as a special exception, the copyright holders give
* permission to link the code of portions of this program with the
* OpenSSL library under certain conditions as described in each
* individual source file, and distribute linked combinations
* including the two.
* You must obey the GNU General Public License in all respects
* for all of the code used other than OpenSSL. If you modify
* file(s) with this exception, you may extend this exception to your
* version of the file(s), but you are not obligated to do so. If you
* do not wish to do so, delete this exception statement from your
* version. If you delete this exception statement from all source
* files in the program, then also delete it here.
*/
/* copyright --> */
#include "GlowFileAllocator.h"
#include "DlAbortEx.h"
#include "TimeA2.h"
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <errno.h>
void GlowFileAllocator::allocate(int fd, int64_t totalLength)
{
struct stat fs;
if(fstat(fd, &fs) < 0) {
throw new DlAbortEx("fstat filed: %s", strerror(errno));
}
if(fs.st_size != lseek(fd, 0, SEEK_END)) {
throw new DlAbortEx("Seek failed: %s", strerror(errno));
}
int32_t bufSize = 4096;
char buf[4096];
memset(buf, 0, bufSize);
int64_t x = (totalLength-fs.st_size+bufSize-1)/bufSize;
fileAllocationMonitor->setMinValue(0);
fileAllocationMonitor->setMaxValue(totalLength);
fileAllocationMonitor->setCurrentValue(fs.st_size);
fileAllocationMonitor->showProgress();
Time cp;
for(int64_t i = 0; i < x; ++i) {
if(write(fd, buf, bufSize) < 0) {
throw new DlAbortEx("Allocation failed: %s", strerror(errno));
}
if(cp.elapsedInMillis(500)) {
fileAllocationMonitor->setCurrentValue(i*bufSize);
fileAllocationMonitor->showProgress();
cp.reset();
}
}
fileAllocationMonitor->setCurrentValue(totalLength);
fileAllocationMonitor->showProgress();
ftruncate(fd, totalLength);
}

51
src/GlowFileAllocator.h Normal file
View File

@ -0,0 +1,51 @@
/* <!-- copyright */
/*
* aria2 - The high speed download utility
*
* Copyright (C) 2006 Tatsuhiro Tsujikawa
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*
* In addition, as a special exception, the copyright holders give
* permission to link the code of portions of this program with the
* OpenSSL library under certain conditions as described in each
* individual source file, and distribute linked combinations
* including the two.
* You must obey the GNU General Public License in all respects
* for all of the code used other than OpenSSL. If you modify
* file(s) with this exception, you may extend this exception to your
* version of the file(s), but you are not obligated to do so. If you
* do not wish to do so, delete this exception statement from your
* version. If you delete this exception statement from all source
* files in the program, then also delete it here.
*/
/* copyright --> */
#ifndef _D_GLOW_FILE_ALLOCATOR_H_
#define _D_GLOW_FILE_ALLOCATOR_H_
#include "FileAllocator.h"
class GlowFileAllocator : public FileAllocator {
public:
GlowFileAllocator() {}
virtual ~GlowFileAllocator() {}
virtual void allocate(int fd, int64_t totalLength);
};
typedef SharedHandle<GlowFileAllocator> GlowFileAllocatorHandle;
#endif // _D_GLOW_FILE_ALLOCATOR_H_

6
src/Makefile.am
View File

@ -61,7 +61,7 @@ SRCS = Socket.h\
BitfieldManFactory.cc BitfieldManFactory.h\
Randomizer.h\
SimpleRandomizer.cc SimpleRandomizer.h\
FileAllocator.cc FileAllocator.h\
FileAllocator.h\
FileAllocationMonitor.cc FileAllocationMonitor.h\
ConsoleFileAllocationMonitor.cc ConsoleFileAllocationMonitor.h\
ChunkChecksumValidator.cc ChunkChecksumValidator.h\
@ -76,7 +76,9 @@ SRCS = Socket.h\
AbstractAuthResolver.h\
DefaultAuthResolver.cc DefaultAuthResolver.h\
NetrcAuthResolver.cc NetrcAuthResolver.h\
RequestFactory.cc RequestFactory.h
RequestFactory.cc RequestFactory.h\
DefaultFileAllocator.cc DefaultFileAllocator.h\
GlowFileAllocator.cc GlowFileAllocator.h
# debug_new.cpp
if ENABLE_ASYNC_DNS

56
src/Makefile.in
View File

@ -210,18 +210,19 @@ am__libaria2c_a_SOURCES_DIST = Socket.h SocketCore.cc SocketCore.h \
UrlRequestInfo.cc UrlRequestInfo.h SpeedCalc.cc SpeedCalc.h \
PeerStat.h BitfieldMan.cc BitfieldMan.h BitfieldManFactory.cc \
BitfieldManFactory.h Randomizer.h SimpleRandomizer.cc \
SimpleRandomizer.h FileAllocator.cc FileAllocator.h \
FileAllocationMonitor.cc FileAllocationMonitor.h \
ConsoleFileAllocationMonitor.cc ConsoleFileAllocationMonitor.h \
ChunkChecksumValidator.cc ChunkChecksumValidator.h \
HttpResponse.cc HttpResponse.h HttpRequest.cc HttpRequest.h \
Range.h AbstractProxyRequestCommand.cc \
AbstractProxyRequestCommand.h AbstractProxyResponseCommand.cc \
AbstractProxyResponseCommand.h Netrc.cc Netrc.h AuthConfig.cc \
AuthConfig.h AuthResolver.h AbstractAuthResolver.h \
DefaultAuthResolver.cc DefaultAuthResolver.h \
NetrcAuthResolver.cc NetrcAuthResolver.h RequestFactory.cc \
RequestFactory.h NameResolver.cc NameResolver.h MetaEntry.h \
SimpleRandomizer.h FileAllocator.h FileAllocationMonitor.cc \
FileAllocationMonitor.h ConsoleFileAllocationMonitor.cc \
ConsoleFileAllocationMonitor.h ChunkChecksumValidator.cc \
ChunkChecksumValidator.h HttpResponse.cc HttpResponse.h \
HttpRequest.cc HttpRequest.h Range.h \
AbstractProxyRequestCommand.cc AbstractProxyRequestCommand.h \
AbstractProxyResponseCommand.cc AbstractProxyResponseCommand.h \
Netrc.cc Netrc.h AuthConfig.cc AuthConfig.h AuthResolver.h \
AbstractAuthResolver.h DefaultAuthResolver.cc \
DefaultAuthResolver.h NetrcAuthResolver.cc NetrcAuthResolver.h \
RequestFactory.cc RequestFactory.h DefaultFileAllocator.cc \
DefaultFileAllocator.h GlowFileAllocator.cc \
GlowFileAllocator.h NameResolver.cc NameResolver.h MetaEntry.h \
Data.cc Data.h Dictionary.cc Dictionary.h List.cc List.h \
MetaFileUtil.cc MetaFileUtil.h MetaEntryVisitor.h \
ShaVisitor.cc ShaVisitor.h PeerConnection.cc PeerConnection.h \
@ -384,13 +385,14 @@ am__objects_4 = SocketCore.$(OBJEXT) Command.$(OBJEXT) \
DownloadEngineFactory.$(OBJEXT) UrlRequestInfo.$(OBJEXT) \
SpeedCalc.$(OBJEXT) BitfieldMan.$(OBJEXT) \
BitfieldManFactory.$(OBJEXT) SimpleRandomizer.$(OBJEXT) \
FileAllocator.$(OBJEXT) FileAllocationMonitor.$(OBJEXT) \
FileAllocationMonitor.$(OBJEXT) \
ConsoleFileAllocationMonitor.$(OBJEXT) \
ChunkChecksumValidator.$(OBJEXT) HttpResponse.$(OBJEXT) \
HttpRequest.$(OBJEXT) AbstractProxyRequestCommand.$(OBJEXT) \
AbstractProxyResponseCommand.$(OBJEXT) Netrc.$(OBJEXT) \
AuthConfig.$(OBJEXT) DefaultAuthResolver.$(OBJEXT) \
NetrcAuthResolver.$(OBJEXT) RequestFactory.$(OBJEXT) \
DefaultFileAllocator.$(OBJEXT) GlowFileAllocator.$(OBJEXT) \
$(am__objects_1) $(am__objects_2) $(am__objects_3)
am_libaria2c_a_OBJECTS = $(am__objects_4)
libaria2c_a_OBJECTS = $(am_libaria2c_a_OBJECTS)
@ -594,18 +596,19 @@ SRCS = Socket.h SocketCore.cc SocketCore.h Command.cc Command.h \
UrlRequestInfo.cc UrlRequestInfo.h SpeedCalc.cc SpeedCalc.h \
PeerStat.h BitfieldMan.cc BitfieldMan.h BitfieldManFactory.cc \
BitfieldManFactory.h Randomizer.h SimpleRandomizer.cc \
SimpleRandomizer.h FileAllocator.cc FileAllocator.h \
FileAllocationMonitor.cc FileAllocationMonitor.h \
ConsoleFileAllocationMonitor.cc ConsoleFileAllocationMonitor.h \
ChunkChecksumValidator.cc ChunkChecksumValidator.h \
HttpResponse.cc HttpResponse.h HttpRequest.cc HttpRequest.h \
Range.h AbstractProxyRequestCommand.cc \
AbstractProxyRequestCommand.h AbstractProxyResponseCommand.cc \
AbstractProxyResponseCommand.h Netrc.cc Netrc.h AuthConfig.cc \
AuthConfig.h AuthResolver.h AbstractAuthResolver.h \
DefaultAuthResolver.cc DefaultAuthResolver.h \
NetrcAuthResolver.cc NetrcAuthResolver.h RequestFactory.cc \
RequestFactory.h $(am__append_1) $(am__append_2) \
SimpleRandomizer.h FileAllocator.h FileAllocationMonitor.cc \
FileAllocationMonitor.h ConsoleFileAllocationMonitor.cc \
ConsoleFileAllocationMonitor.h ChunkChecksumValidator.cc \
ChunkChecksumValidator.h HttpResponse.cc HttpResponse.h \
HttpRequest.cc HttpRequest.h Range.h \
AbstractProxyRequestCommand.cc AbstractProxyRequestCommand.h \
AbstractProxyResponseCommand.cc AbstractProxyResponseCommand.h \
Netrc.cc Netrc.h AuthConfig.cc AuthConfig.h AuthResolver.h \
AbstractAuthResolver.h DefaultAuthResolver.cc \
DefaultAuthResolver.h NetrcAuthResolver.cc NetrcAuthResolver.h \
RequestFactory.cc RequestFactory.h DefaultFileAllocator.cc \
DefaultFileAllocator.h GlowFileAllocator.cc \
GlowFileAllocator.h $(am__append_1) $(am__append_2) \
$(am__append_3)
noinst_LIBRARIES = libaria2c.a
libaria2c_a_SOURCES = $(SRCS)
@ -743,6 +746,7 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DefaultBtProgressInfoFile.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DefaultBtRequestFactory.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DefaultDiskWriter.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DefaultFileAllocator.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DefaultPeerListProcessor.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DefaultPeerStorage.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DefaultPieceStorage.Po@am__quote@
@ -757,7 +761,6 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FeatureConfig.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/File.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FileAllocationMonitor.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FileAllocator.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FileEntry.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FtpConnection.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FtpDownloadCommand.Po@am__quote@
@ -765,6 +768,7 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FtpNegotiationCommand.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FtpTunnelRequestCommand.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FtpTunnelResponseCommand.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/GlowFileAllocator.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/HaveEraseCommand.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/HttpConnection.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/HttpDownloadCommand.Po@am__quote@

21
src/SegmentMan.cc
View File

@ -450,6 +450,27 @@ void SegmentMan::markAllPiecesDone()
}
}
void SegmentMan::markPieceDone(int64_t length)
{
if(bitfield) {
if(length == bitfield->getTotalLength()) {
bitfield->setAllBit();
} else {
int32_t numSegment = length/bitfield->getBlockLength();
int32_t remainingLength = length%bitfield->getBlockLength();
bitfield->setBitRange(0, numSegment-1);
if(remainingLength > 0) {
SegmentHandle segment = new Segment();
segment->index = numSegment;
segment->length = bitfield->getBlockLength(numSegment);
segment->segmentLength = bitfield->getBlockLength();
segment->writtenLength = remainingLength;
usedSegmentEntries.push_back(new SegmentEntry(0, segment));
}
}
}
}
#ifdef ENABLE_MESSAGE_DIGEST
void SegmentMan::checkIntegrity()
{

2
src/SegmentMan.h
View File

@ -285,6 +285,8 @@ public:
void markAllPiecesDone();
void markPieceDone(int64_t length);
#ifdef ENABLE_MESSAGE_DIGEST
void checkIntegrity();

50
src/UrlRequestInfo.cc
View File

@ -41,6 +41,10 @@
#include "FatalException.h"
#include "message.h"
#include "RequestFactory.h"
#include "GlowFileAllocator.h"
#include "File.h"
#include "DefaultDiskWriter.h"
#include "DlAbortEx.h"
std::ostream& operator<<(std::ostream& o, const HeadResult& hr) {
o << "filename = " << hr.filename << ", " << "totalLength = " << hr.totalLength;
@ -184,11 +188,41 @@ RequestInfos UrlRequestInfo::execute() {
}
#endif // ENABLE_MESSAGE_DIGEST
if(e->segmentMan->segmentFileExists()) {
e->segmentMan->load();
e->segmentMan->diskWriter->openExistingFile(e->segmentMan->getFilePath());
#ifdef ENABLE_MESSAGE_DIGEST
if(op->get(PREF_CONTINUE) == V_TRUE && e->segmentMan->fileExists()) {
if(e->segmentMan->totalSize == 0) {
logger->notice("Cannot get file length. Download aborted.");
return RequestInfos();
}
File existingFile(e->segmentMan->getFilePath());
if(e->segmentMan->totalSize < existingFile.size()) {
logger->notice("The local file length is larger than the remote file size. Download aborted.");
return RequestInfos();
}
e->segmentMan->initBitfield(e->option->getAsInt(PREF_SEGMENT_SIZE),
e->segmentMan->totalSize);
e->segmentMan->diskWriter->openExistingFile(e->segmentMan->getFilePath(),
e->segmentMan->totalSize);
if(e->option->get(PREF_CHECK_INTEGRITY) == V_TRUE) {
#ifdef ENABLE_MESSAGE_DIGEST
if(!e->segmentMan->isChunkChecksumValidationReady()) {
throw new DlAbortEx("Chunk checksums are not provided.");
}
e->segmentMan->markAllPiecesDone();
e->segmentMan->checkIntegrity();
#endif // ENABLE_MESSAGE_DIGEST
} else {
e->segmentMan->markPieceDone(existingFile.size());
}
} else if(e->segmentMan->segmentFileExists()) {
e->segmentMan->load();
e->segmentMan->diskWriter->openExistingFile(e->segmentMan->getFilePath(),
e->segmentMan->totalSize);
#ifdef ENABLE_MESSAGE_DIGEST
if(op->get(PREF_CHECK_INTEGRITY) == V_TRUE) {
if(!e->segmentMan->isChunkChecksumValidationReady()) {
throw new DlAbortEx("Chunk checksums are not provided.");
}
e->segmentMan->checkIntegrity();
}
#endif // ENABLE_MESSAGE_DIGEST
@ -202,7 +236,13 @@ RequestInfos UrlRequestInfo::execute() {
e->segmentMan->initBitfield(e->option->getAsInt(PREF_SEGMENT_SIZE),
e->segmentMan->totalSize);
if(e->segmentMan->fileExists() && e->option->get(PREF_CHECK_INTEGRITY) == V_TRUE) {
e->segmentMan->diskWriter->openExistingFile(e->segmentMan->getFilePath());
#ifdef ENABLE_MESSAGE_DIGEST
if(!e->segmentMan->isChunkChecksumValidationReady()) {
throw new DlAbortEx("Chunk checksums are not provided.");
}
#endif // ENABLE_MESSAGE_DIGEST
e->segmentMan->diskWriter->openExistingFile(e->segmentMan->getFilePath(),
e->segmentMan->totalSize);
#ifdef ENABLE_MESSAGE_DIGEST
e->segmentMan->markAllPiecesDone();
e->segmentMan->checkIntegrity();

9
src/main.cc
View File

@ -352,6 +352,7 @@ int main(int argc, char* argv[]) {
op->put(PREF_REALTIME_CHUNK_CHECKSUM, V_TRUE);
op->put(PREF_CHECK_INTEGRITY, V_FALSE);
op->put(PREF_NETRC_PATH, Util::getHomeDir()+"/.netrc");
op->put(PREF_CONTINUE, V_FALSE);
while(1) {
int optIndex = 0;
int lopt;
@ -383,7 +384,8 @@ int main(int argc, char* argv[]) {
{ "file-allocation", required_argument, 0, 'a' },
{ "allow-overwrite", required_argument, &lopt, 202 },
{ "check-integrity", required_argument, &lopt, 203 },
{" realtime-chunk-checksum", required_argument, &lopt, 204 },
{ "realtime-chunk-checksum", required_argument, &lopt, 204 },
{ "continue", no_argument, NULL, 'c' },
#ifdef ENABLE_BITTORRENT
{ "torrent-file", required_argument, NULL, 'T' },
{ "listen-port", required_argument, &lopt, 15 },
@ -411,7 +413,7 @@ int main(int argc, char* argv[]) {
{ "help", no_argument, NULL, 'h' },
{ 0, 0, 0, 0 }
};
c = getopt_long(argc, argv, "Dd:o:l:s:pt:m:vhST:M:C:a:", longOpts, &optIndex);
c = getopt_long(argc, argv, "Dd:o:l:s:pt:m:vhST:M:C:a:c", longOpts, &optIndex);
if(c == -1) {
break;
}
@ -717,6 +719,9 @@ int main(int argc, char* argv[]) {
}
break;
}
case 'c':
op->put(PREF_CONTINUE, V_TRUE);
break;
case 'v':
showVersion();
exit(EXIT_SUCCESS);

2
src/prefs.h
View File

@ -92,6 +92,8 @@
#define PREF_CHECK_INTEGRITY "check_integrity"
// value: string that your file system recognizes as a file name.
#define PREF_NETRC_PATH "netrc_path"
// value:
#define PREF_CONTINUE "continue"
/**
* FTP related preferences

20
test/BitfieldManTest.cc
View File

@ -17,6 +17,7 @@ class BitfieldManTest:public CppUnit::TestFixture {
CPPUNIT_TEST(testGetSparceMissingUnusedIndex);
CPPUNIT_TEST(testIsBitSetOffsetRange);
CPPUNIT_TEST(testGetMissingUnusedLength);
CPPUNIT_TEST(testSetBitRange);
CPPUNIT_TEST_SUITE_END();
private:
RandomizerHandle fixedNumberRandomizer;
@ -39,6 +40,7 @@ public:
void testGetSparceMissingUnusedIndex();
void testIsBitSetOffsetRange();
void testGetMissingUnusedLength();
void testSetBitRange();
};
@ -303,3 +305,21 @@ void BitfieldManTest::testGetMissingUnusedLength()
// from index 1
CPPUNIT_ASSERT_EQUAL((int64_t)3*blockLength, bf.getMissingUnusedLength(1));
}
void BitfieldManTest::testSetBitRange()
{
int32_t blockLength = 1024*1024;
int64_t totalLength = 10*blockLength;
BitfieldMan bf(blockLength, totalLength);
bf.setBitRange(0, 4);
for(int32_t i = 0; i < 5; ++i) {
CPPUNIT_ASSERT(bf.isBitSet(i));
}
for(int32_t i = 5; i < 10; ++i) {
CPPUNIT_ASSERT(!bf.isBitSet(i));
}
CPPUNIT_ASSERT_EQUAL(int64_t(5*blockLength), bf.getCompletedLength());
}

58
test/GlowFileAllocatorTest.cc Normal file
View File

@ -0,0 +1,58 @@
#include "GlowFileAllocator.h"
#include "File.h"
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <fstream>
#include <iomanip>
#include <cppunit/extensions/HelperMacros.h>
class GlowFileAllocatorTest:public CppUnit::TestFixture {
CPPUNIT_TEST_SUITE(GlowFileAllocatorTest);
CPPUNIT_TEST(testAllocate);
CPPUNIT_TEST_SUITE_END();
private:
public:
void setUp() {}
void testAllocate();
};
CPPUNIT_TEST_SUITE_REGISTRATION( GlowFileAllocatorTest );
void GlowFileAllocatorTest::testAllocate()
{
string fn = "/tmp/aria2_GlowFileAllocatorTest_testAllocate";
ofstream of(fn.c_str());
of << "0123456789";
of.close();
File x("/tmp/aria2_GlowFileAllocatorTest_testAllocate");
CPPUNIT_ASSERT_EQUAL((int64_t)10, x.size());
int fd;
if((fd = open(fn.c_str(), O_RDWR, S_IRUSR|S_IWUSR)) < 0) {
CPPUNIT_FAIL("cannot open file");
}
GlowFileAllocator allocator;
allocator.allocate(fd, 4097);
if(close(fd) < 0) {
CPPUNIT_FAIL("cannot close file");
}
ifstream is(fn.c_str());
char buf[11];
is >> std::setw(sizeof(buf)) >> buf;
CPPUNIT_ASSERT(strcmp("0123456789", buf) == 0);
File f(fn);
CPPUNIT_ASSERT_EQUAL((int64_t)4097, f.size());
}

5
test/Makefile.am
View File

@ -1,13 +1,15 @@
TESTS = aria2c
check_PROGRAMS = $(TESTS)
aria2c_SOURCES = AllTest.cc\
SegmentManTest.cc\
BitfieldManTest.cc\
GlowFileAllocatorTest.cc\
RequestTest.cc\
HttpRequestTest.cc\
NetrcTest.cc\
SingletonHolderTest.cc\
HttpHeaderTest.cc\
HttpResponseTest.cc\
BitfieldManTest.cc\
SharedHandleTest.cc\
ChunkedEncodingTest.cc\
FileTest.cc\
@ -29,7 +31,6 @@ aria2c_SOURCES = AllTest.cc\
FeatureConfigTest.cc\
ShareRatioSeedCriteriaTest.cc\
TimeSeedCriteriaTest.cc\
SegmentManTest.cc\
SpeedCalcTest.cc\
DefaultPeerListProcessorTest.cc\
AnnounceListTest.cc\

19
test/Makefile.in
View File

@ -57,10 +57,11 @@ mkinstalldirs = $(SHELL) $(top_srcdir)/mkinstalldirs
CONFIG_HEADER = $(top_builddir)/config.h
CONFIG_CLEAN_FILES =
am__EXEEXT_1 = aria2c$(EXEEXT)
am_aria2c_OBJECTS = AllTest.$(OBJEXT) RequestTest.$(OBJEXT) \
HttpRequestTest.$(OBJEXT) NetrcTest.$(OBJEXT) \
SingletonHolderTest.$(OBJEXT) HttpHeaderTest.$(OBJEXT) \
HttpResponseTest.$(OBJEXT) BitfieldManTest.$(OBJEXT) \
am_aria2c_OBJECTS = AllTest.$(OBJEXT) SegmentManTest.$(OBJEXT) \
BitfieldManTest.$(OBJEXT) GlowFileAllocatorTest.$(OBJEXT) \
RequestTest.$(OBJEXT) HttpRequestTest.$(OBJEXT) \
NetrcTest.$(OBJEXT) SingletonHolderTest.$(OBJEXT) \
HttpHeaderTest.$(OBJEXT) HttpResponseTest.$(OBJEXT) \
SharedHandleTest.$(OBJEXT) ChunkedEncodingTest.$(OBJEXT) \
FileTest.$(OBJEXT) OptionTest.$(OBJEXT) Base64Test.$(OBJEXT) \
UtilTest.$(OBJEXT) CookieBoxTest.$(OBJEXT) DataTest.$(OBJEXT) \
@ -71,8 +72,8 @@ am_aria2c_OBJECTS = AllTest.$(OBJEXT) RequestTest.$(OBJEXT) \
Xml2MetalinkProcessorTest.$(OBJEXT) MetalinkerTest.$(OBJEXT) \
MetalinkEntryTest.$(OBJEXT) FeatureConfigTest.$(OBJEXT) \
ShareRatioSeedCriteriaTest.$(OBJEXT) \
TimeSeedCriteriaTest.$(OBJEXT) SegmentManTest.$(OBJEXT) \
SpeedCalcTest.$(OBJEXT) DefaultPeerListProcessorTest.$(OBJEXT) \
TimeSeedCriteriaTest.$(OBJEXT) SpeedCalcTest.$(OBJEXT) \
DefaultPeerListProcessorTest.$(OBJEXT) \
AnnounceListTest.$(OBJEXT) TrackerWatcherCommandTest.$(OBJEXT) \
DefaultBtContextTest.$(OBJEXT) \
DefaultPieceStorageTest.$(OBJEXT) \
@ -258,13 +259,15 @@ sysconfdir = @sysconfdir@
target_alias = @target_alias@
TESTS = aria2c
aria2c_SOURCES = AllTest.cc\
SegmentManTest.cc\
BitfieldManTest.cc\
GlowFileAllocatorTest.cc\
RequestTest.cc\
HttpRequestTest.cc\
NetrcTest.cc\
SingletonHolderTest.cc\
HttpHeaderTest.cc\
HttpResponseTest.cc\
BitfieldManTest.cc\
SharedHandleTest.cc\
ChunkedEncodingTest.cc\
FileTest.cc\
@ -286,7 +289,6 @@ aria2c_SOURCES = AllTest.cc\
FeatureConfigTest.cc\
ShareRatioSeedCriteriaTest.cc\
TimeSeedCriteriaTest.cc\
SegmentManTest.cc\
SpeedCalcTest.cc\
DefaultPeerListProcessorTest.cc\
AnnounceListTest.cc\
@ -423,6 +425,7 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DictionaryTest.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FeatureConfigTest.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FileTest.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/GlowFileAllocatorTest.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/HttpHeaderTest.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/HttpRequestTest.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/HttpResponseTest.Po@am__quote@

43
test/SegmentManTest.cc
View File

@ -12,6 +12,8 @@ class SegmentManTest:public CppUnit::TestFixture {
CPPUNIT_TEST(testSaveAndLoad);
CPPUNIT_TEST(testNullBitfield);
CPPUNIT_TEST(testCancelSegmentOnNullBitfield);
CPPUNIT_TEST(testMarkPieceDone);
CPPUNIT_TEST(testMarkPieceDone_usedSegment);
CPPUNIT_TEST_SUITE_END();
private:
@ -22,7 +24,8 @@ public:
void testSaveAndLoad();
void testNullBitfield();
void testCancelSegmentOnNullBitfield();
void testMarkPieceDone();
void testMarkPieceDone_usedSegment();
};
@ -109,3 +112,41 @@ void SegmentManTest::testCancelSegmentOnNullBitfield() {
CPPUNIT_ASSERT(!segmentMan.getSegment(1).isNull());
}
void SegmentManTest::testMarkPieceDone()
{
SegmentMan segmentMan;
int32_t pieceLength = 1024*1024;
int64_t totalLength = 10*pieceLength;
segmentMan.initBitfield(pieceLength, totalLength);
segmentMan.markPieceDone(5*pieceLength);
for(int32_t i = 0; i < 5; ++i) {
CPPUNIT_ASSERT(segmentMan.hasSegment(i));
}
for(int32_t i = 5; i < 10; ++i) {
CPPUNIT_ASSERT(!segmentMan.hasSegment(i));
}
}
void SegmentManTest::testMarkPieceDone_usedSegment()
{
SegmentMan segmentMan;
int32_t pieceLength = 1024*1024;
int64_t totalLength = 10*pieceLength;
segmentMan.initBitfield(pieceLength, totalLength);
segmentMan.markPieceDone(5*pieceLength+100);
for(int32_t i = 0; i < 5; ++i) {
CPPUNIT_ASSERT(segmentMan.hasSegment(i));
}
for(int32_t i = 5; i < 10; ++i) {
CPPUNIT_ASSERT(!segmentMan.hasSegment(i));
}
SegmentHandle segment = segmentMan.getSegment(0, 5);
CPPUNIT_ASSERT(!segment.isNull());
CPPUNIT_ASSERT_EQUAL(5, segment->index);
CPPUNIT_ASSERT_EQUAL(pieceLength, segment->length);
CPPUNIT_ASSERT_EQUAL(pieceLength, segment->segmentLength);
CPPUNIT_ASSERT_EQUAL(100, segment->writtenLength);
}