github
/
aria2
mirror of https://github.com/aria2/aria2
1
0
Fork 0
Code Issues Releases Wiki Activity

Support for gzipped -i and --save-session 

Saved sessions may very large, as in hundreds and even thousands of
megabyte when dealing with large queues.
Add support to save and reload sessions to gzipped files, when libz is
available.

The session serializer will output gzipped contents when the file ends
with .gz, while the input file reader (UriListParser) will accept
whatever is thrown at it.
pull/55/head
Nils Maier 2013-03-01 16:07:16 +01:00
parent b0556a7519
commit f2f8b14047
10 changed files with 328 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
configure.ac
View File

@ -100,6 +100,7 @@ if test "x$with_libz" = "xyes"; then
AC_DEFINE([HAVE_ZLIB], [1], [Define to 1 if you have zlib.])
LIBS="$ZLIB_LIBS $LIBS"
CPPFLAGS="$ZLIB_CFLAGS $CPPFLAGS"
AC_CHECK_FUNCS([gzbuffer gzsetparams])
else
AC_MSG_WARN([$ZLIB_PKG_ERRORS])
if test "x$with_libz_requested" = "xyes"; then

12
src/BufferedFile.cc
View File

@ -59,7 +59,7 @@ BufferedFile::BufferedFile(const char* filename, const char* mode)
{}
BufferedFile::BufferedFile(FILE* fp)
: fp_(fp), open_(true), supportsColor_(fp_ ? isatty(fileno(fp_)) : false)
: fp_(fp), open_(fp_), supportsColor_(fp_ ? isatty(fileno(fp_)) : false)
{}
BufferedFile::~BufferedFile()
@ -69,7 +69,8 @@ BufferedFile::~BufferedFile()
BufferedFile::operator unspecified_bool_type() const
{
return (!open_ || ferror(fp_)) ? 0 : &BufferedFile::good_state;
bool ok = isOpen() && !isError();
return ok ? &BufferedFile::good_state : 0;
}
size_t BufferedFile::read(void* ptr, size_t count)
@ -94,7 +95,7 @@ char* BufferedFile::gets(char* s, int size)
char* BufferedFile::getsn(char* s, int size)
{
char* ptr = fgets(s, size, fp_);
char* ptr = gets(s, size);
if(ptr) {
int len = strlen(ptr);
if(ptr[len-1] == '\n') {
@ -131,14 +132,13 @@ int BufferedFile::close()
if (open_) {
open_ = false;
return fclose(fp_);
} else {
return 0;
}
return 0;
}
bool BufferedFile::eof()
{
return open_ && feof(fp_);
return !isOpen()|| isEOF();
}
size_t BufferedFile::transfer(std::ostream& out)

16
src/BufferedFile.h
View File

@ -56,18 +56,18 @@ public:
// returns false.
operator unspecified_bool_type() const;
// wrapper for fread. Using 1 for 2nd argument of fread.
size_t read(void* ptr, size_t count);
virtual size_t read(void* ptr, size_t count);
// wrapper for fwrite. Using 1 for 2nd argument of fwrite.
size_t write(const void* ptr, size_t count);
virtual size_t write(const void* ptr, size_t count);
virtual size_t write(const char* str);
// wrapper for fgets
char* gets(char* s, int size);
virtual char* gets(char* s, int size);
// wrapper for fgets, but trailing '\n' is replaced with '\0'.
char* getsn(char* s, int size);
// Reads one line and returns it. The last '\n' is removed.
std::string getLine();
// wrapper for fclose
int close();
virtual int close();
// Return true if open_ && feof(fp_) != 0. Otherwise returns false.
bool eof();
// Convenient method. Read data to end of file and write them into
@ -83,15 +83,21 @@ public:
static const char WRITE[];
// Mode for append
static const char APPEND[];
private:
// Don't allow copying
BufferedFile(const BufferedFile&);
BufferedFile& operator=(const BufferedFile&);
FILE* fp_;
// true when file has been opened.
bool open_;
bool supportsColor_;
protected:
virtual bool isError() const { return ferror(fp_); }
virtual bool isEOF() const { return feof(fp_); }
virtual bool isOpen() const { return open_; }
};
} // namespace aria2

161
src/GZipFile.cc Normal file
View File

@ -0,0 +1,161 @@
/* <!-- copyright */
/*
* aria2 - The high speed download utility
*
* Copyright (C) 2013 Nils Maier
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*
* In addition, as a special exception, the copyright holders give
* permission to link the code of portions of this program with the
* OpenSSL library under certain conditions as described in each
* individual source file, and distribute linked combinations
* including the two.
* You must obey the GNU General Public License in all respects
* for all of the code used other than OpenSSL. If you modify
* file(s) with this exception, you may extend this exception to your
* version of the file(s), but you are not obligated to do so. If you
* do not wish to do so, delete this exception statement from your
* version. If you delete this exception statement from all source
* files in the program, then also delete it here.
*/
/* copyright --> */
#include "GZipFile.h"
#include <algorithm>
#include <limits>
#include "a2io.h"
#include "util.h"
namespace aria2 {
GZipFile::GZipFile(const char* filename, const char* mode)
: BufferedFile(0), fp_(0), open_(false)
{
FILE* fp =
#ifdef __MINGW32__
a2fopen(utf8ToWChar(filename).c_str(), utf8ToWChar(mode).c_str());
#else // !__MINGW32__
a2fopen(filename, mode);
#endif // !__MINGW32__
open_ = fp;
if (open_) {
int fd = dup(fileno(fp));
if ((open_ = fd) >= 0) {
open_ = (fp_ = gzdopen(fd, mode));
if (!open_) {
::close(fd);
}
}
if (open_) {
#if HAVE_GZBUFFER
gzbuffer(fp_, 1<<17);
#endif
#if HAVE_GZSETPARAMS
gzsetparams(fp_, 2, Z_DEFAULT_STRATEGY);
#endif
}
fclose(fp);
}
}
int GZipFile::close()
{
if (open_) {
open_ = false;
return gzclose(fp_);
}
return 0;
}
bool GZipFile::isError() const
{
int rv = 0;
const char *e = gzerror(fp_, &rv);
return (e != 0 && *e != 0) || rv != 0;
}
size_t GZipFile::read(void* ptr, size_t count)
{
char *data = reinterpret_cast<char*>(ptr);
size_t read = 0;
while (count) {
size_t len = std::min(count, (size_t)std::numeric_limits<unsigned>::max());
int rv = gzread(fp_, data, len);
if (rv <= 0) {
break;
}
count -= rv;
read += rv;
data += rv;
}
return read;
}
size_t GZipFile::write(const void* ptr, size_t count)
{
const char *data = reinterpret_cast<const char*>(ptr);
size_t written = 0;
while (count) {
size_t len = std::min(count, (size_t)std::numeric_limits<unsigned>::max());
int rv = gzwrite(fp_, data, len);
if (rv <= 0) {
break;
}
count -= rv;
written += rv;
data += rv;
}
return written;
}
char* GZipFile::gets(char* s, int size)
{
return gzgets(fp_, s, size);
}
int GZipFile::flush()
{
return gzflush(fp_, 0);
}
int GZipFile::vprintf(const char* format, va_list va)
{
char *buf = 0;
size_t len;
int rv = ::vasprintf(&buf, format, va);
if (rv <= 0) {
// buf is undefined at this point
// do not attempt to free it
return rv;
}
len = strlen(buf);
if (len) {
rv = gzwrite(fp_, buf, len);
}
if (buf) {
free(buf);
}
return rv;
}
} // namespace aria2

71
src/GZipFile.h Normal file
View File

@ -0,0 +1,71 @@
/* <!-- copyright */
/*
* aria2 - The high speed download utility
*
* Copyright (C) 2013 Nils Maier
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*
* In addition, as a special exception, the copyright holders give
* permission to link the code of portions of this program with the
* OpenSSL library under certain conditions as described in each
* individual source file, and distribute linked combinations
* including the two.
* You must obey the GNU General Public License in all respects
* for all of the code used other than OpenSSL. If you modify
* file(s) with this exception, you may extend this exception to your
* version of the file(s), but you are not obligated to do so. If you
* do not wish to do so, delete this exception statement from your
* version. If you delete this exception statement from all source
* files in the program, then also delete it here.
*/
/* copyright --> */
#ifndef D_GZIP_FILE_H
#define D_GZIP_FILE_H
#include <zlib.h>
#include "BufferedFile.h"
namespace aria2 {
class GZipFile: public BufferedFile {
public:
GZipFile(const char* filename, const char* mode);
virtual ~GZipFile() {}
virtual int close();
virtual size_t read(void* ptr, size_t count);
virtual size_t write(const void* ptr, size_t count);
virtual char* gets(char* s, int size);
virtual int vprintf(const char* format, va_list va);
virtual int flush();
private:
// Don't allow copying
GZipFile(const GZipFile&);
GZipFile& operator=(const GZipFile&);
gzFile fp_;
bool open_;
protected:
virtual bool isError() const;
virtual bool isEOF() const { return gzeof(fp_); }
virtual bool isOpen() const { return open_; }
};
} // namespace aria2
#endif // D_GZIP_FILE_H

3
src/Makefile.am
View File

@ -325,7 +325,8 @@ endif # HAVE_OPENSSL
if HAVE_ZLIB
SRCS += GZipEncoder.cc GZipEncoder.h\
GZipDecodingStreamFilter.cc GZipDecodingStreamFilter.h
GZipDecodingStreamFilter.cc GZipDecodingStreamFilter.h\
GZipFile.cc GzipFile.h
endif # HAVE_ZLIB
if HAVE_SQLITE3

64
src/SessionSerializer.cc
View File

@ -52,6 +52,10 @@
#include "OptionParser.h"
#include "OptionHandler.h"
#if HAVE_ZLIB
#include "GZipFile.h"
#endif
namespace aria2 {
SessionSerializer::SessionSerializer
@ -66,11 +70,20 @@ bool SessionSerializer::save(const std::string& filename) const
std::string tempFilename = filename;
tempFilename += "__temp";
{
BufferedFile fp(tempFilename.c_str(), BufferedFile::WRITE);
if(!fp) {
SharedHandle<BufferedFile> fp;
#if HAVE_ZLIB
if (filename.compare(filename.length() - 3, 3, ".gz") == 0) {
fp.reset(new GZipFile(tempFilename.c_str(), BufferedFile::WRITE));
}
else
#endif
{
fp.reset(new BufferedFile(tempFilename.c_str(), BufferedFile::WRITE));
}
if(!fp || !*fp) {
return false;
}
if(!save(fp) || fp.close() == EOF) {
if(!save(*fp) || fp->close() == EOF) {
return false;
}
}
@ -92,12 +105,36 @@ bool writeOption(BufferedFile& fp, const SharedHandle<Option>& op)
false, false);
for(std::vector<std::string>::const_iterator j = v.begin(),
eoj = v.end(); j != eoj; ++j) {
if(fp.printf(" %s=%s\n", pref->k, (*j).c_str()) < 0) {
if (fp.write(" ", 1) != 1) {
return false;
}
if (fp.write(pref->k) == 0) {
return false;
}
if (fp.write("=", 1) != 1) {
return false;
}
if (!j->empty() && fp.write(j->c_str()) == 0) {
return false;
}
if (fp.write("\n", 1) != 1) {
return false;
}
}
} else {
if(fp.printf(" %s=%s\n", pref->k, op->get(pref).c_str()) < 0) {
if (fp.write(" ", 1) != 1) {
return false;
}
if (fp.write(pref->k) == 0) {
return false;
}
if (fp.write("=", 1) != 1) {
return false;
}
if (fp.write(op->get(pref).c_str()) == 0) {
return false;
}
if (fp.write("\n", 1) != 1) {
return false;
}
}
@ -152,27 +189,30 @@ bool writeDownloadResult
}
for(std::vector<std::string>::const_iterator i = uris.begin(),
eoi = uris.end(); i != eoi; ++i) {
if(fp.printf("%s\t", (*i).c_str()) < 0) {
if (fp.write(i->c_str()) == 0 || fp.write("\t", 1) != 1) {
return false;
}
}
if(fp.write("\n gid=", 6) != 6) {
return false;
}
if(fp.write(dr->gid->toHex().c_str()) == 0) {
return false;
}
if(fp.write("\n", 1) != 1) {
return false;
}
if(fp.printf(" gid=%s\n", dr->gid->toHex().c_str()) < 0) {
return false;
}
} else {
if(metainfoCache.count(mi->getGID()) != 0) {
return true;
} else {
metainfoCache.insert(mi->getGID());
if(fp.printf("%s\n", mi->getUri().c_str()) < 0) {
if (fp.write(mi->getUri().c_str()) == 0 || fp.write("\n", 1) != 1) {
return false;
}
// For downloads generated by metadata (e.g., BitTorrent,
// Metalink), save gid of Metadata download.
if(fp.printf(" gid=%s\n", GroupId::toHex(mi->getGID()).c_str()) < 0) {
if (fp.write(" gid=", 5) != 5 || fp.write(GroupId::toHex(mi->getGID()).c_str()) == 0 || fp.write("\n", 1) != 1) {
return false;
}
}
@ -241,7 +281,7 @@ bool SessionSerializer::save(BufferedFile& fp) const
// PREF_PAUSE was removed from option, so save it here looking
// property separately.
if(rg->isPauseRequested()) {
if(fp.printf(" %s=true\n", PREF_PAUSE->k) < 0) {
if (fp.write(" ", 1) != 1 || fp.write(PREF_PAUSE->k) == 0 || fp.write("\n", 1) != 1) {
return false;
}
}

24
src/UriListParser.cc
View File

@ -45,10 +45,18 @@
#include "BufferedFile.h"
#include "OptionParser.h"
#if HAVE_ZLIB
#include "GZipFile.h"
#endif
namespace aria2 {
UriListParser::UriListParser(const std::string& filename)
: fp_(filename.c_str(), BufferedFile::READ)
#if HAVE_ZLIB
: fp_(new GZipFile(filename.c_str(), BufferedFile::READ))
#else
: fp_(new BufferedFile(filename.c_str(), BufferedFile::READ))
#endif
{}
UriListParser::~UriListParser() {}
@ -63,9 +71,9 @@ void UriListParser::parseNext(std::vector<std::string>& uris, Option& op)
// Read options
std::stringstream ss;
while(1) {
line_ = fp_.getLine();
line_ = fp_->getLine();
if(line_.empty()) {
if(fp_.eof()) {
if(fp_->eof()) {
break;
} else if(!fp_) {
throw DL_ABORT_EX("UriListParser:I/O error.");
@ -84,9 +92,9 @@ void UriListParser::parseNext(std::vector<std::string>& uris, Option& op)
optparser->parse(op, ss);
return;
}
line_ = fp_.getLine();
line_ = fp_->getLine();
if(line_.empty()) {
if(fp_.eof()) {
if(fp_->eof()) {
return;
} else if(!fp_) {
throw DL_ABORT_EX("UriListParser:I/O error.");
@ -97,7 +105,11 @@ void UriListParser::parseNext(std::vector<std::string>& uris, Option& op)
bool UriListParser::hasNext()
{
return !line_.empty() || (fp_ && !fp_.eof());
bool rv = !line_.empty() || (fp_ && *fp_ && !fp_->eof());
if (!rv) {
fp_->close();
}
return rv;
}
} // namespace aria2

3
src/UriListParser.h
View File

@ -43,12 +43,13 @@
#include "Option.h"
#include "BufferedFile.h"
#include "SharedHandle.h"
namespace aria2 {
class UriListParser {
private:
BufferedFile fp_;
SharedHandle<BufferedFile> fp_;
std::string line_;
public:

6
test/Makefile.am
View File

@ -3,7 +3,6 @@ TESTS = aria2c
check_PROGRAMS = $(TESTS)
aria2c_SOURCES = AllTest.cc\
TestUtil.cc TestUtil.h\
GZipDecoder.cc GZipDecoder.h\
SocketCoreTest.cc\
array_funTest.cc\
Base64Test.cc\
@ -100,8 +99,9 @@ aria2c_SOURCES += FallocFileAllocationIteratorTest.cc
endif # HAVE_SOME_FALLOCATE
if HAVE_ZLIB
aria2c_SOURCES += GZipDecoderTest.cc\
GZipEncoderTest.cc\
aria2c_SOURCES += \
GZipDecoder.cc GZipDecoder.h\
GZipDecoderTest.cc GZipEncoderTest.cc\
GZipDecodingStreamFilterTest.cc
endif # HAVE_ZLIB