From f2f8b14047c5b0f3a41338440c63ca3f9a591427 Mon Sep 17 00:00:00 2001 From: Nils Maier Date: Fri, 1 Mar 2013 16:07:16 +0100 Subject: [PATCH] Support for gzipped -i and --save-session Saved sessions may very large, as in hundreds and even thousands of megabyte when dealing with large queues. Add support to save and reload sessions to gzipped files, when libz is available. The session serializer will output gzipped contents when the file ends with .gz, while the input file reader (UriListParser) will accept whatever is thrown at it. --- configure.ac | 1 + src/BufferedFile.cc | 14 ++-- src/BufferedFile.h | 16 ++-- src/GZipFile.cc | 161 +++++++++++++++++++++++++++++++++++++++ src/GZipFile.h | 71 +++++++++++++++++ src/Makefile.am | 3 +- src/SessionSerializer.cc | 62 ++++++++++++--- src/UriListParser.cc | 24 ++++-- src/UriListParser.h | 3 +- test/Makefile.am | 8 +- 10 files changed, 328 insertions(+), 35 deletions(-) create mode 100644 src/GZipFile.cc create mode 100644 src/GZipFile.h diff --git a/configure.ac b/configure.ac index 9d4437d7..226b9dcf 100644 --- a/configure.ac +++ b/configure.ac @@ -100,6 +100,7 @@ if test "x$with_libz" = "xyes"; then AC_DEFINE([HAVE_ZLIB], [1], [Define to 1 if you have zlib.]) LIBS="$ZLIB_LIBS $LIBS" CPPFLAGS="$ZLIB_CFLAGS $CPPFLAGS" + AC_CHECK_FUNCS([gzbuffer gzsetparams]) else AC_MSG_WARN([$ZLIB_PKG_ERRORS]) if test "x$with_libz_requested" = "xyes"; then diff --git a/src/BufferedFile.cc b/src/BufferedFile.cc index d1af1c09..62b8e87d 100644 --- a/src/BufferedFile.cc +++ b/src/BufferedFile.cc @@ -59,7 +59,7 @@ BufferedFile::BufferedFile(const char* filename, const char* mode) {} BufferedFile::BufferedFile(FILE* fp) - : fp_(fp), open_(true), supportsColor_(fp_ ? isatty(fileno(fp_)) : false) + : fp_(fp), open_(fp_), supportsColor_(fp_ ? isatty(fileno(fp_)) : false) {} BufferedFile::~BufferedFile() @@ -69,7 +69,8 @@ BufferedFile::~BufferedFile() BufferedFile::operator unspecified_bool_type() const { - return (!open_ || ferror(fp_)) ? 0 : &BufferedFile::good_state; + bool ok = isOpen() && !isError(); + return ok ? &BufferedFile::good_state : 0; } size_t BufferedFile::read(void* ptr, size_t count) @@ -94,7 +95,7 @@ char* BufferedFile::gets(char* s, int size) char* BufferedFile::getsn(char* s, int size) { - char* ptr = fgets(s, size, fp_); + char* ptr = gets(s, size); if(ptr) { int len = strlen(ptr); if(ptr[len-1] == '\n') { @@ -128,17 +129,16 @@ std::string BufferedFile::getLine() int BufferedFile::close() { - if(open_) { + if (open_) { open_ = false; return fclose(fp_); - } else { - return 0; } + return 0; } bool BufferedFile::eof() { - return open_ && feof(fp_); + return !isOpen()|| isEOF(); } size_t BufferedFile::transfer(std::ostream& out) diff --git a/src/BufferedFile.h b/src/BufferedFile.h index f0d78e07..7eb6a36d 100644 --- a/src/BufferedFile.h +++ b/src/BufferedFile.h @@ -56,18 +56,18 @@ public: // returns false. operator unspecified_bool_type() const; // wrapper for fread. Using 1 for 2nd argument of fread. - size_t read(void* ptr, size_t count); + virtual size_t read(void* ptr, size_t count); // wrapper for fwrite. Using 1 for 2nd argument of fwrite. - size_t write(const void* ptr, size_t count); + virtual size_t write(const void* ptr, size_t count); virtual size_t write(const char* str); // wrapper for fgets - char* gets(char* s, int size); + virtual char* gets(char* s, int size); // wrapper for fgets, but trailing '\n' is replaced with '\0'. char* getsn(char* s, int size); // Reads one line and returns it. The last '\n' is removed. std::string getLine(); // wrapper for fclose - int close(); + virtual int close(); // Return true if open_ && feof(fp_) != 0. Otherwise returns false. bool eof(); // Convenient method. Read data to end of file and write them into @@ -83,15 +83,21 @@ public: static const char WRITE[]; // Mode for append static const char APPEND[]; + private: // Don't allow copying BufferedFile(const BufferedFile&); BufferedFile& operator=(const BufferedFile&); FILE* fp_; - // true when file has been opened. bool open_; bool supportsColor_; + +protected: + virtual bool isError() const { return ferror(fp_); } + virtual bool isEOF() const { return feof(fp_); } + virtual bool isOpen() const { return open_; } + }; } // namespace aria2 diff --git a/src/GZipFile.cc b/src/GZipFile.cc new file mode 100644 index 00000000..8acb1bd9 --- /dev/null +++ b/src/GZipFile.cc @@ -0,0 +1,161 @@ +/* */ + +#include "GZipFile.h" + +#include +#include + +#include "a2io.h" +#include "util.h" + +namespace aria2 { + +GZipFile::GZipFile(const char* filename, const char* mode) + : BufferedFile(0), fp_(0), open_(false) +{ + FILE* fp = +#ifdef __MINGW32__ + a2fopen(utf8ToWChar(filename).c_str(), utf8ToWChar(mode).c_str()); +#else // !__MINGW32__ + a2fopen(filename, mode); +#endif // !__MINGW32__ + + open_ = fp; + if (open_) { + int fd = dup(fileno(fp)); + if ((open_ = fd) >= 0) { + open_ = (fp_ = gzdopen(fd, mode)); + if (!open_) { + ::close(fd); + } + } + if (open_) { +#if HAVE_GZBUFFER + gzbuffer(fp_, 1<<17); +#endif +#if HAVE_GZSETPARAMS + gzsetparams(fp_, 2, Z_DEFAULT_STRATEGY); +#endif + } + fclose(fp); + } +} + +int GZipFile::close() +{ + if (open_) { + open_ = false; + return gzclose(fp_); + } + return 0; +} + +bool GZipFile::isError() const +{ + int rv = 0; + const char *e = gzerror(fp_, &rv); + return (e != 0 && *e != 0) || rv != 0; +} + +size_t GZipFile::read(void* ptr, size_t count) +{ + char *data = reinterpret_cast(ptr); + size_t read = 0; + while (count) { + size_t len = std::min(count, (size_t)std::numeric_limits::max()); + int rv = gzread(fp_, data, len); + if (rv <= 0) { + break; + } + count -= rv; + read += rv; + data += rv; + } + return read; +} + +size_t GZipFile::write(const void* ptr, size_t count) +{ + const char *data = reinterpret_cast(ptr); + size_t written = 0; + while (count) { + size_t len = std::min(count, (size_t)std::numeric_limits::max()); + int rv = gzwrite(fp_, data, len); + if (rv <= 0) { + break; + } + count -= rv; + written += rv; + data += rv; + } + return written; +} + +char* GZipFile::gets(char* s, int size) +{ + return gzgets(fp_, s, size); +} + +int GZipFile::flush() +{ + return gzflush(fp_, 0); +} + +int GZipFile::vprintf(const char* format, va_list va) +{ + char *buf = 0; + size_t len; + + int rv = ::vasprintf(&buf, format, va); + if (rv <= 0) { + // buf is undefined at this point + // do not attempt to free it + return rv; + } + + len = strlen(buf); + if (len) { + rv = gzwrite(fp_, buf, len); + } + + if (buf) { + free(buf); + } + return rv; +} + + +} // namespace aria2 diff --git a/src/GZipFile.h b/src/GZipFile.h new file mode 100644 index 00000000..a447dccc --- /dev/null +++ b/src/GZipFile.h @@ -0,0 +1,71 @@ +/* */ + +#ifndef D_GZIP_FILE_H +#define D_GZIP_FILE_H + +#include + +#include "BufferedFile.h" + +namespace aria2 { + +class GZipFile: public BufferedFile { +public: + GZipFile(const char* filename, const char* mode); + virtual ~GZipFile() {} + virtual int close(); + virtual size_t read(void* ptr, size_t count); + virtual size_t write(const void* ptr, size_t count); + virtual char* gets(char* s, int size); + virtual int vprintf(const char* format, va_list va); + virtual int flush(); +private: + // Don't allow copying + GZipFile(const GZipFile&); + GZipFile& operator=(const GZipFile&); + + gzFile fp_; + bool open_; + +protected: + virtual bool isError() const; + virtual bool isEOF() const { return gzeof(fp_); } + virtual bool isOpen() const { return open_; } +}; + +} // namespace aria2 + +#endif // D_GZIP_FILE_H diff --git a/src/Makefile.am b/src/Makefile.am index 5388ba35..fb5bb20b 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -325,7 +325,8 @@ endif # HAVE_OPENSSL if HAVE_ZLIB SRCS += GZipEncoder.cc GZipEncoder.h\ - GZipDecodingStreamFilter.cc GZipDecodingStreamFilter.h + GZipDecodingStreamFilter.cc GZipDecodingStreamFilter.h\ + GZipFile.cc GzipFile.h endif # HAVE_ZLIB if HAVE_SQLITE3 diff --git a/src/SessionSerializer.cc b/src/SessionSerializer.cc index 519cf338..6c4832f5 100644 --- a/src/SessionSerializer.cc +++ b/src/SessionSerializer.cc @@ -52,6 +52,10 @@ #include "OptionParser.h" #include "OptionHandler.h" +#if HAVE_ZLIB +#include "GZipFile.h" +#endif + namespace aria2 { SessionSerializer::SessionSerializer @@ -66,11 +70,20 @@ bool SessionSerializer::save(const std::string& filename) const std::string tempFilename = filename; tempFilename += "__temp"; { - BufferedFile fp(tempFilename.c_str(), BufferedFile::WRITE); - if(!fp) { + SharedHandle fp; +#if HAVE_ZLIB + if (filename.compare(filename.length() - 3, 3, ".gz") == 0) { + fp.reset(new GZipFile(tempFilename.c_str(), BufferedFile::WRITE)); + } + else +#endif + { + fp.reset(new BufferedFile(tempFilename.c_str(), BufferedFile::WRITE)); + } + if(!fp || !*fp) { return false; } - if(!save(fp) || fp.close() == EOF) { + if(!save(*fp) || fp->close() == EOF) { return false; } } @@ -92,12 +105,36 @@ bool writeOption(BufferedFile& fp, const SharedHandle