From 506bc3db13f6130aca37c1df216a3e8a7967bf99 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 27 Nov 2007 12:27:10 +0000 Subject: [PATCH] 2007-11-27 Tatsuhiro Tsujikawa Rewritten to add content-type support. * src/DownloadHandler.{h, cc} * src/BtPostDownloadHandler.{h, cc} * test/BtPostDownloadHandlerTest.cc * src/MetalinkPostDownloadHandler.{h, cc} * test/MetalinkPostDownloadHandlerTest.cc * src/PostDownloadHandler.{h, cc} * src/DownloadHandlerConstants.{h, cc} * src/RequestGroup.cc * src/HttpResponseCommand.cc * src/FtpNegotiationCommand.cc * src/SingleFileDownloadContext.{h, cc} * src/RequestGroup.h * src/RequestGroupCriteria.h * src/ContentTypeRequestGroupCriteria.h Added 'mem' option value for --follow-metalink, --follow-torrent. If it is give, metalink/torrent file is not written to the disk, but just is kept in memory. Parsing is occurred on memory. * src/MetalinkHelper.{h, cc} * src/MetalinkProcessor.h * src/Xml2MetalinkProcessor.{h, cc} * test/Xml2MetalinkProcessorTest.cc * src/DownloadHandlerFactory.{h, cc} * test/DownloadHandlerFactoryTest.cc * src/PreDownloadHandler.{h, cc} * src/OptionHandlerFactory.cc * src/DefaultBtContext.{h, cc} * test/DefaultBtContextTest.cc * src/version_usage.cc * src/Metalink2RequestGroup.{h, cc} * src/RequestGroup.{h, cc} * src/a2functional.h * test/a2functionalTest.cc * src/MemoryBufferPreDownloadHandler.{h, cc} * src/OptionHandlerImpl.h * src/prefs.h * src/Util.{h, cc} * test/UtilTest.cc Keep DownloadResult rather than RequestGroup after downloads to reduce memory usage. * src/RequestGroupMan.{h, cc} * src/DownloadEngine.cc * src/BtDependency.{h, cc}: Changed the type of dependee from WeakHandle to SharedHandle because WeakHandle could be null. * src/RequestGroup.{h, cc} * src/DownloadEngineFactory.cc * src/DownloadResult.h Set totalLength after download finished * src/UnknownLengthPieceStorage.{h, cc} Keep torrent file specified in metalink in memory. * src/Metalink2RequestGroup.cc * src/BtDependency.cc * src/TrueRequestGroupCriteria.h Fixed the bug: seekg is used where seekp should be used. * src/ByteArrayDiskWriter.cc * test/ByteArraydiskWriterTest.cc --- ChangeLog | 64 ++++++++++++++++ TODO | 3 +- src/BtDependency.cc | 16 +++- src/BtDependency.h | 5 +- src/BtPostDownloadHandler.cc | 43 ++++++++--- src/BtPostDownloadHandler.h | 4 +- src/ByteArrayDiskWriter.cc | 6 +- src/ContentTypeRequestGroupCriteria.cc | 79 ++++++++++++++++++++ src/ContentTypeRequestGroupCriteria.h | 59 +++++++++++++++ src/DefaultBtContext.cc | 18 ++++- src/DefaultBtContext.h | 4 + src/DownloadEngine.cc | 1 + src/DownloadEngineFactory.cc | 1 + src/DownloadHandler.cc | 54 ++++++++++++++ src/DownloadHandler.h | 67 +++++++++++++++++ src/DownloadHandlerConstants.cc | 71 ++++++++++++++++++ src/DownloadHandlerConstants.h | 61 +++++++++++++++ src/DownloadHandlerFactory.cc | 98 +++++++++++++++++++++++++ src/DownloadHandlerFactory.h | 79 ++++++++++++++++++++ src/DownloadResult.h | 76 +++++++++++++++++++ src/FtpNegotiationCommand.cc | 2 +- src/HttpResponseCommand.cc | 1 + src/Makefile.am | 6 +- src/Makefile.in | 87 +++++++++++++--------- src/MemoryBufferPreDownloadHandler.cc | 51 +++++++++++++ src/MemoryBufferPreDownloadHandler.h | 51 +++++++++++++ src/Metalink2RequestGroup.cc | 22 +++++- src/Metalink2RequestGroup.h | 9 +++ src/MetalinkHelper.cc | 14 ++++ src/MetalinkHelper.h | 9 +++ src/MetalinkPostDownloadHandler.cc | 29 ++++++-- src/MetalinkPostDownloadHandler.h | 4 +- src/MetalinkProcessor.h | 5 ++ src/OptionHandlerFactory.cc | 4 +- src/OptionHandlerImpl.h | 11 +++ src/PostDownloadHandler.cc | 10 +-- src/PostDownloadHandler.h | 20 ++--- src/PreDownloadHandler.cc | 40 ++++++++++ src/PreDownloadHandler.h | 52 +++++++++++++ src/RequestGroup.cc | 79 ++++++++++++++++++-- src/RequestGroup.h | 23 +++++- src/RequestGroupCriteria.h | 52 +++++++++++++ src/RequestGroupMan.cc | 97 ++++++++++++------------ src/RequestGroupMan.h | 8 +- src/SingleFileDownloadContext.h | 12 +++ src/TrueRequestGroupCriteria.h | 53 +++++++++++++ src/UnknownLengthPieceStorage.cc | 1 + src/UnknownLengthPieceStorage.h | 4 +- src/Util.cc | 16 ++++ src/Util.h | 5 ++ src/Xml2MetalinkProcessor.cc | 44 +++++++++++ src/Xml2MetalinkProcessor.h | 9 ++- src/a2functional.h | 17 +++++ src/prefs.h | 6 +- src/version_usage.cc | 24 ++++-- test/BtPostDownloadHandlerTest.cc | 47 ++++++++++-- test/ByteArrayDiskWriterTest.cc | 45 +++++------- test/DefaultBtContextTest.cc | 16 ++++ test/DownloadHandlerFactoryTest.cc | 90 +++++++++++++++++++++++ test/Makefile.am | 1 + test/Makefile.in | 7 +- test/MetalinkPostDownloadHandlerTest.cc | 47 ++++++++++-- test/RequestGroupManTest.cc | 1 + test/UtilTest.cc | 15 ++++ test/Xml2MetalinkProcessorTest.cc | 24 ++++++ test/a2functionalTest.cc | 11 +++ 66 files changed, 1785 insertions(+), 205 deletions(-) create mode 100644 src/ContentTypeRequestGroupCriteria.cc create mode 100644 src/ContentTypeRequestGroupCriteria.h create mode 100644 src/DownloadHandler.cc create mode 100644 src/DownloadHandler.h create mode 100644 src/DownloadHandlerConstants.cc create mode 100644 src/DownloadHandlerConstants.h create mode 100644 src/DownloadHandlerFactory.cc create mode 100644 src/DownloadHandlerFactory.h create mode 100644 src/DownloadResult.h create mode 100644 src/MemoryBufferPreDownloadHandler.cc create mode 100644 src/MemoryBufferPreDownloadHandler.h create mode 100644 src/PreDownloadHandler.cc create mode 100644 src/PreDownloadHandler.h create mode 100644 src/RequestGroupCriteria.h create mode 100644 src/TrueRequestGroupCriteria.h create mode 100644 test/DownloadHandlerFactoryTest.cc diff --git a/ChangeLog b/ChangeLog index 15383f3b..6fa7b539 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,67 @@ +2007-11-27 Tatsuhiro Tsujikawa + + Rewritten to add content-type support. + * src/DownloadHandler.{h, cc} + * src/BtPostDownloadHandler.{h, cc} + * test/BtPostDownloadHandlerTest.cc + * src/MetalinkPostDownloadHandler.{h, cc} + * test/MetalinkPostDownloadHandlerTest.cc + * src/PostDownloadHandler.{h, cc} + * src/DownloadHandlerConstants.{h, cc} + * src/RequestGroup.cc + * src/HttpResponseCommand.cc + * src/FtpNegotiationCommand.cc + * src/SingleFileDownloadContext.{h, cc} + * src/RequestGroup.h + * src/RequestGroupCriteria.h + * src/ContentTypeRequestGroupCriteria.h + + Added 'mem' option value for --follow-metalink, --follow-torrent. + If it is give, metalink/torrent file is not written to the disk, but + just is kept in memory. Parsing is occurred on memory. + * src/MetalinkHelper.{h, cc} + * src/MetalinkProcessor.h + * src/Xml2MetalinkProcessor.{h, cc} + * test/Xml2MetalinkProcessorTest.cc + * src/DownloadHandlerFactory.{h, cc} + * test/DownloadHandlerFactoryTest.cc + * src/PreDownloadHandler.{h, cc} + * src/OptionHandlerFactory.cc + * src/DefaultBtContext.{h, cc} + * test/DefaultBtContextTest.cc + * src/version_usage.cc + * src/Metalink2RequestGroup.{h, cc} + * src/RequestGroup.{h, cc} + * src/a2functional.h + * test/a2functionalTest.cc + * src/MemoryBufferPreDownloadHandler.{h, cc} + * src/OptionHandlerImpl.h + * src/prefs.h + * src/Util.{h, cc} + * test/UtilTest.cc + + Keep DownloadResult rather than RequestGroup after downloads to reduce + memory usage. + * src/RequestGroupMan.{h, cc} + * src/DownloadEngine.cc + * src/BtDependency.{h, cc}: Changed the type of dependee from + WeakHandle to SharedHandle because WeakHandle could be null. + * src/RequestGroup.{h, cc} + * src/DownloadEngineFactory.cc + * src/DownloadResult.h + + Set totalLength after download finished + * src/UnknownLengthPieceStorage.{h, cc} + + Keep torrent file specified in metalink in memory. + * src/Metalink2RequestGroup.cc + * src/BtDependency.cc + * src/TrueRequestGroupCriteria.h + + Fixed the bug: seekg is used where seekp should be used. + * src/ByteArrayDiskWriter.cc + * test/ByteArraydiskWriterTest.cc + 2007-11-25 Tatsuhiro Tsujikawa Fixed syntax error diff --git a/TODO b/TODO index f6178774..531967c7 100644 --- a/TODO +++ b/TODO @@ -46,7 +46,6 @@ * Limit the number of opening file to,say,100 in MultiDiskAdaptor. * Implement the feature to treat http/ftp as auxuality download method for BitTorrent * http-seeding(single and multi-file torrent) -* Use content-type for PostDownloadHandler * Rewrite exception usage: RecoverableException DlRetryEx .... Retry using same connection/url. Should be renamed to TemporaryFailureException @@ -55,6 +54,8 @@ FatalException .... Program should abort. * replace strtol with Util::parseInt * do performance tuning against Xml2MetalinkProcessor +* remove header files from Makefile.am +* create MetalinkProcessorFactory and get Xml2MetalinkProcessor from it. -- remaining issues to be implemented for 0.12.0 release * Update translation diff --git a/src/BtDependency.cc b/src/BtDependency.cc index 8be2266e..6c0bda48 100644 --- a/src/BtDependency.cc +++ b/src/BtDependency.cc @@ -40,9 +40,12 @@ #include "RecoverableException.h" #include "message.h" #include "prefs.h" +#include "Util.h" +#include "PieceStorage.h" +#include "DiskAdaptor.h" BtDependency::BtDependency(const RequestGroupWeakHandle& dependant, - const RequestGroupWeakHandle& dependee, + const RequestGroupHandle& dependee, const Option* option): _dependant(dependant), _dependee(dependee), @@ -54,9 +57,16 @@ BtDependency::~BtDependency() {} bool BtDependency::resolve() { if(_dependee->getNumCommand() == 0 && _dependee->downloadFinished()) { + RequestGroupHandle dependee = _dependee; + // cut reference here + _dependee = 0; DefaultBtContextHandle btContext = new DefaultBtContext(); try { - btContext->load(_dependee->getFilePath()); + DiskAdaptorHandle diskAdaptor = dependee->getPieceStorage()->getDiskAdaptor(); + diskAdaptor->openExistingFile(); + string content = Util::toString(diskAdaptor); + btContext->loadFromMemory(content.c_str(), content.size(), + File(dependee->getFilePath()).getBasename()); if(_option->defined(PREF_PEER_ID_PREFIX)) { btContext->setPeerIdPrefix(_option->get(PREF_PEER_ID_PREFIX)); } @@ -74,6 +84,8 @@ bool BtDependency::resolve() return true; } else if(_dependee->getNumCommand() == 0) { // _dependee's download failed. + // cut reference here + _dependee = 0; _logger->debug("BtDependency for GID#%d failed. Go without Bt.", _dependant->getGID()); return true; diff --git a/src/BtDependency.h b/src/BtDependency.h index e9872a9f..4a98ad9c 100644 --- a/src/BtDependency.h +++ b/src/BtDependency.h @@ -39,6 +39,7 @@ class RequestGroup; typedef WeakHandle RequestGroupWeakHandle; +typedef SharedHandle RequestGroupHandle; class Option; class Logger; @@ -46,12 +47,12 @@ class BtDependency : public Dependency { private: RequestGroupWeakHandle _dependant; - RequestGroupWeakHandle _dependee; + RequestGroupHandle _dependee; const Option* _option; const Logger* _logger; public: BtDependency(const RequestGroupWeakHandle& dependant, - const RequestGroupWeakHandle& dependee, + const RequestGroupHandle& dependee, const Option* option); virtual ~BtDependency(); diff --git a/src/BtPostDownloadHandler.cc b/src/BtPostDownloadHandler.cc index 096a0e4c..5aae5311 100644 --- a/src/BtPostDownloadHandler.cc +++ b/src/BtPostDownloadHandler.cc @@ -38,23 +38,44 @@ #include "RequestGroup.h" #include "Option.h" #include "Logger.h" +#include "DownloadHandlerConstants.h" +#include "File.h" +#include "PieceStorage.h" +#include "DiskAdaptor.h" +#include "Util.h" +#include "ContentTypeRequestGroupCriteria.h" -BtPostDownloadHandler::BtPostDownloadHandler(const Option* option): - PostDownloadHandler(".torrent", option) -{} +BtPostDownloadHandler::BtPostDownloadHandler() +{ + setCriteria(new ContentTypeRequestGroupCriteria(DownloadHandlerConstants::getBtContentTypes(), + DownloadHandlerConstants::getBtExtensions())); +} BtPostDownloadHandler::~BtPostDownloadHandler() {} -RequestGroups BtPostDownloadHandler::getNextRequestGroups(const string& path) +RequestGroups BtPostDownloadHandler::getNextRequestGroups(RequestGroup* requestGroup) { - _logger->debug("Generating RequestGroups for Torrent file %s", path.c_str()); - RequestGroupHandle rg = new RequestGroup(_option, Strings()); - DefaultBtContextHandle btContext = new DefaultBtContext(); - btContext->load(path); - if(_option->defined(PREF_PEER_ID_PREFIX)) { - btContext->setPeerIdPrefix(_option->get(PREF_PEER_ID_PREFIX)); + const Option* op = requestGroup->getOption(); + _logger->debug("Generating RequestGroups for Torrent file %s", + requestGroup->getFilePath().c_str()); + RequestGroupHandle rg = new RequestGroup(op, Strings()); + + string content; + try { + requestGroup->getPieceStorage()->getDiskAdaptor()->openExistingFile(); + content = Util::toString(requestGroup->getPieceStorage()->getDiskAdaptor()); + requestGroup->getPieceStorage()->getDiskAdaptor()->closeFile(); + } catch(Exception* e) { + requestGroup->getPieceStorage()->getDiskAdaptor()->closeFile(); + throw; } - btContext->setDir(_option->get(PREF_DIR)); + DefaultBtContextHandle btContext = new DefaultBtContext(); + btContext->loadFromMemory(content.c_str(), content.size(), + File(requestGroup->getFilePath()).getBasename()); + if(op->defined(PREF_PEER_ID_PREFIX)) { + btContext->setPeerIdPrefix(op->get(PREF_PEER_ID_PREFIX)); + } + btContext->setDir(op->get(PREF_DIR)); rg->setDownloadContext(btContext); btContext->setOwnerRequestGroup(rg.get()); diff --git a/src/BtPostDownloadHandler.h b/src/BtPostDownloadHandler.h index f1cb1638..8dab5887 100644 --- a/src/BtPostDownloadHandler.h +++ b/src/BtPostDownloadHandler.h @@ -40,11 +40,11 @@ class BtPostDownloadHandler:public PostDownloadHandler { public: - BtPostDownloadHandler(const Option* option); + BtPostDownloadHandler(); virtual ~BtPostDownloadHandler(); - virtual RequestGroups getNextRequestGroups(const string& path); + virtual RequestGroups getNextRequestGroups(RequestGroup* requestGroup); }; typedef SharedHandle BtPostDownloadHandlerHandle; diff --git a/src/ByteArrayDiskWriter.cc b/src/ByteArrayDiskWriter.cc index 750e0437..e232ac5b 100644 --- a/src/ByteArrayDiskWriter.cc +++ b/src/ByteArrayDiskWriter.cc @@ -68,12 +68,12 @@ void ByteArrayDiskWriter::openExistingFile(const string& filename, void ByteArrayDiskWriter::writeData(const unsigned char* data, int32_t dataLength, int64_t position) { if(size() < position) { - buf.seekg(0, ios::end); - for(int32_t i = size(); i < position; ++i) { + buf.seekp(size(), ios::beg); + for(int64_t i = size(); i < position; ++i) { buf.put('\0'); } } else { - buf.seekg(position, ios::beg); + buf.seekp(position, ios::beg); } buf.write(reinterpret_cast(data), dataLength); } diff --git a/src/ContentTypeRequestGroupCriteria.cc b/src/ContentTypeRequestGroupCriteria.cc new file mode 100644 index 00000000..54291f0f --- /dev/null +++ b/src/ContentTypeRequestGroupCriteria.cc @@ -0,0 +1,79 @@ +/* */ +#include "ContentTypeRequestGroupCriteria.h" +#include "RequestGroup.h" +#include "SingleFileDownloadContext.h" +#include "Util.h" + +ContentTypeRequestGroupCriteria::ContentTypeRequestGroupCriteria(const Strings& contentTypes, + const Strings& extensions): + _contentTypes(contentTypes), + _extensions(extensions) {} + +ContentTypeRequestGroupCriteria::~ContentTypeRequestGroupCriteria() {} + +bool ContentTypeRequestGroupCriteria::match(const RequestGroup* requestGroup) const +{ + if(forwardMatch(requestGroup->getFilePath(), _extensions)) { + return true; + } else { + SingleFileDownloadContextHandle dctx = requestGroup->getDownloadContext(); + if(dctx.isNull()) { + return false; + } else { + return exactMatch(dctx->getContentType(), _contentTypes); + } + } +} + +bool ContentTypeRequestGroupCriteria::forwardMatch(const string& target, const Strings& candidates) const +{ + for(Strings::const_iterator itr = candidates.begin(); itr != candidates.end(); ++itr) { + if(Util::endsWith(target, *itr)) { + return true; + } + } + return false; +} + +bool ContentTypeRequestGroupCriteria::exactMatch(const string& target, const Strings& candidates) const +{ + for(Strings::const_iterator itr = candidates.begin(); itr != candidates.end(); ++itr) { + if(target == *itr) { + return true; + } + } + return false; +} diff --git a/src/ContentTypeRequestGroupCriteria.h b/src/ContentTypeRequestGroupCriteria.h new file mode 100644 index 00000000..5c348d0f --- /dev/null +++ b/src/ContentTypeRequestGroupCriteria.h @@ -0,0 +1,59 @@ +/* */ +#ifndef _D_CONTENT_TYPE_REQUEST_GROUP_CRITERIA_H_ +#define _D_CONTENT_TYPE_REQUEST_GROUP_CRITERIA_H_ + +#include "RequestGroupCriteria.h" + +class ContentTypeRequestGroupCriteria:public RequestGroupCriteria +{ +private: + Strings _contentTypes; + Strings _extensions; + + bool forwardMatch(const string& target, const Strings& candidates) const; + + bool exactMatch(const string& target, const Strings& candidates) const; + +public: + ContentTypeRequestGroupCriteria(const Strings& contentTypes, + const Strings& extensions); + + virtual ~ContentTypeRequestGroupCriteria(); + + virtual bool match(const RequestGroup* requestGroup) const; +}; + +#endif // _D_CONTENT_TYPE_REQUEST_GROUP_CRITERIA_H_ diff --git a/src/DefaultBtContext.cc b/src/DefaultBtContext.cc index c4af71b2..b4e43df1 100644 --- a/src/DefaultBtContext.cc +++ b/src/DefaultBtContext.cc @@ -201,12 +201,26 @@ Strings DefaultBtContext::extractUrlList(const MetaEntry* obj) return uris; } +void DefaultBtContext::loadFromMemory(const char* content, int32_t length, const string& defaultName) +{ + MetaEntry* rootEntry = MetaFileUtil::bdecoding(content, length); + if(!dynamic_cast(rootEntry)) { + throw new DlAbortEx("torrent file does not contain a root dictionary ."); + } + processMetaInfo(rootEntry, defaultName); +} + void DefaultBtContext::load(const string& torrentFile) { - clear(); MetaEntry* rootEntry = MetaFileUtil::parseMetaFile(torrentFile); if(!dynamic_cast(rootEntry)) { throw new DlAbortEx("torrent file does not contain a root dictionary ."); } + processMetaInfo(rootEntry, torrentFile); +} + +void DefaultBtContext::processMetaInfo(const MetaEntry* rootEntry, const string& defaultName) +{ + clear(); SharedHandle rootDic = SharedHandle((Dictionary*)rootEntry); Dictionary* infoDic = (Dictionary*)rootDic->get("info"); @@ -231,7 +245,7 @@ void DefaultBtContext::load(const string& torrentFile) { // see http://www.getright.com/seedtorrent.html Strings urlList = extractUrlList(rootDic->get("url-list")); // retrieve file entries - extractFileEntries(infoDic, torrentFile, urlList); + extractFileEntries(infoDic, defaultName, urlList); // retrieve announce Data* announceData = (Data*)rootDic->get("announce"); List* announceListData = (List*)rootDic->get("announce-list"); diff --git a/src/DefaultBtContext.h b/src/DefaultBtContext.h index 880ae5de..4e3f7ccc 100644 --- a/src/DefaultBtContext.h +++ b/src/DefaultBtContext.h @@ -78,6 +78,8 @@ private: Strings extractUrlList(const MetaEntry* obj); + void processMetaInfo(const MetaEntry* rootEntry, const string& defaultName); + public: DefaultBtContext(); virtual ~DefaultBtContext(); @@ -110,6 +112,8 @@ private: virtual void load(const string& torrentFile); + void loadFromMemory(const char* content, int32_t length, const string& defaultName); + virtual string getName() const; virtual int32_t getPieceLength() const; diff --git a/src/DownloadEngine.cc b/src/DownloadEngine.cc index 2210d17e..d2e6039b 100644 --- a/src/DownloadEngine.cc +++ b/src/DownloadEngine.cc @@ -36,6 +36,7 @@ #include "Socket.h" #include "NameResolver.h" #include "StatCalc.h" +#include "DownloadResult.h" #include "RequestGroup.h" #include "RequestGroupMan.h" #include "FileAllocationMan.h" diff --git a/src/DownloadEngineFactory.cc b/src/DownloadEngineFactory.cc index 87d0a65c..72b1aff8 100644 --- a/src/DownloadEngineFactory.cc +++ b/src/DownloadEngineFactory.cc @@ -49,6 +49,7 @@ #include "FileAllocationDispatcherCommand.h" #include "AutoSaveCommand.h" #include "HaveEraseCommand.h" +#include "DownloadResult.h" DownloadEngineFactory::DownloadEngineFactory(): _logger(LogFactory::getInstance()) {} diff --git a/src/DownloadHandler.cc b/src/DownloadHandler.cc new file mode 100644 index 00000000..1518cf80 --- /dev/null +++ b/src/DownloadHandler.cc @@ -0,0 +1,54 @@ +/* */ +#include "DownloadHandler.h" +#include "LogFactory.h" +#include "RequestGroup.h" +#include "RequestGroupCriteria.h" + +DownloadHandler::DownloadHandler(): + _criteria(0), + _logger(LogFactory::getInstance()) {} + +DownloadHandler::~DownloadHandler() {} + +bool DownloadHandler::canHandle(const RequestGroup* requestGroup) const +{ + return !_criteria.isNull() && _criteria->match(requestGroup); +} + +void DownloadHandler::setCriteria(const RequestGroupCriteriaHandle& criteria) +{ + _criteria = criteria; +} diff --git a/src/DownloadHandler.h b/src/DownloadHandler.h new file mode 100644 index 00000000..e5e75e1d --- /dev/null +++ b/src/DownloadHandler.h @@ -0,0 +1,67 @@ +/* */ +#ifndef _D_DOWNLOAD_HANDLER_H_ +#define _D_DOWNLOAD_HANDLER_H_ + +#include "common.h" + +class RequestGroup; +class Logger; +class RequestGroupCriteria; +typedef SharedHandle RequestGroupCriteriaHandle; + +class DownloadHandler +{ +protected: + RequestGroupCriteriaHandle _criteria; + + const Logger* _logger; + +private: + bool forwardMatch(const string& target, const Strings& candidates) const; + + bool exactMatch(const string& target, const Strings& candidates) const; + +public: + DownloadHandler(); + + virtual ~DownloadHandler(); + + bool canHandle(const RequestGroup* requestGroup) const; + + void setCriteria(const RequestGroupCriteriaHandle& criteria); +}; + +#endif // _D_DOWNLOAD_HANDLER_H_ diff --git a/src/DownloadHandlerConstants.cc b/src/DownloadHandlerConstants.cc new file mode 100644 index 00000000..92a4ecff --- /dev/null +++ b/src/DownloadHandlerConstants.cc @@ -0,0 +1,71 @@ +/* */ +#include "DownloadHandlerConstants.h" + +char* DownloadHandlerConstants::METALINK_EXTENSIONS[] = { ".metalink" }; + +char* DownloadHandlerConstants::METALINK_CONTENT_TYPES[] = { + "application/metalink+xml" +}; + +char* DownloadHandlerConstants::BT_EXTENSIONS[] = { ".torrent" }; + +char* DownloadHandlerConstants::BT_CONTENT_TYPES[] = { + "application/x-bittorrent" +}; + +Strings DownloadHandlerConstants::getMetalinkExtensions() +{ + return Strings(&METALINK_EXTENSIONS[0], + &METALINK_EXTENSIONS[arrayLength(METALINK_EXTENSIONS)]); +} + +Strings DownloadHandlerConstants::getMetalinkContentTypes() +{ + return Strings(&METALINK_CONTENT_TYPES[0], + &METALINK_CONTENT_TYPES[arrayLength(METALINK_CONTENT_TYPES)]); +} + +Strings DownloadHandlerConstants::getBtExtensions() +{ + return Strings(&BT_EXTENSIONS[0], + &BT_EXTENSIONS[arrayLength(BT_EXTENSIONS)]); +} + +Strings DownloadHandlerConstants::getBtContentTypes() +{ + return Strings(&BT_CONTENT_TYPES[0], + &BT_CONTENT_TYPES[arrayLength(BT_CONTENT_TYPES)]); +} diff --git a/src/DownloadHandlerConstants.h b/src/DownloadHandlerConstants.h new file mode 100644 index 00000000..0e4e42df --- /dev/null +++ b/src/DownloadHandlerConstants.h @@ -0,0 +1,61 @@ +/* */ +#ifndef _D_DOWNLOAD_HANDLER_CONSTANTS_H_ +#define _D_DOWNLOAD_HANDLER_CONSTANTS_H_ + +#include "common.h" +#include "a2functional.h" + +class DownloadHandlerConstants +{ +public: + static char* METALINK_EXTENSIONS[]; + + static Strings getMetalinkExtensions(); + + static char* METALINK_CONTENT_TYPES[]; + + static Strings getMetalinkContentTypes(); + + static char* BT_EXTENSIONS[]; + + static Strings getBtExtensions(); + + static char* BT_CONTENT_TYPES[]; + + static Strings getBtContentTypes(); +}; + +#endif // _D_DOWNLOAD_HANDLER_CONSTANTS_H_ diff --git a/src/DownloadHandlerFactory.cc b/src/DownloadHandlerFactory.cc new file mode 100644 index 00000000..ad8c4ed3 --- /dev/null +++ b/src/DownloadHandlerFactory.cc @@ -0,0 +1,98 @@ +/* */ +#include "DownloadHandlerFactory.h" +#include "MemoryBufferPreDownloadHandler.h" +#include "MetalinkPostDownloadHandler.h" +#include "BtPostDownloadHandler.h" +#include "DownloadHandlerConstants.h" +#include "ContentTypeRequestGroupCriteria.h" + +#ifdef ENABLE_METALINK +MemoryBufferPreDownloadHandlerHandle DownloadHandlerFactory::_metalinkPreDownloadHandler = 0; + +MetalinkPostDownloadHandlerHandle DownloadHandlerFactory::_metalinkPostDownloadHandler = 0; +#endif // ENABLE_METALINK + +#ifdef ENABLE_BITTORRENT +MemoryBufferPreDownloadHandlerHandle DownloadHandlerFactory::_btPreDownloadHandler = 0; + +BtPostDownloadHandlerHandle DownloadHandlerFactory::_btPostDownloadHandler = 0; +#endif // ENABLE_BITTORRENT + +#ifdef ENABLE_METALINK +MemoryBufferPreDownloadHandlerHandle DownloadHandlerFactory::getMetalinkPreDownloadHandler() +{ + if(_metalinkPreDownloadHandler.isNull()) { + _metalinkPreDownloadHandler = new MemoryBufferPreDownloadHandler(); + + RequestGroupCriteriaHandle criteria = + new ContentTypeRequestGroupCriteria(DownloadHandlerConstants::getMetalinkContentTypes(), + DownloadHandlerConstants::getMetalinkExtensions()); + _metalinkPreDownloadHandler->setCriteria(criteria); + } + return _metalinkPreDownloadHandler; +} + +MetalinkPostDownloadHandlerHandle DownloadHandlerFactory::getMetalinkPostDownloadHandler() +{ + if(_metalinkPostDownloadHandler.isNull()) { + _metalinkPostDownloadHandler = new MetalinkPostDownloadHandler(); + } + return _metalinkPostDownloadHandler; +} +#endif // ENABLE_METALINK + +#ifdef ENABLE_BITTORRENT +MemoryBufferPreDownloadHandlerHandle DownloadHandlerFactory::getBtPreDownloadHandler() +{ + if(_btPreDownloadHandler.isNull()) { + _btPreDownloadHandler = new MemoryBufferPreDownloadHandler(); + + RequestGroupCriteriaHandle criteria = + new ContentTypeRequestGroupCriteria(DownloadHandlerConstants::getBtContentTypes(), + DownloadHandlerConstants::getBtExtensions()); + _btPreDownloadHandler->setCriteria(criteria); + } + return _btPreDownloadHandler; +} + +BtPostDownloadHandlerHandle DownloadHandlerFactory::getBtPostDownloadHandler() +{ + if(_btPostDownloadHandler.isNull()) { + _btPostDownloadHandler = new BtPostDownloadHandler(); + } + return _btPostDownloadHandler; +} +#endif // ENABLE_BITTORRENT diff --git a/src/DownloadHandlerFactory.h b/src/DownloadHandlerFactory.h new file mode 100644 index 00000000..c8c695eb --- /dev/null +++ b/src/DownloadHandlerFactory.h @@ -0,0 +1,79 @@ +/* */ +#ifndef _D_DOWNLOAD_HANDLER_FACTORY_H_ +#define _D_DOWNLOAD_HANDLER_FACTORY_H_ + +#include "common.h" + +class MemoryBufferPreDownloadHandler; +typedef SharedHandle MemoryBufferPreDownloadHandlerHandle; +#ifdef ENABLE_METALINK +class MetalinkPostDownloadHandler; +typedef SharedHandle MetalinkPostDownloadHandlerHandle; +#endif // ENABLE_METALINK +#ifdef ENABLE_BITTORRENT +class BtPostDownloadHandler; +typedef SharedHandle BtPostDownloadHandlerHandle; +#endif // ENABLE_BITTORRENT + +class DownloadHandlerFactory +{ +private: +#ifdef ENABLE_METALINK + static MemoryBufferPreDownloadHandlerHandle _metalinkPreDownloadHandler; + + static MetalinkPostDownloadHandlerHandle _metalinkPostDownloadHandler; +#endif // ENABLE_METALINK + +#ifdef ENABLE_BITTORRENT + static MemoryBufferPreDownloadHandlerHandle _btPreDownloadHandler; + + static BtPostDownloadHandlerHandle _btPostDownloadHandler; +#endif // ENABLE_BITTORRENT +public: +#ifdef ENABLE_METALINK + static MemoryBufferPreDownloadHandlerHandle getMetalinkPreDownloadHandler(); + + static MetalinkPostDownloadHandlerHandle getMetalinkPostDownloadHandler(); +#endif // ENABLE_METALINK + +#ifdef ENABLE_BITTORRENT + static MemoryBufferPreDownloadHandlerHandle getBtPreDownloadHandler(); + + static BtPostDownloadHandlerHandle getBtPostDownloadHandler(); +#endif // ENABLE_BITTORRENT +}; + +#endif // _D_DOWNLOAD_HANDLER_FACTORY_H_ diff --git a/src/DownloadResult.h b/src/DownloadResult.h new file mode 100644 index 00000000..7350d3d9 --- /dev/null +++ b/src/DownloadResult.h @@ -0,0 +1,76 @@ +/* */ +#ifndef _D_DOWNLOAD_RESULT_H_ +#define _D_DOWNLOAD_RESULT_H_ + +#include "common.h" + +class DownloadResult +{ +public: + enum RESULT { + FINISHED, + NOT_YET, + }; + + int32_t gid; + + string filePath; + + int64_t totalLength; + + string uri; + + int32_t numUri; + + RESULT result; + + DownloadResult(int32_t gid, + const string& filePath, + int64_t totalLength, + const string& uri, + int32_t numUri, + RESULT result): + gid(gid), + filePath(filePath), + totalLength(totalLength), + uri(uri), + numUri(numUri), + result(result) {} +}; + +typedef SharedHandle DownloadResultHandle; + +#endif // _D_DOWNLOAD_RESULT_H_ diff --git a/src/FtpNegotiationCommand.cc b/src/FtpNegotiationCommand.cc index 60759352..b1808534 100644 --- a/src/FtpNegotiationCommand.cc +++ b/src/FtpNegotiationCommand.cc @@ -208,7 +208,7 @@ bool FtpNegotiationCommand::recvSize() { SingleFileDownloadContextHandle dctx = _requestGroup->getDownloadContext(); dctx->setTotalLength(size); dctx->setFilename(Util::urldecode(req->getFile())); - + _requestGroup->preDownloadProcessing(); if(e->_requestGroupMan->isSameFileBeingDownloaded(_requestGroup)) { throw new DownloadFailureException(EX_DUPLICATE_FILE_DOWNLOAD, _requestGroup->getFilePath().c_str()); diff --git a/src/HttpResponseCommand.cc b/src/HttpResponseCommand.cc index f4f7fa42..cc014bbc 100644 --- a/src/HttpResponseCommand.cc +++ b/src/HttpResponseCommand.cc @@ -91,6 +91,7 @@ bool HttpResponseCommand::executeInternal() SingleFileDownloadContextHandle dctx = _requestGroup->getDownloadContext(); dctx->setTotalLength(totalLength); dctx->setFilename(httpResponse->determinFilename()); + _requestGroup->preDownloadProcessing(); if(e->_requestGroupMan->isSameFileBeingDownloaded(_requestGroup)) { throw new DownloadFailureException(EX_DUPLICATE_FILE_DOWNLOAD, _requestGroup->getFilePath().c_str()); diff --git a/src/Makefile.am b/src/Makefile.am index 18e6a9cb..4e6e7b0a 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -125,7 +125,11 @@ SRCS = Socket.h\ NullProgressInfoFile.h\ FileAllocationIterator.h\ SingleFileAllocationIterator.cc SingleFileAllocationIterator.h\ - PostDownloadHandler.cc PostDownloadHandler.h\ + ContentTypeRequestGroupCriteria.cc ContentTypeRequestGroupCriteria.h\ + DownloadHandler.cc DownloadHandler.h\ + DownloadHandlerConstants.cc DownloadHandlerConstants.h\ + DownloadHandlerFactory.cc DownloadHandlerFactory.h\ + MemoryBufferPreDownloadHandler.cc MemoryBufferPreDownloadHandler.h\ HaveEraseCommand.cc HaveEraseCommand.h\ Piece.cc Piece.h\ CheckIntegrityMan.cc CheckIntegrityMan.h\ diff --git a/src/Makefile.in b/src/Makefile.in index 51353e6a..2f36ee31 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -255,20 +255,25 @@ am__libaria2c_a_SOURCES_DIST = Socket.h SocketCore.cc SocketCore.h \ BtProgressInfoFile.h DefaultBtProgressInfoFile.cc \ DefaultBtProgressInfoFile.h NullProgressInfoFile.h \ FileAllocationIterator.h SingleFileAllocationIterator.cc \ - SingleFileAllocationIterator.h PostDownloadHandler.cc \ - PostDownloadHandler.h HaveEraseCommand.cc HaveEraseCommand.h \ - Piece.cc Piece.h CheckIntegrityMan.cc CheckIntegrityMan.h \ - CheckIntegrityEntry.cc CheckIntegrityEntry.h \ - PieceHashCheckIntegrityEntry.cc PieceHashCheckIntegrityEntry.h \ - StreamCheckIntegrityEntry.cc StreamCheckIntegrityEntry.h \ - IteratableValidator.h DiskAdaptor.cc DiskAdaptor.h \ - AbstractSingleDiskAdaptor.cc AbstractSingleDiskAdaptor.h \ - CopyDiskAdaptor.cc CopyDiskAdaptor.h DirectDiskAdaptor.cc \ - DirectDiskAdaptor.h MultiDiskAdaptor.cc MultiDiskAdaptor.h \ - Peer.cc Peer.h BtRegistry.cc BtRegistry.h \ - MultiFileAllocationIterator.cc MultiFileAllocationIterator.h \ - PeerConnection.cc PeerConnection.h \ - IteratableChunkChecksumValidator.cc \ + SingleFileAllocationIterator.h \ + ContentTypeRequestGroupCriteria.cc \ + ContentTypeRequestGroupCriteria.h DownloadHandler.cc \ + DownloadHandler.h DownloadHandlerConstants.cc \ + DownloadHandlerConstants.h DownloadHandlerFactory.cc \ + DownloadHandlerFactory.h MemoryBufferPreDownloadHandler.cc \ + MemoryBufferPreDownloadHandler.h HaveEraseCommand.cc \ + HaveEraseCommand.h Piece.cc Piece.h CheckIntegrityMan.cc \ + CheckIntegrityMan.h CheckIntegrityEntry.cc \ + CheckIntegrityEntry.h PieceHashCheckIntegrityEntry.cc \ + PieceHashCheckIntegrityEntry.h StreamCheckIntegrityEntry.cc \ + StreamCheckIntegrityEntry.h IteratableValidator.h \ + DiskAdaptor.cc DiskAdaptor.h AbstractSingleDiskAdaptor.cc \ + AbstractSingleDiskAdaptor.h CopyDiskAdaptor.cc \ + CopyDiskAdaptor.h DirectDiskAdaptor.cc DirectDiskAdaptor.h \ + MultiDiskAdaptor.cc MultiDiskAdaptor.h Peer.cc Peer.h \ + BtRegistry.cc BtRegistry.h MultiFileAllocationIterator.cc \ + MultiFileAllocationIterator.h PeerConnection.cc \ + PeerConnection.h IteratableChunkChecksumValidator.cc \ IteratableChunkChecksumValidator.h \ IteratableChecksumValidator.cc IteratableChecksumValidator.h \ CheckIntegrityCommand.cc CheckIntegrityCommand.h \ @@ -465,9 +470,12 @@ am__objects_12 = SocketCore.$(OBJEXT) Command.$(OBJEXT) \ UnknownLengthPieceStorage.$(OBJEXT) ConsoleStatCalc.$(OBJEXT) \ TransferStat.$(OBJEXT) DefaultBtProgressInfoFile.$(OBJEXT) \ SingleFileAllocationIterator.$(OBJEXT) \ - PostDownloadHandler.$(OBJEXT) HaveEraseCommand.$(OBJEXT) \ - Piece.$(OBJEXT) CheckIntegrityMan.$(OBJEXT) \ - CheckIntegrityEntry.$(OBJEXT) \ + ContentTypeRequestGroupCriteria.$(OBJEXT) \ + DownloadHandler.$(OBJEXT) DownloadHandlerConstants.$(OBJEXT) \ + DownloadHandlerFactory.$(OBJEXT) \ + MemoryBufferPreDownloadHandler.$(OBJEXT) \ + HaveEraseCommand.$(OBJEXT) Piece.$(OBJEXT) \ + CheckIntegrityMan.$(OBJEXT) CheckIntegrityEntry.$(OBJEXT) \ PieceHashCheckIntegrityEntry.$(OBJEXT) \ StreamCheckIntegrityEntry.$(OBJEXT) DiskAdaptor.$(OBJEXT) \ AbstractSingleDiskAdaptor.$(OBJEXT) CopyDiskAdaptor.$(OBJEXT) \ @@ -747,23 +755,28 @@ SRCS = Socket.h SocketCore.cc SocketCore.h Command.cc Command.h \ BtProgressInfoFile.h DefaultBtProgressInfoFile.cc \ DefaultBtProgressInfoFile.h NullProgressInfoFile.h \ FileAllocationIterator.h SingleFileAllocationIterator.cc \ - SingleFileAllocationIterator.h PostDownloadHandler.cc \ - PostDownloadHandler.h HaveEraseCommand.cc HaveEraseCommand.h \ - Piece.cc Piece.h CheckIntegrityMan.cc CheckIntegrityMan.h \ - CheckIntegrityEntry.cc CheckIntegrityEntry.h \ - PieceHashCheckIntegrityEntry.cc PieceHashCheckIntegrityEntry.h \ - StreamCheckIntegrityEntry.cc StreamCheckIntegrityEntry.h \ - IteratableValidator.h DiskAdaptor.cc DiskAdaptor.h \ - AbstractSingleDiskAdaptor.cc AbstractSingleDiskAdaptor.h \ - CopyDiskAdaptor.cc CopyDiskAdaptor.h DirectDiskAdaptor.cc \ - DirectDiskAdaptor.h MultiDiskAdaptor.cc MultiDiskAdaptor.h \ - Peer.cc Peer.h BtRegistry.cc BtRegistry.h \ - MultiFileAllocationIterator.cc MultiFileAllocationIterator.h \ - PeerConnection.cc PeerConnection.h $(am__append_1) \ - $(am__append_2) $(am__append_3) $(am__append_4) \ - $(am__append_5) $(am__append_6) $(am__append_7) \ - $(am__append_8) $(am__append_9) $(am__append_10) \ - $(am__append_11) + SingleFileAllocationIterator.h \ + ContentTypeRequestGroupCriteria.cc \ + ContentTypeRequestGroupCriteria.h DownloadHandler.cc \ + DownloadHandler.h DownloadHandlerConstants.cc \ + DownloadHandlerConstants.h DownloadHandlerFactory.cc \ + DownloadHandlerFactory.h MemoryBufferPreDownloadHandler.cc \ + MemoryBufferPreDownloadHandler.h HaveEraseCommand.cc \ + HaveEraseCommand.h Piece.cc Piece.h CheckIntegrityMan.cc \ + CheckIntegrityMan.h CheckIntegrityEntry.cc \ + CheckIntegrityEntry.h PieceHashCheckIntegrityEntry.cc \ + PieceHashCheckIntegrityEntry.h StreamCheckIntegrityEntry.cc \ + StreamCheckIntegrityEntry.h IteratableValidator.h \ + DiskAdaptor.cc DiskAdaptor.h AbstractSingleDiskAdaptor.cc \ + AbstractSingleDiskAdaptor.h CopyDiskAdaptor.cc \ + CopyDiskAdaptor.h DirectDiskAdaptor.cc DirectDiskAdaptor.h \ + MultiDiskAdaptor.cc MultiDiskAdaptor.h Peer.cc Peer.h \ + BtRegistry.cc BtRegistry.h MultiFileAllocationIterator.cc \ + MultiFileAllocationIterator.h PeerConnection.cc \ + PeerConnection.h $(am__append_1) $(am__append_2) \ + $(am__append_3) $(am__append_4) $(am__append_5) \ + $(am__append_6) $(am__append_7) $(am__append_8) \ + $(am__append_9) $(am__append_10) $(am__append_11) noinst_LIBRARIES = libaria2c.a libaria2c_a_SOURCES = $(SRCS) aria2c_LDADD = libaria2c.a @LIBINTL@ @ALLOCA@ @LIBGNUTLS_LIBS@\ @@ -897,6 +910,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Command.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/CompactPeerListProcessor.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ConsoleStatCalc.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ContentTypeRequestGroupCriteria.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Cookie.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/CookieBox.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/CookieBoxFactory.Po@am__quote@ @@ -925,6 +939,9 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DownloadCommand.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DownloadEngine.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DownloadEngineFactory.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DownloadHandler.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DownloadHandlerConstants.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DownloadHandlerFactory.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Exception.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FeatureConfig.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/File.Po@am__quote@ @@ -958,6 +975,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/IteratableChunkChecksumValidator.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/List.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/LogFactory.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/MemoryBufferPreDownloadHandler.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/MessageDigestHelper.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/MetaFileUtil.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Metalink2RequestGroup.Po@am__quote@ @@ -991,7 +1009,6 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/PieceHashCheckIntegrityEntry.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/PiecedSegment.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Platform.Po@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/PostDownloadHandler.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/RealtimeCommand.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Request.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/RequestGroup.Po@am__quote@ diff --git a/src/MemoryBufferPreDownloadHandler.cc b/src/MemoryBufferPreDownloadHandler.cc new file mode 100644 index 00000000..fe906820 --- /dev/null +++ b/src/MemoryBufferPreDownloadHandler.cc @@ -0,0 +1,51 @@ +/* */ +#include "MemoryBufferPreDownloadHandler.h" +#include "RequestGroup.h" +#include "ByteArrayDiskWriterFactory.h" +#include "DownloadContext.h" + +MemoryBufferPreDownloadHandler::MemoryBufferPreDownloadHandler() {} + +MemoryBufferPreDownloadHandler::~MemoryBufferPreDownloadHandler() {} + +void MemoryBufferPreDownloadHandler::execute(RequestGroup* requestGroup) +{ + requestGroup->setDiskWriterFactory(new ByteArrayDiskWriterFactory()); + requestGroup->setFileAllocationEnabled(false); + requestGroup->setPreLocalFileCheckEnabled(false); + + requestGroup->getDownloadContext()->setDir("[MEMORY]"); +} diff --git a/src/MemoryBufferPreDownloadHandler.h b/src/MemoryBufferPreDownloadHandler.h new file mode 100644 index 00000000..2ea6f7b3 --- /dev/null +++ b/src/MemoryBufferPreDownloadHandler.h @@ -0,0 +1,51 @@ +/* */ +#ifndef _D_MEMORY_BUFFER_PRE_DOWNLOAD_HANDLER_H_ +#define _D_MEMORY_BUFFER_PRE_DOWNLOAD_HANDLER_H_ + +#include "PreDownloadHandler.h" + +class MemoryBufferPreDownloadHandler:public PreDownloadHandler +{ +public: + MemoryBufferPreDownloadHandler(); + + virtual ~MemoryBufferPreDownloadHandler(); + + virtual void execute(RequestGroup* requestGroup); +}; + +typedef SharedHandle MemoryBufferPreDownloadHandlerHandle; +#endif // _D_MEMORY_BUFFER_PRE_DOWNLOAD_HANDLER_H_ diff --git a/src/Metalink2RequestGroup.cc b/src/Metalink2RequestGroup.cc index 3962c3b6..2e36f514 100644 --- a/src/Metalink2RequestGroup.cc +++ b/src/Metalink2RequestGroup.cc @@ -42,6 +42,9 @@ #include "message.h" #include "SingleFileDownloadContext.h" #include "MetalinkHelper.h" +#include "BinaryStream.h" +#include "MemoryBufferPreDownloadHandler.h" +#include "TrueRequestGroupCriteria.h" #ifdef ENABLE_BITTORRENT # include "BtDependency.h" #endif // ENABLE_BITTORRENT @@ -97,6 +100,18 @@ RequestGroups Metalink2RequestGroup::generate(const string& metalinkFile) { MetalinkEntries entries = MetalinkHelper::parseAndQuery(metalinkFile, _option); + return createRequestGroup(entries); +} + +RequestGroups Metalink2RequestGroup::generate(const BinaryStreamHandle& binaryStream) +{ + MetalinkEntries entries = MetalinkHelper::parseAndQuery(binaryStream, + _option); + return createRequestGroup(entries); +} + +RequestGroups Metalink2RequestGroup::createRequestGroup(MetalinkEntries entries) +{ if(entries.size() == 0) { _logger->notice(EX_NO_RESULT_WITH_YOUR_PREFS); return RequestGroups(); @@ -142,9 +157,14 @@ RequestGroups Metalink2RequestGroup::generate(const string& metalinkFile) new SingleFileDownloadContext(_option->getAsInt(PREF_SEGMENT_SIZE), 0, ""); - dctx->setDir(_option->get(PREF_DIR)); + //dctx->setDir(_option->get(PREF_DIR)); torrentRg->setDownloadContext(dctx); + torrentRg->clearPreDowloadHandler(); torrentRg->clearPostDowloadHandler(); + // make it in-memory download + PreDownloadHandlerHandle preh = new MemoryBufferPreDownloadHandler(); + preh->setCriteria(new TrueRequestGroupCriteria()); + torrentRg->addPreDownloadHandler(preh); groups.push_back(torrentRg); } #endif // ENABLE_BITTORRENT diff --git a/src/Metalink2RequestGroup.h b/src/Metalink2RequestGroup.h index f739a702..d9c82cd3 100644 --- a/src/Metalink2RequestGroup.h +++ b/src/Metalink2RequestGroup.h @@ -42,18 +42,27 @@ class Logger; class RequestGroup; typedef SharedHandle RequestGroupHandle; typedef deque RequestGroups; +class BinaryStream; +typedef SharedHandle BinaryStreamHandle; +class MetalinkEntry; +typedef SharedHandle MetalinkEntryHandle; +typedef deque MetalinkEntries; class Metalink2RequestGroup { private: const Option* _option; const Logger* _logger; + + RequestGroups createRequestGroup(MetalinkEntries entries); public: Metalink2RequestGroup(const Option* option); ~Metalink2RequestGroup(); RequestGroups generate(const string& metalinkFile); + + RequestGroups generate(const BinaryStreamHandle& binaryStream); }; #endif // _D_METALINK_2_REQUEST_GROUP_H_ diff --git a/src/MetalinkHelper.cc b/src/MetalinkHelper.cc index 00597844..0f225107 100644 --- a/src/MetalinkHelper.cc +++ b/src/MetalinkHelper.cc @@ -39,6 +39,7 @@ #include "Metalinker.h" #include "prefs.h" #include "DlAbortEx.h" +#include "BinaryStream.h" MetalinkHelper::MetalinkHelper() {} @@ -49,6 +50,19 @@ MetalinkEntries MetalinkHelper::parseAndQuery(const string& filename, const Opti Xml2MetalinkProcessor proc; MetalinkerHandle metalinker = proc.parseFile(filename); + return query(metalinker, option); +} + +MetalinkEntries MetalinkHelper::parseAndQuery(const BinaryStreamHandle& binaryStream, const Option* option) +{ + Xml2MetalinkProcessor proc; + + MetalinkerHandle metalinker = proc.parseFromBinaryStream(binaryStream); + return query(metalinker, option); +} + +MetalinkEntries MetalinkHelper::query(const MetalinkerHandle& metalinker, const Option* option) +{ if(metalinker->entries.empty()) { throw new DlAbortEx("No file entry found. Probably, the metalink file is not configured properly or broken."); } diff --git a/src/MetalinkHelper.h b/src/MetalinkHelper.h index 809ef680..d15c23af 100644 --- a/src/MetalinkHelper.h +++ b/src/MetalinkHelper.h @@ -41,14 +41,23 @@ class Option; class MetalinkEntry; typedef SharedHandle MetalinkEntryHandle; typedef deque MetalinkEntries; +class BinaryStream; +typedef SharedHandle BinaryStreamHandle; +class Metalinker; +typedef SharedHandle MetalinkerHandle; class MetalinkHelper { private: MetalinkHelper(); ~MetalinkHelper(); + + static MetalinkEntries query(const MetalinkerHandle& metalinker, const Option* option); + public: static MetalinkEntries parseAndQuery(const string& filename, const Option* option); + + static MetalinkEntries parseAndQuery(const BinaryStreamHandle& binaryStream, const Option* option); }; #endif // _D_METALINK_HELPER_H_ diff --git a/src/MetalinkPostDownloadHandler.cc b/src/MetalinkPostDownloadHandler.cc index 56f24b63..a0f305c4 100644 --- a/src/MetalinkPostDownloadHandler.cc +++ b/src/MetalinkPostDownloadHandler.cc @@ -36,15 +36,32 @@ #include "RequestGroup.h" #include "Metalink2RequestGroup.h" #include "Logger.h" +#include "DiskAdaptor.h" +#include "PieceStorage.h" +#include "DownloadHandlerConstants.h" +#include "ContentTypeRequestGroupCriteria.h" -MetalinkPostDownloadHandler::MetalinkPostDownloadHandler(const Option* option): - PostDownloadHandler(".metalink", option) -{} +MetalinkPostDownloadHandler::MetalinkPostDownloadHandler() +{ + setCriteria(new ContentTypeRequestGroupCriteria(DownloadHandlerConstants::getMetalinkContentTypes(), + DownloadHandlerConstants::getMetalinkExtensions())); +} MetalinkPostDownloadHandler::~MetalinkPostDownloadHandler() {} -RequestGroups MetalinkPostDownloadHandler::getNextRequestGroups(const string& path) +RequestGroups MetalinkPostDownloadHandler::getNextRequestGroups(RequestGroup* requestGroup) { - _logger->debug("Generating RequestGroups for Metalink file %s", path.c_str()); - return Metalink2RequestGroup(_option).generate(path); + const Option* op = requestGroup->getOption(); + _logger->debug("Generating RequestGroups for Metalink file %s", + requestGroup->getFilePath().c_str()); + DiskAdaptorHandle diskAdaptor = requestGroup->getPieceStorage()->getDiskAdaptor(); + try { + diskAdaptor->openExistingFile(); + RequestGroups rgs = Metalink2RequestGroup(op).generate(diskAdaptor); + diskAdaptor->closeFile(); + return rgs; + } catch(Exception* e) { + diskAdaptor->closeFile(); + throw; + } } diff --git a/src/MetalinkPostDownloadHandler.h b/src/MetalinkPostDownloadHandler.h index 068f44c2..a4af00a8 100644 --- a/src/MetalinkPostDownloadHandler.h +++ b/src/MetalinkPostDownloadHandler.h @@ -40,11 +40,11 @@ class MetalinkPostDownloadHandler:public PostDownloadHandler { public: - MetalinkPostDownloadHandler(const Option* option); + MetalinkPostDownloadHandler(); virtual ~MetalinkPostDownloadHandler(); - virtual RequestGroups getNextRequestGroups(const string& path); + virtual RequestGroups getNextRequestGroups(RequestGroup* requestGroup); }; typedef SharedHandle MetalinkPostDownloadHandlerHandle; diff --git a/src/MetalinkProcessor.h b/src/MetalinkProcessor.h index f3f5b6c8..adc6aa21 100644 --- a/src/MetalinkProcessor.h +++ b/src/MetalinkProcessor.h @@ -38,11 +38,16 @@ #include "Metalinker.h" #include "common.h" +class BinaryStream; +typedef SharedHandle BinaryStreamHandle; + class MetalinkProcessor { public: virtual ~MetalinkProcessor() {} virtual MetalinkerHandle parseFile(const string& filename) = 0; + + virtual MetalinkerHandle parseFromBinaryStream(const BinaryStreamHandle& binaryStream) = 0; }; #endif // _D_METALINK_PROCESSOR_H_ diff --git a/src/OptionHandlerFactory.cc b/src/OptionHandlerFactory.cc index 22d93c45..aa4b7294 100644 --- a/src/OptionHandlerFactory.cc +++ b/src/OptionHandlerFactory.cc @@ -56,7 +56,7 @@ OptionHandlers OptionHandlerFactory::createOptionHandlers() handlers.push_back(new ParameterOptionHandler(PREF_HTTP_PROXY_METHOD, V_GET, V_TUNNEL)); handlers.push_back(new IntegerRangeOptionHandler(PREF_LISTEN_PORT, 1024, UINT16_MAX)); - handlers.push_back(new BooleanOptionHandler(PREF_FOLLOW_TORRENT)); + handlers.push_back(new ParameterOptionHandler(PREF_FOLLOW_TORRENT, V_TRUE, V_MEM, V_FALSE)); handlers.push_back(new BooleanOptionHandler(PREF_NO_PREALLOCATION)); handlers.push_back(new BooleanOptionHandler(PREF_DIRECT_FILE_MAPPING)); handlers.push_back(new IntegerRangeOptionHandler(PREF_SELECT_FILE, 1, INT32_MAX)); @@ -66,7 +66,7 @@ OptionHandlers OptionHandlerFactory::createOptionHandlers() handlers.push_back(new DefaultOptionHandler(PREF_METALINK_VERSION)); handlers.push_back(new DefaultOptionHandler(PREF_METALINK_LANGUAGE)); handlers.push_back(new DefaultOptionHandler(PREF_METALINK_OS)); - handlers.push_back(new BooleanOptionHandler(PREF_FOLLOW_METALINK)); + handlers.push_back(new ParameterOptionHandler(PREF_FOLLOW_METALINK, V_TRUE, V_MEM, V_FALSE)); handlers.push_back(new DefaultOptionHandler(PREF_METALINK_LOCATION)); handlers.push_back(new UnitNumberOptionHandler(PREF_LOWEST_SPEED_LIMIT, 0)); handlers.push_back(new UnitNumberOptionHandler(PREF_MAX_DOWNLOAD_LIMIT, 0)); diff --git a/src/OptionHandlerImpl.h b/src/OptionHandlerImpl.h index 4f60a78e..c48333f6 100644 --- a/src/OptionHandlerImpl.h +++ b/src/OptionHandlerImpl.h @@ -209,6 +209,17 @@ public: _validParamValues.push_back(validParamValue1); _validParamValues.push_back(validParamValue2); } + + ParameterOptionHandler(const string& optName, + const string& validParamValue1, + const string& validParamValue2, + const string& validParamValue3): + NameMatchOptionHandler(optName) + { + _validParamValues.push_back(validParamValue1); + _validParamValues.push_back(validParamValue2); + _validParamValues.push_back(validParamValue3); + } virtual ~ParameterOptionHandler() {} diff --git a/src/PostDownloadHandler.cc b/src/PostDownloadHandler.cc index 9afb549f..7fd7e43b 100644 --- a/src/PostDownloadHandler.cc +++ b/src/PostDownloadHandler.cc @@ -33,14 +33,8 @@ */ /* copyright --> */ #include "PostDownloadHandler.h" -#include "Util.h" -#include "LogFactory.h" +#include "RequestGroupCriteria.h" -PostDownloadHandler::PostDownloadHandler(const string& extension, const Option* option):_extension(extension), _option(option), _logger(LogFactory::getInstance()) {} +PostDownloadHandler::PostDownloadHandler(const RequestGroupCriteriaHandle& criteria):DownloadHandler(criteria) {} PostDownloadHandler::~PostDownloadHandler() {} - -bool PostDownloadHandler::canHandle(const string& path) -{ - return Util::endsWith(path, _extension); -} diff --git a/src/PostDownloadHandler.h b/src/PostDownloadHandler.h index 0a630f40..824f7379 100644 --- a/src/PostDownloadHandler.h +++ b/src/PostDownloadHandler.h @@ -35,29 +35,19 @@ #ifndef _D_POST_DOWNLOAD_HANDLER_H_ #define _D_POST_DOWNLOAD_HANDLER_H_ -#include "common.h" +#include "DownloadHandler.h" -class Option; -class RequestGroup; typedef SharedHandle RequestGroupHandle; typedef deque RequestGroups; -class Logger; -class PostDownloadHandler +class PostDownloadHandler:public DownloadHandler { -private: - string _extension; -protected: - const Option* _option; - const Logger* _logger; public: - PostDownloadHandler(const string& extension, const Option* option); + PostDownloadHandler() {} - virtual ~PostDownloadHandler(); + virtual ~PostDownloadHandler() {} - bool canHandle(const string& path); - - virtual RequestGroups getNextRequestGroups(const string& path) = 0; + virtual RequestGroups getNextRequestGroups(RequestGroup* requestGroup) = 0; }; typedef SharedHandle PostDownloadHandlerHandle; diff --git a/src/PreDownloadHandler.cc b/src/PreDownloadHandler.cc new file mode 100644 index 00000000..46de27fb --- /dev/null +++ b/src/PreDownloadHandler.cc @@ -0,0 +1,40 @@ +/* */ +#include "PreDownloadHandler.h" +#include "RequestGroupCriteria.h" + +PreDownloadHandler::PreDownloadHandler(const RequestGroupCriteriaHandle& criteria):DownloadHandler(criteria) {} + +PreDownloadHandler::~PreDownloadHandler() {} diff --git a/src/PreDownloadHandler.h b/src/PreDownloadHandler.h new file mode 100644 index 00000000..082a59e6 --- /dev/null +++ b/src/PreDownloadHandler.h @@ -0,0 +1,52 @@ +/* */ +#ifndef _D_PRE_DOWNLOAD_HANDLER_H_ +#define _D_PRE_DOWNLOAD_HANDLER_H_ + +#include "DownloadHandler.h" + +class PreDownloadHandler:public DownloadHandler +{ +public: + PreDownloadHandler() {} + + virtual ~PreDownloadHandler() {} + + virtual void execute(RequestGroup* requestGroup) = 0; +}; + +typedef SharedHandle PreDownloadHandlerHandle; +typedef deque PreDownloadHandlers; +#endif // _D_PRE_DOWNLOAD_HANDLER_H_ diff --git a/src/RequestGroup.cc b/src/RequestGroup.cc index 7f16ac54..2ae83294 100644 --- a/src/RequestGroup.cc +++ b/src/RequestGroup.cc @@ -58,7 +58,10 @@ #include "RequestGroupMan.h" #include "DefaultBtProgressInfoFile.h" #include "DefaultPieceStorage.h" -#include "PostDownloadHandler.h" +#include "DownloadResult.h" +#include "DownloadHandlerFactory.h" +#include "MemoryBufferPreDownloadHandler.h" +#include "DownloadHandlerConstants.h" #ifdef ENABLE_MESSAGE_DIGEST # include "CheckIntegrityCommand.h" #endif // ENABLE_MESSAGE_DIGEST @@ -100,6 +103,7 @@ RequestGroup::RequestGroup(const Option* option, } else { _fileAllocationEnabled = false; } + initializePreDownloadHandler(); initializePostDownloadHandler(); } @@ -555,14 +559,34 @@ void RequestGroup::releaseRuntimeResource() } } +void RequestGroup::preDownloadProcessing() +{ + _logger->debug("Finding PreDownloadHandler for path %s.", getFilePath().c_str()); + try { + for(PreDownloadHandlers::const_iterator itr = _preDownloadHandlers.begin(); + itr != _preDownloadHandlers.end(); ++itr) { + if((*itr)->canHandle(this)) { + (*itr)->execute(this); + break; + } + } + } catch(RecoverableException* ex) { + _logger->error(EX_EXCEPTION_CAUGHT, ex); + delete ex; + return; + } + _logger->debug("No PreDownloadHandler found."); + return; +} + RequestGroups RequestGroup::postDownloadProcessing() { _logger->debug("Finding PostDownloadHandler for path %s.", getFilePath().c_str()); try { for(PostDownloadHandlers::const_iterator itr = _postDownloadHandlers.begin(); itr != _postDownloadHandlers.end(); ++itr) { - if((*itr)->canHandle(getFilePath())) { - return (*itr)->getNextRequestGroups(getFilePath()); + if((*itr)->canHandle(this)) { + return (*itr)->getNextRequestGroups(this); } } } catch(RecoverableException* ex) { @@ -574,16 +598,32 @@ RequestGroups RequestGroup::postDownloadProcessing() return RequestGroups(); } -void RequestGroup::initializePostDownloadHandler() +void RequestGroup::initializePreDownloadHandler() { #ifdef ENABLE_BITTORRENT - if(_option->get(PREF_FOLLOW_TORRENT) == V_TRUE) { - _postDownloadHandlers.push_back(new BtPostDownloadHandler(_option)); + if(_option->get(PREF_FOLLOW_TORRENT) == V_MEM) { + _preDownloadHandlers.push_back(DownloadHandlerFactory::getBtPreDownloadHandler()); } #endif // ENABLE_BITTORRENT #ifdef ENABLE_METALINK - if(_option->get(PREF_FOLLOW_METALINK) == V_TRUE) { - _postDownloadHandlers.push_back(new MetalinkPostDownloadHandler(_option)); + if(_option->get(PREF_FOLLOW_METALINK) == V_MEM) { + _preDownloadHandlers.push_back(DownloadHandlerFactory::getMetalinkPreDownloadHandler()); + } +#endif // ENABLE_METALINK +} + +void RequestGroup::initializePostDownloadHandler() +{ +#ifdef ENABLE_BITTORRENT + if(_option->get(PREF_FOLLOW_TORRENT) == V_TRUE || + _option->get(PREF_FOLLOW_TORRENT) == V_MEM) { + _postDownloadHandlers.push_back(DownloadHandlerFactory::getBtPostDownloadHandler()); + } +#endif // ENABLE_BITTORRENT +#ifdef ENABLE_METALINK + if(_option->get(PREF_FOLLOW_METALINK) == V_TRUE || + _option->get(PREF_FOLLOW_METALINK) == V_MEM) { + _postDownloadHandlers.push_back(DownloadHandlerFactory::getMetalinkPostDownloadHandler()); } #endif // ENABLE_METALINK } @@ -628,11 +668,21 @@ void RequestGroup::addPostDownloadHandler(const PostDownloadHandlerHandle& handl _postDownloadHandlers.push_back(handler); } +void RequestGroup::addPreDownloadHandler(const PreDownloadHandlerHandle& handler) +{ + _preDownloadHandlers.push_back(handler); +} + void RequestGroup::clearPostDowloadHandler() { _postDownloadHandlers.clear(); } +void RequestGroup::clearPreDowloadHandler() +{ + _preDownloadHandlers.clear(); +} + SegmentManHandle RequestGroup::getSegmentMan() const { return _segmentMan; @@ -674,3 +724,16 @@ bool RequestGroup::needsFileAllocation() const _option->getAsLLInt(PREF_NO_FILE_ALLOCATION_LIMIT) <= getTotalLength() && !_pieceStorage->getDiskAdaptor()->fileAllocationIterator()->finished(); } + +DownloadResultHandle RequestGroup::createDownloadResult() const +{ + Strings uris = getUris(); + return new DownloadResult(_gid, + getFilePath(), + getTotalLength(), + uris.empty() ? "":uris.front(), + uris.size(), + downloadFinished()? + DownloadResult::FINISHED : + DownloadResult::NOT_YET); +} diff --git a/src/RequestGroup.h b/src/RequestGroup.h index f74fd171..b8cef64f 100644 --- a/src/RequestGroup.h +++ b/src/RequestGroup.h @@ -54,6 +54,9 @@ typedef SharedHandle BtProgressInfoFileHandle; class Dependency; typedef SharedHandle DependencyHandle; class DlAbortEx; +class PreDownloadHandler; +typedef SharedHandle PreDownloadHandlerHandle; +typedef deque PreDownloadHandlers; class PostDownloadHandler; typedef SharedHandle PostDownloadHandlerHandle; typedef deque PostDownloadHandlers; @@ -66,7 +69,8 @@ typedef SharedHandle RequestGroupHandle; typedef deque RequestGroups; class CheckIntegrityEntry; typedef SharedHandle CheckIntegrityEntryHandle; - +class DownloadResult; +typedef SharedHandle DownloadResultHandle; class RequestGroup { private: @@ -105,6 +109,8 @@ private: bool _haltRequested; + PreDownloadHandlers _preDownloadHandlers; + PostDownloadHandlers _postDownloadHandlers; const Option* _option; @@ -117,6 +123,8 @@ private: void validateTotalLength(int64_t expectedTotalLength, int64_t actualTotalLength) const; + void initializePreDownloadHandler(); + void initializePostDownloadHandler(); bool tryAutoFileRenaming(); @@ -271,6 +279,12 @@ public: void clearPostDowloadHandler(); + void preDownloadProcessing(); + + void addPreDownloadHandler(const PreDownloadHandlerHandle& handler); + + void clearPreDowloadHandler(); + Commands processCheckIntegrityEntry(const CheckIntegrityEntryHandle& entry, DownloadEngine* e); void initPieceStorage(); @@ -280,6 +294,13 @@ public: void loadAndOpenFile(const BtProgressInfoFileHandle& progressInfoFile); void shouldCancelDownloadForSafety(); + + DownloadResultHandle createDownloadResult() const; + + const Option* getOption() const + { + return _option; + } }; typedef SharedHandle RequestGroupHandle; diff --git a/src/RequestGroupCriteria.h b/src/RequestGroupCriteria.h new file mode 100644 index 00000000..5e193fef --- /dev/null +++ b/src/RequestGroupCriteria.h @@ -0,0 +1,52 @@ +/* */ +#ifndef _D_REQUEST_GROUP_CRITERIA_H_ +#define _D_REQUEST_GROUP_CRITERIA_H_ + +#include "common.h" + +class RequestGroup; + +class RequestGroupCriteria +{ +public: + virtual ~RequestGroupCriteria() {} + + virtual bool match(const RequestGroup* requestGroup) const = 0; +}; + +typedef SharedHandle RequestGroupCriteriaHandle; + +#endif // _D_REQUEST_GROUP_CRITERIA_H_ diff --git a/src/RequestGroupMan.cc b/src/RequestGroupMan.cc index f064a8ba..1515cafe 100644 --- a/src/RequestGroupMan.cc +++ b/src/RequestGroupMan.cc @@ -40,6 +40,7 @@ #include "DownloadEngine.h" #include "message.h" #include "a2functional.h" +#include "DownloadResult.h" #include #include #include @@ -103,31 +104,31 @@ void RequestGroupMan::removeStoppedGroup() if((*itr)->getNumCommand() > 0) { temp.push_back(*itr); } else { - (*itr)->closeFile(); - if((*itr)->downloadFinished()) { - _logger->notice(MSG_FILE_DOWNLOAD_COMPLETED, - (*itr)->getFilePath().c_str()); - if((*itr)->allDownloadFinished()) { - (*itr)->getProgressInfoFile()->removeFile(); + try { + (*itr)->closeFile(); + if((*itr)->downloadFinished()) { + _logger->notice(MSG_FILE_DOWNLOAD_COMPLETED, + (*itr)->getFilePath().c_str()); + if((*itr)->allDownloadFinished()) { + (*itr)->getProgressInfoFile()->removeFile(); + } else { + (*itr)->getProgressInfoFile()->save(); + } + RequestGroups nextGroups = (*itr)->postDownloadProcessing(); + if(nextGroups.size() > 0) { + _logger->debug("Adding %d RequestGroups as a result of PostDownloadHandler.", (int32_t)nextGroups.size()); + copy(nextGroups.rbegin(), nextGroups.rend(), front_inserter(_reservedGroups)); + } } else { (*itr)->getProgressInfoFile()->save(); } - RequestGroups nextGroups = (*itr)->postDownloadProcessing(); - if(nextGroups.size() > 0) { - _logger->debug("Adding %d RequestGroups as a result of PostDownloadHandler.", (int32_t)nextGroups.size()); - copy(nextGroups.rbegin(), nextGroups.rend(), front_inserter(_reservedGroups)); - } - } else { - try { - (*itr)->getProgressInfoFile()->save(); - } catch(RecoverableException* ex) { - _logger->error(EX_EXCEPTION_CAUGHT, ex); - delete ex; - } + } catch(RecoverableException* ex) { + _logger->error(EX_EXCEPTION_CAUGHT, ex); + delete ex; } (*itr)->releaseRuntimeResource(); ++count; - _spentGroups.push_back(*itr); + _downloadResults.push_back((*itr)->createDownloadResult()); } } _requestGroups = temp; @@ -145,11 +146,11 @@ void RequestGroupMan::fillRequestGroupFromReserver(DownloadEngine* e) num > 0 && _reservedGroups.size() > 0; --num) { RequestGroupHandle groupToAdd = _reservedGroups.front(); _reservedGroups.pop_front(); - if(!groupToAdd->isDependencyResolved()) { - temp.push_front(groupToAdd); - continue; - } try { + if(!groupToAdd->isDependencyResolved()) { + temp.push_front(groupToAdd); + continue; + } _requestGroups.push_back(groupToAdd); Commands commands = groupToAdd->createInitialCommand(e); ++count; @@ -157,6 +158,7 @@ void RequestGroupMan::fillRequestGroupFromReserver(DownloadEngine* e) } catch(RecoverableException* ex) { _logger->error(EX_EXCEPTION_CAUGHT, ex); delete ex; + _downloadResults.push_back(groupToAdd->createDownloadResult()); } } copy(temp.begin(), temp.end(), front_inserter(_reservedGroups)); @@ -170,20 +172,21 @@ Commands RequestGroupMan::getInitialCommands(DownloadEngine* e) Commands commands; for(RequestGroups::iterator itr = _requestGroups.begin(); itr != _requestGroups.end();) { - if((*itr)->isDependencyResolved()) { - try { + try { + if((*itr)->isDependencyResolved()) { Commands nextCommands = (*itr)->createInitialCommand(e); copy(nextCommands.begin(), nextCommands.end(), back_inserter(commands)); ++itr; - } catch(RecoverableException* e) { - _logger->error(EX_EXCEPTION_CAUGHT, e); - delete e; + } else { + _reservedGroups.push_front((*itr)); itr = _requestGroups.erase(itr); } - } else { - _reservedGroups.push_front((*itr)); + } catch(RecoverableException* e) { + _logger->error(EX_EXCEPTION_CAUGHT, e); + delete e; + _downloadResults.push_back((*itr)->createDownloadResult()); itr = _requestGroups.erase(itr); - } + } } return commands; } @@ -223,35 +226,37 @@ void RequestGroupMan::showDownloadResults(ostream& o) const << " (OK):download completed.(ERR):error occurred.(INPR):download in-progress." << "\n" << "gid|stat|path/URI" << "\n" << "===+====+======================================================================" << "\n"; - for(RequestGroups::const_iterator itr = _spentGroups.begin(); - itr != _spentGroups.end(); ++itr) { - o << formatDownloadResult((*itr)->downloadFinished()?"OK":"ERR", *itr) << "\n"; + for(DownloadResults::const_iterator itr = _downloadResults.begin(); + itr != _downloadResults.end(); ++itr) { + string status = (*itr)->result == DownloadResult::FINISHED ? "OK" : "ERR"; + o << formatDownloadResult(status, *itr) << "\n"; } for(RequestGroups::const_iterator itr = _requestGroups.begin(); itr != _requestGroups.end(); ++itr) { - o << formatDownloadResult((*itr)->downloadFinished()?"OK":"INPR", *itr) << "\n"; + DownloadResultHandle result = (*itr)->createDownloadResult(); + string status = result->result == DownloadResult::FINISHED ? "OK" : "INPR"; + o << formatDownloadResult(status, result) << "\n"; } } -string RequestGroupMan::formatDownloadResult(const string& status, const RequestGroupHandle& requestGroup) const +string RequestGroupMan::formatDownloadResult(const string& status, const DownloadResultHandle& downloadResult) const { stringstream o; - o << setw(3) << requestGroup->getGID() << "|" + o << setw(3) << downloadResult->gid << "|" << setw(4) << status << "|"; - if(requestGroup->downloadFinished()) { - o << requestGroup->getFilePath(); + if(downloadResult->result == DownloadResult::FINISHED) { + o << downloadResult->filePath; } else { - Strings uris = requestGroup->getUris(); - if(uris.empty()) { - if(requestGroup->getFilePath().empty()) { + if(downloadResult->numUri == 0) { + if(downloadResult->filePath.empty()) { o << "n/a"; } else { - o << requestGroup->getFilePath(); + o << downloadResult->filePath; } } else { - o << uris.front(); - if(uris.size() > 1) { - o << " (" << uris.size()-1 << "more)"; + o << downloadResult->uri; + if(downloadResult->numUri > 1) { + o << " (" << downloadResult->numUri-1 << "more)"; } } } diff --git a/src/RequestGroupMan.h b/src/RequestGroupMan.h index bb6a2646..96691ebe 100644 --- a/src/RequestGroupMan.h +++ b/src/RequestGroupMan.h @@ -45,17 +45,21 @@ typedef deque RequestGroups; class Command; typedef deque Commands; class Logger; +class DownloadResult; +typedef SharedHandle DownloadResultHandle; +typedef deque DownloadResults; class RequestGroupMan { private: RequestGroups _requestGroups; RequestGroups _reservedGroups; - RequestGroups _spentGroups; + DownloadResults _downloadResults; const Logger* _logger; int32_t _maxSimultaneousDownloads; int32_t _gidCounter; - string formatDownloadResult(const string& status, const RequestGroupHandle& requestGroup) const; + string formatDownloadResult(const string& status, + const DownloadResultHandle& downloadResult) const; public: RequestGroupMan(const RequestGroups& requestGroups, int32_t maxSimultaneousDownloads = 1); diff --git a/src/SingleFileDownloadContext.h b/src/SingleFileDownloadContext.h index b0f19a6e..32520b92 100644 --- a/src/SingleFileDownloadContext.h +++ b/src/SingleFileDownloadContext.h @@ -53,6 +53,8 @@ private: string _filename; string _ufilename; + string _contentType; + Strings _pieceHashes; string _pieceHashAlgo; @@ -185,6 +187,16 @@ public: { _pieceHashAlgo = algo; } + + void setContentType(const string& contentType) + { + _contentType = contentType; + } + + const string& getContentType() + { + return _contentType; + } }; typedef SharedHandle SingleFileDownloadContextHandle; diff --git a/src/TrueRequestGroupCriteria.h b/src/TrueRequestGroupCriteria.h new file mode 100644 index 00000000..28ea8911 --- /dev/null +++ b/src/TrueRequestGroupCriteria.h @@ -0,0 +1,53 @@ +/* */ +#ifndef _D_TRUE_REQUEST_GROUP_CRITERIA_H_ +#define _D_TRUE_REQUEST_GROUP_CRITERIA_H_ + +#include "RequestGroupCriteria.h" + +class TrueRequestGroupCriteria:public RequestGroupCriteria +{ +public: + TrueRequestGroupCriteria() {} + + virtual ~TrueRequestGroupCriteria() {} + + virtual bool match(const RequestGroup* requestGroup) const + { + return true; + } +}; + +#endif // _D_TRUE_REQUEST_GROUP_CRITERIA_H_ diff --git a/src/UnknownLengthPieceStorage.cc b/src/UnknownLengthPieceStorage.cc index 3955c889..853cb648 100644 --- a/src/UnknownLengthPieceStorage.cc +++ b/src/UnknownLengthPieceStorage.cc @@ -108,6 +108,7 @@ void UnknownLengthPieceStorage::completePiece(const PieceHandle& piece) if(_piece == piece) { _downloadFinished = true; _totalLength = _piece->getLength(); + _diskAdaptor->setTotalLength(_totalLength); _piece = 0; } } diff --git a/src/UnknownLengthPieceStorage.h b/src/UnknownLengthPieceStorage.h index 2aec4ede..bea423ba 100644 --- a/src/UnknownLengthPieceStorage.h +++ b/src/UnknownLengthPieceStorage.h @@ -42,6 +42,8 @@ class DownloadContext; typedef SharedHandle DownloadContextHandle; class DiskWriterFactory; typedef SharedHandle DiskWriterFactoryHandle; +class DirectDiskAdaptor; +typedef SharedHandle DirectDiskAdaptorHandle; class UnknownLengthPieceStorage:public PieceStorage { private: @@ -49,7 +51,7 @@ private: const Option* _option; - DiskAdaptorHandle _diskAdaptor; + DirectDiskAdaptorHandle _diskAdaptor; DiskWriterFactoryHandle _diskWriterFactory; diff --git a/src/Util.cc b/src/Util.cc index ab8568eb..5363e81a 100644 --- a/src/Util.cc +++ b/src/Util.cc @@ -41,12 +41,14 @@ #include "DlAbortEx.h" #include "BitfieldMan.h" #include "DefaultDiskWriter.h" +#include "BinaryStream.h" #include #include #include #include #include #include +#include #ifndef HAVE_SLEEP # ifdef HAVE_WINSOCK_H @@ -833,3 +835,17 @@ void Util::convertBitfield(BitfieldMan* dest, const BitfieldMan* src) } } } + +string Util::toString(const BinaryStreamHandle& binaryStream) +{ + stringstream strm; + char data[2048]; + while(1) { + int32_t dataLength = binaryStream->readData((unsigned char*)data, sizeof(data), strm.tellp()); + strm.write(data, dataLength); + if(dataLength == 0) { + break; + } + } + return strm.str(); +} diff --git a/src/Util.h b/src/Util.h index 7bf1a7a8..12120264 100644 --- a/src/Util.h +++ b/src/Util.h @@ -46,6 +46,8 @@ class Randomizer; typedef SharedHandle RandomizerHandle; class BitfieldMan; +class BinaryStream; +typedef SharedHandle BinaryStreamHandle; #define STRTOLL(X) strtoll(X, (char**)NULL, 10) @@ -163,6 +165,9 @@ public: static void mkdirs(const string& dirpath); static void convertBitfield(BitfieldMan* dest, const BitfieldMan* src); + + // binaryStream has to be opened before calling this function. + static string toString(const BinaryStreamHandle& binaryStream); }; #endif // _D_UTIL_H_ diff --git a/src/Xml2MetalinkProcessor.cc b/src/Xml2MetalinkProcessor.cc index 87c34c33..2eae4f3d 100644 --- a/src/Xml2MetalinkProcessor.cc +++ b/src/Xml2MetalinkProcessor.cc @@ -35,6 +35,7 @@ #include "Xml2MetalinkProcessor.h" #include "DlAbortEx.h" #include "Util.h" +#include "BinaryStream.h" #include #include #include @@ -62,6 +63,49 @@ MetalinkerHandle Xml2MetalinkProcessor::parseFile(const string& filename) { if(!doc) { throw new DlAbortEx("Cannot parse metalink file %s", filename.c_str()); } + return processDoc(doc); +} + +MetalinkerHandle Xml2MetalinkProcessor::parseFromBinaryStream(const BinaryStreamHandle& binaryStream) { + release(); + int32_t bufSize = 4096; + unsigned char buf[bufSize]; + + int32_t res = binaryStream->readData(buf, 4, 0); + if(res != 4) { + throw new DlAbortEx("Too small data for metalink parsing."); + } + + xmlParserCtxtPtr ctx = xmlCreatePushParserCtxt(0, 0, (const char*)buf, res, 0); + try { + int64_t readOffset = res; + while(1) { + int32_t res = binaryStream->readData(buf, bufSize, readOffset); + if(res == 0) { + break; + } + if(xmlParseChunk(ctx, (const char*)buf, res, 0) != 0) { + throw new DlAbortEx("Cannot parse metalink file"); + } + readOffset += res; + } + xmlParseChunk(ctx, (const char*)buf, 0, 1); + doc = ctx->myDoc; + + xmlFreeParserCtxt(ctx); + } catch(Exception* e) { + xmlFreeParserCtxt(ctx); + throw; + } + + if(!doc) { + throw new DlAbortEx("Cannot parse metalink file"); + } + return processDoc(doc); +} + +MetalinkerHandle Xml2MetalinkProcessor::processDoc(xmlDocPtr doc) +{ context = xmlXPathNewContext(doc); if(!context) { throw new DlAbortEx("Cannot create new xpath context"); diff --git a/src/Xml2MetalinkProcessor.h b/src/Xml2MetalinkProcessor.h index 2d1c9b28..2a5cdbea 100644 --- a/src/Xml2MetalinkProcessor.h +++ b/src/Xml2MetalinkProcessor.h @@ -39,6 +39,9 @@ #include #include +class BinaryStream; +typedef SharedHandle BinaryStreamHandle; + class Xml2MetalinkProcessor : public MetalinkProcessor { private: xmlDocPtr doc; @@ -59,12 +62,16 @@ private: bool xpathExists(const string& xpath); void release(); + + MetalinkerHandle processDoc(xmlDocPtr doc); + public: Xml2MetalinkProcessor(); virtual ~Xml2MetalinkProcessor(); virtual MetalinkerHandle parseFile(const string& filename); - + + virtual MetalinkerHandle parseFromBinaryStream(const BinaryStreamHandle& binaryStream); }; #endif // _D_XML2_METALINK_PROCESSOR_H_ diff --git a/src/a2functional.h b/src/a2functional.h index 963399be..8382b572 100644 --- a/src/a2functional.h +++ b/src/a2functional.h @@ -107,3 +107,20 @@ adopt2nd(const BinaryOp& binaryOp, const UnaryOp& unaryOp) { return adopt2nd_t(binaryOp, unaryOp); }; + +template +char (&char_array_ref(T (&)[N]))[N]; + +template +std::size_t +arrayLength(T (&a)[N]) +{ + return sizeof(char_array_ref(a)); +} + +template +std::size_t +arrayLength(T (&a)[0u]) +{ + return 0; +} diff --git a/src/prefs.h b/src/prefs.h index 897f603c..a705f73f 100644 --- a/src/prefs.h +++ b/src/prefs.h @@ -46,7 +46,7 @@ #define V_FALSE "false" #undef V_NONE #define V_NONE "none" - +#define V_MEM "mem" /** * General preferences */ @@ -189,7 +189,7 @@ #define PREF_TORRENT_FILE "torrent-file" // values: 1*digit #define PREF_LISTEN_PORT "listen-port" -// values: true | false +// values: true | false | mem #define PREF_FOLLOW_TORRENT "follow-torrent" // values: 1*digit *( (,|-) 1*digit) #define PREF_SELECT_FILE "select-file" @@ -219,7 +219,7 @@ #define PREF_METALINK_LOCATION "metalink-location" // values: 1*digit #define PREF_METALINK_SERVERS "metalink-servers" -// values: true | false +// values: true | false | mem #define PREF_FOLLOW_METALINK "follow-metalink" #endif // _D_PREFS_H_ diff --git a/src/version_usage.cc b/src/version_usage.cc index 55c813ad..3390e815 100644 --- a/src/version_usage.cc +++ b/src/version_usage.cc @@ -235,9 +235,15 @@ void showUsage() { #endif // ENABLE_BITTORRENT || ENABLE_METALINK #ifdef ENABLE_BITTORRENT cout << _(" -T, --torrent-file=TORRENT_FILE The path to the .torrent file.") << endl; - cout << _(" --follow-torrent=true|false Set to false to prevent aria2 from\n" - " entering BitTorrent mode even if the filename of\n" - " the downloaded file ends with .torrent.\n" + cout << _(" --follow-torrent=true|false|mem If true or mem is specified, when a file\n" + " whose suffix is .metaink or content type is\n" + " application/x-bittorrent is downloaded, aria2\n" + " parses it as a torrent file and downloads files\n" + " mentioned in it.\n" + " If mem is specified, a metalink file is not\n" + " written to the disk, but is just kept in memory.\n" + " If false is specified, the action mentioned above\n" + " is not taken.\n" " Default: true") << endl; cout << _(" --direct-file-mapping=true|false Directly read from and write to each file\n" " mentioned in .torrent file.\n" @@ -281,9 +287,15 @@ void showUsage() { cout << _(" --metalink-location=LOCATION[,...] The location of the preferred server.\n" " A comma-deliminated list of locations is\n" " acceptable.") << endl; - cout << _(" --follow-metalink=true|false Set to false to prevent aria2 from\n" - " entering Metalink mode even if the filename of\n" - " the downloaded file ends with .metalink.\n" + cout << _(" --follow-metalink=true|false|mem If true or mem is specified, when a file\n" + " whose suffix is .metaink or content type is\n" + " application/metalink+xml is downloaded, aria2\n" + " parses it as a metalink file and downloads files\n" + " mentioned in it.\n" + " If mem is specified, a metalink file is not\n" + " written to the disk, but is just kept in memory.\n" + " If false is specified, the action mentioned above\n" + " is not taken.\n" " Default: true") << endl; #endif // ENABLE_METALINK cout << _(" -v, --version Print the version number and exit.") << endl; diff --git a/test/BtPostDownloadHandlerTest.cc b/test/BtPostDownloadHandlerTest.cc index b4228ee2..ec469ee4 100644 --- a/test/BtPostDownloadHandlerTest.cc +++ b/test/BtPostDownloadHandlerTest.cc @@ -2,12 +2,14 @@ #include "BtContext.h" #include "RequestGroup.h" #include "Option.h" +#include "SingleFileDownloadContext.h" #include class BtPostDownloadHandlerTest:public CppUnit::TestFixture { CPPUNIT_TEST_SUITE(BtPostDownloadHandlerTest); - CPPUNIT_TEST(testCanHandle); + CPPUNIT_TEST(testCanHandle_extension); + CPPUNIT_TEST(testCanHandle_contentType); CPPUNIT_TEST(testGetNextRequestGroups); CPPUNIT_TEST_SUITE_END(); private: @@ -15,26 +17,55 @@ private: public: void setUp() {} - void testCanHandle(); + void testCanHandle_extension(); + void testCanHandle_contentType(); void testGetNextRequestGroups(); }; CPPUNIT_TEST_SUITE_REGISTRATION( BtPostDownloadHandlerTest ); -void BtPostDownloadHandlerTest::testCanHandle() +void BtPostDownloadHandlerTest::testCanHandle_extension() { Option op; - BtPostDownloadHandler handler(&op); - CPPUNIT_ASSERT(!handler.canHandle(".torrent!!")); - CPPUNIT_ASSERT(handler.canHandle(".torrent")); + SingleFileDownloadContextHandle dctx = new SingleFileDownloadContext(0, 0, "test.torrent"); + RequestGroup rg(&op, Strings()); + rg.setDownloadContext(dctx); + + BtPostDownloadHandler handler; + + CPPUNIT_ASSERT(handler.canHandle(&rg)); + + dctx->setFilename("test.torrent2"); + CPPUNIT_ASSERT(!handler.canHandle(&rg)); +} + +void BtPostDownloadHandlerTest::testCanHandle_contentType() +{ + Option op; + SingleFileDownloadContextHandle dctx = new SingleFileDownloadContext(0, 0, "test"); + dctx->setContentType("application/x-bittorrent"); + RequestGroup rg(&op, Strings()); + rg.setDownloadContext(dctx); + + BtPostDownloadHandler handler; + + CPPUNIT_ASSERT(handler.canHandle(&rg)); + + dctx->setContentType("application/octet-stream"); + CPPUNIT_ASSERT(!handler.canHandle(&rg)); } void BtPostDownloadHandlerTest::testGetNextRequestGroups() { Option op; - BtPostDownloadHandler handler(&op); - RequestGroups groups = handler.getNextRequestGroups("test.torrent"); + SingleFileDownloadContextHandle dctx = new SingleFileDownloadContext(0, 0, "test.torrent"); + RequestGroup rg(&op, Strings()); + rg.setDownloadContext(dctx); + rg.initPieceStorage(); + + BtPostDownloadHandler handler; + RequestGroups groups = handler.getNextRequestGroups(&rg); CPPUNIT_ASSERT_EQUAL((size_t)1, groups.size()); BtContextHandle btctx = groups.front()->getDownloadContext(); CPPUNIT_ASSERT(!btctx.isNull()); diff --git a/test/ByteArrayDiskWriterTest.cc b/test/ByteArrayDiskWriterTest.cc index 18ea5e0d..6e12417e 100644 --- a/test/ByteArrayDiskWriterTest.cc +++ b/test/ByteArrayDiskWriterTest.cc @@ -26,41 +26,36 @@ CPPUNIT_TEST_SUITE_REGISTRATION( ByteArrayDiskWriterTest ); void ByteArrayDiskWriterTest::testWriteAndRead() { ByteArrayDiskWriter bw; - string msg1 = "Hello world!"; + string msg1 = "Hello"; bw.writeData((const unsigned char*)msg1.c_str(), msg1.size(), 0); - + // write at the end of stream + string msg2 = " World"; + bw.writeData((const unsigned char*)msg2.c_str(), msg2.size(), 5); + // write at the end of stream +1 + string msg3 = "!!"; + bw.writeData((const unsigned char*)msg3.c_str(), msg3.size(), 12); + // write space at the 'hole' + string msg4 = " "; + bw.writeData((const unsigned char*)msg4.c_str(), msg4.size(), 11); + char buf[100]; - int32_t c = bw.readData((unsigned char*)buf, sizeof(buf), 0); + int32_t c = bw.readData((unsigned char*)buf, sizeof(buf), 1); buf[c] = '\0'; - CPPUNIT_ASSERT_EQUAL(msg1, string(buf)); - - // second call - memset(buf, '\0', sizeof(buf)); - - c = bw.readData((unsigned char*)buf, sizeof(buf), 0); - buf[c] = '\0'; - - CPPUNIT_ASSERT_EQUAL(msg1, string(buf)); + CPPUNIT_ASSERT_EQUAL(string("ello World !!"), string(buf)); } void ByteArrayDiskWriterTest::testWriteAndRead2() { ByteArrayDiskWriter bw; - string msg1 = "Hello world!"; - bw.writeData((const unsigned char*)msg1.c_str(), msg1.size(), 16); - + string msg1 = "Hello World"; + bw.writeData((const unsigned char*)msg1.c_str(), msg1.size(), 0); + string msg2 = "From Mars"; + bw.writeData((const unsigned char*)msg2.c_str(), msg2.size(), 6); + char buf[100]; - int32_t c = bw.readData((unsigned char*)buf, sizeof(buf), 16); + int32_t c = bw.readData((unsigned char*)buf, sizeof(buf), 0); buf[c] = '\0'; - CPPUNIT_ASSERT_EQUAL(msg1, string(buf)); - - // second call - memset(buf, '\0', sizeof(buf)); - - c = bw.readData((unsigned char*)buf, sizeof(buf), 16); - buf[c] = '\0'; - - CPPUNIT_ASSERT_EQUAL(msg1, string(buf)); + CPPUNIT_ASSERT_EQUAL(string("Hello From Mars"), string(buf)); } diff --git a/test/DefaultBtContextTest.cc b/test/DefaultBtContextTest.cc index 0a77fb42..41694c17 100644 --- a/test/DefaultBtContextTest.cc +++ b/test/DefaultBtContextTest.cc @@ -28,6 +28,7 @@ class DefaultBtContextTest:public CppUnit::TestFixture { CPPUNIT_TEST(testComputeFastSet); CPPUNIT_TEST(testGetFileEntries_multiFileUrlList); CPPUNIT_TEST(testGetFileEntries_singleFileUrlList); + CPPUNIT_TEST(testLoadFromMemory); CPPUNIT_TEST_SUITE_END(); public: void setUp() { @@ -51,6 +52,7 @@ public: void testComputeFastSet(); void testGetFileEntries_multiFileUrlList(); void testGetFileEntries_singleFileUrlList(); + void testLoadFromMemory(); }; @@ -309,3 +311,17 @@ void DefaultBtContextTest::testGetFileEntries_singleFileUrlList() { CPPUNIT_ASSERT_EQUAL(string("http://localhost/dist/aria2.tar.bz2"), uris1[0]); } + +void DefaultBtContextTest::testLoadFromMemory() +{ + string memory = "d8:announce36:http://aria.rednoah.com/announce.php13:announce-listll16:http://tracker1 el15:http://tracker2el15:http://tracker3ee7:comment17:REDNOAH.COM RULES13:creation datei1123456789e4:infod5:filesld6:lengthi284e4:pathl5:aria23:src6:aria2ceed6:lengthi100e4:pathl19:aria2-0.2.2.tar.bz2eee4:name10:aria2-test12:piece lengthi128e6:pieces60:AAAAAAAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCee"; + + DefaultBtContext btContext; + btContext.loadFromMemory(memory.c_str(), memory.size(), "default"); + + string correctHash = "248d0a1cd08284299de78d5c1ed359bb46717d8c"; + + CPPUNIT_ASSERT_EQUAL((int32_t)20, btContext.getInfoHashLength()); + CPPUNIT_ASSERT_EQUAL(correctHash, Util::toHex(btContext.getInfoHash(), + btContext.getInfoHashLength())); +} diff --git a/test/DownloadHandlerFactoryTest.cc b/test/DownloadHandlerFactoryTest.cc new file mode 100644 index 00000000..5039045c --- /dev/null +++ b/test/DownloadHandlerFactoryTest.cc @@ -0,0 +1,90 @@ +#include "DownloadHandlerFactory.h" +#include "RequestGroup.h" +#include "Option.h" +#include "SingleFileDownloadContext.h" +#include "MemoryBufferPreDownloadHandler.h" +#include + +class DownloadHandlerFactoryTest:public CppUnit::TestFixture { + + CPPUNIT_TEST_SUITE(DownloadHandlerFactoryTest); + CPPUNIT_TEST(testGetMetalinkPreDownloadHandler_extension); + CPPUNIT_TEST(testGetMetalinkPreDownloadHandler_contentType); + CPPUNIT_TEST(testGetBtPreDownloadHandler_extension); + CPPUNIT_TEST(testGetBtPreDownloadHandler_contentType); + CPPUNIT_TEST_SUITE_END(); +private: + +public: + void setUp() {} + + void testGetMetalinkPreDownloadHandler_extension(); + void testGetMetalinkPreDownloadHandler_contentType(); + void testGetBtPreDownloadHandler_extension(); + void testGetBtPreDownloadHandler_contentType(); +}; + + +CPPUNIT_TEST_SUITE_REGISTRATION( DownloadHandlerFactoryTest ); + +void DownloadHandlerFactoryTest::testGetMetalinkPreDownloadHandler_extension() +{ + Option op; + SingleFileDownloadContextHandle dctx = new SingleFileDownloadContext(0, 0, "test.metalink"); + RequestGroup rg(&op, Strings()); + rg.setDownloadContext(dctx); + + PreDownloadHandlerHandle handler = DownloadHandlerFactory::getMetalinkPreDownloadHandler(); + + CPPUNIT_ASSERT(handler->canHandle(&rg)); + + dctx->setFilename("test.metalink2"); + CPPUNIT_ASSERT(!handler->canHandle(&rg)); +} + +void DownloadHandlerFactoryTest::testGetMetalinkPreDownloadHandler_contentType() +{ + Option op; + SingleFileDownloadContextHandle dctx = new SingleFileDownloadContext(0, 0, "test"); + dctx->setContentType("application/metalink+xml"); + RequestGroup rg(&op, Strings()); + rg.setDownloadContext(dctx); + + PreDownloadHandlerHandle handler = DownloadHandlerFactory::getMetalinkPreDownloadHandler(); + + CPPUNIT_ASSERT(handler->canHandle(&rg)); + + dctx->setContentType("application/octet-stream"); + CPPUNIT_ASSERT(!handler->canHandle(&rg)); +} + +void DownloadHandlerFactoryTest::testGetBtPreDownloadHandler_extension() +{ + Option op; + SingleFileDownloadContextHandle dctx = new SingleFileDownloadContext(0, 0, "test.torrent"); + RequestGroup rg(&op, Strings()); + rg.setDownloadContext(dctx); + + PreDownloadHandlerHandle handler = DownloadHandlerFactory::getBtPreDownloadHandler(); + + CPPUNIT_ASSERT(handler->canHandle(&rg)); + + dctx->setFilename("test.torrent2"); + CPPUNIT_ASSERT(!handler->canHandle(&rg)); +} + +void DownloadHandlerFactoryTest::testGetBtPreDownloadHandler_contentType() +{ + Option op; + SingleFileDownloadContextHandle dctx = new SingleFileDownloadContext(0, 0, "test"); + dctx->setContentType("application/x-bittorrent"); + RequestGroup rg(&op, Strings()); + rg.setDownloadContext(dctx); + + PreDownloadHandlerHandle handler = DownloadHandlerFactory::getBtPreDownloadHandler(); + + CPPUNIT_ASSERT(handler->canHandle(&rg)); + + dctx->setContentType("application/octet-stream"); + CPPUNIT_ASSERT(!handler->canHandle(&rg)); +} diff --git a/test/Makefile.am b/test/Makefile.am index a8cf166f..603c350f 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -94,6 +94,7 @@ aria2c_SOURCES += BtAllowedFastMessageTest.cc\ BtRegistryTest.cc\ BtDependencyTest.cc\ BtPostDownloadHandlerTest.cc\ + DownloadHandlerFactoryTest.cc\ TimeSeedCriteriaTest.cc endif # ENABLE_BITTORRENT diff --git a/test/Makefile.in b/test/Makefile.in index 16b05206..012b343a 100644 --- a/test/Makefile.in +++ b/test/Makefile.in @@ -82,6 +82,7 @@ check_PROGRAMS = $(am__EXEEXT_1) @ENABLE_BITTORRENT_TRUE@ BtRegistryTest.cc\ @ENABLE_BITTORRENT_TRUE@ BtDependencyTest.cc\ @ENABLE_BITTORRENT_TRUE@ BtPostDownloadHandlerTest.cc\ +@ENABLE_BITTORRENT_TRUE@ DownloadHandlerFactoryTest.cc\ @ENABLE_BITTORRENT_TRUE@ TimeSeedCriteriaTest.cc @ENABLE_METALINK_TRUE@am__append_3 = MetalinkerTest.cc\ @@ -152,8 +153,8 @@ am__aria2c_SOURCES_DIST = AllTest.cc SequenceTest.cc \ MetaFileUtilTest.cc ByteArrayDiskWriterTest.cc PeerTest.cc \ PeerMessageUtilTest.cc ShareRatioSeedCriteriaTest.cc \ BtRegistryTest.cc BtDependencyTest.cc \ - BtPostDownloadHandlerTest.cc TimeSeedCriteriaTest.cc \ - MetalinkerTest.cc MetalinkEntryTest.cc \ + BtPostDownloadHandlerTest.cc DownloadHandlerFactoryTest.cc \ + TimeSeedCriteriaTest.cc MetalinkerTest.cc MetalinkEntryTest.cc \ Xml2MetalinkProcessorTest.cc Metalink2RequestGroupTest.cc \ MetalinkPostDownloadHandlerTest.cc MetalinkHelperTest.cc @ENABLE_MESSAGE_DIGEST_TRUE@am__objects_1 = \ @@ -198,6 +199,7 @@ am__aria2c_SOURCES_DIST = AllTest.cc SequenceTest.cc \ @ENABLE_BITTORRENT_TRUE@ BtRegistryTest.$(OBJEXT) \ @ENABLE_BITTORRENT_TRUE@ BtDependencyTest.$(OBJEXT) \ @ENABLE_BITTORRENT_TRUE@ BtPostDownloadHandlerTest.$(OBJEXT) \ +@ENABLE_BITTORRENT_TRUE@ DownloadHandlerFactoryTest.$(OBJEXT) \ @ENABLE_BITTORRENT_TRUE@ TimeSeedCriteriaTest.$(OBJEXT) @ENABLE_METALINK_TRUE@am__objects_3 = MetalinkerTest.$(OBJEXT) \ @ENABLE_METALINK_TRUE@ MetalinkEntryTest.$(OBJEXT) \ @@ -544,6 +546,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DefaultPeerStorageTest.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DefaultPieceStorageTest.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DictionaryTest.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DownloadHandlerFactoryTest.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FeatureConfigTest.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FileEntryTest.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FileTest.Po@am__quote@ diff --git a/test/MetalinkPostDownloadHandlerTest.cc b/test/MetalinkPostDownloadHandlerTest.cc index 2c711cf1..69caf68b 100644 --- a/test/MetalinkPostDownloadHandlerTest.cc +++ b/test/MetalinkPostDownloadHandlerTest.cc @@ -1,12 +1,14 @@ #include "MetalinkPostDownloadHandler.h" #include "RequestGroup.h" #include "Option.h" +#include "SingleFileDownloadContext.h" #include class MetalinkPostDownloadHandlerTest:public CppUnit::TestFixture { CPPUNIT_TEST_SUITE(MetalinkPostDownloadHandlerTest); - CPPUNIT_TEST(testCanHandle); + CPPUNIT_TEST(testCanHandle_extension); + CPPUNIT_TEST(testCanHandle_contentType); CPPUNIT_TEST(testGetNextRequestGroups); CPPUNIT_TEST_SUITE_END(); private: @@ -14,26 +16,55 @@ private: public: void setUp() {} - void testCanHandle(); + void testCanHandle_extension(); + void testCanHandle_contentType(); void testGetNextRequestGroups(); }; CPPUNIT_TEST_SUITE_REGISTRATION( MetalinkPostDownloadHandlerTest ); -void MetalinkPostDownloadHandlerTest::testCanHandle() +void MetalinkPostDownloadHandlerTest::testCanHandle_extension() { Option op; - MetalinkPostDownloadHandler handler(&op); - CPPUNIT_ASSERT(!handler.canHandle(".metalink!!")); - CPPUNIT_ASSERT(handler.canHandle(".metalink")); + SingleFileDownloadContextHandle dctx = new SingleFileDownloadContext(0, 0, "test.metalink"); + RequestGroup rg(&op, Strings()); + rg.setDownloadContext(dctx); + + MetalinkPostDownloadHandler handler; + + CPPUNIT_ASSERT(handler.canHandle(&rg)); + + dctx->setFilename("test.metalink2"); + CPPUNIT_ASSERT(!handler.canHandle(&rg)); +} + +void MetalinkPostDownloadHandlerTest::testCanHandle_contentType() +{ + Option op; + SingleFileDownloadContextHandle dctx = new SingleFileDownloadContext(0, 0, "test"); + dctx->setContentType("application/metalink+xml"); + RequestGroup rg(&op, Strings()); + rg.setDownloadContext(dctx); + + MetalinkPostDownloadHandler handler; + + CPPUNIT_ASSERT(handler.canHandle(&rg)); + + dctx->setContentType("application/octet-stream"); + CPPUNIT_ASSERT(!handler.canHandle(&rg)); } void MetalinkPostDownloadHandlerTest::testGetNextRequestGroups() { Option op; - MetalinkPostDownloadHandler handler(&op); - RequestGroups groups = handler.getNextRequestGroups("test.xml"); + SingleFileDownloadContextHandle dctx = new SingleFileDownloadContext(0, 0, "test.xml"); + RequestGroup rg(&op, Strings()); + rg.setDownloadContext(dctx); + rg.initPieceStorage(); + + MetalinkPostDownloadHandler handler; + RequestGroups groups = handler.getNextRequestGroups(&rg); #ifdef ENABLE_BITTORRENT CPPUNIT_ASSERT_EQUAL((size_t)6/* 5 + 1 torrent file download */, groups.size()); #else diff --git a/test/RequestGroupManTest.cc b/test/RequestGroupManTest.cc index 6b88e63b..ef7eb707 100644 --- a/test/RequestGroupManTest.cc +++ b/test/RequestGroupManTest.cc @@ -4,6 +4,7 @@ #include "SingleFileDownloadContext.h" #include "RequestGroup.h" #include "Option.h" +#include "DownloadResult.h" #include using namespace std; diff --git a/test/UtilTest.cc b/test/UtilTest.cc index 7a92b9e7..4a50ccac 100644 --- a/test/UtilTest.cc +++ b/test/UtilTest.cc @@ -2,6 +2,7 @@ #include "FixedNumberRandomizer.h" #include "DlAbortEx.h" #include "BitfieldMan.h" +#include "ByteArrayDiskWriter.h" #include #include @@ -36,6 +37,7 @@ class UtilTest:public CppUnit::TestFixture { CPPUNIT_TEST(testParseIntRange_invalidRange); CPPUNIT_TEST(testParseInt); CPPUNIT_TEST(testParseLLInt); + CPPUNIT_TEST(testToString_binaryStream); CPPUNIT_TEST_SUITE_END(); private: @@ -69,6 +71,7 @@ public: void testParseIntRange_invalidRange(); void testParseInt(); void testParseLLInt(); + void testToString_binaryStream(); }; @@ -580,3 +583,15 @@ void UtilTest::testParseLLInt() delete e; } } + +void UtilTest::testToString_binaryStream() +{ + DiskWriterHandle dw = new ByteArrayDiskWriter(); + string data = string(16*1024+256, 'a'); + dw->initAndOpenFile("dummy"); + dw->writeData((const unsigned char*)data.c_str(), data.size(), 0); + + string readData = Util::toString(dw); + + CPPUNIT_ASSERT_EQUAL(data, readData); +} diff --git a/test/Xml2MetalinkProcessorTest.cc b/test/Xml2MetalinkProcessorTest.cc index ddafb861..8e4d0444 100644 --- a/test/Xml2MetalinkProcessorTest.cc +++ b/test/Xml2MetalinkProcessorTest.cc @@ -1,5 +1,6 @@ #include "Xml2MetalinkProcessor.h" #include "Exception.h" +#include "DefaultDiskWriter.h" #include using namespace std; @@ -8,6 +9,7 @@ class Xml2MetalinkProcessorTest:public CppUnit::TestFixture { CPPUNIT_TEST_SUITE(Xml2MetalinkProcessorTest); CPPUNIT_TEST(testParseFile); + CPPUNIT_TEST(testParseFromBinaryStream); CPPUNIT_TEST_SUITE_END(); private: @@ -20,6 +22,7 @@ public: } void testParseFile(); + void testParseFromBinaryStream(); }; @@ -112,3 +115,24 @@ void Xml2MetalinkProcessorTest::testParseFile() { delete e; } } + +void Xml2MetalinkProcessorTest::testParseFromBinaryStream() { + Xml2MetalinkProcessor proc; + try { + DefaultDiskWriterHandle dw = new DefaultDiskWriter(); + dw->openExistingFile("test.xml"); + + MetalinkerHandle metalinker = proc.parseFromBinaryStream(dw); + + dw->closeFile(); + + MetalinkEntries::iterator entryItr = metalinker->entries.begin(); + MetalinkEntryHandle entry1 = *entryItr; + CPPUNIT_ASSERT_EQUAL(string("aria2-0.5.2.tar.bz2"), entry1->getPath()); + } catch(Exception* e) { + cerr << *e; + CPPUNIT_FAIL(e->getMsg()); + delete e; + throw; + } +} diff --git a/test/a2functionalTest.cc b/test/a2functionalTest.cc index 0a936e82..c021b1a2 100644 --- a/test/a2functionalTest.cc +++ b/test/a2functionalTest.cc @@ -10,10 +10,12 @@ class a2functionalTest:public CppUnit::TestFixture { CPPUNIT_TEST_SUITE(a2functionalTest); CPPUNIT_TEST(testMemFunSh); CPPUNIT_TEST(testAdopt2nd); + CPPUNIT_TEST(testArrayLength); CPPUNIT_TEST_SUITE_END(); public: void testMemFunSh(); void testAdopt2nd(); + void testArrayLength(); class Greeting { public: @@ -62,3 +64,12 @@ void a2functionalTest::testAdopt2nd() CPPUNIT_ASSERT_EQUAL(string("A Japanese said:HAROO WAARUDO"), adopt2nd(plus(), mem_fun_sh(&Greeting::sayGreeting))("A Japanese said:", greeting)); } + +void a2functionalTest::testArrayLength() +{ + int64_t ia[] = { 1, 2, 3, 4, 5 }; + int64_t zeroLengthArray[] = {}; + + CPPUNIT_ASSERT_EQUAL((size_t)5, arrayLength(ia)); + CPPUNIT_ASSERT_EQUAL((size_t)0, arrayLength(zeroLengthArray)); +}