diff options
author | Andreas Baumann <abaumann@yahoo.com> | 2012-08-18 10:46:56 +0200 |
---|---|---|
committer | Andreas Baumann <abaumann@yahoo.com> | 2012-08-18 10:46:56 +0200 |
commit | d1ec47b36afeb73ac96bb9794e5ad4105b62466e (patch) | |
tree | cbc8a6ff2189ea7eefd5df909a90971fcd5969d4 /src/modules | |
parent | a6eafa7a0a3813ba52f88b9dd939608c3177f017 (diff) | |
download | crawler-d1ec47b36afeb73ac96bb9794e5ad4105b62466e.tar.gz crawler-d1ec47b36afeb73ac96bb9794e5ad4105b62466e.tar.bz2 |
some code cleanup in fetchers
Diffstat (limited to 'src/modules')
5 files changed, 15 insertions, 14 deletions
diff --git a/src/modules/fetcher/libfetch/LibFetchFetcher.cpp b/src/modules/fetcher/libfetch/LibFetchFetcher.cpp index 9cbc926..ee4649e 100644 --- a/src/modules/fetcher/libfetch/LibFetchFetcher.cpp +++ b/src/modules/fetcher/libfetch/LibFetchFetcher.cpp @@ -3,6 +3,17 @@ #include "fetch.h" +LibFetchFetcher::LibFetchFetcher( ) +{ + // TODO: connection pool parameters from instantiation + fetchConnectionCacheInit( 1024, 16 ); +} + +LibFetchFetcher::~LibFetchFetcher( ) +{ + fetchConnectionCacheClose( ); +} + RewindInputStream *LibFetchFetcher::fetch( const URL url ) { LibFetchRewindInputStream *s = new LibFetchRewindInputStream( url ); diff --git a/src/modules/fetcher/libfetch/LibFetchFetcher.hpp b/src/modules/fetcher/libfetch/LibFetchFetcher.hpp index 1103612..a99791f 100755 --- a/src/modules/fetcher/libfetch/LibFetchFetcher.hpp +++ b/src/modules/fetcher/libfetch/LibFetchFetcher.hpp @@ -7,11 +7,9 @@ class LibFetchFetcher : public Fetcher { public: - LibFetchFetcher( ) { - } + LibFetchFetcher( ); - virtual ~LibFetchFetcher( ) { - } + virtual ~LibFetchFetcher( ); virtual RewindInputStream *fetch( const URL url ); }; diff --git a/src/modules/fetcher/libfetch/LibFetchRewindInputStream.cpp b/src/modules/fetcher/libfetch/LibFetchRewindInputStream.cpp index ee606fe..1b4ab20 100644 --- a/src/modules/fetcher/libfetch/LibFetchRewindInputStream.cpp +++ b/src/modules/fetcher/libfetch/LibFetchRewindInputStream.cpp @@ -1,18 +1,11 @@ #include "LibFetchRewindInputStream.hpp" -#include "Logger.hpp" #include "SpoolRewindInputStream.hpp" -#include <string> -#include <stdexcept> -#include <cstring> -#include <errno.h> - using namespace std; class libfetch_buffer : public spool_streambuf { public: - explicit libfetch_buffer( fetchIO *io, size_t bufSize = 256, size_t putBack = 1, size_t spoolBufSize = 8192 ); protected: @@ -49,6 +42,7 @@ LibFetchRewindInputStream::LibFetchRewindInputStream( const URL &url ) setstate( badbit ); return; } + m_buf = new libfetch_buffer( m_io ); rdbuf( m_buf ); } diff --git a/src/modules/fetcher/winhttp/WinHttpRewindInputStream.cpp b/src/modules/fetcher/winhttp/WinHttpRewindInputStream.cpp index 92d0998..5ab372c 100755 --- a/src/modules/fetcher/winhttp/WinHttpRewindInputStream.cpp +++ b/src/modules/fetcher/winhttp/WinHttpRewindInputStream.cpp @@ -1,7 +1,6 @@ #include "WinHttpRewindInputStream.hpp" #include "WinHttpFetcher.hpp" #include "SpoolRewindInputStream.hpp" -#include "Logger.hpp" #include "win32/errormsg.hpp" #include "win32/stringutils.hpp" @@ -12,7 +11,7 @@ class winhttp_buffer : public spool_streambuf { public: explicit winhttp_buffer( HINTERNET connect, HINTERNET request, size_t bufSize = 256, size_t putBack = 1, size_t spoolBufSize = 8192 ); - + protected: virtual streambuf::int_type readFromSource( ); diff --git a/src/modules/fetcher/winhttp/WinHttpRewindInputStream.hpp b/src/modules/fetcher/winhttp/WinHttpRewindInputStream.hpp index 5ab00f8..41f0523 100755 --- a/src/modules/fetcher/winhttp/WinHttpRewindInputStream.hpp +++ b/src/modules/fetcher/winhttp/WinHttpRewindInputStream.hpp @@ -1,7 +1,6 @@ #ifndef __WINHTTP_REWIND_INPUT_STREAM_H #define __WINHTTP_REWIND_INPUT_STREAM_H -#include "RewindInputStream.hpp" #include "SpoolRewindInputStream.hpp" #define WIN32_MEAN_AND_LEAN |