summaryrefslogtreecommitdiff
path: root/src/modules
diff options
context:
space:
mode:
authorAndreas Baumann <abaumann@yahoo.com>2012-08-18 10:46:56 +0200
committerAndreas Baumann <abaumann@yahoo.com>2012-08-18 10:46:56 +0200
commitd1ec47b36afeb73ac96bb9794e5ad4105b62466e (patch)
treecbc8a6ff2189ea7eefd5df909a90971fcd5969d4 /src/modules
parenta6eafa7a0a3813ba52f88b9dd939608c3177f017 (diff)
downloadcrawler-d1ec47b36afeb73ac96bb9794e5ad4105b62466e.tar.gz
crawler-d1ec47b36afeb73ac96bb9794e5ad4105b62466e.tar.bz2
some code cleanup in fetchers
Diffstat (limited to 'src/modules')
-rw-r--r--src/modules/fetcher/libfetch/LibFetchFetcher.cpp11
-rwxr-xr-xsrc/modules/fetcher/libfetch/LibFetchFetcher.hpp6
-rw-r--r--src/modules/fetcher/libfetch/LibFetchRewindInputStream.cpp8
-rwxr-xr-xsrc/modules/fetcher/winhttp/WinHttpRewindInputStream.cpp3
-rwxr-xr-xsrc/modules/fetcher/winhttp/WinHttpRewindInputStream.hpp1
5 files changed, 15 insertions, 14 deletions
diff --git a/src/modules/fetcher/libfetch/LibFetchFetcher.cpp b/src/modules/fetcher/libfetch/LibFetchFetcher.cpp
index 9cbc926..ee4649e 100644
--- a/src/modules/fetcher/libfetch/LibFetchFetcher.cpp
+++ b/src/modules/fetcher/libfetch/LibFetchFetcher.cpp
@@ -3,6 +3,17 @@
#include "fetch.h"
+LibFetchFetcher::LibFetchFetcher( )
+{
+ // TODO: connection pool parameters from instantiation
+ fetchConnectionCacheInit( 1024, 16 );
+}
+
+LibFetchFetcher::~LibFetchFetcher( )
+{
+ fetchConnectionCacheClose( );
+}
+
RewindInputStream *LibFetchFetcher::fetch( const URL url )
{
LibFetchRewindInputStream *s = new LibFetchRewindInputStream( url );
diff --git a/src/modules/fetcher/libfetch/LibFetchFetcher.hpp b/src/modules/fetcher/libfetch/LibFetchFetcher.hpp
index 1103612..a99791f 100755
--- a/src/modules/fetcher/libfetch/LibFetchFetcher.hpp
+++ b/src/modules/fetcher/libfetch/LibFetchFetcher.hpp
@@ -7,11 +7,9 @@
class LibFetchFetcher : public Fetcher
{
public:
- LibFetchFetcher( ) {
- }
+ LibFetchFetcher( );
- virtual ~LibFetchFetcher( ) {
- }
+ virtual ~LibFetchFetcher( );
virtual RewindInputStream *fetch( const URL url );
};
diff --git a/src/modules/fetcher/libfetch/LibFetchRewindInputStream.cpp b/src/modules/fetcher/libfetch/LibFetchRewindInputStream.cpp
index ee606fe..1b4ab20 100644
--- a/src/modules/fetcher/libfetch/LibFetchRewindInputStream.cpp
+++ b/src/modules/fetcher/libfetch/LibFetchRewindInputStream.cpp
@@ -1,18 +1,11 @@
#include "LibFetchRewindInputStream.hpp"
-#include "Logger.hpp"
#include "SpoolRewindInputStream.hpp"
-#include <string>
-#include <stdexcept>
-#include <cstring>
-#include <errno.h>
-
using namespace std;
class libfetch_buffer : public spool_streambuf
{
public:
-
explicit libfetch_buffer( fetchIO *io, size_t bufSize = 256, size_t putBack = 1, size_t spoolBufSize = 8192 );
protected:
@@ -49,6 +42,7 @@ LibFetchRewindInputStream::LibFetchRewindInputStream( const URL &url )
setstate( badbit );
return;
}
+
m_buf = new libfetch_buffer( m_io );
rdbuf( m_buf );
}
diff --git a/src/modules/fetcher/winhttp/WinHttpRewindInputStream.cpp b/src/modules/fetcher/winhttp/WinHttpRewindInputStream.cpp
index 92d0998..5ab372c 100755
--- a/src/modules/fetcher/winhttp/WinHttpRewindInputStream.cpp
+++ b/src/modules/fetcher/winhttp/WinHttpRewindInputStream.cpp
@@ -1,7 +1,6 @@
#include "WinHttpRewindInputStream.hpp"
#include "WinHttpFetcher.hpp"
#include "SpoolRewindInputStream.hpp"
-#include "Logger.hpp"
#include "win32/errormsg.hpp"
#include "win32/stringutils.hpp"
@@ -12,7 +11,7 @@ class winhttp_buffer : public spool_streambuf
{
public:
explicit winhttp_buffer( HINTERNET connect, HINTERNET request, size_t bufSize = 256, size_t putBack = 1, size_t spoolBufSize = 8192 );
-
+
protected:
virtual streambuf::int_type readFromSource( );
diff --git a/src/modules/fetcher/winhttp/WinHttpRewindInputStream.hpp b/src/modules/fetcher/winhttp/WinHttpRewindInputStream.hpp
index 5ab00f8..41f0523 100755
--- a/src/modules/fetcher/winhttp/WinHttpRewindInputStream.hpp
+++ b/src/modules/fetcher/winhttp/WinHttpRewindInputStream.hpp
@@ -1,7 +1,6 @@
#ifndef __WINHTTP_REWIND_INPUT_STREAM_H
#define __WINHTTP_REWIND_INPUT_STREAM_H
-#include "RewindInputStream.hpp"
#include "SpoolRewindInputStream.hpp"
#define WIN32_MEAN_AND_LEAN