diff options
author | Andreas Baumann <mail@andreasbaumann.cc> | 2014-10-03 20:23:24 +0200 |
---|---|---|
committer | Andreas Baumann <mail@andreasbaumann.cc> | 2014-10-03 20:23:24 +0200 |
commit | 992919c4be2336c5b8f2f0549712c51bd7fb9c03 (patch) | |
tree | 45db77d91baa8509edb8a46356d2a42aefe3faa7 /src/modules | |
parent | 7af671da43dee7dd07333cab38c181106a328832 (diff) | |
download | crawler-992919c4be2336c5b8f2f0549712c51bd7fb9c03.tar.gz crawler-992919c4be2336c5b8f2f0549712c51bd7fb9c03.tar.bz2 |
fixed setting data from source in fetcher modules
Diffstat (limited to 'src/modules')
-rw-r--r-- | src/modules/fetcher/libcurl/LibCurlRewindInputStream.cpp | 70 |
1 files changed, 3 insertions, 67 deletions
diff --git a/src/modules/fetcher/libcurl/LibCurlRewindInputStream.cpp b/src/modules/fetcher/libcurl/LibCurlRewindInputStream.cpp index 7d75123..19098e8 100644 --- a/src/modules/fetcher/libcurl/LibCurlRewindInputStream.cpp +++ b/src/modules/fetcher/libcurl/LibCurlRewindInputStream.cpp @@ -35,8 +35,10 @@ size_t libcurl_buffer::write_data( char *ptr, size_t s, size_t n ) size_t data_spooled = spoolSourceData( ptr, chunk_size ); - return data_spooled; + // otherwise we must use CURL_WRITEFUNC_PAUSE + assert( data_spooled == n ); + return data_spooled; } streambuf::int_type libcurl_buffer::readFromSource( ) @@ -84,69 +86,3 @@ string LibCurlRewindInputStream::lastErrMsg( ) const // TODO: fetch m_curl state and return error return "ERROR"; } - -#if 0 - -#include <iostream> -#include <sys/types.h> -#include <cassert> -#include <cstring> -#include <cstdlib> - -using namespace std; - -class MemoryWriter { - static const size_t initial_size = 100; - - protected: - char *c; - size_t c_size; - - public: - MemoryWriter( ) { - c = (char *)malloc( initial_size ); - c_size = 0; - } - - ~MemoryWriter( ) { - if( c != NULL ) - free( c ); - } - - size_t f( char *ptr, size_t s, size_t n ) { - size_t part_size = s * n; - - c = (char *)realloc( c, c_size + part_size + 1 ); - - assert( c != NULL ); - - memcpy( &c[c_size], ptr, part_size ); - c_size += part_size; - c[c_size] = '\0'; - - return part_size; - } - - char *str( ) { - return c; - } - - size_t size( ) { - return c_size; - } -}; - -int main( int argc, char *argv[] ) { - Easy curl; - MemoryWriter w; - - - cout << cURLpp::Infos::ResponseCode::get( curl ) << endl - << w.size( ) << " " - << url << endl; - - cout << w.str( ) << endl; - - return EXIT_SUCCESS; -} -#endif |