summaryrefslogtreecommitdiff
path: root/src/modules
diff options
context:
space:
mode:
authorAndreas Baumann <mail@andreasbaumann.cc>2014-10-03 20:23:24 +0200
committerAndreas Baumann <mail@andreasbaumann.cc>2014-10-03 20:23:24 +0200
commit992919c4be2336c5b8f2f0549712c51bd7fb9c03 (patch)
tree45db77d91baa8509edb8a46356d2a42aefe3faa7 /src/modules
parent7af671da43dee7dd07333cab38c181106a328832 (diff)
downloadcrawler-992919c4be2336c5b8f2f0549712c51bd7fb9c03.tar.gz
crawler-992919c4be2336c5b8f2f0549712c51bd7fb9c03.tar.bz2
fixed setting data from source in fetcher modules
Diffstat (limited to 'src/modules')
-rw-r--r--src/modules/fetcher/libcurl/LibCurlRewindInputStream.cpp70
1 files changed, 3 insertions, 67 deletions
diff --git a/src/modules/fetcher/libcurl/LibCurlRewindInputStream.cpp b/src/modules/fetcher/libcurl/LibCurlRewindInputStream.cpp
index 7d75123..19098e8 100644
--- a/src/modules/fetcher/libcurl/LibCurlRewindInputStream.cpp
+++ b/src/modules/fetcher/libcurl/LibCurlRewindInputStream.cpp
@@ -35,8 +35,10 @@ size_t libcurl_buffer::write_data( char *ptr, size_t s, size_t n )
size_t data_spooled = spoolSourceData( ptr, chunk_size );
- return data_spooled;
+ // otherwise we must use CURL_WRITEFUNC_PAUSE
+ assert( data_spooled == n );
+ return data_spooled;
}
streambuf::int_type libcurl_buffer::readFromSource( )
@@ -84,69 +86,3 @@ string LibCurlRewindInputStream::lastErrMsg( ) const
// TODO: fetch m_curl state and return error
return "ERROR";
}
-
-#if 0
-
-#include <iostream>
-#include <sys/types.h>
-#include <cassert>
-#include <cstring>
-#include <cstdlib>
-
-using namespace std;
-
-class MemoryWriter {
- static const size_t initial_size = 100;
-
- protected:
- char *c;
- size_t c_size;
-
- public:
- MemoryWriter( ) {
- c = (char *)malloc( initial_size );
- c_size = 0;
- }
-
- ~MemoryWriter( ) {
- if( c != NULL )
- free( c );
- }
-
- size_t f( char *ptr, size_t s, size_t n ) {
- size_t part_size = s * n;
-
- c = (char *)realloc( c, c_size + part_size + 1 );
-
- assert( c != NULL );
-
- memcpy( &c[c_size], ptr, part_size );
- c_size += part_size;
- c[c_size] = '\0';
-
- return part_size;
- }
-
- char *str( ) {
- return c;
- }
-
- size_t size( ) {
- return c_size;
- }
-};
-
-int main( int argc, char *argv[] ) {
- Easy curl;
- MemoryWriter w;
-
-
- cout << cURLpp::Infos::ResponseCode::get( curl ) << endl
- << w.size( ) << " "
- << url << endl;
-
- cout << w.str( ) << endl;
-
- return EXIT_SUCCESS;
-}
-#endif