added an experimental curl fetcher

1 files changed, 59 insertions, 23 deletions

diff --git a/src/libcrawler/SpoolRewindInputStream.cpp b/src/libcrawler/SpoolRewindInputStream.cpp
index 9135741..13ab105 100644
--- a/src/libcrawler/SpoolRewindInputStream.cpp
+++ b/src/libcrawler/SpoolRewindInputStream.cpp
@@ -34,6 +34,61 @@ spool_streambuf::~spool_streambuf( )
 	}
 }
 
+streambuf::int_type spool_streambuf::spoolSourceData( char *data, size_t n )
+{
+	size_t data_len = m_buf.size( ) - ( m_start - m_base ) ;
+	if( n < data_len ) {
+		data_len = n;
+	}
+
+	m_base = &m_buf.front( );
+	m_start = m_base;
+
+	memcpy( m_start, data, data_len );
+
+	spoolData( data_len );
+
+	return data_len;
+}
+
+void spool_streambuf::spoolData( size_t n )
+{
+	switch( m_state ) {
+		case TO_SPOOL_MEMORY:
+			// as long we can "spool" to memory, do so..
+			if( m_spoolBufPos + n <= m_spoolBufSize ) {
+				m_spoolBuf.insert( m_spoolBuf.begin( ) + m_spoolBufPos, m_start, m_start + n );
+				m_spoolBufPos += n;
+			} else {
+				// ..otherwise start spooling to disk, write
+				// current memory spool buffer first..
+				LOG( logWARNING ) << "Spooling spool buffer exceeded (>" << m_spoolBufSize << ")";
+				m_spoolFile.open( "/tmp/spool.tmp", ios::binary | ios::out | ios::trunc );
+				assert( m_spoolFile.good( ) );
+				m_spoolFile.write( &m_spoolBuf.front( ), m_spoolBufSize );
+				assert( m_spoolFile.good( ) );
+				m_state = TO_SPOOL_FILE;
+				m_spoolFile.write( m_start, n );
+				assert( m_spoolFile.good( ) );
+			}
+			break;
+
+		case TO_SPOOL_FILE:
+			// we are appending to the spool file
+			assert( m_spoolFile.good( ) );
+			m_spoolFile.write( m_start, n );
+			assert( m_spoolFile.good( ) );
+			break;
+
+		case FROM_SPOOL_MEMORY:
+		case FROM_SPOOL_FILE:
+			throw logic_error( "Still getting data from source after rewind!" );
+
+		default:
+			throw logic_error( "Illegal state!" );
+	}
+}
+
 streambuf::int_type spool_streambuf::underflow( )
 {
 	// check if buffer is exhausted, if not, return current character
@@ -62,29 +117,7 @@ streambuf::int_type spool_streambuf::underflow( )
 				return traits_type::eof( );
 			}
 
-			if( m_state == TO_SPOOL_MEMORY ) {
-				// as long we can "spool" to memory, do so..
-				if( m_spoolBufPos + n <= m_spoolBufSize ) {
-					m_spoolBuf.insert( m_spoolBuf.begin( ) + m_spoolBufPos, m_start, m_start + n );
-					m_spoolBufPos += n;
-				} else {
-					// ..otherwise start spooling to disk, write
-					// current memory spool buffer first..
-					LOG( logWARNING ) << "Spooling spool buffer exceeded (>" << m_spoolBufSize << ")";
-					m_spoolFile.open( "/tmp/spool.tmp", ios::binary | ios::out | ios::trunc );
-					assert( m_spoolFile.good( ) );
-					m_spoolFile.write( &m_spoolBuf.front( ), m_spoolBufSize );
-					assert( m_spoolFile.good( ) );
-					m_state = TO_SPOOL_FILE;
-					m_spoolFile.write( m_start, n );
-					assert( m_spoolFile.good( ) );
-				}
-			} else {
-				// we are appending to the spool file
-				assert( m_spoolFile.good( ) );
-				m_spoolFile.write( m_start, n );
-				assert( m_spoolFile.good( ) );
-			}
+			spoolData( n );
 		
 			break;
 		
@@ -115,6 +148,9 @@ streambuf::int_type spool_streambuf::underflow( )
 			}
 
 			break;
+
+		default:
+			throw logic_error( "Illegal state!" );
 	}
 	
 	// set pointers