added a common base class for spooling rewind input stream, adapted

libfetch rewind input stream to use that one

2 files changed, 17 insertions, 182 deletions

diff --git a/src/modules/fetcher/libfetch/LibFetchRewindInputStream.cpp b/src/modules/fetcher/libfetch/LibFetchRewindInputStream.cpp
index 19c0f60..ee606fe 100644
--- a/src/modules/fetcher/libfetch/LibFetchRewindInputStream.cpp
+++ b/src/modules/fetcher/libfetch/LibFetchRewindInputStream.cpp
@@ -1,192 +1,48 @@
 #include "LibFetchRewindInputStream.hpp"
 #include "Logger.hpp"
+#include "SpoolRewindInputStream.hpp"
 
-#include <streambuf>
-#include <vector>
-#include <algorithm>
 #include <string>
-#include <cstring>
 #include <stdexcept>
-#include <cassert>
-#include <fstream>
 #include <cstring>
 #include <errno.h>
 
 using namespace std;
 
-class libfetch_buffer : public streambuf
+class libfetch_buffer : public spool_streambuf
 {
 	public:
+	
 		explicit libfetch_buffer( fetchIO *io, size_t bufSize = 256, size_t putBack = 1, size_t spoolBufSize = 8192 );
-
-		~libfetch_buffer( );
- 
-		void rewind( );
 		
-	private:
-		int_type underflow( );
+	protected:
 	
+		virtual streambuf::int_type readFromSource( );
+
 	private:
 		fetchIO *m_io;
-		const size_t m_putBack;
-		vector<char> m_buf;
-		vector<char> m_spoolBuf;
-		size_t m_spoolBufPos;
-		size_t m_spoolBufSize;
-		fstream m_spoolFile;
-		enum { TO_SPOOL_MEMORY = 1, TO_SPOOL_FILE = 2, FROM_SPOOL_MEMORY = 3, FROM_SPOOL_FILE = 4 } m_state;
 };
 
 libfetch_buffer::libfetch_buffer( fetchIO *io, size_t bufSize, size_t putBack, size_t spoolBufSize )
-	: m_io( io ), m_putBack( max( putBack, size_t( 1 ) ) ),
-	  m_buf( max( bufSize, putBack ) + putBack ),
-	  m_spoolBuf( spoolBufSize ), m_spoolBufPos( 0 ),
-	  m_spoolBufSize( 0 ), m_state( TO_SPOOL_MEMORY )
+	: spool_streambuf( bufSize, putBack, spoolBufSize ), m_io( io )
 {
-	char *end = &m_buf.front( ) + m_buf.size( );
-	setg( end, end, end );
-}
-
-libfetch_buffer::~libfetch_buffer( )
-{
-	switch( m_state ) {
-		case TO_SPOOL_MEMORY:
-		case FROM_SPOOL_MEMORY:
-			// memory only, nothing to clean up
-			break;
-
-		case TO_SPOOL_FILE:
-		case FROM_SPOOL_FILE:
-			m_spoolFile.close( );
-			(void)remove( "/tmp/spool.tmp" );
-			break;
-	}
 }
 
-streambuf::int_type libfetch_buffer::underflow( )
+streambuf::int_type libfetch_buffer::readFromSource( )
 {
-	// check if buffer is exhausted, if not, return current character
-	if( gptr( ) < egptr( ) ) 
-		return traits_type::to_int_type( *gptr( ) );
-	
-	char *base = &m_buf.front( );
-	char *start = base;
-	
-	// move put back away
-	if( eback( ) == base ) {
-		memmove( base, egptr( ) - m_putBack, m_putBack );
-		start += m_putBack;
-	}
-	
-	// read from source or spool (depends on calling rewind)	
 	ssize_t n;
-	switch( m_state ) {
-		case TO_SPOOL_MEMORY:
-		case TO_SPOOL_FILE:
-			n = fetchIO_read( m_io, start, m_buf.size( ) - ( start - base ) );
-			if( n == 0 ) {
-				return traits_type::eof( );
-			} else if( n < 0 ) {
-				// TODO handle error
-			}
-
-			if( m_state == TO_SPOOL_MEMORY ) {
-				// as long we can "spool" to memory, do so..
-				if( m_spoolBufSize + n <= m_spoolBuf.size( ) ) {
-					m_spoolBuf.insert( m_spoolBuf.begin( ) + m_spoolBufPos, start, start + n );
-					m_spoolBufPos += n;
-					m_spoolBufSize += n;
-				} else {
-					// ..otherwise start spooling to disk, write
-					// current memory spool buffer first..
-					LOG( logWARNING ) << "Spooling spool buffer exceeded (>" << m_spoolBuf.size( ) << ")";
-					m_spoolFile.open( "/tmp/spool.tmp", ios::binary | ios::out | ios::trunc );
-					assert( m_spoolFile.good( ) );
-					m_spoolFile.write( &m_spoolBuf.front( ), m_spoolBufSize );
-					assert( m_spoolFile.good( ) );
-					m_state = TO_SPOOL_FILE;
-					m_spoolFile.write( start, n );
-					assert( m_spoolFile.good( ) );
-				}
-			} else {
-				// we are appending to the spool file
-				assert( m_spoolFile.good( ) );
-				m_spoolFile.write( start, n );
-				assert( m_spoolFile.good( ) );
-			}
-		
-			break;
-		
-		case FROM_SPOOL_MEMORY:
-			n = min( m_buf.size( ) - ( start - base ), m_spoolBufSize - m_spoolBufPos );
-			if( n == 0 ) {
-				return traits_type::eof( );
-			}
-			
-			copy(	m_spoolBuf.begin( ) + m_spoolBufPos,
-				m_spoolBuf.begin( ) + m_spoolBufPos + n,
-				m_buf.begin( ) + ( start - base ) );
-
-			m_spoolBufPos += n;
-
-			break;
 
-		case FROM_SPOOL_FILE:
-
-			n = min( m_buf.size( ) - ( start - base ), m_spoolBufSize - m_spoolBufPos );
-			m_spoolFile.read( start, n );
-			m_spoolBufPos += n;
-			if( m_spoolBufPos > m_spoolBufSize ) {
-				return traits_type::eof( );
-			}
-			if( n == 0 || m_spoolFile.eof( ) ) {
-				return traits_type::eof( );
-			}
-
-			break;
-	}
-	
-	// set pointers
-	setg( base, start, start + n );
-	
-	return traits_type::to_int_type( *gptr( ) );
-}
-
-void libfetch_buffer::rewind( )
-{
-	switch( m_state ) {
-		case TO_SPOOL_MEMORY:
-			m_spoolBufPos = 0;
-			m_state = FROM_SPOOL_MEMORY;
-			break;
-
-		case TO_SPOOL_FILE:
-			m_spoolFile.close( );
-			m_spoolFile.open( "/tmp/spool.tmp", ios::binary | ios::in );
-			m_spoolFile.seekg( 0, ios::end );
-			m_spoolBufSize = m_spoolFile.tellg( );
-			m_spoolFile.seekg( 0, ios::beg );
-			m_spoolBufPos = 0;
-			m_state = FROM_SPOOL_FILE;
-			break;
-		
-		case FROM_SPOOL_MEMORY:
-			m_spoolBufPos = 0;
-			break;
-
-		case FROM_SPOOL_FILE:
-			m_spoolBufPos = 0;
-			m_spoolFile.seekg( 0, ios::beg );
-			break;
+	n = fetchIO_read( m_io, m_start, m_buf.size( ) - ( m_start - m_base ) );
+	if( n == 0 ) {
+		return traits_type::eof( );
+	} else if( n < 0 ) {
+		// TODO handle error
 	}
-	
-	char *end = &m_buf.front( ) + m_buf.size( );
-	setg( end, end, end );
-	pubseekpos( 0, ios_base::in );
+	return n;
 }
 
 LibFetchRewindInputStream::LibFetchRewindInputStream( const URL &url )
-	: RewindInputStream( url ), m_io( 0 ), m_buf( 0 )
+	: SpoolRewindInputStream( url ), m_io( 0 )
 {
 	m_io = fetchGetURL( url.str( ).c_str( ), "" );
 	if( m_io == NULL ) {
@@ -203,21 +59,6 @@ LibFetchRewindInputStream::~LibFetchRewindInputStream( )
 	if( m_io ) fetchIO_close( m_io );
 }
 
-void LibFetchRewindInputStream::rewind( )
-{
-	// consume rest of web request, force spooling in streambuf
-	enum { CHUNKSIZE = 1024 };
-	char buf[CHUNKSIZE];
-	
-	while( good( ) && !eof( ) ) {
-		read( buf, CHUNKSIZE );
-	}
-	
-	ios::clear( );
-	assert( m_buf != 0 );
-	m_buf->rewind( );
-}
-
 string LibFetchRewindInputStream::lastErrMsg( ) const
 {
 	return fetchLastErrString;
diff --git a/src/modules/fetcher/libfetch/LibFetchRewindInputStream.hpp b/src/modules/fetcher/libfetch/LibFetchRewindInputStream.hpp
index 099c0ae..9347b77 100755
--- a/src/modules/fetcher/libfetch/LibFetchRewindInputStream.hpp
+++ b/src/modules/fetcher/libfetch/LibFetchRewindInputStream.hpp
@@ -1,26 +1,20 @@
 #ifndef __LIBFETCH_REWIND_INPUT_STREAM_H
 #define __LIBFETCH_REWIND_INPUT_STREAM_H
 
-#include "RewindInputStream.hpp"
-#include "URL.hpp"
+#include "SpoolRewindInputStream.hpp"
 
 #include "fetch.h"
 
-class libfetch_buffer;
-
-class LibFetchRewindInputStream : public RewindInputStream
+class LibFetchRewindInputStream : public SpoolRewindInputStream
 {
 	public:
 		LibFetchRewindInputStream( const URL &url );
 		virtual ~LibFetchRewindInputStream( );
 	
-		virtual void rewind( );
-
 		virtual std::string lastErrMsg( ) const;
 		
 	private:
 		fetchIO *m_io;
-		libfetch_buffer *m_buf;
 };
 
 #endif