more splitting into libcrawl, crawl binary

moved more public header to 'include'
changed approach for dynamic linking on Windows

55 files changed, 367 insertions, 221 deletions

diff --git a/.gitignore b/.gitignore
index 8d86f6e..72d32b9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -25,6 +25,6 @@
 tests/*/test1
 tests/*/test2
 tests/*/test3
-src/crawl
+src/crawl/crawl
 makefiles/gmake/platform.mk.vars
 makefiles/gmake/platform.vars
diff --git a/include/crawler/CrawlerExportable.hpp b/include/crawler/CrawlerExportable.hpp
new file mode 100755
index 0000000..5b89108
--- /dev/null
+++ b/include/crawler/CrawlerExportable.hpp
@@ -0,0 +1,26 @@
+#ifndef __CRAWLER_EXPORTABLE_H
+#define __CRAWLER_EXPORTABLE_H
+
+#ifndef _WIN32
+
+#define CRAWLER_DLL_VISIBLE
+
+#else
+
+#ifdef SHARED
+
+#ifdef BUILDING_CRAWLER
+#define CRAWLER_DLL_VISIBLE __declspec(dllexport)
+#else
+#define CRAWLER_DLL_VISIBLE __declspec(dllimport)
+#endif
+
+#else
+
+#define CRAWLER_DLL_VISIBLE
+
+#endif // BUILDING_CRAWLER
+
+#endif // _WIN32
+
+#endif
diff --git a/src/DNSResolver.hpp b/include/crawler/DNSResolver.hpp
index 8f79734..8f79734 100644
--- a/src/DNSResolver.hpp
+++ b/include/crawler/DNSResolver.hpp
diff --git a/src/Deduper.hpp b/include/crawler/Deduper.hpp
index 3cb33c1..3cb33c1 100644
--- a/src/Deduper.hpp
+++ b/include/crawler/Deduper.hpp
diff --git a/src/Fetcher.hpp b/include/crawler/Fetcher.hpp
index 40f1c7a..40f1c7a 100755
--- a/src/Fetcher.hpp
+++ b/include/crawler/Fetcher.hpp
diff --git a/src/Frontier.hpp b/include/crawler/Frontier.hpp
index 54c0dd6..54c0dd6 100644
--- a/src/Frontier.hpp
+++ b/include/crawler/Frontier.hpp
diff --git a/src/MIMEType.hpp b/include/crawler/MIMEType.hpp
index 3a628ca..3a628ca 100644
--- a/src/MIMEType.hpp
+++ b/include/crawler/MIMEType.hpp
diff --git a/src/Processor.hpp b/include/crawler/Processor.hpp
index bc17ec0..bc17ec0 100644
--- a/src/Processor.hpp
+++ b/include/crawler/Processor.hpp
diff --git a/src/RewindInputStream.hpp b/include/crawler/RewindInputStream.hpp
index 39d7b6e..6bbe80c 100755
--- a/src/RewindInputStream.hpp
+++ b/include/crawler/RewindInputStream.hpp
@@ -1,6 +1,7 @@
 #ifndef __REWIND_INPUT_STREAM_H
 #define __REWIND_INPUT_STREAM_H
 
+#include "CrawlerExportable.hpp"
 #include "URL.hpp"
 
 #include <iostream>
@@ -13,7 +14,7 @@ class RewindInputStream : public std::istream {
 			return m_baseUrl;
 		}
 		
-		virtual void rewind( ) = 0;
+		CRAWLER_DLL_VISIBLE virtual void rewind( ) = 0;
 
 		virtual std::string lastErrMsg( ) const = 0;
 
diff --git a/src/SpoolRewindInputStream.hpp b/include/crawler/SpoolRewindInputStream.hpp
index aff593d..f065271 100755
--- a/src/SpoolRewindInputStream.hpp
+++ b/include/crawler/SpoolRewindInputStream.hpp
@@ -10,17 +10,17 @@
 class spool_streambuf : public std::streambuf
 {
 	public:
-		explicit spool_streambuf( size_t bufSize = 256, size_t putBack = 1, size_t spoolBufSize = 8192 );
+		explicit CRAWLER_DLL_VISIBLE spool_streambuf( size_t bufSize = 256, size_t putBack = 1, size_t spoolBufSize = 8192 );
 
-		~spool_streambuf( );
+		CRAWLER_DLL_VISIBLE ~spool_streambuf( );
  
-		void rewind( );
+		CRAWLER_DLL_VISIBLE void rewind( );
 
 	protected:
-		virtual std::streambuf::int_type readFromSource( ) = 0;
+		CRAWLER_DLL_VISIBLE virtual std::streambuf::int_type readFromSource( ) = 0;
 		
 	private:
-		int_type underflow( );
+		CRAWLER_DLL_VISIBLE int_type underflow( );
 
 	private:
 		const size_t m_putBack;
@@ -39,10 +39,10 @@ class spool_streambuf : public std::streambuf
 class SpoolRewindInputStream : public RewindInputStream
 {
 	public:
-		SpoolRewindInputStream( const URL &url );
-		virtual ~SpoolRewindInputStream( );
+		CRAWLER_DLL_VISIBLE SpoolRewindInputStream( const URL &url );
+		CRAWLER_DLL_VISIBLE virtual ~SpoolRewindInputStream( );
 	
-		virtual void rewind( );
+		CRAWLER_DLL_VISIBLE virtual void rewind( );
 		
 	protected:
 		spool_streambuf *m_buf;
diff --git a/src/URL.hpp b/include/crawler/URL.hpp
index 5cbd733..255a2db 100644..100755
--- a/src/URL.hpp
+++ b/include/crawler/URL.hpp
@@ -1,6 +1,8 @@
 #ifndef __URL_H
 #define __URL_H
 
+#include "CrawlerExportable.hpp"
+
 #include <string>
 #include <iostream>
 #include <sstream>
@@ -81,7 +83,7 @@ class URL {
 			return os.str( );
 		}
 		
-		static URL Null;
+		static URL CRAWLER_DLL_VISIBLE Null;
 		
 		bool operator!=( const URL &other ) const
 		{
diff --git a/src/URLFilter.hpp b/include/crawler/URLFilter.hpp
index 2136009..2136009 100644
--- a/src/URLFilter.hpp
+++ b/include/crawler/URLFilter.hpp
diff --git a/src/URLNormalizer.hpp b/include/crawler/URLNormalizer.hpp
index af1781a..af1781a 100644
--- a/src/URLNormalizer.hpp
+++ b/include/crawler/URLNormalizer.hpp
diff --git a/src/URLSeen.hpp b/include/crawler/URLSeen.hpp
index 742c863..742c863 100644
--- a/src/URLSeen.hpp
+++ b/include/crawler/URLSeen.hpp
diff --git a/include/crawler/win32/errormsg.hpp b/include/crawler/win32/errormsg.hpp
new file mode 100755
index 0000000..f1ceb93
--- /dev/null
+++ b/include/crawler/win32/errormsg.hpp
@@ -0,0 +1,10 @@
+#ifndef __ERRORMSG_H
+#define __ERRORMSG_H
+
+#include <string>
+
+#include "CrawlerExportable.hpp"
+
+CRAWLER_DLL_VISIBLE std::string getLastError( );
+
+#endif
diff --git a/include/crawler/win32/stringutils.hpp b/include/crawler/win32/stringutils.hpp
new file mode 100755
index 0000000..b0404cd
--- /dev/null
+++ b/include/crawler/win32/stringutils.hpp
@@ -0,0 +1,10 @@
+#ifndef __STRINGUTILS_H
+#define __STRINGUTILS_H
+
+#include <string>
+
+#include "CrawlerExportable.hpp"
+
+CRAWLER_DLL_VISIBLE std::wstring s2ws( const std::string &s );
+
+#endif
diff --git a/include/logger/ConsoleLogSink.hpp b/include/logger/ConsoleLogSink.hpp
index 4380df8..75dea72 100755
--- a/include/logger/ConsoleLogSink.hpp
+++ b/include/logger/ConsoleLogSink.hpp
@@ -8,7 +8,7 @@ class ConsoleLogSink : public LogSink
 	public:
 		ConsoleLogSink( const LogLevel level ) : LogSink( level ) { }
 		
-		DLL_VISIBLE virtual void log( const LogLevel level, const std::string &msg );
+		LOGGER_DLL_VISIBLE virtual void log( const LogLevel level, const std::string &msg );
 };
 
 #endif
diff --git a/include/logger/FileLogSink.hpp b/include/logger/FileLogSink.hpp
index 0acb6b9..5ab9b86 100755
--- a/include/logger/FileLogSink.hpp
+++ b/include/logger/FileLogSink.hpp
@@ -13,7 +13,7 @@ class FileLogSink : public LogSink
 		
 		~FileLogSink( );
 		
-		DLL_VISIBLE virtual void log( const LogLevel level, const std::string &msg );
+		LOGGER_DLL_VISIBLE virtual void log( const LogLevel level, const std::string &msg );
 	
 	private:
 		std::string m_filename;
diff --git a/include/logger/LogSink.hpp b/include/logger/LogSink.hpp
index 4d07df0..06113f9 100755
--- a/include/logger/LogSink.hpp
+++ b/include/logger/LogSink.hpp
@@ -1,7 +1,7 @@
 #ifndef __LOGSINK_H
 #define __LOGSINK_H
 
-#include "Exportable.hpp"
+#include "LoggerExportable.hpp"
 #include "Logger.hpp"
 
 class LogSink
@@ -11,7 +11,7 @@ class LogSink
 		
 		virtual ~LogSink( ) { }
 		
-		DLL_VISIBLE virtual void log( const LogLevel level, const std::string &msg ) = 0;
+		LOGGER_DLL_VISIBLE virtual void log( const LogLevel level, const std::string &msg ) = 0;
 	
 		void setReportingLevel( const LogLevel level ) { m_level = level; }
 		
diff --git a/include/logger/Logger.hpp b/include/logger/Logger.hpp
index f0c0012..451ca45 100755
--- a/include/logger/Logger.hpp
+++ b/include/logger/Logger.hpp
@@ -1,7 +1,7 @@
 #ifndef __LOGGER_H
 #define __LOGGER_H
 
-#include "Exportable.hpp"
+#include "LoggerExportable.hpp"
 #include "Singleton.hpp"
 #include "ScopedPtr.hpp"
 
@@ -30,34 +30,32 @@ class Logger : public Singleton< Logger >
 	public:
 		DECLARE_SINGLETON( Logger )
 		
-		DLL_VISIBLE void addSink( LogSink *sink );
-		DLL_VISIBLE void removeSink( LogSink *sink );
-		DLL_VISIBLE void log( const LogLevel level, const std::string &msg );
+		LOGGER_DLL_VISIBLE void addSink( LogSink *sink );
+		LOGGER_DLL_VISIBLE void removeSink( LogSink *sink );
+		LOGGER_DLL_VISIBLE void log( const LogLevel level, const std::string &msg );
 
-		DLL_VISIBLE static std::string toString( const LogLevel level );
-		DLL_VISIBLE static LogLevel fromString( const std::string &s );
+		LOGGER_DLL_VISIBLE static std::string toString( const LogLevel level );
+		LOGGER_DLL_VISIBLE static LogLevel fromString( const std::string &s );
 		
-		DLL_VISIBLE void openConsoleLog( const LogLevel level );
-		DLL_VISIBLE void openFileLog( const LogLevel level, const std::string &filename );
-		DLL_VISIBLE void openSyslog( const LogLevel level, const std::string &ident, const std::string &facility );
-		DLL_VISIBLE void openWinDbgLog( const LogLevel level );
+		LOGGER_DLL_VISIBLE void openConsoleLog( const LogLevel level );
+		LOGGER_DLL_VISIBLE void openFileLog( const LogLevel level, const std::string &filename );
+		LOGGER_DLL_VISIBLE void openSyslog( const LogLevel level, const std::string &ident, const std::string &facility );
+		LOGGER_DLL_VISIBLE void openWinDbgLog( const LogLevel level );
 		
 	protected:
-		DLL_VISIBLE Logger( );
-		DLL_VISIBLE virtual ~Logger( );
+		LOGGER_DLL_VISIBLE Logger( );
+		LOGGER_DLL_VISIBLE virtual ~Logger( );
 			
 	private:
 		scopedPtr< LoggerImpl > m_impl;
 };
 
-DEFINE_SINGLETON( Logger )
-
 class LogStream : private noncopyable, public std::ostringstream
 {
 	public:
-		DLL_VISIBLE LogStream( Logger &logger, const LogLevel level );
-		DLL_VISIBLE ~LogStream( );
-		DLL_VISIBLE std::ostream &get( );
+		LOGGER_DLL_VISIBLE LogStream( Logger &logger, const LogLevel level );
+		LOGGER_DLL_VISIBLE ~LogStream( );
+		LOGGER_DLL_VISIBLE std::ostream &get( );
 	
 	private:
 		LogStream( );
diff --git a/include/logger/LoggerExportable.hpp b/include/logger/LoggerExportable.hpp
new file mode 100755
index 0000000..0445213
--- /dev/null
+++ b/include/logger/LoggerExportable.hpp
@@ -0,0 +1,26 @@
+#ifndef __LOGGER_EXPORTABLE_H
+#define __LOGGER_EXPORTABLE_H
+
+#ifndef _WIN32
+
+#define LOGGER_DLL_VISIBLE
+
+#else
+
+#ifdef SHARED
+
+#ifdef BUILDING_LOGGER
+#define LOGGER_DLL_VISIBLE __declspec(dllexport)
+#else
+#define LOGGER_DLL_VISIBLE __declspec(dllimport)
+#endif
+
+#else
+
+#define LOGGER_DLL_VISIBLE
+
+#endif // BUILDING_LOGGER
+
+#endif // _WIN32
+
+#endif
diff --git a/include/logger/SyslogLogSink.hpp b/include/logger/SyslogLogSink.hpp
index f5824a6..058e126 100755
--- a/include/logger/SyslogLogSink.hpp
+++ b/include/logger/SyslogLogSink.hpp
@@ -12,10 +12,10 @@ class SyslogLogSink : public LogSink
 		
 		~SyslogLogSink( );
 		
-		DLL_VISIBLE virtual void log( const LogLevel level, const std::string &msg );
+		LOGGER_DLL_VISIBLE virtual void log( const LogLevel level, const std::string &msg );
 		
-		static int levelToSyslogLevel( const LogLevel level );
-		static int facilityFromString( const std::string &facility );
+		LOGGER_DLL_VISIBLE static int levelToSyslogLevel( const LogLevel level );
+		LOGGER_DLL_VISIBLE static int facilityFromString( const std::string &facility );
 		
 	private:
 		std::string m_ident;
diff --git a/include/logger/WinDbgLogSink.hpp b/include/logger/WinDbgLogSink.hpp
index d073cdc..2fe8a43 100755
--- a/include/logger/WinDbgLogSink.hpp
+++ b/include/logger/WinDbgLogSink.hpp
@@ -8,7 +8,7 @@ class WinDbgLogSink : public LogSink
 	public:
 		WinDbgLogSink( const LogLevel level ) : LogSink( level ) { }
 		
-		DLL_VISIBLE virtual void log( const LogLevel level, const std::string &msg );
+		LOGGER_DLL_VISIBLE virtual void log( const LogLevel level, const std::string &msg );
 };
 
 #endif
diff --git a/include/util/Exportable.hpp b/include/util/Exportable.hpp
deleted file mode 100755
index fc8269f..0000000
--- a/include/util/Exportable.hpp
+++ /dev/null
@@ -1,26 +0,0 @@
-#ifndef __EXPORTABLE_H
-#define __EXPORTABLE_H
-
-#ifndef _WIN32
-
-#define DLL_VISIBLE
-#define DLL_EXTERN
-
-#else
-
-#ifdef NODLL
-#define DLL_VISIBLE
-#define DLL_EXTERN
-#else
-#ifdef SHARED
-#define DLL_VISIBLE __declspec(dllexport)
-#define DLL_EXTERN
-#else
-#define DLL_VISIBLE __declspec(dllimport)
-#define DLL_EXTERN
-#endif
-#endif
-
-#endif // _WIN32
-
-#endif
diff --git a/include/util/NonCopyable.hpp b/include/util/NonCopyable.hpp
index 99804d2..22fdaf9 100755
--- a/include/util/NonCopyable.hpp
+++ b/include/util/NonCopyable.hpp
@@ -1,12 +1,12 @@
 #ifndef __NONCOPYABLE_H
 #define __NONCOPYABLE_H
 
-#include "Exportable.hpp"
+#include "UtilExportable.hpp"
 
 namespace __dont_touch
 {
 
-class DLL_VISIBLE noncopyable
+class UTIL_DLL_VISIBLE noncopyable
 {
 	protected:
 		noncopyable( ) { }
diff --git a/include/util/ScopedPtr.hpp b/include/util/ScopedPtr.hpp
index 991409d..5fdd1f2 100755
--- a/include/util/ScopedPtr.hpp
+++ b/include/util/ScopedPtr.hpp
@@ -2,9 +2,10 @@
 #define __SCOPEDPTR_H
 
 #include "NonCopyable.hpp"
+#include "UtilExportable.hpp"
 
 template< typename T >
-class DLL_VISIBLE scopedPtr : private noncopyable
+class UTIL_DLL_VISIBLE scopedPtr : private noncopyable
 {
 	public:
 		explicit scopedPtr( T *p = 0 ) : m_p( p ) { }
diff --git a/include/util/Singleton.hpp b/include/util/Singleton.hpp
index f291435..80ab5e8 100755
--- a/include/util/Singleton.hpp
+++ b/include/util/Singleton.hpp
@@ -3,7 +3,7 @@
 
 #include "ScopedPtr.hpp"
 #include "NonCopyable.hpp"
-#include "Exportable.hpp"
+#include "UtilExportable.hpp"
 #include "Noreturn.hpp"
 
 #include <cstdlib>
@@ -13,20 +13,11 @@
 	friend class Singleton< T >; \
 	friend class scopedPtr< T >;
 	
-#define DEFINE_SINGLETON( T )
-
-#if 0
-#define DECLARE_SINGLETON( T ) \
-
-#define DEFINE_SINGLETON( T ) \
-	DLL_EXTERN template class DLL_VISIBLE Singleton< T >; 
-#endif
-
 template< class T >
 class Singleton : private noncopyable
 {
 	public:
-		DLL_VISIBLE static T& instance( )
+		UTIL_DLL_VISIBLE static T& instance( )
 		{
 			if( destroyed ) {
 				onDeadReference( );
diff --git a/include/util/UtilExportable.hpp b/include/util/UtilExportable.hpp
new file mode 100755
index 0000000..f9598f1
--- /dev/null
+++ b/include/util/UtilExportable.hpp
@@ -0,0 +1,26 @@
+#ifndef __UTIL_EXPORTABLE_H
+#define __UTIL_EXPORTABLE_H
+
+#ifndef _WIN32
+
+#define UTIL_DLL_VISIBLE
+
+#else
+
+#ifdef SHARED
+
+#ifdef BUILDING_UTIL
+#define UTIL_DLL_VISIBLE __declspec(dllexport)
+#else
+#define UTIL_DLL_VISIBLE __declspec(dllimport)
+#endif
+
+#else
+
+#define UTIL_DLL_VISIBLE
+
+#endif
+
+#endif
+
+#endif
diff --git a/src/GNUmakefile b/src/GNUmakefile
index 97b934a..7367ce2 100755
--- a/src/GNUmakefile
+++ b/src/GNUmakefile
@@ -1,69 +1,17 @@
 TOPDIR = ..
 
-SUBDIRS = logger modules
-
--include $(TOPDIR)/makefiles/gmake/platform.mk
-
-INCLUDE_CPPFLAGS = \
-
-
-INCLUDE_DIRS = \
-	-I. \
-	-I$(TOPDIR)/include/logger \
-	-I$(TOPDIR)/include/util \
-	-I$(TOPDIR)/include/module
-
-INCLUDE_LDFLAGS = \
-	-L$(TOPDIR)/src/logger
-
-INCLUDE_LIBS = \
-	-llogger
-
-# openssl
-ifeq ($(WITH_SSL),1)
-
-INCLUDE_CFLAGS += \
-	-DWITH_SSL
-
-INCLUDE_LIBS += \
-	$(OPENSSL_LIBS)
-endif
-
-STATIC_LIB = libcrawler.a
-
-DYNAMIC_LIB = libcrawler.so
-DYNAMIC_LIB_MAJOR = 0
-DYNAMIC_LIB_MINOR = 0
-DYNAMIC_LIB_PATCH = 0
-
-CPP_OBJS = \
-	URL.o \
-	MIMEType.o \
-	SpoolRewindInputStream.o
-
-CPP_BINS = \
-	crawl$(EXE)
+SUBDIRS = logger libcrawler modules crawl
 
 -include $(TOPDIR)/makefiles/gmake/sub.mk
 
 local_all:
 
-modules: $(DYNAMIC_LIB)
-
 local_clean:
-	@-rm -f $(LOCAL_STATIC_LIB)
 
 local_distclean:
 
 local_install:
-	$(INSTALL) -d -m 0755 $(DESTDIR)$(bindir)
-	$(INSTALL) -m 0775 crawl$(EXE) $(DESTDIR)$(bindir)
 
 local_uninstall:
-	@-rm -f $(DESTDIR)$(bindir)/crawl
-	@-rmdir $(DESTDIR)$(bindir)
 
 local_test:
-
-run:
-	@LD_LIBRARY_PATH=$(TOPDIR)/src:$(TOPDIR)/src/logger:$(TOPDIR)/googleurl:$(TOPDIR)/libfetch:$(TOPDIR)/streamhtmlparser ./crawl
diff --git a/src/Makefile.W32 b/src/Makefile.W32
index ffcbe27..f3702cf 100755
--- a/src/Makefile.W32
+++ b/src/Makefile.W32
@@ -1,58 +1,12 @@
 TOPDIR = ..
 
-SUBDIRS = logger modules
-
-!INCLUDE $(TOPDIR)\makefiles\nmake\platform.mk
-
-INCLUDE_CXXFLAGS = \
-	/D_WIN32_WINNT=0x504
-
-INCLUDE_DIRS = \
-	/I. \
-	/I$(TOPDIR)\include\logger \
-	/I$(TOPDIR)\include\module \
-	/I$(TOPDIR)\include\util
-
-INCLUDE_LDFLAGS = \
-
-INCLUDE_LIBS = \
-	$(TOPDIR)\src\logger\logger.lib \
-	Ws2_32.lib
-
-LOCAL_STATIC_LIB_OBJS = \
-	win32\errormsg.obj \
-	win32\stringutils.obj \
-	URL.obj \
-	MIMEType.obj \
-	SpoolRewindInputStream.obj
-
-LOCAL_STATIC_LIB = \
-	crawler.lib
-
-CPP_OBJS = \
-	$(LOCAL_STATIC_LIB_OBJS)
-
-CPP_BINS = \
-	crawl.exe
+SUBDIRS = logger libcrawler modules crawl
 
 !INCLUDE $(TOPDIR)\makefiles\nmake\sub.mk
 
-all: dependencies $(CPP_OBJS) $(CPP_BINS)
-
-dependencies:
-	cd logger & $(MAKE) /nologo /f Makefile.w32 all
-
-crawl.exe: crawl.obj $(CPP_OBJS)
-
-$(LOCAL_STATIC_LIB): $(LOCAL_STATIC_LIB_OBJS)
-	$(LINK) /lib /nologo /out:$@ $(STATIC_LDFLAGS) $?
-
-local_all: $(LOCAL_STATIC_LIB) $(CPP_BINS)
+local_all:
 
 local_clean:
-	@-erase $(LOCAL_STATIC_LIB) 2>NUL
-	@-erase $(CPP_OBJS) win32\*.obj 2>NUL
-	@-erase test.bat 2>NUL
 
 local_distclean:
 
@@ -62,6 +16,7 @@ copy_prereq:
 	@-copy "$(ICU_DIR)\bin\icuuc49.dll" . >NUL
  	@-copy "$(ICU_DIR)\bin\icudt49.dll" . >NUL
 	@-copy "$(TOPDIR)\src\logger\logger.dll" . >NUL
+	@-copy "$(TOPDIR)\src\libcrawler\crawler.dll" . >NUL
 
 run: copy_prereq
-	@-crawl.exe
+	@-crawl\crawl.exe
diff --git a/src/crawl/GNUmakefile b/src/crawl/GNUmakefile
new file mode 100755
index 0000000..6899fde
--- /dev/null
+++ b/src/crawl/GNUmakefile
@@ -0,0 +1,55 @@
+TOPDIR = ../..
+
+SUBDIRS =
+
+-include $(TOPDIR)/makefiles/gmake/platform.mk
+
+INCLUDE_CPPFLAGS = \
+
+INCLUDE_DIRS = \
+	-I. \
+	-I$(TOPDIR)/include/logger \
+	-I$(TOPDIR)/include/util \
+	-I$(TOPDIR)/include/module
+
+INCLUDE_LDFLAGS = \
+	-L$(TOPDIR)/src/logger
+
+INCLUDE_LIBS = \
+	-llogger
+
+# openssl
+ifeq ($(WITH_SSL),1)
+
+INCLUDE_CFLAGS += \
+	-DWITH_SSL
+
+INCLUDE_LIBS += \
+	$(OPENSSL_LIBS)
+endif
+
+CPP_OBJS = \
+
+CPP_BINS = \
+	crawl$(EXE)
+
+-include $(TOPDIR)/makefiles/gmake/sub.mk
+
+local_all:
+
+local_clean:
+
+local_distclean:
+
+local_install:
+	$(INSTALL) -d -m 0755 $(DESTDIR)$(bindir)
+	$(INSTALL) -m 0775 crawl$(EXE) $(DESTDIR)$(bindir)
+
+local_uninstall:
+	@-rm -f $(DESTDIR)$(bindir)/crawl
+	@-rmdir $(DESTDIR)$(bindir)
+
+local_test:
+
+run:
+	@LD_LIBRARY_PATH=$(TOPDIR)/src:$(TOPDIR)/src/logger:$(TOPDIR)/googleurl:$(TOPDIR)/libfetch:$(TOPDIR)/streamhtmlparser ./crawl
diff --git a/src/crawl/Makefile.W32 b/src/crawl/Makefile.W32
new file mode 100755
index 0000000..74442dc
--- /dev/null
+++ b/src/crawl/Makefile.W32
@@ -0,0 +1,39 @@
+TOPDIR = ..\..
+
+SUBDIRS =
+
+!INCLUDE $(TOPDIR)\makefiles\nmake\platform.mk
+
+INCLUDE_CXXFLAGS = \
+	/D_WIN32_WINNT=0x504 \
+	/DSHARED
+
+INCLUDE_DIRS = \
+	/I. \
+	/I$(TOPDIR)\include\logger \
+	/I$(TOPDIR)\include\module \
+	/I$(TOPDIR)\include\util \
+	/I$(TOPDIR)\include\crawler
+
+INCLUDE_LDFLAGS = \
+
+INCLUDE_LIBS = \
+	$(TOPDIR)\src\logger\logger.lib \
+	$(TOPDIR)\src\libcrawler\crawler.lib
+
+CPP_OBJS = \
+
+CPP_BINS = \
+	crawl.exe
+
+!INCLUDE $(TOPDIR)\makefiles\nmake\sub.mk
+
+crawl.exe: crawl.obj
+
+local_all: $(CPP_BINS)
+
+local_clean:
+
+local_distclean:
+
+local_test:
diff --git a/src/crawl.cpp b/src/crawl/crawl.cpp
index 823ed02..823ed02 100755
--- a/src/crawl.cpp
+++ b/src/crawl/crawl.cpp
diff --git a/src/libcrawler/GNUmakefile b/src/libcrawler/GNUmakefile
new file mode 100755
index 0000000..c1e7a7f
--- /dev/null
+++ b/src/libcrawler/GNUmakefile
@@ -0,0 +1,42 @@
+TOPDIR = ../..
+
+SUBDIRS =
+
+-include $(TOPDIR)/makefiles/gmake/platform.mk
+
+INCLUDE_CPPFLAGS = \
+
+INCLUDE_LDFLAGS = \
+
+INCLUDE_DIRS = \
+	-I. \
+	-I$(TOPDIR)/include/logger \
+	-I$(TOPDIR)/include/util
+
+INCLUDE_LIBS = \
+
+STATIC_LIB = libcrawler.a
+
+DYNAMIC_LIB = libcrawler.so
+DYNAMIC_LIB_MAJOR = 0
+DYNAMIC_LIB_MINOR = 0
+DYNAMIC_LIB_PATCH = 0
+
+CPP_OBJS = \
+	URL.o \
+	MIMEType.o \
+	SpoolRewindInputStream.o
+
+-include $(TOPDIR)/makefiles/gmake/sub.mk
+
+local_all:
+
+local_clean:
+
+local_distclean:
+
+local_install:
+
+local_uninstall:
+
+local_test:
diff --git a/src/MIMEType.cpp b/src/libcrawler/MIMEType.cpp
index 25dc20c..25dc20c 100644
--- a/src/MIMEType.cpp
+++ b/src/libcrawler/MIMEType.cpp
diff --git a/src/libcrawler/Makefile.W32 b/src/libcrawler/Makefile.W32
new file mode 100755
index 0000000..ab18d2c
--- /dev/null
+++ b/src/libcrawler/Makefile.W32
@@ -0,0 +1,45 @@
+TOPDIR = ..\..
+
+SUBDIRS =
+
+!INCLUDE $(TOPDIR)\makefiles\nmake\platform.mk
+
+INCLUDE_CXXFLAGS = \
+	/D_WIN32_WINNT=0x504 \
+	/DBUILDING_CRAWLER
+
+INCLUDE_DIRS = \
+	/I. \
+	/I$(TOPDIR)\include\crawler \
+	/I$(TOPDIR)\include\logger \
+	/I$(TOPDIR)\include\util
+
+INCLUDE_LDFLAGS = \
+
+INCLUDE_LIBS = \
+	$(TOPDIR)\src\logger\logger.lib
+
+CPP_OBJS = \
+	win32\errormsg.dllobj \
+	win32\stringutils.dllobj \
+	URL.dllobj \
+	MIMEType.dllobj \
+	SpoolRewindInputStream.dllobj
+	
+DYNAMIC_LIB = \
+	crawler.dll
+
+!INCLUDE $(TOPDIR)\makefiles\nmake\sub.mk
+
+$(DYNAMIC_LIB): $(CPP_OBJS)
+	$(LINK) /nologo /dll /out:$@ $(LDFLAGS) $(LIBS) $?
+
+local_all: $(DYNAMIC_LIB)
+
+local_clean:
+	@-erase $(DYNAMIC_LIB) 2>NUL
+	@-erase win32\*.obj 2>NUL
+
+local_distclean:
+
+local_test:
diff --git a/src/SpoolRewindInputStream.cpp b/src/libcrawler/SpoolRewindInputStream.cpp
index 9135741..9135741 100644
--- a/src/SpoolRewindInputStream.cpp
+++ b/src/libcrawler/SpoolRewindInputStream.cpp
diff --git a/src/URL.cpp b/src/libcrawler/URL.cpp
index f208500..f208500 100644
--- a/src/URL.cpp
+++ b/src/libcrawler/URL.cpp
diff --git a/src/win32/errormsg.cpp b/src/libcrawler/win32/errormsg.cpp
index 1b58ea3..c0a65d8 100755
--- a/src/win32/errormsg.cpp
+++ b/src/libcrawler/win32/errormsg.cpp
@@ -1,4 +1,4 @@
-#include "errormsg.hpp"
+#include "win32/errormsg.hpp"
 
 using namespace std;
 
diff --git a/src/win32/stringutils.cpp b/src/libcrawler/win32/stringutils.cpp
index a82dd7a..607735c 100755
--- a/src/win32/stringutils.cpp
+++ b/src/libcrawler/win32/stringutils.cpp
@@ -1,4 +1,4 @@
-#include "errormsg.hpp"
+#include "win32/stringutils.hpp"
 
 using namespace std;
 
diff --git a/src/logger/Makefile.W32 b/src/logger/Makefile.W32
index 3f9352d..d5516ca 100755
--- a/src/logger/Makefile.W32
+++ b/src/logger/Makefile.W32
@@ -6,7 +6,7 @@ SUBDIRS =
 
 INCLUDE_CXXFLAGS = \
 	/D_WIN32_WINNT=0x504 \
-	/DSHARED
+	/DSHARED /DBUILDING_LOGGER /DBUILDING_UTIL
 
 INCLUDE_DIRS = \
 	/I. \
diff --git a/src/modules/deduper/null/Makefile.W32 b/src/modules/deduper/null/Makefile.W32
index e01235f..e0d6c60 100755
--- a/src/modules/deduper/null/Makefile.W32
+++ b/src/modules/deduper/null/Makefile.W32
@@ -11,12 +11,21 @@ INCLUDE_DIRS = \
 	/I. \
 	/I$(TOPDIR)\src \
 	/I$(TOPDIR)\include\module \
-	/I$(TOPDIR)\include\util
+	/I$(TOPDIR)\include\util \
+	/I$(TOPDIR)\include\crawler
+
+copy_prereq:
+	@-copy "$(ICU_DIR)\bin\icuuc49.dll" . >NUL
+ 	@-copy "$(ICU_DIR)\bin\icudt49.dll" . >NUL
+	@-copy "$(TOPDIR)\src\logger\logger.dll" . >NUL
+
+run: copy_prereq
+	@-crawl.exe
 
 INCLUDE_LDFLAGS = \
 
 INCLUDE_LIBS = \
-	$(TOPDIR)\src\crawler.lib
+	$(TOPDIR)\src\libcrawler\crawler.lib
 
 DYNAMIC_MODULE = \
 	mod_deduper_null.dll
diff --git a/src/modules/fetcher/file/Makefile.W32 b/src/modules/fetcher/file/Makefile.W32
index 3203d6d..a77daa6 100755
--- a/src/modules/fetcher/file/Makefile.W32
+++ b/src/modules/fetcher/file/Makefile.W32
@@ -11,12 +11,13 @@ INCLUDE_DIRS = \
 	/I. \
 	/I$(TOPDIR)\src \
 	/I$(TOPDIR)\include\module \
-	/I$(TOPDIR)\include\util
+	/I$(TOPDIR)\include\util \
+	/I$(TOPDIR)\include\crawler
 
 INCLUDE_LDFLAGS = \
 
 INCLUDE_LIBS = \
-	$(TOPDIR)\src\crawler.lib
+	$(TOPDIR)\src\libcrawler\crawler.lib
 
 DYNAMIC_MODULE = \
 	mod_fetcher_file.dll
diff --git a/src/modules/fetcher/winhttp/Makefile.W32 b/src/modules/fetcher/winhttp/Makefile.W32
index b46aa88..4cedb9c 100755
--- a/src/modules/fetcher/winhttp/Makefile.W32
+++ b/src/modules/fetcher/winhttp/Makefile.W32
@@ -12,12 +12,13 @@ INCLUDE_DIRS = \
 	/I$(TOPDIR)\src \
 	/I$(TOPDIR)\include\module \
 	/I$(TOPDIR)\include\util \
-	/I$(TOPDIR)\include\logger
+	/I$(TOPDIR)\include\logger \
+	/I$(TOPDIR)\include\crawler
 
 INCLUDE_LDFLAGS = \
 
 INCLUDE_LIBS = \
-	$(TOPDIR)\src\crawler.lib \
+	$(TOPDIR)\src\libcrawler\crawler.lib \
 	$(TOPDIR)\src\logger\logger.lib \
 	WinHttp.lib
 
diff --git a/src/modules/frontier/memory/Makefile.W32 b/src/modules/frontier/memory/Makefile.W32
index b44d95f..b12e4ca 100755
--- a/src/modules/frontier/memory/Makefile.W32
+++ b/src/modules/frontier/memory/Makefile.W32
@@ -12,12 +12,13 @@ INCLUDE_DIRS = \
 	/I$(TOPDIR)\src \
 	/I$(TOPDIR)\include\module \
 	/I$(TOPDIR)\include\util \
-	/I$(TOPDIR)\include\logger
+	/I$(TOPDIR)\include\logger \
+	/I$(TOPDIR)\include\crawler
 
 INCLUDE_LDFLAGS = \
 
 INCLUDE_LIBS = \
-	$(TOPDIR)\src\crawler.lib \
+	$(TOPDIR)\src\libcrawler\crawler.lib \
 	$(TOPDIR)\src\logger\logger.lib
 
 DYNAMIC_MODULE = \
diff --git a/src/modules/urlfilter/chain/Makefile.W32 b/src/modules/urlfilter/chain/Makefile.W32
index 5a766ab..d3ad373 100755
--- a/src/modules/urlfilter/chain/Makefile.W32
+++ b/src/modules/urlfilter/chain/Makefile.W32
@@ -12,12 +12,12 @@ INCLUDE_DIRS = \
 	/I$(TOPDIR)\src \
 	/I$(TOPDIR)\include\module \
 	/I$(TOPDIR)\include\util \
-	/I$(TOPDIR)\include\logger
+	/I$(TOPDIR)\include\crawler
 
 INCLUDE_LDFLAGS = \
 
 INCLUDE_LIBS = \
-	$(TOPDIR)\src\crawler.lib
+	$(TOPDIR)\src\libcrawler\crawler.lib
 
 DYNAMIC_MODULE = \
 	mod_urlfilter_chain.dll
diff --git a/src/modules/urlfilter/host/Makefile.W32 b/src/modules/urlfilter/host/Makefile.W32
index 3b99125..024dc67 100755
--- a/src/modules/urlfilter/host/Makefile.W32
+++ b/src/modules/urlfilter/host/Makefile.W32
@@ -12,12 +12,13 @@ INCLUDE_DIRS = \
 	/I$(TOPDIR)\src \
 	/I$(TOPDIR)\include\module \
 	/I$(TOPDIR)\include\util \
-	/I$(TOPDIR)\include\logger
+	/I$(TOPDIR)\include\logger \
+	/I$(TOPDIR)\include\crawler
 
 INCLUDE_LDFLAGS = \
 
 INCLUDE_LIBS = \
-	$(TOPDIR)\src\crawler.lib \
+	$(TOPDIR)\src\libcrawler\crawler.lib \
 	$(TOPDIR)\src\logger\logger.lib
 
 DYNAMIC_MODULE = \
diff --git a/src/modules/urlfilter/protocol/Makefile.W32 b/src/modules/urlfilter/protocol/Makefile.W32
index 747d714..e859829 100755
--- a/src/modules/urlfilter/protocol/Makefile.W32
+++ b/src/modules/urlfilter/protocol/Makefile.W32
@@ -12,12 +12,13 @@ INCLUDE_DIRS = \
 	/I$(TOPDIR)\src \
 	/I$(TOPDIR)\include\module \
 	/I$(TOPDIR)\include\util \
-	/I$(TOPDIR)\include\logger
+	/I$(TOPDIR)\include\logger \
+	/I$(TOPDIR)\include\crawler
 
 INCLUDE_LDFLAGS = \
 
 INCLUDE_LIBS = \
-	$(TOPDIR)\src\crawler.lib \
+	$(TOPDIR)\src\libcrawler\crawler.lib \
 	$(TOPDIR)\src\logger\logger.lib
 
 DYNAMIC_MODULE = \
diff --git a/src/modules/urlnormalizer/googleurl/Makefile.W32 b/src/modules/urlnormalizer/googleurl/Makefile.W32
index 4cc09c1..a906404 100755
--- a/src/modules/urlnormalizer/googleurl/Makefile.W32
+++ b/src/modules/urlnormalizer/googleurl/Makefile.W32
@@ -12,13 +12,14 @@ INCLUDE_DIRS = \
 	/I$(TOPDIR)\src \
 	/I$(TOPDIR)\include\module \
 	/I$(TOPDIR)\include\util \
+	/I$(TOPDIR)\include\crawler \
 	/I$(TOPDIR)\googleurl
 
 INCLUDE_LDFLAGS = \
 
 INCLUDE_LIBS = \
 	$(TOPDIR)\googleurl\googleurl.lib \
-	$(TOPDIR)\src\crawler.lib \
+	$(TOPDIR)\src\libcrawler\crawler.lib \
 	"$(ICU_DIR)\lib\icuuc.lib"
 
 DYNAMIC_MODULE = \
diff --git a/src/modules/urlnormalizer/simpleurl/Makefile.W32 b/src/modules/urlnormalizer/simpleurl/Makefile.W32
index 69e732a..2a26e9c 100755
--- a/src/modules/urlnormalizer/simpleurl/Makefile.W32
+++ b/src/modules/urlnormalizer/simpleurl/Makefile.W32
@@ -11,12 +11,13 @@ INCLUDE_DIRS = \
 	/I. \
 	/I$(TOPDIR)\src \
 	/I$(TOPDIR)\include\module \
-	/I$(TOPDIR)\include\util
+	/I$(TOPDIR)\include\util \
+	/I$(TOPDIR)\include\crawler
 
 INCLUDE_LDFLAGS = \
 
 INCLUDE_LIBS = \
-	$(TOPDIR)\src\crawler.lib
+	$(TOPDIR)\src\libcrawler\crawler.lib
 
 DYNAMIC_MODULE = \
 	mod_urlnormalizer_simple.dll
diff --git a/src/win32/errormsg.hpp b/src/win32/errormsg.hpp
deleted file mode 100755
index 443e27d..0000000
--- a/src/win32/errormsg.hpp
+++ /dev/null
@@ -1,8 +0,0 @@
-#ifndef __ERRORMSG_H
-#define __ERRORMSG_H
-
-#include <string>
-
-std::string getLastError( );
-
-#endif
diff --git a/src/win32/stringutils.hpp b/src/win32/stringutils.hpp
deleted file mode 100755
index 6d4bd80..0000000
--- a/src/win32/stringutils.hpp
+++ /dev/null
@@ -1,8 +0,0 @@
-#ifndef __STRINGUTILS_H
-#define __STRINGUTILS_H
-
-#include <string>
-
-std::wstring s2ws( const std::string &s );
-
-#endif
diff --git a/tests/logger/Makefile.W32 b/tests/logger/Makefile.W32
index e499e28..783add0 100755
--- a/tests/logger/Makefile.W32
+++ b/tests/logger/Makefile.W32
@@ -5,7 +5,8 @@ SUBDIRS =
 !INCLUDE $(TOPDIR)\makefiles\nmake\platform.mk
 
 INCLUDE_CXXFLAGS = \
-	/D_WIN32_WINNT=0x504
+	/D_WIN32_WINNT=0x504 \
+	/DSHARED
 
 INCLUDE_DIRS = \
 	/I. \
@@ -16,7 +17,6 @@ INCLUDE_DIRS = \
 INCLUDE_LDFLAGS = \
 
 INCLUDE_LIBS = \
-	$(TOPDIR)\src\crawler.lib \
 	$(TOPDIR)\src\logger\logger.lib \
 	Ws2_32.lib
 
diff --git a/tests/utils/Makefile.W32 b/tests/utils/Makefile.W32
index 03ea7ae..ec6f9e7 100755
--- a/tests/utils/Makefile.W32
+++ b/tests/utils/Makefile.W32
@@ -5,9 +5,8 @@ SUBDIRS =
 !INCLUDE $(TOPDIR)\makefiles\nmake\platform.mk
 
 INCLUDE_CXXFLAGS = \
-	/D_WIN32_WINNT=0x504 \
-	/DNODLL
-
+	/D_WIN32_WINNT=0x504
+	
 INCLUDE_DIRS = \
 	/I. \
 	/I$(TOPDIR)\src \
diff --git a/tests/utils/test3.cpp b/tests/utils/test3.cpp
index 2a9a37f..a07365a 100755
--- a/tests/utils/test3.cpp
+++ b/tests/utils/test3.cpp
@@ -30,8 +30,6 @@ class Logger : public Singleton< Logger >
 		}
 };
 
-DEFINE_SINGLETON( Logger )
-
 /* this works, and two loggers can coexist, but they have different type 
 class DerivedLogger : public Logger
 {