From 2b15ea93c0afd872b60cea94fa3c9e585919305c Mon Sep 17 00:00:00 2001 From: Andreas Baumann Date: Fri, 10 Aug 2012 18:19:17 +0200 Subject: some more windows modules --- Makefile.W32 | 2 +- sqlite3/Makefile.W32 | 30 +++ src/ModuleLoader.hpp | 6 - src/crawlingwolf.cpp | 246 +++++++++++++---------- src/modules/Makefile.W32 | 3 +- src/modules/deduper/Makefile.W32 | 13 ++ src/modules/deduper/null/Makefile.W32 | 44 ++++ src/modules/fetcher/Makefile.W32 | 14 ++ src/modules/fetcher/file/Makefile.W32 | 45 +++++ src/modules/frontier/Makefile.W32 | 13 ++ src/modules/frontier/memory/Makefile.W32 | 44 ++++ src/modules/urlfilter/Makefile.W32 | 13 ++ src/modules/urlfilter/chain/Makefile.W32 | 44 ++++ src/modules/urlfilter/host/Makefile.W32 | 44 ++++ src/modules/urlfilter/protocol/Makefile.W32 | 44 ++++ src/modules/urlnormalizer/simpleurl/Makefile.W32 | 1 - src/modules/urlseen/Makefile.W32 | 13 ++ src/modules/urlseen/memory/Makefile.W32 | 44 ++++ 18 files changed, 544 insertions(+), 119 deletions(-) create mode 100644 sqlite3/Makefile.W32 create mode 100755 src/modules/deduper/Makefile.W32 create mode 100755 src/modules/deduper/null/Makefile.W32 create mode 100755 src/modules/fetcher/Makefile.W32 create mode 100755 src/modules/fetcher/file/Makefile.W32 create mode 100755 src/modules/frontier/Makefile.W32 create mode 100755 src/modules/frontier/memory/Makefile.W32 create mode 100755 src/modules/urlfilter/Makefile.W32 create mode 100755 src/modules/urlfilter/chain/Makefile.W32 create mode 100755 src/modules/urlfilter/host/Makefile.W32 create mode 100755 src/modules/urlfilter/protocol/Makefile.W32 create mode 100755 src/modules/urlseen/Makefile.W32 create mode 100755 src/modules/urlseen/memory/Makefile.W32 diff --git a/Makefile.W32 b/Makefile.W32 index cd0942c..a22e0e3 100755 --- a/Makefile.W32 +++ b/Makefile.W32 @@ -1,7 +1,7 @@ TOPDIR = . #SUBDIRS = libfetch streamhtmlparser googleurl sqlite3 src tests -SUBDIRS = utils src tests +SUBDIRS = utils sqlite3 src tests PACKAGE_NAME = CrawlingWolf PACKAGE_VERSION = 0.0.1 diff --git a/sqlite3/Makefile.W32 b/sqlite3/Makefile.W32 new file mode 100644 index 0000000..ae19513 --- /dev/null +++ b/sqlite3/Makefile.W32 @@ -0,0 +1,30 @@ +TOPDIR = .. + +SUBDIRS = + +!INCLUDE $(TOPDIR)\makefiles\nmake\platform.mk + +INCLUDE_CXXFLAGS = \ + /D_WIN32_WINNT=0x504 + +INCLUDE_DIRS = \ + /I. + +OBJS = \ + sqlite3.obj + +LIBRARIES = \ + sqlite3.lib + +!INCLUDE $(TOPDIR)\makefiles\nmake\sub.mk + +sqlite3.lib: $(OBJS) + $(LINK) /lib /out:$@ $(STATIC_LDFLAGS) $? + +local_all: $(LIBRARIES) + +local_clean: + +local_distclean: + +local_test: diff --git a/src/ModuleLoader.hpp b/src/ModuleLoader.hpp index 0f41548..8b00750 100755 --- a/src/ModuleLoader.hpp +++ b/src/ModuleLoader.hpp @@ -246,10 +246,4 @@ class ModuleLoader< Interface, TYPELIST_4( T1, T2, T3, T4 ) > : public BaseModul } }; -#ifndef _WIN32 -#define MODULE_EXT ".so" -#else -#define MODULE_EXT ".dll" -#endif - #endif diff --git a/src/crawlingwolf.cpp b/src/crawlingwolf.cpp index 67d6383..23b6b83 100755 --- a/src/crawlingwolf.cpp +++ b/src/crawlingwolf.cpp @@ -54,126 +54,152 @@ BOOL WINAPI termHandler( DWORD ctrlType ) int main( void ) { - FILELog::reportingLevel( ) = logINFO; + try { + FILELog::reportingLevel( ) = logINFO; #ifndef _WIN32 - struct sigaction sa; - memset( &sa, 0, sizeof( struct sigaction ) ); - sa.sa_handler = terminate_func; - sa.sa_flags = SA_RESTART; - if( sigaction( SIGINT, &sa, NULL ) < 0 ) { - cerr << "Unable to install termianation signal handler" << endl; - } + struct sigaction sa; + memset( &sa, 0, sizeof( struct sigaction ) ); + sa.sa_handler = terminate_func; + sa.sa_flags = SA_RESTART; + if( sigaction( SIGINT, &sa, NULL ) < 0 ) { + cerr << "Unable to install termianation signal handler" << endl; + } #else - SetConsoleCtrlHandler( termHandler, TRUE ); + SetConsoleCtrlHandler( termHandler, TRUE ); #endif - LOG( logNOTICE ) << "Loading modules"; - - vector normalizerModules; - normalizerModules.push_back( "./modules/urlnormalizer/simpleurl/mod_urlnormalizer_simple" MODULE_EXT ); - normalizerModules.push_back( "./modules/urlnormalizer/googleurl/mod_urlnormalizer_googleurl" MODULE_EXT ); - ModuleLoader urlNormalizers( normalizerModules ); - - vector filterModules; - filterModules.push_back( "./modules/urlfilter/protocol/mod_urlfilter_protocol" MODULE_EXT ); - filterModules.push_back( "./modules/urlfilter/host/mod_urlfilter_host" MODULE_EXT ); - ModuleLoader ) > urlFilters( filterModules ); - - vector filterChainModules; - filterChainModules.push_back( "./modules/urlfilter/chain/mod_urlfilter_chain" MODULE_EXT ); - ModuleLoader ) > urlChainFilter( filterChainModules ); - - vector frontierModules; - frontierModules.push_back( "./modules/frontier/memory/mod_frontier_memory" MODULE_EXT ); - ModuleLoader frontiers( frontierModules ); - - vector fetcherModules; - fetcherModules.push_back( "./modules/fetcher/libfetch/mod_fetcher_libfetch" MODULE_EXT ); - ModuleLoader fetchers( fetcherModules ); - - vector urlseenModules; - urlseenModules.push_back( "./modules/urlseen/memory/mod_urlseen_memory" MODULE_EXT ); - ModuleLoader urlSeens( urlseenModules ); - - vector deduperModules; - deduperModules.push_back( "./modules/deduper/null/mod_deduper_null" MODULE_EXT ); - ModuleLoader dedupers( deduperModules ); - - vector processorModules; - processorModules.push_back( "./modules/processor/htmllinkextract/mod_processor_htmllinkextract" MODULE_EXT ); - ModuleLoader processors( processorModules ); - - vector typeDetectModules; - typeDetectModules.push_back( "./modules/typedetect/libmagic/mod_typedetect_libmagic" MODULE_EXT ); - ModuleLoader typeDetectors( typeDetectModules ); - - Frontier *frontier = frontiers.create( "memory" ); - Fetcher *fetcher = fetchers.create( "libfetch" ); - Deduper *deduper = dedupers.create( "null" ); - URLSeen *urlSeen = urlSeens.create( "memory" ); - TypeDetect *typeDetect = typeDetectors.create( "libmagic" ); - - set protocols; - protocols.insert( "http" ); - protocols.insert( "https" ); - URLFilter *protocolFilter = urlFilters.create( "protocol", protocols ); - - set hosts; - hosts.insert( "www.andreasbaumann.cc" ); - URLFilter *hostFilter = urlFilters.create( "host", hosts ); - - list filters; - filters.push_back( hostFilter ); - filters.push_back( protocolFilter ); - URLFilter *chainFilter = urlChainFilter.create( "chain", filters ); + LOG( logNOTICE ) << "Loading modules"; + + vector normalizerModules; +#ifndef _WIN32 + normalizerModules.push_back( "./modules/urlnormalizer/simpleurl/mod_urlnormalizer_simple.so" ); + normalizerModules.push_back( "./modules/urlnormalizer/googleurl/mod_urlnormalizer_googleurl.so" ); +#else + normalizerModules.push_back( ".\\modules\\urlnormalizer\\simpleurl\\mod_urlnormalizer_simple.dll" ); +#endif + ModuleLoader urlNormalizers( normalizerModules ); - URLNormalizer *normalizer = urlNormalizers.create( "google" ); - - Processor *htmlParser = processors.create( "htmllinkextract", - normalizer, frontier, chainFilter, urlSeen ); - - LOG( logNOTICE ) << "Crawler started.."; - - frontier->addUrl( normalizer->parseUrl( "http://www.andreasbaumann.cc" ) ); - - URL url; - while( !term && ( url = frontier->getNextUrl( ) ) != URL::Null ) { - LOG( logINFO ) << "Got URL " << url; - RewindInputStream *s = fetcher->fetch( url ); + vector filterModules; +#ifndef _WIN32 + filterModules.push_back( "./modules/urlfilter/protocol/mod_urlfilter_protocol.so" ); + filterModules.push_back( "./modules/urlfilter/host/mod_urlfilter_host.so" ); +#endif + ModuleLoader ) > urlFilters( filterModules ); + + vector filterChainModules; +#ifndef _WIN32 + filterChainModules.push_back( "./modules/urlfilter/chain/mod_urlfilter_chain.so" ); +#endif + ModuleLoader ) > urlChainFilter( filterChainModules ); - if( deduper->contentSeen( url, s ) ) { - LOG( logINFO ) << "URL " << url << " is a duplicate, content already seen"; - delete s; - continue; - } + vector frontierModules; +#ifndef _WIN32 + frontierModules.push_back( "./modules/frontier/memory/mod_frontier_memory.so" ); +#endif + ModuleLoader frontiers( frontierModules ); + + vector fetcherModules; +#ifndef _WIN32 + fetcherModules.push_back( "./modules/fetcher/libfetch/mod_fetcher_libfetch.so" ); +#endif + ModuleLoader fetchers( fetcherModules ); + + vector urlseenModules; +#ifndef _WIN32 + urlseenModules.push_back( "./modules/urlseen/memory/mod_urlseen_memory.so" ); +#endif + ModuleLoader urlSeens( urlseenModules ); - MIMEType mimeType = typeDetect->detect( s ); - if( mimeType != MIMEType::Null ) { - if( mimeType == "text/html" ) { - s->rewind( ); - htmlParser->process( s ); - } else if( mimeType == "application/octet-stream" ) { - s->rewind( ); - LOG( logINFO ) << "Storing binary content " << url; + vector deduperModules; +#ifndef _WIN32 + deduperModules.push_back( "./modules/deduper/null/mod_deduper_null.so" ); +#endif + ModuleLoader dedupers( deduperModules ); + + vector processorModules; +#ifndef _WIN32 + processorModules.push_back( "./modules/processor/htmllinkextract/mod_processor_htmllinkextract.so" ); +#endif + ModuleLoader processors( processorModules ); + + vector typeDetectModules; +#ifndef _WIN32 + typeDetectModules.push_back( "./modules/typedetect/libmagic/mod_typedetect_libmagic.so" ); +#endif + ModuleLoader typeDetectors( typeDetectModules ); + + Frontier *frontier = frontiers.create( "memory" ); + Fetcher *fetcher = fetchers.create( "libfetch" ); + Deduper *deduper = dedupers.create( "null" ); + URLSeen *urlSeen = urlSeens.create( "memory" ); + TypeDetect *typeDetect = typeDetectors.create( "libmagic" ); + + set protocols; + protocols.insert( "http" ); + protocols.insert( "https" ); + URLFilter *protocolFilter = urlFilters.create( "protocol", protocols ); + + set hosts; + hosts.insert( "www.andreasbaumann.cc" ); + URLFilter *hostFilter = urlFilters.create( "host", hosts ); + + list filters; + filters.push_back( hostFilter ); + filters.push_back( protocolFilter ); + URLFilter *chainFilter = urlChainFilter.create( "chain", filters ); + +// URLNormalizer *normalizer = urlNormalizers.create( "google" ); + URLNormalizer *normalizer = urlNormalizers.create( "simple" ); + + Processor *htmlParser = processors.create( "htmllinkextract", + normalizer, frontier, chainFilter, urlSeen ); + + LOG( logNOTICE ) << "Crawler started.."; + + frontier->addUrl( normalizer->parseUrl( "http://www.andreasbaumann.cc" ) ); + + URL url; + while( !term && ( url = frontier->getNextUrl( ) ) != URL::Null ) { + LOG( logINFO ) << "Got URL " << url; + RewindInputStream *s = fetcher->fetch( url ); + + if( deduper->contentSeen( url, s ) ) { + LOG( logINFO ) << "URL " << url << " is a duplicate, content already seen"; + delete s; + continue; + } + + MIMEType mimeType = typeDetect->detect( s ); + if( mimeType != MIMEType::Null ) { + if( mimeType == "text/html" ) { + s->rewind( ); + htmlParser->process( s ); + } else if( mimeType == "application/octet-stream" ) { + s->rewind( ); + LOG( logINFO ) << "Storing binary content " << url; + } } + + delete s; } - delete s; + processors.destroy( htmlParser ); + urlNormalizers.destroy( normalizer ); + urlChainFilter.destroy( chainFilter ); + urlFilters.destroy( protocolFilter ); + urlFilters.destroy( hostFilter ); + typeDetectors.destroy( typeDetect ); + urlSeens.destroy( urlSeen ); + dedupers.destroy( deduper ); + fetchers.destroy( fetcher ); + frontiers.destroy( frontier ); + + LOG( logNOTICE ) << "Crawler stopped.. normal shutdown.."; + + return 0; + } catch( exception &e ) { + LOG( logFATAL ) << "Crawler stopped: " << e.what( ); + return 1; } - - processors.destroy( htmlParser ); - urlNormalizers.destroy( normalizer ); - urlChainFilter.destroy( chainFilter ); - urlFilters.destroy( protocolFilter ); - urlFilters.destroy( hostFilter ); - typeDetectors.destroy( typeDetect ); - urlSeens.destroy( urlSeen ); - dedupers.destroy( deduper ); - fetchers.destroy( fetcher ); - frontiers.destroy( frontier ); - - LOG( logNOTICE ) << "Crawler stopped.."; - - return 0; } diff --git a/src/modules/Makefile.W32 b/src/modules/Makefile.W32 index fe00419..ddd6338 100644 --- a/src/modules/Makefile.W32 +++ b/src/modules/Makefile.W32 @@ -1,7 +1,8 @@ TOPDIR = ..\.. SUBDIRS = \ - urlnormalizer + urlnormalizer urlfilter frontier fetcher urlseen \ + deduper #SUBDIRS = \ # urlnormalizer urlfilter frontier fetcher urlseen \ diff --git a/src/modules/deduper/Makefile.W32 b/src/modules/deduper/Makefile.W32 new file mode 100755 index 0000000..b0520c5 --- /dev/null +++ b/src/modules/deduper/Makefile.W32 @@ -0,0 +1,13 @@ +TOPDIR = ..\..\.. + +SUBDIRS = null + +!INCLUDE $(TOPDIR)\makefiles\nmake\sub.mk + +local_all: + +local_clean: + +local_distclean: + +local_test: diff --git a/src/modules/deduper/null/Makefile.W32 b/src/modules/deduper/null/Makefile.W32 new file mode 100755 index 0000000..e115ad2 --- /dev/null +++ b/src/modules/deduper/null/Makefile.W32 @@ -0,0 +1,44 @@ +TOPDIR = ..\..\..\.. + +SUBDIRS = + +!INCLUDE $(TOPDIR)\makefiles\nmake\platform.mk + +INCLUDE_CXXFLAGS = \ + /D_WIN32_WINNT=0x504 /DSHARED + +INCLUDE_DIRS = \ + /I. \ + /I$(TOPDIR)\src + +INCLUDE_LDFLAGS = \ + +INCLUDE_LIBS = \ + $(TOPDIR)\src\crawlingwolf.lib + +DYNAMIC_MODULE = \ + mod_deduper_null.dll + +STATIC_LIB = \ + nulldeduper.lib + +CPP_OBJS = \ + NullDeduper.obj + +!INCLUDE $(TOPDIR)\makefiles\nmake\sub.mk + +$(STATIC_LIB): $(CPP_OBJS) + $(LINK) /lib /nologo /out:$@ $(STATIC_LDFLAGS) $? + +$(DYNAMIC_MODULE): $(CPP_OBJS) + $(LINK) /dll /nologo /out:$@ $(LDFLAGS) $(LIBS) $? + +local_all: $(STATIC_LIB) $(DYNAMIC_MODULE) + +local_clean: + @-erase $(LOCAL_STATIC_LIB) 2>NUL + @-erase $(CPP_OBJS) 2>NUL + +local_distclean: + +local_test: diff --git a/src/modules/fetcher/Makefile.W32 b/src/modules/fetcher/Makefile.W32 new file mode 100755 index 0000000..044cb38 --- /dev/null +++ b/src/modules/fetcher/Makefile.W32 @@ -0,0 +1,14 @@ +TOPDIR = ..\..\.. + +SUBDIRS = file +#libfetch + +!INCLUDE $(TOPDIR)\makefiles\nmake\sub.mk + +local_all: + +local_clean: + +local_distclean: + +local_test: diff --git a/src/modules/fetcher/file/Makefile.W32 b/src/modules/fetcher/file/Makefile.W32 new file mode 100755 index 0000000..4145c84 --- /dev/null +++ b/src/modules/fetcher/file/Makefile.W32 @@ -0,0 +1,45 @@ +TOPDIR = ..\..\..\.. + +SUBDIRS = + +!INCLUDE $(TOPDIR)\makefiles\nmake\platform.mk + +INCLUDE_CXXFLAGS = \ + /D_WIN32_WINNT=0x504 /DSHARED + +INCLUDE_DIRS = \ + /I. \ + /I$(TOPDIR)\src + +INCLUDE_LDFLAGS = \ + +INCLUDE_LIBS = \ + $(TOPDIR)\src\crawlingwolf.lib + +DYNAMIC_MODULE = \ + mod_fetcher_file.dll + +STATIC_LIB = \ + filefetcher.lib + +CPP_OBJS = \ + FileFetcher.obj \ + FileRewindInputStream.obj + +!INCLUDE $(TOPDIR)\makefiles\nmake\sub.mk + +$(STATIC_LIB): $(CPP_OBJS) + $(LINK) /lib /nologo /out:$@ $(STATIC_LDFLAGS) $? + +$(DYNAMIC_MODULE): $(CPP_OBJS) + $(LINK) /dll /nologo /out:$@ $(LDFLAGS) $(LIBS) $? + +local_all: $(STATIC_LIB) $(DYNAMIC_MODULE) + +local_clean: + @-erase $(LOCAL_STATIC_LIB) 2>NUL + @-erase $(CPP_OBJS) 2>NUL + +local_distclean: + +local_test: diff --git a/src/modules/frontier/Makefile.W32 b/src/modules/frontier/Makefile.W32 new file mode 100755 index 0000000..f9988be --- /dev/null +++ b/src/modules/frontier/Makefile.W32 @@ -0,0 +1,13 @@ +TOPDIR = ..\..\.. + +SUBDIRS = memory + +!INCLUDE $(TOPDIR)\makefiles\nmake\sub.mk + +local_all: + +local_clean: + +local_distclean: + +local_test: diff --git a/src/modules/frontier/memory/Makefile.W32 b/src/modules/frontier/memory/Makefile.W32 new file mode 100755 index 0000000..751db28 --- /dev/null +++ b/src/modules/frontier/memory/Makefile.W32 @@ -0,0 +1,44 @@ +TOPDIR = ..\..\..\.. + +SUBDIRS = + +!INCLUDE $(TOPDIR)\makefiles\nmake\platform.mk + +INCLUDE_CXXFLAGS = \ + /D_WIN32_WINNT=0x504 /DSHARED + +INCLUDE_DIRS = \ + /I. \ + /I$(TOPDIR)\src + +INCLUDE_LDFLAGS = \ + +INCLUDE_LIBS = \ + $(TOPDIR)\src\crawlingwolf.lib + +DYNAMIC_MODULE = \ + mod_frontier_memory.dll + +STATIC_LIB = \ + frontiermemory.lib + +CPP_OBJS = \ + MemoryFrontier.obj + +!INCLUDE $(TOPDIR)\makefiles\nmake\sub.mk + +$(STATIC_LIB): $(CPP_OBJS) + $(LINK) /lib /nologo /out:$@ $(STATIC_LDFLAGS) $? + +$(DYNAMIC_MODULE): $(CPP_OBJS) + $(LINK) /dll /nologo /out:$@ $(LDFLAGS) $(LIBS) $? + +local_all: $(STATIC_LIB) $(DYNAMIC_MODULE) + +local_clean: + @-erase $(LOCAL_STATIC_LIB) 2>NUL + @-erase $(CPP_OBJS) 2>NUL + +local_distclean: + +local_test: diff --git a/src/modules/urlfilter/Makefile.W32 b/src/modules/urlfilter/Makefile.W32 new file mode 100755 index 0000000..5e01806 --- /dev/null +++ b/src/modules/urlfilter/Makefile.W32 @@ -0,0 +1,13 @@ +TOPDIR = ..\..\.. + +SUBDIRS = protocol host chain + +!INCLUDE $(TOPDIR)\makefiles\nmake\sub.mk + +local_all: + +local_clean: + +local_distclean: + +local_test: diff --git a/src/modules/urlfilter/chain/Makefile.W32 b/src/modules/urlfilter/chain/Makefile.W32 new file mode 100755 index 0000000..e66db24 --- /dev/null +++ b/src/modules/urlfilter/chain/Makefile.W32 @@ -0,0 +1,44 @@ +TOPDIR = ..\..\..\.. + +SUBDIRS = + +!INCLUDE $(TOPDIR)\makefiles\nmake\platform.mk + +INCLUDE_CXXFLAGS = \ + /D_WIN32_WINNT=0x504 /DSHARED + +INCLUDE_DIRS = \ + /I. \ + /I$(TOPDIR)\src + +INCLUDE_LDFLAGS = \ + +INCLUDE_LIBS = \ + $(TOPDIR)\src\crawlingwolf.lib + +DYNAMIC_MODULE = \ + mod_urlfilter_chain.dll + +STATIC_LIB = \ + chainurlfilter.lib + +CPP_OBJS = \ + ChainURLFilter.obj + +!INCLUDE $(TOPDIR)\makefiles\nmake\sub.mk + +$(STATIC_LIB): $(CPP_OBJS) + $(LINK) /lib /nologo /out:$@ $(STATIC_LDFLAGS) $? + +$(DYNAMIC_MODULE): $(CPP_OBJS) + $(LINK) /dll /nologo /out:$@ $(LDFLAGS) $(LIBS) $? + +local_all: $(STATIC_LIB) $(DYNAMIC_MODULE) + +local_clean: + @-erase $(LOCAL_STATIC_LIB) 2>NUL + @-erase $(CPP_OBJS) 2>NUL + +local_distclean: + +local_test: diff --git a/src/modules/urlfilter/host/Makefile.W32 b/src/modules/urlfilter/host/Makefile.W32 new file mode 100755 index 0000000..fb6ba2e --- /dev/null +++ b/src/modules/urlfilter/host/Makefile.W32 @@ -0,0 +1,44 @@ +TOPDIR = ..\..\..\.. + +SUBDIRS = + +!INCLUDE $(TOPDIR)\makefiles\nmake\platform.mk + +INCLUDE_CXXFLAGS = \ + /D_WIN32_WINNT=0x504 /DSHARED + +INCLUDE_DIRS = \ + /I. \ + /I$(TOPDIR)\src + +INCLUDE_LDFLAGS = \ + +INCLUDE_LIBS = \ + $(TOPDIR)\src\crawlingwolf.lib + +DYNAMIC_MODULE = \ + mod_urlfilter_host.dll + +STATIC_LIB = \ + hosturlfilter.lib + +CPP_OBJS = \ + HostURLFilter.obj + +!INCLUDE $(TOPDIR)\makefiles\nmake\sub.mk + +$(STATIC_LIB): $(CPP_OBJS) + $(LINK) /lib /nologo /out:$@ $(STATIC_LDFLAGS) $? + +$(DYNAMIC_MODULE): $(CPP_OBJS) + $(LINK) /dll /nologo /out:$@ $(LDFLAGS) $(LIBS) $? + +local_all: $(STATIC_LIB) $(DYNAMIC_MODULE) + +local_clean: + @-erase $(LOCAL_STATIC_LIB) 2>NUL + @-erase $(CPP_OBJS) 2>NUL + +local_distclean: + +local_test: diff --git a/src/modules/urlfilter/protocol/Makefile.W32 b/src/modules/urlfilter/protocol/Makefile.W32 new file mode 100755 index 0000000..e3959d9 --- /dev/null +++ b/src/modules/urlfilter/protocol/Makefile.W32 @@ -0,0 +1,44 @@ +TOPDIR = ..\..\..\.. + +SUBDIRS = + +!INCLUDE $(TOPDIR)\makefiles\nmake\platform.mk + +INCLUDE_CXXFLAGS = \ + /D_WIN32_WINNT=0x504 /DSHARED + +INCLUDE_DIRS = \ + /I. \ + /I$(TOPDIR)\src + +INCLUDE_LDFLAGS = \ + +INCLUDE_LIBS = \ + $(TOPDIR)\src\crawlingwolf.lib + +DYNAMIC_MODULE = \ + mod_urlfilter_protocol.dll + +STATIC_LIB = \ + protocolurlfilter.lib + +CPP_OBJS = \ + ProtocolURLFilter.obj + +!INCLUDE $(TOPDIR)\makefiles\nmake\sub.mk + +$(STATIC_LIB): $(CPP_OBJS) + $(LINK) /lib /nologo /out:$@ $(STATIC_LDFLAGS) $? + +$(DYNAMIC_MODULE): $(CPP_OBJS) + $(LINK) /dll /nologo /out:$@ $(LDFLAGS) $(LIBS) $? + +local_all: $(STATIC_LIB) $(DYNAMIC_MODULE) + +local_clean: + @-erase $(LOCAL_STATIC_LIB) 2>NUL + @-erase $(CPP_OBJS) 2>NUL + +local_distclean: + +local_test: diff --git a/src/modules/urlnormalizer/simpleurl/Makefile.W32 b/src/modules/urlnormalizer/simpleurl/Makefile.W32 index b12c7d9..259bcc4 100755 --- a/src/modules/urlnormalizer/simpleurl/Makefile.W32 +++ b/src/modules/urlnormalizer/simpleurl/Makefile.W32 @@ -15,7 +15,6 @@ INCLUDE_LDFLAGS = \ INCLUDE_LIBS = \ $(TOPDIR)\src\crawlingwolf.lib -# kernel32.lib advapi32.lib Ws2_32.lib DYNAMIC_MODULE = \ mod_urlnormalizer_simple.dll diff --git a/src/modules/urlseen/Makefile.W32 b/src/modules/urlseen/Makefile.W32 new file mode 100755 index 0000000..f9988be --- /dev/null +++ b/src/modules/urlseen/Makefile.W32 @@ -0,0 +1,13 @@ +TOPDIR = ..\..\.. + +SUBDIRS = memory + +!INCLUDE $(TOPDIR)\makefiles\nmake\sub.mk + +local_all: + +local_clean: + +local_distclean: + +local_test: diff --git a/src/modules/urlseen/memory/Makefile.W32 b/src/modules/urlseen/memory/Makefile.W32 new file mode 100755 index 0000000..26a94ae --- /dev/null +++ b/src/modules/urlseen/memory/Makefile.W32 @@ -0,0 +1,44 @@ +TOPDIR = ..\..\..\.. + +SUBDIRS = + +!INCLUDE $(TOPDIR)\makefiles\nmake\platform.mk + +INCLUDE_CXXFLAGS = \ + /D_WIN32_WINNT=0x504 /DSHARED + +INCLUDE_DIRS = \ + /I. \ + /I$(TOPDIR)\src + +INCLUDE_LDFLAGS = \ + +INCLUDE_LIBS = \ + $(TOPDIR)\src\crawlingwolf.lib + +DYNAMIC_MODULE = \ + mod_urlseen_memory.dll + +STATIC_LIB = \ + urlseenmemory.lib + +CPP_OBJS = \ + MemoryURLSeen.obj + +!INCLUDE $(TOPDIR)\makefiles\nmake\sub.mk + +$(STATIC_LIB): $(CPP_OBJS) + $(LINK) /lib /nologo /out:$@ $(STATIC_LDFLAGS) $? + +$(DYNAMIC_MODULE): $(CPP_OBJS) + $(LINK) /dll /nologo /out:$@ $(LDFLAGS) $(LIBS) $? + +local_all: $(STATIC_LIB) $(DYNAMIC_MODULE) + +local_clean: + @-erase $(LOCAL_STATIC_LIB) 2>NUL + @-erase $(CPP_OBJS) 2>NUL + +local_distclean: + +local_test: -- cgit v1.2.3-54-g00ecf