sitemap processing (work in progress)

8 files changed, 219 insertions, 7 deletions

diff --git a/src/modules/processor/GNUmakefile b/src/modules/processor/GNUmakefile
index 8b91967..48a733a 100644
--- a/src/modules/processor/GNUmakefile
+++ b/src/modules/processor/GNUmakefile
@@ -1,6 +1,6 @@
 TOPDIR = ../../..
 
-SUBDIRS = htmllinkextract robotstxt
+SUBDIRS = htmllinkextract robotstxt sitemap
 
 -include $(TOPDIR)/makefiles/gmake/sub.mk
 
diff --git a/src/modules/processor/Makefile.W32 b/src/modules/processor/Makefile.W32
index 530fd98..0888560 100755
--- a/src/modules/processor/Makefile.W32
+++ b/src/modules/processor/Makefile.W32
@@ -1,6 +1,6 @@
 TOPDIR = ..\..\..
 
-SUBDIRS = htmllinkextract robotstxt
+SUBDIRS = htmllinkextract robotstxt sitemap
 
 !INCLUDE $(TOPDIR)\makefiles\nmake\sub.mk
 
diff --git a/src/modules/processor/robotstxt/RobotsTxtProcessor.cpp b/src/modules/processor/robotstxt/RobotsTxtProcessor.cpp
index 7a91465..ec5bf20 100644
--- a/src/modules/processor/robotstxt/RobotsTxtProcessor.cpp
+++ b/src/modules/processor/robotstxt/RobotsTxtProcessor.cpp
@@ -8,7 +8,8 @@
 
 using namespace std;
 
-RobotsTxtProcessor::RobotsTxtProcessor( )
+RobotsTxtProcessor::RobotsTxtProcessor( URLNormalizer *normalizer, Frontier *frontier, URLFilter *urlFilter, URLSeen *urlSeen )
+	: m_normalizer( normalizer ), m_frontier( frontier ), m_urlFilter( urlFilter ), m_urlSeen( urlSeen )
 {
 }
 
@@ -53,7 +54,11 @@ void RobotsTxtProcessor::handleLine( const string &line )
 		case SitemapKeyword: {
 			skipSpaces( s, line.end( ) );
 			string sitemap = string( s, line.end( ) );
-			LOG( logINFO ) << "Found Sitemap '" << sitemap << "'";
+			LOG( logINFO ) << "Found sitemap '" << sitemap << "'";
+			URL sitemapLink = m_normalizer->parseUrl( sitemap );
+			if( !m_urlSeen->seen( sitemapLink ) ) {
+				m_frontier->addUrl( sitemapLink );
+			}
 			}
 			break;
 			
@@ -81,4 +86,4 @@ void RobotsTxtProcessor::process( RewindInputStream *s )
 	}
 }
 
-REGISTER_MODULE( "robotstxt_processor", Processor, RobotsTxtProcessor )
+REGISTER_MODULE_4( "robotstxt_processor", Processor, RobotsTxtProcessor, URLNormalizer *, Frontier *, URLFilter *, URLSeen * )
diff --git a/src/modules/processor/robotstxt/RobotsTxtProcessor.hpp b/src/modules/processor/robotstxt/RobotsTxtProcessor.hpp
index 532c741..7db6302 100644
--- a/src/modules/processor/robotstxt/RobotsTxtProcessor.hpp
+++ b/src/modules/processor/robotstxt/RobotsTxtProcessor.hpp
@@ -3,10 +3,14 @@
 
 #include "Processor.hpp"
 #include "ModuleRegistry.hpp"
+#include "URLNormalizer.hpp"
+#include "Frontier.hpp"
+#include "URLFilter.hpp"
+#include "URLSeen.hpp"
 
 class RobotsTxtProcessor : public Processor {
 	public:
-		RobotsTxtProcessor( );
+		RobotsTxtProcessor( URLNormalizer *normalizer, Frontier *frontier, URLFilter *urlFilter, URLSeen *urlSeen );
 		virtual ~RobotsTxtProcessor( );
 		virtual void process( RewindInputStream *s );
 
@@ -19,12 +23,18 @@ class RobotsTxtProcessor : public Processor {
 		NoKeyword
 	} KeywordType;
 	
+	protected:
+		URLNormalizer *m_normalizer;
+		Frontier *m_frontier;
+		URLFilter *m_urlFilter;
+		URLSeen *m_urlSeen;
+	
 	private:
 		KeywordType getKeyword( string::const_iterator &it, string::const_iterator end );
 		void skipSpaces( string::const_iterator &it, string::const_iterator end );
 		void handleLine( const std::string &s );
 };
 
-DECLARE_MODULE( Processor )
+DECLARE_MODULE_4( Processor, URLNormalizer *, Frontier *, URLFilter *, URLSeen * )
 
 #endif
diff --git a/src/modules/processor/sitemap/GNUmakefile b/src/modules/processor/sitemap/GNUmakefile
new file mode 100644
index 0000000..f206b35
--- /dev/null
+++ b/src/modules/processor/sitemap/GNUmakefile
@@ -0,0 +1,44 @@
+TOPDIR = ../../../..
+
+SUBDIRS =
+
+-include $(TOPDIR)/makefiles/gmake/platform.mk
+
+INCLUDE_DIRS = \
+	-I. -I$(TOPDIR)/src \
+	-I$(TOPDIR)/include/logger \
+	-I$(TOPDIR)/include/util \
+	-I$(TOPDIR)/include/module \
+	-I$(TOPDIR)/include/crawler
+
+INCLUDE_CXXFLAGS = \
+
+INCLUDE_LDFLAGS = \
+	-L$(TOPDIR)/src/libcrawler
+
+INCLUDE_LIBS = \
+	-lcrawler
+
+DYNAMIC_MODULE = \
+	mod_processor_sitemap.so
+
+STATIC_LIB = \
+	libsitemapprocessor.a
+	
+CPP_OBJS = \
+	SitemapProcessor.o
+
+-include $(TOPDIR)/makefiles/gmake/sub.mk
+
+local_all:
+
+local_clean:
+
+local_distclean:
+
+local_install:
+
+local_uninstall:
+
+local_test:
+
diff --git a/src/modules/processor/sitemap/Makefile.W32 b/src/modules/processor/sitemap/Makefile.W32
new file mode 100755
index 0000000..115b4c8
--- /dev/null
+++ b/src/modules/processor/sitemap/Makefile.W32
@@ -0,0 +1,52 @@
+TOPDIR = ..\..\..\..
+
+SUBDIRS =
+
+!INCLUDE $(TOPDIR)\makefiles\nmake\platform.mk
+
+INCLUDE_CXXFLAGS = \
+	/D_WIN32_WINNT=0x504
+
+INCLUDE_DIRS = \
+	/I. \
+	/I$(TOPDIR)\src \
+	/I$(TOPDIR)\include\module \
+	/I$(TOPDIR)\include\util \
+	/I$(TOPDIR)\include\logger \
+	/I$(TOPDIR)\include\crawler
+
+INCLUDE_LDFLAGS = \
+
+INCLUDE_LIBS = \
+	$(TOPDIR)\src\libcrawler\crawler.lib \
+	$(TOPDIR)\src\liblogger\logger.lib
+
+DYNAMIC_MODULE = \
+	mod_processor_sitemap.dll
+
+STATIC_LIB = \
+	sitemapprocessor.lib
+
+CPP_OBJS = \
+	SitemapProcessor.obj
+
+SHARED_CPP_OBJS = \
+	HTMLLinkExtractProcessor.dllobj
+
+!INCLUDE $(TOPDIR)\makefiles\nmake\sub.mk
+
+$(STATIC_LIB): $(CPP_OBJS)
+	$(LINK) /lib /nologo /out:$@ $(STATIC_LDFLAGS) $?
+
+$(DYNAMIC_MODULE): $(SHARED_CPP_OBJS)
+	$(LINK) /dll /nologo /out:$@ $(LDFLAGS) $(LIBS) $?
+
+local_all: $(STATIC_LIB) $(DYNAMIC_MODULE)
+
+local_clean:
+	@-erase $(LOCAL_STATIC_LIB) 2>NUL
+	@-erase $(CPP_OBJS) 2>NUL
+
+local_distclean:
+
+local_test:
diff --git a/src/modules/processor/sitemap/SitemapProcessor.cpp b/src/modules/processor/sitemap/SitemapProcessor.cpp
new file mode 100644
index 0000000..f575a0c
--- /dev/null
+++ b/src/modules/processor/sitemap/SitemapProcessor.cpp
@@ -0,0 +1,71 @@
+#include "HTMLLinkExtractProcessor.hpp"
+#include "Logger.hpp"
+
+#include <string>
+#include <cstring>
+
+using namespace std;
+using namespace streamhtmlparser;
+
+HTMLLinkExtractProcessor::HTMLLinkExtractProcessor( URLNormalizer *normalizer, Frontier *frontier, URLFilter *filter, URLSeen *urlSeen )
+	: m_normalizer( normalizer ), m_frontier( frontier ), m_filter( filter ), m_urlSeen( urlSeen ),
+	m_parser( ), m_baseUrl( URL::Null )
+{
+}
+
+HTMLLinkExtractProcessor::~HTMLLinkExtractProcessor( )
+{
+}
+
+void HTMLLinkExtractProcessor::process( RewindInputStream *s )
+{
+	string link;
+	char buf[1] = {0};
+	bool in_link = false;
+	
+	m_baseUrl = s->getBaseUrl( );
+	
+	while( s->good( ) && !s->eof( ) ) {
+		buf[0] = s->get( );
+		m_parser.Parse( buf, 1 );
+				
+		if( m_parser.state( ) == HtmlParser::STATE_VALUE && m_parser.tag( ) != NULL && m_parser.attribute( ) != NULL && m_parser.value( ) != NULL ) {
+			if( 	strcmp( m_parser.tag( ), "base" ) == 0 &&
+				strcmp( m_parser.attribute( ), "href" ) == 0 ) {
+				m_baseUrl = m_normalizer->parseUrl( m_parser.value( ) );
+			}
+			if( 	( ( 	strcmp( m_parser.tag( ), "a" ) == 0 ||
+					strcmp( m_parser.tag( ), "area" ) == 0 ||
+					strcmp( m_parser.tag( ), "link" ) == 0 ) &&
+					strcmp( m_parser.attribute( ), "href" ) == 0 ) || 
+				( (	strcmp( m_parser.tag( ), "img" ) == 0 ||
+					strcmp( m_parser.tag( ), "frame" ) == 0 ||
+					strcmp( m_parser.tag( ), "iframe" ) == 0 ||
+					strcmp( m_parser.tag( ), "embed" ) == 0 ) &&
+					strcmp( m_parser.attribute( ), "src" ) == 0 )
+				) {				
+				link = m_parser.value( );
+				in_link = true;
+			}
+		} else if( in_link && m_parser.state( ) == HtmlParser::STATE_TAG ) {
+			URL absoluteLink = m_normalizer->normalize( m_baseUrl, link );
+			if( m_filter->filter( absoluteLink ) ) {
+				if( !m_urlSeen->seen( absoluteLink ) ) {
+					m_frontier->addUrl( absoluteLink );
+				}
+			}
+			
+			link.clear( );
+			in_link = false;
+		} else if( m_parser.state( ) == HtmlParser::STATE_ERROR ) {
+			// TODO: proper error handling
+			cerr << endl << "ERROR at " << endl;
+			m_parser.Reset( );
+			return;
+		}
+	}
+
+	m_parser.Reset( );
+}
+
+REGISTER_MODULE_4( "htmllinkextract_processor", Processor, HTMLLinkExtractProcessor, URLNormalizer *, Frontier *, URLFilter *, URLSeen * )
diff --git a/src/modules/processor/sitemap/SitemapProcessor.hpp b/src/modules/processor/sitemap/SitemapProcessor.hpp
new file mode 100644
index 0000000..d5c6bc6
--- /dev/null
+++ b/src/modules/processor/sitemap/SitemapProcessor.hpp
@@ -0,0 +1,30 @@
+#ifndef __HTML_LINK_EXTRACT_PROCESSOR_H
+#define __HTML_LINK_EXTRACT_PROCESSOR_H
+
+#include "Processor.hpp"
+#include "URLNormalizer.hpp"
+#include "Frontier.hpp"
+#include "URLFilter.hpp"
+#include "URLSeen.hpp"
+#include "ModuleRegistry.hpp"
+
+#include "htmlparser_cpp.h"
+
+class HTMLLinkExtractProcessor : public Processor {
+	public:
+		HTMLLinkExtractProcessor( URLNormalizer *normalizer, Frontier *frontier, URLFilter *filter, URLSeen *urlSeen );
+		virtual ~HTMLLinkExtractProcessor( );
+		virtual void process( RewindInputStream *s );
+	
+	protected:
+		URLNormalizer *m_normalizer;
+		Frontier *m_frontier;
+		URLFilter *m_filter;
+		URLSeen *m_urlSeen;
+		streamhtmlparser::HtmlParser m_parser;
+		URL m_baseUrl;
+};
+
+DECLARE_MODULE_4( Processor, URLNormalizer *, Frontier *, URLFilter *, URLSeen * )
+
+#endif