1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
|
#include "SitemapProcessor.hpp"
#include "Logger.hpp"
#include "textwolf.hpp"
#include "textwolf/istreamiterator.hpp"
#include <string>
#include <cstring>
using namespace std;
using namespace textwolf;
SitemapProcessor::SitemapProcessor( URLNormalizer *normalizer, Frontier *frontier, URLFilter *filter, URLSeen *urlSeen )
: m_normalizer( normalizer ), m_frontier( frontier ), m_filter( filter ), m_urlSeen( urlSeen )
{
}
SitemapProcessor::~SitemapProcessor( )
{
}
void SitemapProcessor::process( RewindInputStream *s )
{
IStreamIterator isitr( *s );
typedef enum {
SITEMAPINDEX_XMLNS = 1,
SITEMAPINDEX_SITEMAP_LOC = 2,
URLSET_URL_LOC = 3
} XmlNodes;
typedef XMLPathSelectAutomaton<charset::UTF8> Automaton;
Automaton atm;
// /sitemapindex@xmlns
(*atm)["sitemapindex"]("xmlns") = SITEMAPINDEX_XMLNS;
// /sitemapindex/sitemap/loc::text
(*atm)["sitemapindex"]["sitemap"]["loc"]( ) = SITEMAPINDEX_SITEMAP_LOC;
// /urlset/url/loc::text
(*atm)["urlset"]["url"]["loc"]( ) = URLSET_URL_LOC;
typedef XMLPathSelect<charset::UTF8> PathSelect;
typedef XMLScanner<IStreamIterator, charset::UTF8, charset::UTF8, std::string> Scanner;
Scanner xsc( isitr );
PathSelect xsel( &atm );
Scanner::iterator ci, ce;
for( ci = xsc.begin( ), ce = xsc.end( ); ci != ce; ci++ ) {
if( ci->type( ) == Scanner::ErrorOccurred ) {
throw std::runtime_error( ci->content( ) );
}
PathSelect::iterator itr = xsel.push( ci->type( ), ci->content( ), ci->size( ) );
PathSelect::iterator end = xsel.end( );
for( ; itr != end; itr++ ) {
switch( *itr ) {
case SITEMAPINDEX_XMLNS: {
string schema = ci->content( );
string expectedSchema( "http://www.sitemaps.org/schemas/sitemap/0.9" );
if( schema.compare( expectedSchema ) != 0 ) {
LOG( logERROR ) << "Sitemap XML has invalid doctype '" << schema << "' (expected '"
<< expectedSchema << "')";
return;
}
break;
}
case SITEMAPINDEX_SITEMAP_LOC: {
string sitemap = ci->content( );
LOG( logINFO ) << "Found sitemap '" << sitemap << "'";
URL sitemapLink = m_normalizer->parseUrl( sitemap );
if( !m_urlSeen->seen( sitemapLink ) ) {
m_frontier->addUrl( sitemapLink );
}
break;
}
case URLSET_URL_LOC: {
string urlString = ci->content( );
LOG( logINFO ) << "Found URL in sitemap '" << urlString << "'";
URL url = m_normalizer->parseUrl( urlString );
if( !m_urlSeen->seen( url ) ) {
m_frontier->addUrl( url );
}
break;
}
default:
LOG( logERROR ) << "Unexpected value in XML parser: " << *itr;
}
}
}
}
REGISTER_MODULE_4( "sitemap_processor", 0, 0, Processor, SitemapProcessor, URLNormalizer *, Frontier *, URLFilter *, URLSeen * )
|
