blob: 532c741428f94fb2ee774a16cef8909829ae32f7 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
|
#ifndef __ROBOTS_TXT_PROCESSOR_H
#define __ROBOTS_TXT_PROCESSOR_H
#include "Processor.hpp"
#include "ModuleRegistry.hpp"
class RobotsTxtProcessor : public Processor {
public:
RobotsTxtProcessor( );
virtual ~RobotsTxtProcessor( );
virtual void process( RewindInputStream *s );
typedef enum {
UserAgentKeyword,
DisallowKeyword,
CrawlDelayKeyword,
SitemapKeyword,
UnknownKeyword,
NoKeyword
} KeywordType;
private:
KeywordType getKeyword( string::const_iterator &it, string::const_iterator end );
void skipSpaces( string::const_iterator &it, string::const_iterator end );
void handleLine( const std::string &s );
};
DECLARE_MODULE( Processor )
#endif
|
