src/modules/processor/robotstxt/RobotsTxtProcessor.cpp


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30

#include "RobotsTxtProcessor.hpp"
#include "Logger.hpp"

#include <string>
#include <cstring>

using namespace std;

RobotsTxtProcessor::RobotsTxtProcessor( )
{
}

RobotsTxtProcessor::~RobotsTxtProcessor( )
{
}

void RobotsTxtProcessor::process( RewindInputStream *s )
{
	char buf[2] = {0, 0};
	URL url = s->getBaseUrl( );
	
	while( s->good( ) && !s->eof( ) ) {
		buf[0] = s->get( );
		if( buf[0] ) {
			cout << buf;
		}
	}
}

REGISTER_MODULE( "robotstxt_processor", Processor, RobotsTxtProcessor )