#ifndef __ROBOTS_TXT_PROCESSOR_H
#define __ROBOTS_TXT_PROCESSOR_H

#include "Processor.hpp"
#include "ModuleRegistry.hpp"

class RobotsTxtProcessor : public Processor {
	public:
		RobotsTxtProcessor( );
		virtual ~RobotsTxtProcessor( );
		virtual void process( RewindInputStream *s );

	typedef enum {
		UserAgentKeyword,
		DisallowKeyword,
		CrawlDelayKeyword,
		SitemapKeyword,
		UnknownKeyword,
		NoKeyword
	} KeywordType;
	
	private:
		KeywordType getKeyword( string::const_iterator &it, string::const_iterator end );
		void skipSpaces( string::const_iterator &it, string::const_iterator end );
		void handleLine( const std::string &s );
};

DECLARE_MODULE( Processor )

#endif