src/crawl/crawl.conf


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61

-- global settings

crawler = {
	-- stop after N documents
	stop_after_N_operations = 10,
	
	module_path = "modules",
	
	modules_search_recursive = true
}

logger = {
	level = "DEBUG"
}

modules = {
	urlnormalizers = {
		"mod_urlnormalizer_simple",
		"mod_urlnormalizer_googleurl"
	}
}

-- seeds: URLS which are fed in the beginning to the URL frontier

seeds = {
	"http://www.wolframe.net",
	"http://wolframe.net"
}

filters = {
	-- allowed protocols to be fetched
	protocols = {
		"http",
		"https"
	},
	
	-- allowed hosts to be crawled (FQDN)
	hosts = {
		"www.wolframe.net",
		"wolframe.net"
	}
}


function init( )
	io.write( "Init..\n" )
	normalizer = GoogleURLNormalizer:new( )
end

function destroy( )
	io.write( "Destroy..\n" )
	normalizer:delete( )
end

function crawl( )
	io.write( "Crawling..\n" )
	local baseUrl = normalizer:parseUrl( "http://www.base.com" )
	io.write( "base URL is: " .. baseUrl:str( ) .. "\n" )
	local url = normalizer:normalize( baseUrl, "/relativedir/relativefile.html" )
	io.write( "URL is: " .. url:str( ) .. "\n" )
end