summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAndreas Baumann <mail@andreasbaumann.cc>2014-10-16 12:47:49 +0200
committerAndreas Baumann <mail@andreasbaumann.cc>2014-10-16 12:47:49 +0200
commit09dbbdcd142d3538858cbd086fa57d2d4f04bc07 (patch)
treee68d29b00cdf66871d8ae0c1bf775a4346dbbed2
parent2e97443290938f320d8c47d803dd5c9720ffc67a (diff)
downloadcrawler-09dbbdcd142d3538858cbd086fa57d2d4f04bc07.tar.gz
crawler-09dbbdcd142d3538858cbd086fa57d2d4f04bc07.tar.bz2
loading url normalizers now via Lua mechano
-rwxr-xr-xsrc/crawl/crawl.cpp19
1 files changed, 6 insertions, 13 deletions
diff --git a/src/crawl/crawl.cpp b/src/crawl/crawl.cpp
index 1b409bc..7d6622e 100755
--- a/src/crawl/crawl.cpp
+++ b/src/crawl/crawl.cpp
@@ -100,31 +100,24 @@ int main( int /* argc */, char *argv[] )
vector<string> entries = directory_entries( modulePath, true, modulesSearchRecursive );
vector<string> modules = luaVm.getStringArray( "modules.urlnormalizers" );
+ vector<string> normalizerModules;
vector<string>::const_iterator it2, end2 = modules.end( );
for( it2 = modules.begin( ); it2 != end2; it2++ ) {
#ifndef _WIN32
- string module = (*it2) + ".so";
+ string module = *it2 + ".so";
#else
- string module = (*it2) + ".dll";
+ string module = *it2 + ".dll";
#endif
cout << "Searching for module '" << module << "'" << endl;
vector<string>::const_iterator it, end = entries.end( );
for( it = entries.begin( ); it != end; it++ ) {
- if( endswith( (*it), module ) ) {
- cout << " Found in file '" << (*it) << "'" << endl;
+ if( endswith( *it, module ) ) {
+ cout << " Found in file '" << *it << "'" << endl;
+ normalizerModules.push_back( *it );
}
}
}
-
- vector<string> normalizerModules;
-#ifndef _WIN32
- normalizerModules.push_back( "./modules/urlnormalizer/simpleurl/mod_urlnormalizer_simple.so" );
- normalizerModules.push_back( "./modules/urlnormalizer/googleurl/mod_urlnormalizer_googleurl.so" );
-#else
- normalizerModules.push_back( ".\\modules\\urlnormalizer\\simpleurl\\mod_urlnormalizer_simple.dll" );
- normalizerModules.push_back( ".\\modules\\urlnormalizer\\googleurl\\mod_urlnormalizer_googleurl.dll" );
-#endif
ModuleLoader<URLNormalizer> urlNormalizers( normalizerModules, CLOSE_DEFERRED, (void *)&luaVm );
// initialize crawler function