From 0fdd066c566564a786e9070fbfb82a0b70bd1790 Mon Sep 17 00:00:00 2001 From: Andreas Baumann Date: Tue, 14 Oct 2014 11:45:25 +0200 Subject: adapted google url normlazier Lua binding for Windows adapting cralwer to link work with Lua on windows (work in progress) --- makefiles/nmake/config.mk | 12 ------- makefiles/nmake/platform.mk | 12 +++++++ src/Makefile.W32 | 6 ++-- src/crawl/Makefile.W32 | 24 +++++++++++++ src/libcrawler/Makefile.W32 | 42 ++++++++++++++++++++++- src/modules/urlnormalizer/googleurl/GNUmakefile | 2 +- src/modules/urlnormalizer/googleurl/Makefile.W32 | 43 ++++++++++++++++++++++++ tests/url/Makefile.W32 | 4 +-- 8 files changed, 127 insertions(+), 18 deletions(-) diff --git a/makefiles/nmake/config.mk b/makefiles/nmake/config.mk index cb15837..5593036 100755 --- a/makefiles/nmake/config.mk +++ b/makefiles/nmake/config.mk @@ -68,15 +68,3 @@ DOXYGEN = C:\Program Files\Doxygen\bin\doxygen.exe ########################################################################## HHC_LOCATION = C:\Program Files\HTML Help Workshop\hhc.exe - -# tolua and Lua -############### - -TOLUA_DIR = $(TOPDIR)\tolua -LUA_DIR = $(TOPDIR)\lua -TOLUA = $(TOLUA_DIR)\bin\tolua -TOLUA_CFLAGS = /DSHARED /DLUA_LIB -TOLUA_INCLUDES = /I$(TOLUA_DIR)\include /I$(LUA_DIR)\src -TOLUA_LDFLAGS = -TOLUA_LIBS = $(TOLUA_DIR)\lib\tolua.lib $(LUA_DIR)\src\lua52.lib - diff --git a/makefiles/nmake/platform.mk b/makefiles/nmake/platform.mk index ea15202..0bda6dd 100755 --- a/makefiles/nmake/platform.mk +++ b/makefiles/nmake/platform.mk @@ -32,3 +32,15 @@ XSLTPROC = $(LIBXSLT_DIR)\bin\xsltproc.exe !IFDEF WITH_ICU ICU_DIR=C:\develop\icu4c-52_1-src\icu !ENDIF + +# tolua and Lua +############### + +TOLUA_DIR = $(TOPDIR)\tolua +LUA_DIR = $(TOPDIR)\lua +TOLUA = $(TOLUA_DIR)\bin\tolua +TOLUA_CFLAGS = /DSHARED /DLUA_LIB +TOLUA_INCLUDES = /I$(TOLUA_DIR)\include /I$(LUA_DIR)\src +TOLUA_LDFLAGS = +TOLUA_LIBS = $(TOLUA_DIR)\lib\tolua.lib $(LUA_DIR)\src\lua52.lib + diff --git a/src/Makefile.W32 b/src/Makefile.W32 index 1b9e068..427e285 100755 --- a/src/Makefile.W32 +++ b/src/Makefile.W32 @@ -13,13 +13,15 @@ local_distclean: local_test: copy_prereq: - @-copy "$(ICU_DIR)\bin\icuuc49.dll" . >NUL - @-copy "$(ICU_DIR)\bin\icudt49.dll" . >NUL + @-copy "$(ICU_DIR)\bin\icuuc52d.dll" . >NUL + @-copy "$(ICU_DIR)\bin\icudt52.dll" . >NUL @-copy "$(TOPDIR)\src\libutil\util.dll" . >NUL @-copy "$(TOPDIR)\src\liblogger\logger.dll" . >NUL @-copy "$(TOPDIR)\src\libcrawler\crawler.dll" . >NUL @-copy "$(TOPDIR)\src\libluaglue\luaglue.dll" . >NUL +!IFDEF WITH_LUA @-copy "$(TOPDIR)\lua\src\lua52.dll" . >NUL +!ENDIF run: copy_prereq @-crawl\crawl.exe crawl\crawl.conf diff --git a/src/crawl/Makefile.W32 b/src/crawl/Makefile.W32 index 237d189..68c7e24 100755 --- a/src/crawl/Makefile.W32 +++ b/src/crawl/Makefile.W32 @@ -8,6 +8,13 @@ INCLUDE_CXXFLAGS = \ /D_WIN32_WINNT=0x504 \ /DSHARED +!IFDEF WITH_LUA +INCLUDE_CXXFLAGS = $(INCLUDE_CXXFLAGS) \ + /DWITH_LUA \ + $(TOLUA_CFLAGS) \ + /DTOLUA_API=__declspec(dllimport) +!ENDIF + INCLUDE_DIRS = \ /I. \ /I$(TOPDIR)\include \ @@ -17,9 +24,20 @@ INCLUDE_DIRS = \ /I$(TOPDIR)\include\crawler \ /I$(TOPDIR)\include\luaglue \ /I$(TOPDIR)\lua\src + +!IFDEF WITH_LUA +INCLUDE_DIRS = $(INCLUDE_DIRS) \ + $(TOLUA_INCLUDES) \ + /I$(TOPDIR)\src\libcrawler +!ENDIF INCLUDE_LDFLAGS = \ +!IFDEF WITH_LUA +INCLUDE_LDFLAGS = $(INCLUDE_LDFLAGS) \ + $(TOLUA_LDFLAGS) +!ENDIF + INCLUDE_LIBS = \ $(TOPDIR)\src\libutil\util.lib \ $(TOPDIR)\src\liblogger\logger.lib \ @@ -27,6 +45,12 @@ INCLUDE_LIBS = \ $(TOPDIR)\src\libluaglue\luaglue.lib \ $(TOPDIR)\lua\src\lua52.lib +!IFDEF WITH_LUA +INCLUDE_LIBS = $(INCLUDE_LIBS) \ + $(TOPDIR)\src\libluaglue\luaglue.lib \ + $(TOLUA_LIBS) +!ENDIF + CPP_OBJS = \ CPP_BINS = \ diff --git a/src/libcrawler/Makefile.W32 b/src/libcrawler/Makefile.W32 index 2d31f47..d3cd06c 100755 --- a/src/libcrawler/Makefile.W32 +++ b/src/libcrawler/Makefile.W32 @@ -8,27 +8,60 @@ INCLUDE_CXXFLAGS = \ /D_WIN32_WINNT=0x504 \ /DBUILDING_CRAWLER +!IFDEF WITH_LUA +INCLUDE_CXXFLAGS = $(INCLUDE_CXXFLAGS) \ + /DWITH_LUA \ + $(TOLUA_CFLAGS) +!ENDIF + INCLUDE_DIRS = \ /I. \ /I$(TOPDIR)\include\crawler \ /I$(TOPDIR)\include\logger \ /I$(TOPDIR)\include\util -INCLUDE_LDFLAGS = \ +!IFDEF WITH_LUA +INCLUDE_DIRS = $(INCLUDE_DIRS) \ + -I$(TOPDIR)\include\luaglue \ + $(TOLUA_INCLUDES) +!ENDIF + +INCLUDE_LDFLAGS = + +!IFDEF WITH_LUA +INCLUDE_LDFLAGS = $(INCLUDE_LDFLAGS) \ + $(TOLUA_LDFLAGS) +!ENDIF INCLUDE_LIBS = \ $(TOPDIR)\src\liblogger\logger.lib +!IFDEF WITH_LUA +INCLUDE_LIBS = $(INCLUDE_LIBS) \ + $(TOPDIR)\src\libluaglue\luaglue.lib \ + $(TOLUA_LIBS) +!ENDIF + CPP_OBJS = \ URL.obj \ MIMEType.obj \ SpoolRewindInputStream.obj +!IFDEF WITH_LUA +CPP_OBJS = $(CPP_OBJS) \ + URLLua.obj +!ENDIF + DYNAMIC_CPP_OBJS = \ URL.dllobj \ MIMEType.dllobj \ SpoolRewindInputStream.dllobj +!IFDEF WITH_LUA +DYNAMIC_CPP_OBJS = $(DYNAMIC_CPP_OBJS) \ + URLLua.dllobj +!ENDIF + STATIC_LIB = \ crawlerstatic.lib @@ -37,6 +70,13 @@ DYNAMIC_LIB = \ !INCLUDE $(TOPDIR)\makefiles\nmake\sub.mk +!IFDEF WITH_LUA +URLLua.cpp: URLLua.hpp + +URLLua.hpp: URL.pkg + $(TOLUA) -H URLLua.hpp -o URLLua.cpp URL.pkg +!ENDIF + $(DYNAMIC_LIB): $(DYNAMIC_CPP_OBJS) $(LINK) /nologo /dll /out:$@ $(LDFLAGS) $(LIBS) $? diff --git a/src/modules/urlnormalizer/googleurl/GNUmakefile b/src/modules/urlnormalizer/googleurl/GNUmakefile index 499e02b..e92f061 100644 --- a/src/modules/urlnormalizer/googleurl/GNUmakefile +++ b/src/modules/urlnormalizer/googleurl/GNUmakefile @@ -9,7 +9,7 @@ INCLUDE_DIRS = \ -I$(TOPDIR)/include/module \ -I$(TOPDIR)/include/util \ -I$(TOPDIR)/include/crawler \ - -I$(TOPDIR)/googleurl \ + -I$(TOPDIR)/googleurl ifeq ($(WITH_LUA),1) INCLUDE_DIRS += \ diff --git a/src/modules/urlnormalizer/googleurl/Makefile.W32 b/src/modules/urlnormalizer/googleurl/Makefile.W32 index fb0a76b..6934972 100755 --- a/src/modules/urlnormalizer/googleurl/Makefile.W32 +++ b/src/modules/urlnormalizer/googleurl/Makefile.W32 @@ -7,6 +7,12 @@ SUBDIRS = INCLUDE_CXXFLAGS = \ /D_WIN32_WINNT=0x504 /DWIN32 +!IFDEF WITH_LUA +INCLUDE_CXXFLAGS = $(INCLUDE_CXXFLAGS) \ + /DWITH_LUA \ + $(TOLUA_CFLAGS) +!ENDIF + INCLUDE_DIRS = \ /I. \ /I$(TOPDIR)\src \ @@ -15,12 +21,29 @@ INCLUDE_DIRS = \ /I$(TOPDIR)\include\crawler \ /I$(TOPDIR)\googleurl +!IFDEF WITH_LUA +INCLUDE_DIRS = $(INCLUDE_DIRS) \ + -I$(TOPDIR)\include\luaglue \ + $(TOLUA_INCLUDES) +!ENDIF + INCLUDE_LDFLAGS = \ +!IFDEF WITH_LUA +INCLUDE_LDFLAGS = $(INCLUDE_LDFLAGS) \ + $(TOLUA_LDFLAGS) +!ENDIF + INCLUDE_LIBS = \ $(TOPDIR)\googleurl\googleurl.lib \ $(TOPDIR)\src\libcrawler\crawler.lib +!IFDEF WITH_LUA +INCLUDE_LIBS = $(INCLUDE_LIBS) \ + $(TOPDIR)\src\libluaglue\luaglue.lib \ + $(TOLUA_LIBS) +!ENDIF + !IFDEF DEBUG INCLUDE_LIBS = $(INCLUDE_LIBS) \ "$(ICU_DIR)\lib\icuucd.lib" @@ -38,11 +61,28 @@ STATIC_LIB = \ CPP_OBJS = \ GoogleURLNormalizer.obj +!IFDEF WITH_LUA +CPP_OBJS = $(CPP_OBJS) \ + GoogleURLNormalizerLua.obj +!ENDIF + SHARED_CPP_OBJS = \ GoogleURLNormalizer.dllobj +!IFDEF WITH_LUA +SHARED_CPP_OBJS = $(SHARED_CPP_OBJS) \ + GoogleURLNormalizerLua.dllobj +!ENDIF + !INCLUDE $(TOPDIR)\makefiles\nmake\sub.mk +!IFDEF WITH_LUA +GoogleURLNormalizer.cpp: GoogleURLNormalizerLua.hpp + +GoogleURLNormalizerLua.hpp: GoogleURLNormalizer.pkg + $(TOLUA) -H GoogleURLNormalizerLua.hpp -o GoogleURLNormalizerLua.cpp GoogleURLNormalizer.pkg +!ENDIF + $(STATIC_LIB): $(CPP_OBJS) $(LINK) /lib /nologo /out:$@ $(STATIC_LDFLAGS) $? @@ -54,6 +94,9 @@ local_all: $(STATIC_LIB) $(DYNAMIC_MODULE) local_clean: @-erase $(LOCAL_STATIC_LIB) 2>NUL @-erase $(CPP_OBJS) 2>NUL +!IFDEF WITH_LUA + @-erase GoogleURLNormalizerLua.cpp GoogleURLNormalizerLua.hpp 2>NUL +!ENDIF local_distclean: diff --git a/tests/url/Makefile.W32 b/tests/url/Makefile.W32 index 93e6494..b4d51b4 100755 --- a/tests/url/Makefile.W32 +++ b/tests/url/Makefile.W32 @@ -52,8 +52,8 @@ local_clean: local_distclean: local_test: - @-copy "$(ICU_DIR)\bin\icuuc49.dll" . >NUL - @-copy "$(ICU_DIR)\bin\icudt49.dll" . >NUL + @-copy "$(ICU_DIR)\bin\icuuc52d.dll" . >NUL + @-copy "$(ICU_DIR)\bin\icudt52.dll" . >NUL @-for %%m in ( simple_urlnormalizer google_urlnormalizer ) do \ @echo Using URL normalizer '%m'.. & \ @exec_test test1 test1 "parse illegal protocol" %m parse www.andreasbaumann.cc & \ -- cgit v1.2.3-54-g00ecf