initial load

6 files changed, 554 insertions, 0 deletions

diff --git a/LICENSE.txt b/LICENSE.txt
new file mode 100644
index 0000000..d645695
--- /dev/null
+++ b/LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/README.txt b/README.txt
new file mode 100644
index 0000000..bd43d27
--- /dev/null
+++ b/README.txt
@@ -0,0 +1,5 @@
+Required software:
+- lucene-core (comes in pom.xml)
+- jargs
+    Install it into the local Maven repository:
+    mvn install:install-file -Dfile=dist/jargs-1.0.0.jar -DgroupId=jargs -DartifactId=jargs -Dversion=1.0.0 -Dpackaging=jar
diff --git a/dist/jargs-1.0.0.jar b/dist/jargs-1.0.0.jar
new file mode 100644
index 0000000..cdbc80b
--- /dev/null
+++ b/dist/jargs-1.0.0.jar
diff --git a/pom.xml b/pom.xml
new file mode 100644
index 0000000..e15d4c2
--- /dev/null
+++ b/pom.xml
@@ -0,0 +1,57 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>luceneanalyzer</groupId>
+  <artifactId>luceneanalyzer</artifactId>
+  <version>0.0.1</version>
+  <description>Lucene Index Analysis Tool</description>
+    
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.lucene</groupId>
+      <artifactId>lucene-core</artifactId>
+      <version>2.3.2</version>
+    </dependency>
+    <dependency>
+      <groupId>jargs</groupId>
+      <artifactId>jargs</artifactId>
+      <version>1.0.0</version>
+    </dependency>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <version>3.8.1</version>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+  
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-compiler-plugin</artifactId>
+        <configuration>
+          <source>1.5</source>
+          <target>1.5</target>
+        </configuration>
+      </plugin>
+      <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>dependency-maven-plugin</artifactId>
+        <version>1.0</version>
+      	<executions>
+          <execution>
+            <id>copy-dependencies</id>
+            <phase>package</phase>
+            <goals>
+              <goal>copy-dependencies</goal>
+            </goals>
+            <configuration>
+              <outputDirectory>${project.build.directory}/lib</outputDirectory>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+
+</project>
diff --git a/run.sh b/run.sh
new file mode 100755
index 0000000..bb604c5
--- /dev/null
+++ b/run.sh
@@ -0,0 +1,6 @@
+#!/bin/sh
+
+LOCATION=`dirname $0`
+java \
+	-classpath `find $LOCATION/lib -type f -name '*.jar' | tr '\n' ':'` \
+ 	org.dyndns.andreasbaumann.LuceneAnalyzer $*
diff --git a/src/main/java/org/dyndns/andreasbaumann/LuceneAnalyzer.java b/src/main/java/org/dyndns/andreasbaumann/LuceneAnalyzer.java
new file mode 100644
index 0000000..aab2862
--- /dev/null
+++ b/src/main/java/org/dyndns/andreasbaumann/LuceneAnalyzer.java
@@ -0,0 +1,284 @@
+/*
+ *   LuceneAnalyzer - Lucene Index Analyzer
+ *
+ *   Copyright (C) 2006  Andreas Baumann
+ *
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation; either version 2 of the License, or
+ *   (at your option) any later version.
+ *
+ *   This program is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *   GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with this program; if not, write to the Free Software
+ *   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+ 
+package org.dyndns.andreasbaumann;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.ArrayList;
+import java.util.List;
+
+import jargs.gnu.CmdLineParser;
+import jargs.gnu.CmdLineParser.Option;
+import jargs.gnu.CmdLineParser.OptionException;
+
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexReader.FieldOption;
+import org.apache.lucene.index.TermEnum;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermDocs;
+import org.apache.lucene.index.TermPositions;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+
+/**
+ * Lucene index analyzer. Works for file system indexes only (not
+ * for indexes fully in RAM or in different persistence systems as
+ * a JDBCDirectory.
+ *
+ * Note: requires lucene 1.9.1
+ *
+ * @author Andreas Baumann, <abaumann@yahoo.com>
+ * @version $Id$
+ */
+
+public class LuceneAnalyzer
+{
+	private static final String versionString = "0.0.2";
+	
+	private static void printGlobalInfo( IndexReader indexReader ) throws IOException
+	{
+		System.out.println( "Global Information:" );
+		System.out.println( "===================" );
+		
+		System.out.println( "\tnumber of documents: " + indexReader.numDocs( ) );
+
+		// we should get the number of features differently, this is inefficient, but Lucene
+		// has no notion of global statistics (because the default weighting schema doesn't
+		// make use of it!)
+		int nofFeatures = 0;
+		int nofTokens = 0;
+		TermEnum terms = indexReader.terms( );
+		while( terms.next( ) ) {
+		   Term term = terms.term( );
+			int df = terms.docFreq( );
+			nofFeatures++;
+			nofTokens += df;
+		}
+		System.out.println( "\ttotal number of features: " + nofFeatures );
+		System.out.println( "\ttotal number of tokens: " + nofTokens );
+
+		System.out.println( "\tversion: " + indexReader.getVersion( ) );
+		System.out.println( "\tstill current: " + indexReader.isCurrent( ) );
+		
+		//TODO: we don't get segment information!
+		//System.out.println( "is optimized:" + segmentInfos.size( ) == 1 && !indexReader.hasDeletions( ) );
+		System.out.println( "\tmaximal document number: " + indexReader.maxDoc( ) );
+		System.out.println( "\thas deletions: " + indexReader.hasDeletions( ) );
+
+		System.out.println( "" );
+	}
+	
+	private static void printFieldInfoPerFieldOption( IndexReader indexReader, IndexReader.FieldOption fieldOption )
+	{
+		System.out.println( "Fields of type '" + fieldOption + "':" );
+		Collection fields = indexReader.getFieldNames( fieldOption );
+		Iterator fieldIterator = fields.iterator( );
+		while( fieldIterator.hasNext( ) ) {
+			String field = (String)fieldIterator.next( );
+			if( field != null && !field.equals( "" ) ) {
+				// TODO: define data type here!
+				System.out.println( "\t" + field.toString( ) );
+			}
+		}
+	}
+	
+	private static void printFieldInfo( IndexReader indexReader ) throws IOException
+	{
+		System.out.println( "Field Information:" );
+		System.out.println( "==================" );
+		
+		// very bad design, this field types!
+		printFieldInfoPerFieldOption( indexReader, IndexReader.FieldOption.ALL );
+		printFieldInfoPerFieldOption( indexReader, IndexReader.FieldOption.INDEXED );
+		printFieldInfoPerFieldOption( indexReader, IndexReader.FieldOption.INDEXED_NO_TERMVECTOR );
+		printFieldInfoPerFieldOption( indexReader, IndexReader.FieldOption.INDEXED_WITH_TERMVECTOR );
+		printFieldInfoPerFieldOption( indexReader, IndexReader.FieldOption.TERMVECTOR );
+		printFieldInfoPerFieldOption( indexReader, IndexReader.FieldOption.TERMVECTOR_WITH_OFFSET );
+		printFieldInfoPerFieldOption( indexReader, IndexReader.FieldOption.TERMVECTOR_WITH_POSITION );
+		printFieldInfoPerFieldOption( indexReader, IndexReader.FieldOption.TERMVECTOR_WITH_POSITION_OFFSET );
+		printFieldInfoPerFieldOption( indexReader, IndexReader.FieldOption.UNINDEXED );		
+		System.out.println( "" );
+	}
+	
+	private static void printTerms( IndexReader indexReader,
+	                                boolean printDocNumbers,
+                                   boolean printPositions ) throws IOException
+	{
+		System.out.println( "Terms:" );
+		System.out.println( "======" );
+		TermEnum terms = indexReader.terms( );
+		while( terms.next( ) ) {
+		   Term term = terms.term( );
+			// the df is stored in the iterator and not in the term, weird...
+			int df = terms.docFreq( );
+			if( !printDocNumbers && !printPositions ) {
+				System.out.print( term.field( ) + "\t" + term.text( ) + "\t" + df );
+			} else {
+				System.out.print( term.field( ) + "\t" + term.text( ) );
+			}
+
+			if( printDocNumbers ) {
+				TermDocs termDocs = indexReader.termDocs( term );
+				boolean first = true;
+				while( termDocs.next( ) ) {
+					if( first ) {
+						System.out.print( "\t" + termDocs.doc( ) );
+						first = false;
+					} else {
+						System.out.print( "," + termDocs.doc( ) );
+					}
+				}
+				termDocs.close( );
+			} else if( printPositions ) {
+				TermPositions termPositions = indexReader.termPositions( term );
+				boolean first = true;
+				while( termPositions.next( ) ) {
+					if( first ) {
+						System.out.print( "\t" + termPositions.doc( ) );
+						first = false;
+					} else {
+						System.out.print( "," + termPositions.doc( ) );
+					}
+					
+					for( int i = 0; i < termPositions.freq( ); i++ ) {
+						int position = termPositions.nextPosition( );
+						if( i == 0 ) {
+							System.out.print( "[" );
+						}
+						System.out.print( position );
+						if( i > 0 && i < termPositions.freq( ) - 1 ) {
+							System.out.print( "," );
+						}
+						if( i == termPositions.freq( ) - 1 ) {
+							System.out.print( "]" );
+						}
+					}
+				}
+				termPositions.close( );
+			}
+			
+			System.out.println( "" );
+		}
+		System.out.println( "" );
+	}
+
+ 	private static List optionHelpStrings = new ArrayList();
+	
+	private static Option addHelp( Option option, String helpString )
+	{
+		if( option.shortForm( ) != null ) {
+			optionHelpStrings.add( " -" + option.shortForm( ) + "/--" + option.longForm( ) + ": " + helpString  );
+		} else {
+			optionHelpStrings.add( " --" + option.longForm( ) + ": " + helpString  );
+		}
+		return option;
+	}
+
+	private static void printUsage()
+	{
+		System.err.println( "Usage: java " + LuceneAnalyzer.class.getName( ) + " <lucene index dir>" );
+		for( Iterator i = optionHelpStrings.iterator( ); i.hasNext( ); ) {
+			System.err.println( i.next( ) );
+		}
+	}
+	
+	private static void printVersion()
+	{
+		System.out.println( "Version " + LuceneAnalyzer.class.getName( ) + " " + versionString );
+	}
+                	
+	public static void main( String[] args ) throws IOException
+	{
+		CmdLineParser parser = new CmdLineParser( );
+		
+		// default options, well-known, should always be around
+		Option verbose = addHelp( parser.addBooleanOption( 'v', "verbose" ),
+		                          "print extra verbosity information" );
+		Option help =    addHelp( parser.addBooleanOption( 'h', "help" ),
+		                          "print this help message" );
+		Option version = addHelp( parser.addBooleanOption( "version" ),
+		                          "print version information" );
+
+		// read the command line options
+		try {
+			parser.parse( args );
+		} catch( OptionException e ) {
+			System.err.println( e.getMessage( ) );
+			printUsage( );
+			System.exit( 1 );
+		}
+		
+		if( (Boolean)parser.getOptionValue( help, Boolean.FALSE ) ) {
+			printUsage( );
+			System.exit( 0 );
+		}
+		
+		if( (Boolean)parser.getOptionValue( version, Boolean.FALSE ) ) {
+			printVersion( );
+			System.exit( 0 );
+		}
+
+		// verbosity as a level, increased with -vvv
+		int verbosity = 0;
+		while( true ) {
+			Boolean verboseValue = (Boolean)parser.getOptionValue( verbose );
+			if( verboseValue == null ) {
+				break;
+			} else {
+				verbosity++;
+			}
+		}
+		
+		// read command line arguments
+		String[] otherArgs = parser.getRemainingArgs( );
+		
+		if( otherArgs.length != 1 ) {
+			System.err.println( "Missing a lucene index directory as first argument" );
+			printUsage( );
+			System.exit( 1 );
+		}
+		
+		File indexDir = new File( otherArgs[0] );
+		if( !indexDir.exists( ) ) {
+			System.err.println( indexDir + " doesn't exist" );
+			System.exit( 1 );
+		}
+		if( !indexDir.isDirectory( ) ) {
+			System.err.println( indexDir + " is not a directory" );
+			System.exit( 1 );
+		}
+		
+		Directory luceneDirectory = FSDirectory.getDirectory( indexDir, false );
+		IndexReader indexReader = IndexReader.open( luceneDirectory );
+		
+		printGlobalInfo( indexReader );
+		printFieldInfo( indexReader );
+		printTerms( indexReader, verbosity == 1, verbosity == 2 );
+		
+		indexReader.close( );
+		
+		System.exit( 0 );
+	}
+}