Index: ocean/build.xml =================================================================== --- ocean/build.xml (revision 0) +++ ocean/build.xml (revision 0) @@ -0,0 +1,43 @@ + + + + + + + + Ocean realtime search on Lucene + + + + + + + + + + + + + + + + + Index: ocean/lib/commons-io-1.3.2.jar =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Property changes on: ocean\lib\commons-io-1.3.2.jar ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Index: ocean/lib/commons-lang-2.3.jar =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Property changes on: ocean\lib\commons-lang-2.3.jar ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Index: ocean/lib/jdom.jar =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Property changes on: ocean\lib\jdom.jar ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Index: ocean/lib/slf4j-api-1.5.2.jar =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Property changes on: ocean\lib\slf4j-api-1.5.2.jar ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Index: ocean/lib/slf4j-simple-1.5.2.jar =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Property changes on: ocean\lib\slf4j-simple-1.5.2.jar ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Index: ocean/src/net/sourceforge/jsorter/FieldUtil.java =================================================================== --- ocean/src/net/sourceforge/jsorter/FieldUtil.java (revision 0) +++ ocean/src/net/sourceforge/jsorter/FieldUtil.java (revision 0) @@ -0,0 +1,75 @@ +/* + * FieldUtil.java + * + * Created on February 3, 2006, 5:24 PM + */ + +package net.sourceforge.jsorter; + +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * + * @author administrator + * @version + */ +public class FieldUtil { + private static Map fieldMap = new ConcurrentHashMap(); + + public static void setFieldValue(String fieldName, Object object, Object value) { + try { + Field field = getField(object.getClass(), fieldName); + field.setAccessible(true); + field.set(object, value); + } catch (Exception ex) { ex.printStackTrace(); } + } + + public static Object getFieldValue(String fieldName, Object object) { + try { + Field field = getField(object.getClass(), fieldName); + return field.get(object); + } catch (Exception ex) { ex.printStackTrace(); return null; } + } + + public static Field getField(Class clazz, String fieldName) { + Field[] fields = getFields(clazz); + for (int x=0; x < fields.length; x++) { + if (fields[x].getName().equals(fieldName)) { + return fields[x]; + } + } + return null; + } + + public static Field[] getFields(Class clazz) { + Field[] fields = (Field[])fieldMap.get(clazz); + if (fields != null) return fields; + List list = new ArrayList(); + getFields(clazz, list); + + fields = new Field[list.size()]; + for (int x=0; x < fields.length; x++) { + fields[x] = (Field)list.get(x); + } + fieldMap.put(clazz, fields); + return fields; + } + + public static void getFields(Class clazz, List list) { + Class superClass = clazz.getSuperclass(); + if (superClass != null) getFields(superClass, list); + Field[] fields = clazz.getDeclaredFields(); + for (int x=0; x < fields.length; x++) { + int modifiers = fields[x].getModifiers(); + if (Modifier.isPublic(modifiers) && !Modifier.isStatic(modifiers) + && !Modifier.isTransient(modifiers) && !Modifier.isFinal(modifiers)) + list.add(fields[x]); + } + } +} + Index: ocean/src/net/sourceforge/jsorter/MethodUtil.java =================================================================== --- ocean/src/net/sourceforge/jsorter/MethodUtil.java (revision 0) +++ ocean/src/net/sourceforge/jsorter/MethodUtil.java (revision 0) @@ -0,0 +1,38 @@ +package net.sourceforge.jsorter; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; + +/** + * + * @author Jason Rutherglen + */ +public class MethodUtil { + + public MethodUtil() { + } + + public static Object call(String methodName, Object parameters, Object object) throws IllegalAccessException, InvocationTargetException, NoSuchMethodException { + Object[] parameterArray = null; + Class[] classArray = null; + if (parameters != null) { + if (!parameters.getClass().isArray()) { + parameterArray = new Object[] { parameters }; + classArray = new Class[] { parameters.getClass() }; + } else { + classArray = new Class[parameterArray.length]; + for (int x = 0; x < parameterArray.length; x++) { + if (parameterArray[x] != null) { + classArray[x] = parameterArray.getClass(); + } + } + } + } else { + parameterArray = new Object[0]; + classArray = new Class[0]; + } + + Method method = object.getClass().getMethod(methodName, classArray); + return method.invoke(object, parameterArray); + } +} Index: ocean/src/net/sourceforge/jsorter/ReflectColumns.java =================================================================== --- ocean/src/net/sourceforge/jsorter/ReflectColumns.java (revision 0) +++ ocean/src/net/sourceforge/jsorter/ReflectColumns.java (revision 0) @@ -0,0 +1,66 @@ + + +package net.sourceforge.jsorter; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +/** + * + * @author Jason Rutherglen + */ +public class ReflectColumns { + public List list = new ArrayList(); + + public ReflectColumns() { + } + + public SortReflect[] getSortReflects() { + SortReflect[] sortReflects = new SortReflect[list.size()]; + int count = 0; + Iterator iterator = list.iterator(); + while (iterator.hasNext()) { + SortableColumnReflect sortableColumnReflect = (SortableColumnReflect)iterator.next(); + sortReflects[count] = sortableColumnReflect.sortReflect; + + count++; + } + return sortReflects; + } + + public List getColumns() { + return list; + } + + public class SortableColumnReflect implements SortableColumn { + int position; + String name; + SortReflect sortReflect; + int order; + + public SortableColumnReflect(int position, String name, SortReflect sortReflect, int order) { + this.position = position; + this.name = name; + this.sortReflect = sortReflect; + this.order = order; + } + + public String getColumnName() { + return name; + } + + public int getColumnOrder() { + return order; + } + + public int getColumnPosition() { + return position; + } + } + + public void add(String name, SortReflect sortReflect, int order) { + SortableColumnReflect sortableColumnReflect = new SortableColumnReflect(list.size(), name, sortReflect, order); + list.add(sortableColumnReflect); + } +} Index: ocean/src/net/sourceforge/jsorter/SortableBoolean.java =================================================================== --- ocean/src/net/sourceforge/jsorter/SortableBoolean.java (revision 0) +++ ocean/src/net/sourceforge/jsorter/SortableBoolean.java (revision 0) @@ -0,0 +1,161 @@ +/* + * Copyright 2002-2005 Robert Breidecker. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.sourceforge.jsorter; + +/** + * This class is a wrapper on top of the Boolean class. Its purpose is to + * provide a class that allows for Boolean values to be sorted, because the + * Boolean class does not implement the Comparable interface. + * + * @author Robert Breidecker + */ +public final class SortableBoolean implements Comparable { + /** + * The Boolean value being wrapped. + */ + private Boolean booleanValue = null; + + /** + * Creates a new SortableBoolean object from the input boolean value. + * + * @param value + * The boolean value used to create the new SortableBoolean. + */ + public SortableBoolean(final boolean value) { + booleanValue = new Boolean(value); + } + + /** + * Creates a new SortableBoolean object from the input Boolean. + */ + public SortableBoolean(final Boolean value) { + booleanValue = value; + } + + /** + * Creates a new SortableBoolean object from the input String. The new + * object will have a true value if the input value is "true", otherwise it + * will have a false value. + */ + public SortableBoolean(final String value) { + booleanValue = new Boolean(value); + } + + /** + * Returns the primitive boolean value for this object. + * + * @return The primitive boolean value for this object. + */ + public boolean booleanValue() { + return booleanValue.booleanValue(); + } + + /** + * Compares this object with the specified object for order. Returns a + * negative integer, zero, or a positive integer as this object is less + * than, equal to, or greater than the specified object. + * + * @param object + * The object to compare this object to. + * + * @return A negative integer, zero, or a positive integer as this object is + * less than, equal to, or greater than the specified object. + */ + public int compareTo(final Object object) { + int returnValue = -1; + + if (object == null) { + throw new IllegalArgumentException( + "This object can not be compared " + "to a null value."); + } + + if (!(object instanceof SortableBoolean)) { + throw new IllegalArgumentException("The input object must be an " + + "instance of SortableBoolean."); + } + + final SortableBoolean compareToBoolean = (SortableBoolean) object; + + if (booleanValue() == false && compareToBoolean.booleanValue() == true) { + returnValue = -1; + } else if (booleanValue() == false + && compareToBoolean.booleanValue() == false) { + returnValue = 0; + } else if (booleanValue() == true + && compareToBoolean.booleanValue() == true) { + returnValue = 0; + } else if (booleanValue() == true + && compareToBoolean.booleanValue() == false) { + returnValue = 1; + } + + return returnValue; + } + + /** + * Returns true if and only if the argument is not null and is a + * SortableBoolean object that represents the same boolean value as this + * object. + * + * @param object + * The object to compare this object to. + * + * @return Returns true if the specified object represents the same value as + * this object. + */ + public boolean equals(final Object object) { + if (object == null) { + return false; + } + + if (!(object instanceof SortableBoolean)) { + return false; + } + + return booleanValue.equals(object); + } + + /** + * Returns a hash code for this object. This method calls the hashCode + * method on the Boolean object it is wrapping. + * + * @return The hash code for the Boolean object wrapped by this object. + */ + public int hashCode() { + return booleanValue.hashCode(); + } + + /** + * Returns a String object representing this object's value. If this object + * represents the value true, a string equal to "true" is returned. + * Otherwise, a string equal to "false" is returned. + * + * @return A string representation of this object. + */ + public String toString() { + return booleanValue.toString(); + } + + /** + * Returns the Boolean wrapped by this object. + * + * @return The Boolean wrapped by this object. + */ + public Boolean getBoolean() { + return booleanValue; + } +} \ No newline at end of file Index: ocean/src/net/sourceforge/jsorter/SortableColumn.java =================================================================== --- ocean/src/net/sourceforge/jsorter/SortableColumn.java (revision 0) +++ ocean/src/net/sourceforge/jsorter/SortableColumn.java (revision 0) @@ -0,0 +1,46 @@ +/* + * Copyright 2002-2005 Robert Breidecker. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.sourceforge.jsorter; + +/** + * This class represents a column that will be sorted by Sorter, SwingSorter, or + * SortComparator. + * + * @author Robert Breidecker + */ +public interface SortableColumn { + /** + * Returns the number position of the column to sort with. + * + * @return The number position of the column to sort with. + */ + public int getColumnPosition(); + + /** + * Returns the order to sort the column by. + * + * @return The order to sort the column by. + */ + public int getColumnOrder(); + + /** + * Returns the name of this column. + * + * @return The name of this column. + */ + public String getColumnName(); +} \ No newline at end of file Index: ocean/src/net/sourceforge/jsorter/SortableColumnImpl.java =================================================================== --- ocean/src/net/sourceforge/jsorter/SortableColumnImpl.java (revision 0) +++ ocean/src/net/sourceforge/jsorter/SortableColumnImpl.java (revision 0) @@ -0,0 +1,113 @@ +/* + * Copyright 2002-2005 Robert Breidecker. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.sourceforge.jsorter; + +/** + * This class represents a column that will be sorted by Sorter, SwingSorter, or + * SortComparator. + * + * @author Robert Breidecker + */ +public class SortableColumnImpl implements SortableColumn { + /** + * The number position of the column to sort with. + */ + private int columnPosition = SorterConstants.FIRST_COLUMN_POSITION; + + /** + * The order to sort the column by. + */ + private int columnOrder = SorterConstants.ASCENDING_ORDER; + + /** + * The name of the sort column. + */ + private String columnName = null; + + /** + * SortColumn constructor. The constructor will set both the column number + * and the column order. + * + * @param columnPosition + * The number position of the column in the data to sort by. + * Column numbers start at zero for the first column. + * + * @param columnOrder + * This value will be used for specifying the order in which the + * column sorted by. Sort order can either be ascending or + * descending. The ASCENDING_ORDER and DESCENDING_ORDER constants + * in the Sorter class should be used for this value. + */ + public SortableColumnImpl(final int columnPosition, final int columnOrder) { + this(columnPosition, columnOrder, null); + } + + /** + * SortColumn constructor. The constructor will set both the column number, + * the column order and the column name. + * + * @param columnPosition + * The number position of the column in the data to sort by. + * Column numbers start at zero for the first column. + * + * @param columnOrder + * This value will be used for specifying the order in which the + * column sorted by. Sort order can either be ascending or + * descending. The ASCENDING_ORDER and DESCENDING_ORDER constants + * in the Sorter class should be used for this value. + * + * @param columnName + * A name or description for the column. This field only needs to + * be specified if you are planning to display information about + * the column to the user or want to use the name as an + * identifier for the column. If you do not want to use this + * field, you can pass null in as a value. + */ + public SortableColumnImpl(final int columnPosition, final int columnOrder, + final String columnName) { + this.columnPosition = columnPosition; + this.columnOrder = columnOrder; + this.columnName = columnName; + } + + /** + * Returns the number position of the column to sort with. + * + * @return The number position of the column to sort with. + */ + public int getColumnPosition() { + return columnPosition; + } + + /** + * Returns the order to sort the column by. + * + * @return The order to sort the column by. + */ + public int getColumnOrder() { + return columnOrder; + } + + /** + * Returns the name of this column. + * + * @return The name of this column. + */ + public String getColumnName() { + return columnName; + } +} \ No newline at end of file Index: ocean/src/net/sourceforge/jsorter/SortableSwingComponent.java =================================================================== --- ocean/src/net/sourceforge/jsorter/SortableSwingComponent.java (revision 0) +++ ocean/src/net/sourceforge/jsorter/SortableSwingComponent.java (revision 0) @@ -0,0 +1,34 @@ +/* + * Copyright 2002-2005 Robert Breidecker. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.sourceforge.jsorter; + +import java.util.List; + +/** + * This class represents a component that will be sorted by SwingSorter. + * + * @author Robert Breidecker + */ +public interface SortableSwingComponent { + /** + * Returns a lists of lists that contain the component's data values. Each + * inner list contained in the outer list represents a row of data. + * + * @return The data list for this component. + */ + public List getDataList(); +} \ No newline at end of file Index: ocean/src/net/sourceforge/jsorter/SortComparator.java =================================================================== --- ocean/src/net/sourceforge/jsorter/SortComparator.java (revision 0) +++ ocean/src/net/sourceforge/jsorter/SortComparator.java (revision 0) @@ -0,0 +1,230 @@ +/* + * Copyright 2002-2005 Robert Breidecker. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.sourceforge.jsorter; + +import java.util.Comparator; +import java.util.List; + +/** + * The default comparator for the Sort and SwingSorter classes. This comparator + * is used for sorting multiple columns. + * + * @author Robert Breidecker + */ +public class SortComparator implements Comparator { + /** + * The list of sort columns. + */ + private List sortColumns = null; + + /** + * Indicate how to treat null data values. + */ + private int nullBehavior = SorterConstants.NULLS_ARE_INVALID; + + /** + * Comparator constructor. + * + * @param sortColumns + * A list of SortColumns that represent the positions of columns + * in your data that you want to be evaluated in the sort. The + * sort will start with the first column position in the list. + * Column positions not in the list will not be used in the sort. + * The number of items in this list should equal the number of + * items in the columnOrders list. + * + * @param nullBehavior + * An integer representing the constant that indicates how null + * data values should behave while being sorted. See the null + * behavior constants in this class. + */ + public SortComparator(final List sortColumns, final int nullBehavior) { + this.sortColumns = sortColumns; + this.nullBehavior = nullBehavior; + } + + /** + * Overrides the java.util.Comparator compare method. This method is used + * for comparing two dimensional data. See the standard JDK documention for + * more information. + * + * @param one + * Object - The first object used in the compare. This field + * should be a list of lists containing objects which implement + * the Comparable interface. Some of these object types include + * String, Integer, Long, Short, Float, Byte, Double and Date. + * See the standard JDK documention for Comparator for a complete + * list. The object type for each column of data must be + * consistent or a ClassCaseException will be thrown. + * + * @param two + * Object - The second object used in the compare. This field + * should be a list of lists containing objects which implement + * the Comparable interface. Some of these object types include + * String, Integer, Long, Short, Float, Byte, Double and Date. + * See the standard JDK documention for Comparator for a complete + * list. The object type for each column of data must be + * consistent or a ClassCaseException will be thrown. + * + * @return A negative integer, zero, or a positive integer as the first + * argument is less than, equal to, or greater than the second. + * + * @exception ClassCastException + * Data in a column in not all of the same data type. + */ + public int compare(final Object one, final Object two) throws ClassCastException { + // The number of columns in the table. + int numColumns; + + // The return value. + int rtn = 0; + + // Used for counting the number of real fields in the data. + int ctr = 0; + + // Holds the type of sort order being used. + int columnOrder; + + // Used for counting the number of values in the sort columns. + int compareCtr; + + // One row of data; + final List listOne = (List)one; + + if (sortColumns == null) { + numColumns = listOne.size(); + } else { + numColumns = sortColumns.size(); + } + + while (rtn == 0 && ctr < numColumns) { + // The first object to compare. + Comparable comparableOne; + + // The second object to compare. + Comparable comparableTwo; + + // Make sure compare column is within range. + if (sortColumns == null) { + compareCtr = ctr; + } else { + compareCtr = ((SortableColumn)sortColumns.get(ctr)).getColumnPosition(); + } + + if (compareCtr <= listOne.size()) { + // Another row of data; + final List listTwo = (List) two; + + // Get the field to use in the compare. + if (sortColumns == null) { + comparableOne = (Comparable)listOne.get(compareCtr); + comparableTwo = (Comparable)listTwo.get(compareCtr); + } else { + comparableOne = (Comparable)listOne.get(((SortableColumn) sortColumns.get(ctr)).getColumnPosition()); + comparableTwo = (Comparable)listTwo.get(((SortableColumn) sortColumns.get(ctr)).getColumnPosition()); + } + + // Get the sort type that goes with the sort column. + if (sortColumns == null) { + // If no sort columns were specified, then use ascending + // order. + columnOrder = SorterConstants.ASCENDING_ORDER; + } else { + columnOrder = ((SortableColumn)sortColumns.get(ctr)).getColumnOrder(); + } + + // Compare the objects. + if (comparableOne != null && comparableTwo != null) { + if (columnOrder == SorterConstants.ASCENDING_ORDER) { + try { + rtn = comparableOne.compareTo(comparableTwo); + } catch (ClassCastException exception) { + throw exception; + } + } else { + try { + rtn = comparableTwo.compareTo(comparableOne); + } catch (ClassCastException exception) { + throw exception; + } + } + } else { + if (nullBehavior == SorterConstants.NULLS_ARE_INVALID) { + throw new IllegalStateException("Null data values are not valid."); + } else if (comparableOne == null && comparableTwo != null) { + if (columnOrder == SorterConstants.ASCENDING_ORDER) { + rtn = -1; + } else { + rtn = 1; + } + + if (nullBehavior == SorterConstants.NULLS_ARE_GREATEST) { + rtn = rtn * -1; + } + } else if (comparableOne != null && comparableTwo == null) { + if (columnOrder == SorterConstants.ASCENDING_ORDER) { + rtn = 1; + } else { + rtn = -1; + } + + if (nullBehavior == SorterConstants.NULLS_ARE_GREATEST) { + rtn = rtn * -1; + } + } else { + rtn = 0; + } + } + } + ctr++; + } + return rtn; + } + + /** + * Returns the null behavior for this object. + * + * @return An integer representing the constant that indicates how null data + * values should behave while being sorted. See the null behavior + * constants in this class. The default value for this class is + * NULLS_ARE_INVALID. + */ + public int getNullBehavior() { + return nullBehavior; + } + + /** + * Set the null behavior for this object. + * + * @param nullBehavior + * An integer representing the constant that indicates how null + * data values should behave while being sorted. See the null + * behavior constants in this class. + * + * @throws IllegalArgumentException + * Thrown if the null behavior value is not valid. + */ + public void setNullBehavior(final int nullBehavior) { + if (nullBehavior != SorterConstants.NULLS_ARE_GREATEST + && nullBehavior != SorterConstants.NULLS_ARE_INVALID + && nullBehavior != SorterConstants.NULLS_ARE_LEAST) { + throw new IllegalArgumentException("Invalid null behavior."); + } + + this.nullBehavior = nullBehavior; + } +} \ No newline at end of file Index: ocean/src/net/sourceforge/jsorter/Sorter.java =================================================================== --- ocean/src/net/sourceforge/jsorter/Sorter.java (revision 0) +++ ocean/src/net/sourceforge/jsorter/Sorter.java (revision 0) @@ -0,0 +1,383 @@ +/* + * Copyright 2002-2005 Robert Breidecker. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.sourceforge.jsorter; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; +import java.util.Vector; + +import net.sourceforge.jsorter.SortReflect.MethodException; + +/** + * Used for sorting lists of objects. This class is particularly good for + * sorting table data and multi-column lists. + * + * Note: This is class is not thread safe. + * + * @author Robert Breidecker + */ +public class Sorter { + /** + * Holds the data used for sorting. + */ + private List table = null; + + /** + * The columns to sort by. + */ + private List sortColumns = null; + + /** + * Indicate how to treat null data values. + */ + private int nullBehavior = SorterConstants.NULLS_ARE_INVALID; + + public List getReflectResults() { + List results = new ArrayList(table.size()); + Iterator iterator = table.iterator(); + while (iterator.hasNext()) { + List row = (List)iterator.next(); + SortHolder sortHolder = (SortHolder)row.get(0); + results.add(sortHolder.object); + } + return results; + } + + public static Sorter createReflect(ReflectColumns reflectColumns, List objects, int nullBehavior) throws MethodException { + SortReflect[] sortReflects = reflectColumns.getSortReflects(); + List table = SortHolder.getTable(sortReflects, objects, nullBehavior); + Sorter sorter = new Sorter(table, reflectColumns.getColumns()); + sorter.setNullBehavior(nullBehavior); + return sorter; + } + + /** + * Sorter constructor. This version of the constructor does not use any + * parameters. If you use this constructor, you should use the setter + * methods to set the values you want the sort routine to use. + */ + public Sorter() { + super(); + } + + /** + * Sorter constructor. This version of the contructor takes only a table as + * a parameter. Since no sort columns are specified, this will cause the + * sort routine to sort the table using all of the columns in the table in + * ascending order. To change the table or sort columns, you must use the + * appropriate "set" methods. + * + * @param table + * This field should be an object that implements the List + * interface. This list we will call the "table". Classes that + * implement the List interface include ArrayList, Vector and + * LinkedList. Each item in the table list should itself be an + * object that implements the List interface. These lists we will + * call "columns". Column lists should only contain objects that + * implement the Comparable interface. Classes that implement the + * Comparable interface include String, Integer, Long, Short, + * Float, Byte, Double and Date. See the standard JDK documention + * for Comparator for a complete list. The class type for each + * object in a column list must be consistent or a + * ClassCaseException will be thrown during the sort. + */ + public Sorter(final List table) { + // Set the values for the new sorter object. + setTable(table); + } + + /** + * Sorter constructor. This version of the contructor takes a table and a + * column order as parameters. A sort column for each column in the data + * will be created with the column order specified. This will cause the sort + * routine to sort the table using all of the columns in the table (position + * 0 and up or left to right) in the order specified in the column order. To + * override the table or sort columns, you must use the appropriate "set" + * methods. A vector will be used for storing the sort columns that are + * dynamically created. + * + * @param table + * This field should be an object that implements the List + * interface. This list we will call the "table". Classes that + * implement the List interface include ArrayList, Vector and + * LinkedList. Each item in the table list should itself be an + * object that implements the List interface. These lists we will + * call "columns". Column lists should only contain objects that + * implement the Comparable interface. Classes that implement the + * Comparable interface include String, Integer, Long, Short, + * Float, Byte, Double and Date. See the standard JDK documention + * for Comparator for a complete list. The class type for each + * object in a column list must be consistent or a + * ClassCaseException will be thrown during the sort. + * + * @param columnOrder + * This value will be used for specifying the order in which the + * columns in the table are sorted. Sort order can either be + * ascending or descending. The ASCENDING_ORDER and + * DESCENDING_ORDER constants in this class should be used for + * this value. + */ + public Sorter(final List table, final int columnOrder) { + // Get the first row. + List firstRow = null; + if (table.size() > 0) { + firstRow = (List)table.get(0); + } + + // Build the sort columns. + List sortColumns = null; + if (firstRow != null) { + sortColumns = new Vector(); + final int numColumns = firstRow.size(); + for (int columnCtr = 0; columnCtr < numColumns; columnCtr++) { + final SortableColumn sortColumn = new SortableColumnImpl(columnCtr, columnOrder); + + sortColumns.add(sortColumn); + } + } + + // Set the values for the new sorter object. + setTable(table); + setSortColumns(sortColumns); + } + + /** + * Sorter constructor. This version of the contructor takes a table and a + * list of sort columns as parameters. To override the table or sort + * columns, you must use the appropriate "set" methods. + * + * @param table + * This field should be an object that implements the List + * interface. This list we will call the "table". Classes that + * implement the List interface include ArrayList, Vector and + * LinkedList. Each item in the table list should itself be an + * object that implements the List interface. These lists we will + * call "columns". Column lists should only contain objects that + * implement the Comparable interface. Classes that implement the + * Comparable interface include String, Integer, Long, Short, + * Float, Byte, Double and Date. See the standard JDK documention + * for Comparator for a complete list. The class type for each + * object in a column list must be consistent or a + * ClassCaseException will be thrown during the sort. + * + * @param sortColumns + * Sort columns are a list of numbers specifying the colums to + * sort the table by. Each number in the list should be an + * instance of the SortColumn class represents a position of a + * column in the table. Column position start at zero just as + * they do in the standard Collection classes and Java arrays. + * When this field is null, there must be only one and only one + * column order specified. The sort routine will then sort using + * every column in the table starting from position 0 and up or + * left to right in ascending order. + */ + public Sorter(final List table, final List sortColumns) { + // Set the values for the new sorter object. + setTable(table); + setSortColumns(sortColumns); + } + + /** + * Returns the list of columns to sort by. + * + * @return The list of columns to sort by. + */ + public List getSortColumns() { + return sortColumns; + } + + /** + * Returns the table of sort data. + * + * @return The table of sort data. + */ + public List getTable() { + return table; + } + + /** + * Updates the list of sort columns. + * + * @param sortColumns + * Sort columns are a list of numbers specifying the colums to + * sort the table by. Each number in the list should be an + * instance of the SortColumn class represents a position of a + * column in the table. Column position start at zero just as + * they do in the standard Collection classes and Java arrays. + * When this field is null, there must be only one and only one + * column order specified. The sort routine will then sort using + * every column in the table starting from position 0 and up or + * left to right in ascending order. + */ + public void setSortColumns(final List sortColumns) { + validateSortColumns(sortColumns); + this.sortColumns = sortColumns; + } + + /** + * Updates the table of sort data. + * + * @param table + * This field should be an object that implements the List + * interface. This list we will call the "table". Classes that + * implement the List interface include ArrayList, Vector and + * LinkedList. Each item in the table list should itself be an + * object that implements the List interface. These lists we will + * call "columns". Column lists should only contain objects that + * implement the Comparable interface. Classes that implement the + * Comparable interface include String, Integer, Long, Short, + * Float, Byte, Double and Date. See the standard JDK documention + * for Comparator for a complete list. The class type for each + * object in a column list must be consistent or a + * ClassCaseException will be thrown during the sort. + */ + public void setTable(final List table) { + validateTable(table); + this.table = table; + } + + /** + * Returns the null behavior for this object. + * + * @return An integer representing the constant that indicates how null data + * values should behave while being sorted. See the null behavior + * constants in this class. The default value for this class is + * NULLS_ARE_INVALID. + */ + public int getNullBehavior() { + return nullBehavior; + } + + /** + * Set the null behavior for this object. + * + * @param nullBehavior + * An integer representing the constant that indicates how null + * data values should behave while being sorted. See the null + * behavior constants in this class. + * + * @throws IllegalArgumentException + * Thrown if the null behavior value is not valid. + */ + public void setNullBehavior(final int nullBehavior) { + if (nullBehavior != SorterConstants.NULLS_ARE_GREATEST + && nullBehavior != SorterConstants.NULLS_ARE_INVALID + && nullBehavior != SorterConstants.NULLS_ARE_LEAST) { + throw new IllegalArgumentException("Invalid null behavior."); + } + + this.nullBehavior = nullBehavior; + } + + /** + * This routine sorts the table of data. The sort uses the sort columns to + * determine how to sort the data. + * + * @exception IllegalStateException + * The data in this class is in an invalid state. + */ + public void sort() { + // Sort the data. + List sortColumns = getSortColumns(); + //System.out.println("sortColumns: "+sortColumns); + Collections.sort(table, new SortComparator(getSortColumns(), getNullBehavior())); + } + + public List sortReflect() { + sort(); + return getReflectResults(); + } + + /** + * This routine sorts the table of data using a comparator provided in the + * parameters to do the sorting. The sort columns for this class will not be + * used unless the input comparator has been coded to do so. + * + * @param comparator + * A comparator to use for comparing the data rows in the table + * that has already been set on this class. + * + * @exception IllegalStateException + * The data in Sorter is in an invalid state. + */ + public void sort(final Comparator comparator) { + // Sort the data. + Collections.sort(getTable(), comparator); + } + + /** + * Validates the list of sort columns. + * + * @param sortColumns + * The list of sort columns to validate. + * + * @exception IllegalArgumentException + * Input data is invalid. + */ + private void validateSortColumns(final List sortColumns) { + if (sortColumns != null) { + // Validate size. + if (sortColumns.size() < 1) { + throw new IllegalArgumentException("Sort columns can "+ "not be empty."); + } + + for (int ctr = 0, size = sortColumns.size(); ctr < size; ctr++) { + // Validate for SortColumns. + if (!(sortColumns.get(ctr) instanceof SortableColumn)) { + throw new IllegalArgumentException("The list of sort "+ "columns does not contain all SortColumn objects."); + } + + // Validate for greater than or equal to zero. + if (((SortableColumn) sortColumns.get(ctr)).getColumnPosition() < 0) { + throw new IllegalArgumentException("A sort column number is less than zero."); + } + + // Validate for invalid column order. + if (((SortableColumn)sortColumns.get(ctr)).getColumnOrder() != SorterConstants.ASCENDING_ORDER + && ((SortableColumn)sortColumns.get(ctr)).getColumnOrder() != SorterConstants.DESCENDING_ORDER) { + throw new IllegalArgumentException("A sort column order is invalid."); + } + } + } + } + + /** + * Validates the table of sort data. + * + * @param table + * The table of sort data to validate. + * + * @exception IllegalArgumentException + * Input data is invalid. + */ + private void validateTable(final List table) { + // Validate for null. + if (table == null) { + throw new IllegalArgumentException("The table of sort data "+ "can not be null."); + } + + // Validate for Lists. + if (table.size() > 0) { + if (!(table.get(0) instanceof List)) { + throw new IllegalArgumentException("The table does not implement " + "the List interface."); + } + } + } +} \ No newline at end of file Index: ocean/src/net/sourceforge/jsorter/SorterConstants.java =================================================================== --- ocean/src/net/sourceforge/jsorter/SorterConstants.java (revision 0) +++ ocean/src/net/sourceforge/jsorter/SorterConstants.java (revision 0) @@ -0,0 +1,57 @@ +/* + * Copyright 2002-2005 Robert Breidecker. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.sourceforge.jsorter; + +/** + * This class contains constants used in the JSorter project. + * + * @author Robert Breidecker + */ +public class SorterConstants { + /** + * The constant for the first column position. + */ + public static final int FIRST_COLUMN_POSITION = 0; + + /** + * The constant for ascending order. + */ + public static final int ASCENDING_ORDER = 1; + + /** + * The constant for descending order. + */ + public static final int DESCENDING_ORDER = 0; + + /** + * The constant for stating that null data values are invalid. This is the + * default value. + */ + public static final int NULLS_ARE_INVALID = 0; + + /** + * The constant for stating that null data values are valid and should be + * treated as the least of possible values when sorting. + */ + public static final int NULLS_ARE_LEAST = 1; + + /** + * The constant for stating that null data values are valid and should be + * treated as the greatest of possible values when sorting. + */ + public static final int NULLS_ARE_GREATEST = 2; +} \ No newline at end of file Index: ocean/src/net/sourceforge/jsorter/SortHolder.java =================================================================== --- ocean/src/net/sourceforge/jsorter/SortHolder.java (revision 0) +++ ocean/src/net/sourceforge/jsorter/SortHolder.java (revision 0) @@ -0,0 +1,90 @@ + + +package net.sourceforge.jsorter; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; + +import net.sourceforge.jsorter.SortReflect.MethodException; + + +/** + * + * @author Jason Rutherglen + */ +public class SortHolder implements Comparable { + public Comparable attribute; + public T object; + public int nullBehavior = SorterConstants.NULLS_ARE_INVALID; + + public SortHolder(Comparable attribute, T object, int nullBehavior) { + this.attribute = attribute; + this.object = object; + this.nullBehavior = nullBehavior; + } + + public String toString() { + if (attribute == null) return "null"; + return attribute.toString(); + } + + public static List getTable(SortReflect[] sortReflect, Collection collection, int nullBehavior) throws MethodException { + List table = new ArrayList(collection.size()); + Iterator iterator = collection.iterator(); + while (iterator.hasNext()) { + Object value = iterator.next(); + List row = new ArrayList(sortReflect.length); + for (int x=0; x < sortReflect.length; x++) { + Object attribute = sortReflect[x].get(value); + if (attribute instanceof java.net.URL) { + attribute = ((java.net.URL)attribute).toString(); + } + Comparable attributeComparable = (Comparable)attribute; + SortHolder sortHolder = new SortHolder(attributeComparable, value, nullBehavior); + row.add(sortHolder); + } + table.add(row); + } + return table; + } + + public int compareTo(Object obj) { + Comparable comparableOne = (Comparable)attribute; + Comparable comparableTwo = (Comparable)obj; + if (obj instanceof SortHolder) { + SortHolder sortHolder = (SortHolder)obj; + comparableTwo = (Comparable)sortHolder.attribute; + } + + int rtn; + + if (comparableOne != null && comparableTwo != null) { + try { + rtn = comparableOne.compareTo(comparableTwo); + } catch (ClassCastException exception) { + throw exception; + } + } else { + if (nullBehavior == SorterConstants.NULLS_ARE_INVALID) { + throw new IllegalStateException("Null data values are not valid."); + } else if (comparableOne == null && comparableTwo != null) { + rtn = -1; + + if (nullBehavior == SorterConstants.NULLS_ARE_GREATEST) { + rtn = rtn * -1; + } + } else if (comparableOne != null && comparableTwo == null) { + rtn = 1; + + if (nullBehavior == SorterConstants.NULLS_ARE_GREATEST) { + rtn = rtn * -1; + } + } else { + rtn = 0; + } + } + return rtn; + } +} Index: ocean/src/net/sourceforge/jsorter/SortReflect.java =================================================================== --- ocean/src/net/sourceforge/jsorter/SortReflect.java (revision 0) +++ ocean/src/net/sourceforge/jsorter/SortReflect.java (revision 0) @@ -0,0 +1,63 @@ +package net.sourceforge.jsorter; + +import java.lang.reflect.InvocationTargetException; + +/** + * + * @author Jason Rutherglen + */ +public class SortReflect { + public static final int METHOD = 1; + public static final int FIELD = 2; + public int type; + public String methodName; + public String fieldName; + public Object[] params; + + private SortReflect() { + } + + public static class MethodException extends Exception { + public MethodException(Throwable throwable) { + super(throwable); + } + } + + public Object get(Object value) throws MethodException { + if (type == METHOD) { + try { + return MethodUtil.call(methodName, params, value); + } catch (InvocationTargetException invocationTargetException) { + throw new MethodException(invocationTargetException.getCause()); + } catch (Exception exception) { + throw new MethodException(exception); + } + } else if (type == FIELD) { + return FieldUtil.getFieldValue(fieldName, value); + } + throw new RuntimeException("type is invalid"); + } + + public static SortReflect field(String fieldName) { + SortReflect sortReflect = new SortReflect(); + sortReflect.type = FIELD; + sortReflect.fieldName = fieldName; + return sortReflect; + } + + public static SortReflect method(String methodName) { + SortReflect sortReflect = new SortReflect(); + sortReflect.type = METHOD; + sortReflect.methodName = methodName; + return sortReflect; + } + + public static SortReflect method(String methodName, Object[] params) { + SortReflect sortReflect = new SortReflect(); + sortReflect.type = METHOD; + sortReflect.methodName = methodName; + sortReflect.params = params; + return sortReflect; + } + +} Index: ocean/src/net/sourceforge/jsorter/SwingSorter.java =================================================================== --- ocean/src/net/sourceforge/jsorter/SwingSorter.java (revision 0) +++ ocean/src/net/sourceforge/jsorter/SwingSorter.java (revision 0) @@ -0,0 +1,580 @@ +/* + * Copyright 2002-2005 Robert Breidecker. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.sourceforge.jsorter; + +import java.util.List; +import java.util.Vector; + +import javax.swing.DefaultComboBoxModel; +import javax.swing.DefaultListModel; +import javax.swing.JComboBox; +import javax.swing.JList; +import javax.swing.JTable; +import javax.swing.table.DefaultTableModel; + +/** + * Used for sorting data in the Swing objects JTable, JList, JComboBox, + * DefaultTableModel, DefaultListModel, DefaultComboBoxModel and + * SortableComponent. + * + * Note: This is class is not thread safe. + * + * @author Robert Breidecker + */ +public class SwingSorter { + /** + * Indicate how to treat null data values. + */ + private int nullBehavior = SorterConstants.NULLS_ARE_INVALID; + + /** + * Sorts a custom component that implements the SortableComponent interface. + * This method assumes that the object type in the display column is made up + * entirely of the same type and that the type implements the Comparable + * interface. For example, the data in the display column is entirely made + * of String or Integers. Some of the objects that implement the Comparable + * interface include String, Integer, Long, Short, Float, Byte, Double and + * Date. + * + * @param component + * The component to be sorted in ascending order. The component + * model must implement the SortableComponent interface. + */ + public void sortComponent(final SortableSwingComponent component) { + sortComponent(component, SorterConstants.ASCENDING_ORDER); + } + + /** + * Sorts a custom component that implements the SortableComponent interface. + * This method assumes that the object type in the display column is made up + * entirely of the same type and that the type implements the Comparable + * interface. For example, the data in the display column is entirely made + * of String or Integers. Some of the objects that implement the Comparable + * interface include String, Integer, Long, Short, Float, Byte, Double and + * Date. + * + * @param component + * The component to be sorted in ascending order. The component + * model must implement the SortableComponent interface. + * + * @param columnOrder + * An integer that represents the type of ordering to be used + * when sorting by only the first column in the data. The type of + * sort can either be ascending or descending. The + * ASCENDING_ORDER and DESCENDING_ORDER constants in this class + * should be used. + */ + public void sortComponent(final SortableSwingComponent component, + final int columnOrder) { + final SortableColumn sortColumn = new SortableColumnImpl(0, columnOrder); + final List sortColumns = new Vector(); + sortColumns.add(sortColumn); + + sortComponent(component, sortColumns); + } + + /** + * Sorts a custom component that implements the SortableComponent interface. + * This method assumes that the object type in the display column is made up + * entirely of the same type and that the type implements the Comparable + * interface. For example, the data in the display column is entirely made + * of String or Integers. Some of the objects that implement the Comparable + * interface include String, Integer, Long, Short, Float, Byte, Double and + * Date. + * + * @param component + * The component to be sorted in ascending order. The component + * model must implement the SortableComponent interface. + * + * @param sortColumns + * Sort columns are a list of numbers specifying the colums to + * sort the table by. Each number in the list should be an + * instance of the SortColumn class represents a position of a + * column in the table. Column position start at zero just as + * they do in the standard Collection classes and Java arrays. + * When this field is null, there must be only one and only one + * column order specified. The sort routine will then sort using + * every column in the table starting from position 0 and up or + * left to right in ascending order. + */ + public void sortComponent(final SortableSwingComponent component, + final List sortColumns) { + // Create a new Sorter object. + final Sorter sorter = new Sorter(component.getDataList(), sortColumns); + sorter.setNullBehavior(nullBehavior); + + // Sort the data! + sorter.sort(); + } + + /** + * Sorts a combo box using Sorter. This method assumes that the object type + * in the display column is made up entirely of the same type and that the + * type implements the Comparable interface. For example, the data in the + * display column is entirely made of String or Integers. Some of the + * objects that implement the Comparable interface include String, Integer, + * Long, Short, Float, Byte, Double and Date. + * + * @param comboBox + * The combo box to be sorted in ascending order. The combo box + * model must be an instance or a descendent of + * DefaultComboBoxModel. + */ + public void sortComboBox(final JComboBox comboBox) { + sortComboBox(comboBox, SorterConstants.ASCENDING_ORDER); + } + + /** + * Sorts a combo box using Sorter. This method assumes that the object type + * in the display column is made up entirely of the same type and that the + * type implements the Comparable interface. For example, the data in the + * display column is entirely made of String or Integers. Some of the + * objects that implement the Comparable interface include String, Integer, + * Long, Short, Float, Byte, Double and Date. + * + * @param comboBox + * The combo box to be sorted. The combo box model must be an + * instance or a descendent of DefaultComboBoxModel or implement + * the SortableComponent. + * + * @param columnOrder + * An integer that represents the type of ordering to be used + * when sorting by only the first column in the data. The type of + * sort can either be ascending or descending. The + * ASCENDING_ORDER and DESCENDING_ORDER constants in this class + * should be used. + * + * @exception IllegalArgumentException + * Input data is invalid. + */ + public void sortComboBox(final JComboBox comboBox, final int columnOrder) { + if (comboBox.getModel() instanceof DefaultComboBoxModel) { + final DefaultComboBoxModel model = (DefaultComboBoxModel) comboBox + .getModel(); + + sortComboBoxModel(model, columnOrder); + } else if (comboBox.getModel() instanceof SortableSwingComponent) { + final SortableSwingComponent component = (SortableSwingComponent) comboBox + .getModel(); + + sortComponent(component, columnOrder); + } else { + throw new IllegalArgumentException( + "ComboBox model must be an " + + "instance of decendent of DefaultComboBoxModel or implement the " + + "SortableComponent interface."); + } + } + + /** + * Sorts a combo box model using Sorter. This method assumes that the object + * type in the display column is made up entirely of the same type and that + * the type implements the Comparable interface. For example, the data in + * the display column is entirely made of String or Integers. Some of the + * objects that implement the Comparable interface include String, Integer, + * Long, Short, Float, Byte, Double and Date. + * + * @param model + * The combo box model to be sorted in ascending order. The combo + * box model must be an instance or a descendent of + * DefaultComboBoxModel. + */ + public void sortComboBoxModel(final DefaultComboBoxModel model) { + sortComboBoxModel(model, SorterConstants.ASCENDING_ORDER); + } + + /** + * Sorts a combo box model using Sorter. This method assumes that the object + * type in the display column is made up entirely of the same type and that + * the type implements the Comparable interface. For example, the data in + * the display column is entirely made of String or Integers. Some of the + * objects that implement the Comparable interface include String, Integer, + * Long, Short, Float, Byte, Double and Date. + * + * @param model + * The combo box model to be sorted. The combo box model must be + * an instance or a descendent of DefaultComboBoxModel. + * + * @param columnOrder + * An integer that represents the type of ordering to be used + * when sorting by only the first column in the data. The type of + * sort can either be ascending or descending. The + * ASCENDING_ORDER and DESCENDING_ORDER constants in this class + * should be used. + */ + public void sortComboBoxModel(final DefaultComboBoxModel model, + final int columnOrder) { + // Create the table of sort data. + final List table = new Vector(); + + // Get the model data. + for (int ctr = 0, size = model.getSize(); ctr < size; ctr++) { + // Create a new row. + final List row = new Vector(); + row.add(model.getElementAt(ctr)); + table.add(row); + } + + // Create a new Sorter object. + final Sorter sorter = new Sorter(table, columnOrder); + sorter.setNullBehavior(nullBehavior); + + // Sort the data! + sorter.sort(); + + // Clear the model data. + model.removeAllElements(); + + // Re-add the sorted data to the model. + for (int ctr = 0, size = table.size(); ctr < size; ctr++) { + final List row = (List) table.get(ctr); + + // Get the first element from the row, because a list + // only has one column. + model.addElement(row.get(0)); + } + } + + /** + * Sorts a list model using Sorter. This method assumes that the object type + * in the display column is made up entirely of the same type and that the + * type implements the Comparable interface. For example, the data in the + * display column is entirely made of String or Integers. Some of the + * objects that implement the Comparable interface include String, Integer, + * Long, Short, Float, Byte, Double and Date. + * + * @param model + * The list model to be sorted in ascending order. Must be a + * DefaultListModel or a descendent of DefaultListModel. + */ + public void sortListModel(final DefaultListModel model) { + sortListModel(model, SorterConstants.ASCENDING_ORDER); + } + + /** + * Sorts a list using Sorter. This method assumes that the object type in + * the display column is made up entirely of the same type and that the type + * implements the Comparable interface. For example, the data in the display + * column is entirely made of String or Integers. Some of the objects that + * implement the Comparable interface include String, Integer, Long, Short, + * Float, Byte, Double and Date. + * + * @param list + * The list to be sorted in ascending order. The list model used + * must be a DefaultListBoxModel or a descendent of + * DefaultListModel. + */ + public void sortList(final JList list) { + sortList(list, SorterConstants.ASCENDING_ORDER); + } + + /** + * Sorts a list using Sorter. This method assumes that the object type in + * the display column is made up entirely of the same type and that the type + * implements the Comparable interface. For example, the data in the display + * column is entirely made of String or Integers. Some of the objects that + * implement the Comparable interface include String, Integer, Long, Short, + * Float, Byte, Double and Date. + * + * @param list + * The list to be sorted. The list model used must be a + * DefaultListModel or a descendent of DefaultListModel or + * implement the SortableComponent interface. + * + * @param columnOrder + * An integer that represents the type of ordering to be used + * when sorting by only the first column in the data. The type of + * sort can either be ascending or descending. The + * ASCENDING_ORDER and DESCENDING_ORDER constants in this class + * should be used. + * + * @exception IllegalArgumentException + * Input data is invalid. + */ + public void sortList(final JList list, final int columnOrder) { + if (list.getModel() instanceof DefaultListModel) { + final DefaultListModel model = (DefaultListModel) list.getModel(); + + sortListModel(model, columnOrder); + } else if (list.getModel() instanceof SortableSwingComponent) { + final SortableSwingComponent model = (SortableSwingComponent) list + .getModel(); + + sortComponent(model, columnOrder); + } else { + throw new IllegalArgumentException( + "List model must be an " + + "instance of decendent of DefaultListModel or implement the " + + "SortableComponent interface."); + } + } + + /** + * Sorts a list model using Sorter. This method assumes that the object type + * in the display column is made up entirely of the same type and that the + * type implements the Comparable interface. For example, the data in the + * display column is entirely made of String or Integers. Some of the + * objects that implement the Comparable interface include String, Integer, + * Long, Short, Float, Byte, Double and Date. + * + * @param model + * The list model to be sorted. Must be a DefaultListModel or a + * descendent of DefaultListModel. + * + * @param newSortOrder + * An integer that represents the type of ordering to be used + * when sorting by only the first column in the data. The type of + * sort can either be ascending or descending. The + * ASCENDING_ORDER and DESCENDING_ORDER constants in this class + * should be used. + */ + public void sortListModel(final DefaultListModel model, + final int newSortOrder) { + // Create the table of sort data. + final Vector table = new Vector(); + + // Get the model data. + for (int ctr = 0, size = model.getSize(); ctr < size; ctr++) { + // Create a new row. + final Vector row = new Vector(); + row.add(model.getElementAt(ctr)); + table.add(row); + } + + // Create a new Sorter. + final Sorter sorter = new Sorter(table, newSortOrder); + sorter.setNullBehavior(nullBehavior); + + // Sort the vector data. + sorter.sort(); + + // Clear the model data. + model.removeAllElements(); + + // Re-add the sorted data to the model. + for (int ctr = 0, size = table.size(); ctr < size; ctr++) { + final List row = (List) table.get(ctr); + + // Get the first element from the row, because a list + // only has one column. + model.addElement(row.get(0)); + } + } + + /** + * Sorts a table using Sorter. This method assumes that the object type in + * each column is made up entirely of the same type and that the type + * implements the Comparable interface. For example, the data in column one + * can be entirely made of String types and column two can be entirely made + * up of Integers. Some of the objects that implement the Comparable + * interface include String, Integer, Long, Short, Float, Byte, Double and + * Date. This method will sort using all columns in ascending order. + * + * @param table + * JTable The table to be sorted. The table model of the table + * must be an instance or a descendent of DefaultTableModel or + * must implement the SortableComponent interface. + */ + public void sortTable(final JTable table) { + // Sort the table. + sortTable(table); + } + + /** + * Sorts a table using Sorter. This method assumes that the object type in + * each column is made up entirely of the same type and that the type + * implements the Comparable interface. For example, the data in column one + * can be entirely made of String types and column two can be entirely made + * up of Integers. Some of the objects that implement the Comparable + * interface include String, Integer, Long, Short, Float, Byte, Double and + * Date. This method will sort using all columns in the order specified. + * + * @param table + * JTable The table to be sorted. The table model of the table + * must be an instance or a descendent of DefaultTableModel or + * must implement the SortableComponent interface. + * + * @param columnOrder + * An integer that represents the type of ordering to be used + * when sorting by only the first column in the data. The type of + * sort can either be ascending or descending. The + * ASCENDING_ORDER and DESCENDING_ORDER constants in this class + * should be used. + */ + public void sortTable(final JTable table, final int columnOrder) { + // Sort the table! + sortTable(table, columnOrder); + } + + /** + * Sorts a table using Sorter. This method assumes that the object type in + * each column is made up entirely of the same type and that the type + * implements the Comparable interface. For example, the data in column one + * can be entirely made of String types and column two is can be entriely + * made up of Integers. Some of the objects that implement the Comparable + * interface include String, Integer, Long, Short, Float, Byte, Double and + * Date. + * + * @param table + * JTable The table to be sorted. The table model of the table + * must be an instance or a descendent of DefaultTableModel or + * must implement the SortableComponent interface. + * + * @param sortColumns + * Sort columns are a list of numbers specifying the colums to + * sort the table by. Each number in the list should be an + * instance of the SortColumn class represents a position of a + * column in the table. Column position start at zero just as + * they do in the standard Collection classes and Java arrays. + * When this field is null, there must be only one and only one + * column order specified. The sort routine will then sort using + * every column in the table starting from position 0 and up or + * left to right in ascending order. + * + * @exception IllegalArgumentException + * Input data is invalid. + */ + public void sortTable(final JTable table, final List sortColumns) { + if (table.getModel() instanceof DefaultTableModel) { + final DefaultTableModel model = (DefaultTableModel) table + .getModel(); + + sortTableModel(model, sortColumns); + } else if (table.getModel() instanceof SortableSwingComponent) { + final SortableSwingComponent model = (SortableSwingComponent) table + .getModel(); + + sortComponent(model, sortColumns); + } else { + throw new IllegalArgumentException( + "Table model must be an " + + "instance of decendent of DefaultTableModel or implement the " + + "SortableComponent interface."); + } + } + + /** + * Sorts a table model using Sorter. This method assumes that the object + * type in each column is made up entirely of the same type and that the + * type implements the Comparable interface. For example, the data in column + * one can be entirely made of String types and column two is can be + * entriely made up of Integers. Some of the objects that implement the + * Comparable interface include String, Integer, Long, Short, Float, Byte, + * Double and Date. + * + * @param model + * The table model to be sorted. The table model must be an + * instance or a descendent of DefaultTableModel. + */ + public void sortTableModel(final DefaultTableModel model) { + // Sort the data. + sortTableModel(model); + } + + /** + * Sorts a table model using Sorter. This method assumes that the object + * type in each column is made up entirely of the same type and that the + * type implements the Comparable interface. For example, the data in column + * one can be entirely made of String types and column two is can be + * entriely made up of Integers. Some of the objects that implement the + * Comparable interface include String, Integer, Long, Short, Float, Byte, + * Double and Date. + * + * @param model + * The table model to be sorted. The table model must be an + * instance or a descendent of DefaultTableModel. + * + * @param columnOrder + * An integer that represents the type of ordering to be used + * when sorting by only the first column in the data. The type of + * sort can either be ascending or descending. The + * ASCENDING_ORDER and DESCENDING_ORDER constants in this class + * should be used. + */ + public void sortTableModel(final DefaultTableModel model, + final int columnOrder) { + // Sort the table. + sortTableModel(model, columnOrder); + } + + /** + * Sorts a table model using Sorter. This method assumes that the object + * type in each column is made up entirely of the same type and that the + * type implements the Comparable interface. For example, the data in column + * one can be entirely made of String types and column two is can be + * entriely made up of Integers. Some of the objects that implement the + * Comparable interface include String, Integer, Long, Short, Float, Byte, + * Double and Date. + * + * @param model + * The table model to be sorted. The table model must be an + * instance or a descendent of DefaultTableModel. + * + * @param sortColumns + * Sort columns are a list of numbers specifying the colums to + * sort the table by. Each number in the list should be an + * instance of the SortColumn class represents a position of a + * column in the table. Column position start at zero just as + * they do in the standard Collection classes and Java arrays. + * When this field is null, there must be only one and only one + * column order specified. The sort routine will then sort using + * every column in the table starting from position 0 and up or + * left to right in ascending order. + */ + public void sortTableModel(final DefaultTableModel model, + final List sortColumns) { + // Create a new Sorter. + final Sorter sorter = new Sorter(model.getDataVector(), sortColumns); + sorter.setNullBehavior(nullBehavior); + + // Sort the data. + sorter.sort(); + } + + /** + * Returns the null behavior for this object. + * + * @return An integer representing the constant that indicates how null data + * values should behave while being sorted. See the null behavior + * constants in this class. The default value for this class is + * NULLS_ARE_INVALID. + */ + public int getNullBehavior() { + return nullBehavior; + } + + /** + * Set the null behavior for this object. + * + * @param nullBehavior + * An integer representing the constant that indicates how null + * data values should behave while being sorted. See the null + * behavior constants in this class. + * + * @throws IllegalArgumentException + * Thrown if the null behavior value is not valid. + */ + public void setNullBehavior(final int nullBehavior) { + if (nullBehavior != SorterConstants.NULLS_ARE_GREATEST + && nullBehavior != SorterConstants.NULLS_ARE_INVALID + && nullBehavior != SorterConstants.NULLS_ARE_LEAST) { + throw new IllegalArgumentException("Invalid null behavior."); + } + + this.nullBehavior = nullBehavior; + } +} \ No newline at end of file Index: ocean/src/org/apache/lucene/index/OceanSegmentReader.java =================================================================== --- ocean/src/org/apache/lucene/index/OceanSegmentReader.java (revision 0) +++ ocean/src/org/apache/lucene/index/OceanSegmentReader.java (revision 0) @@ -0,0 +1,71 @@ +package org.apache.lucene.index; + +import java.io.IOException; + +import org.apache.lucene.ocean.util.BytesPool; +import org.apache.lucene.util.BitVector; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class OceanSegmentReader extends SegmentReader { + final static Logger LOG = LoggerFactory.getLogger(OceanSegmentReader.class); + private BytesPool bytesPool; + + public OceanSegmentReader() { + bytesPool = BytesPool.getInstance(); + allowCloneWithChanges = true; + } + + public synchronized void close(boolean flush) throws IOException { + super.close(flush); + } + + protected void doClose() throws IOException { + // return deletedDocs bytes to bytesPool + if (deletedDocs != null) { + if (deletedDocsCopyOnWriteRef == null || deletedDocsCopyOnWriteRef.refCount() == 1) { + bytesPool.returnBytes(deletedDocs.getBits()); + } + } + super.doClose(); + } + /** + protected synchronized void decRef() throws IOException { + if (getRefCount() == 1) { + hasChanges = false; + } + super.decRef(); + } + **/ + /** + protected BitVector cloneDeletedDocs(BitVector deletedDocs) { + byte[] bits = deletedDocs.getBits(); + byte[] cloneBits = bytesPool.getBytes(bits.length); + System.arraycopy(bits, 0, cloneBits, 0, bits.length); + BitVector cloned = new BitVector(cloneBits, deletedDocs.size(), -1); + if (cloned.count() != deletedDocs.count()) { + BitVector first = new BitVector(bits, deletedDocs.size(), -1); + int firstCount = first.count(); + LOG.info("firstCount: "+firstCount+" cloned.count: "+cloned.count()+" deletedDocs.count: "+deletedDocs.count()); + assert cloned.count() == deletedDocs.count(); + } + // + return cloned; + } + **/ + /** + public Object clone() { + boolean origHasChanges = hasChanges; + hasChanges = false; + Object clone = super.clone(); + hasChanges = origHasChanges; + return clone; + } + **/ + // TODO: need an in memory write lock that does not check the segment version + protected void acquireWriteLock() throws IOException { + //if (!isCurrent()) { + // throw new IOException("reader is not current"); + //} + } +} Index: ocean/src/org/apache/lucene/ocean/AbstractTransaction.java =================================================================== --- ocean/src/org/apache/lucene/ocean/AbstractTransaction.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/AbstractTransaction.java (revision 0) @@ -0,0 +1,230 @@ +package org.apache.lucene.ocean; + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Callable; + +import org.apache.commons.lang.exception.ExceptionUtils; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.ocean.Batch.MasterBatch; +import org.apache.lucene.ocean.Index.IndexSnapshot; +import org.apache.lucene.ocean.WriteableMemoryIndex.MemoryIndexSnapshot; +import org.apache.lucene.store.RAMDirectory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public abstract class AbstractTransaction extends Transaction implements Callable { + final static Logger LOG = LoggerFactory.getLogger(AbstractTransaction.class); + protected Batch batch; + protected List failures = new ArrayList(); + protected List deletesResults = new ArrayList(); + protected List newIndexSnapshots = new ArrayList(); + protected Long id; + protected TransactionSystem system; + //protected Snapshot newSnapshot; + protected Snapshot previousSnapshot; + protected int numDocsAdded = 0; + protected final List documentIds; + + public AbstractTransaction(Long id, List documentIds, Snapshot previousSnapshot, Batch batch, TransactionSystem system) throws Exception { + this.id = id; + this.documentIds = documentIds; + this.previousSnapshot = previousSnapshot; + this.batch = batch; + this.system = system; + } + + //void addDocsAdded(int value) { + // numDocsAdded += value; + //} + + public Snapshot getPreviousSnapshot() { + return previousSnapshot; + } + + public Long getId() { + return id; + } + + protected CommitResult createCommitResult() throws Exception { + if (failures.size() == 0) { + List indexSnapshots = new ArrayList(previousSnapshot.getDeleteOnlyIndexSnapshots()); + indexSnapshots.addAll(newIndexSnapshots); + WriteableMemoryIndex writeableIndex = (WriteableMemoryIndex) previousSnapshot.getWriteableSnapshot().getIndex(); + Snapshot newSnapshot = new Snapshot(new BigDecimal(id), writeableIndex.getLatestIndexSnapshot(), indexSnapshots, system, System + .currentTimeMillis()); + for (DeletesResult deletesResult : deletesResults) { + numDocsAdded += deletesResult.getNumAdded(); + } + CommitResult commitResult = new CommitResult(newSnapshot, documentIds, deletesResults, numDocsAdded, writeableIndex.getId()); + return commitResult; + } else { + // rollback indexes + LOG.info("rolling back snapshot: " + id); + for (IndexSnapshot indexSnapshot : previousSnapshot.getIndexSnapshots()) { + indexSnapshot.getIndex().rollback(id); + } + // TODO: need assertion that rollbacks worked + throw new Exception("transaction failed " + failures); + } + } + + public boolean go() { + return true; + } + + public void ready(Index index) { + } + + protected void handleMasterBatch() { + try { + if (batch instanceof MasterBatch) { + MasterBatch masterBatch = (MasterBatch) batch; + if (masterBatch.hasDeletes()) { + int numDocIds = 0; + for (DeletesResult deletesResult : deletesResults) { + numDocIds += deletesResult.getDocIds().size(); + } + long[] docIds = new long[numDocIds]; + int x = 0; + for (DeletesResult deletesResult : deletesResults) { + for (Long docId : deletesResult.getDocIds()) { + docIds[x] = docId; + x++; + } + } + masterBatch.getDeletes().setDocIds(docIds); + } + system.getTransactionLog().writeMasterBatch(id, masterBatch); + } + } catch (Exception exception) { + failures.add(new LogFailure(exception)); + } + } + + public void failed(Index index, Throwable throwable) { + failures.add(new IndexFailure(index, throwable)); + } + + public class AddRamIndexDocumentsTask implements Callable { + private RAMDirectory ramDirectory; + private WriteableMemoryIndex writeableIndex; + + public AddRamIndexDocumentsTask(RAMDirectory ramDirectory, WriteableMemoryIndex writeableIndex) { + this.ramDirectory = ramDirectory; + this.writeableIndex = writeableIndex; + } + + public DeletesResult call() throws Exception { + // TODO: create new ramindex + IndexReader indexReader = IndexReader.open(ramDirectory); + if (indexReader.maxDoc() > 30) { + long indexIdNum = system.getNextRamIndexId(); + IndexID indexId = new IndexID(indexIdNum, "ram"); + Analyzer analyzer = batch.getAnalyzer(); + RamIndex ramIndex = new RamIndex(indexId, id, null, ramDirectory, system); + IndexSnapshot indexSnapshot = ramIndex.commitIndex(AbstractTransaction.this); + newIndexSnapshots.add(indexSnapshot); + DeletesResult deletesResult = new DeletesResult(indexId); + addDeletesResult(deletesResult); + return deletesResult; + } else { + MemoryIndexSnapshot latestIndexSnapshot = writeableIndex.getLatestIndexSnapshot(); + writeableIndex.createIndexSnapshot(id, latestIndexSnapshot.cloneReaders(true)); + //MemoryIndexSnapshot memoryIndexSnapshot = latestIndexSnapshot.newIndexSnapshot(id, indexReader); + DeletesResult deletesResult = new DeletesResult(writeableIndex.getId()); + addDeletesResult(deletesResult); + return deletesResult; + } + // writeableIndex.commitNothing(Transaction.this); + //return deletesResult; + } + } + + public class AddWriteableMemoryDocumentsTask implements Callable { + private Documents documents; + private Analyzer analyzer; + private Deletes deletes; + private WriteableMemoryIndex writeableIndex; + + public AddWriteableMemoryDocumentsTask(Documents documents, Analyzer analyzer, Deletes deletes, WriteableMemoryIndex writeableIndex) { + this.documents = documents; + this.analyzer = analyzer; + this.deletes = deletes; + this.writeableIndex = writeableIndex; + } + + public DeletesResult call() throws Exception { + DeletesResult deletesResult = writeableIndex.commitChanges(documents, deletes, analyzer, AbstractTransaction.this); + addDeletesResult(deletesResult); + return deletesResult; + } + } + + void addDeletesResult(DeletesResult deletesResult) { + assert deletesResult != null; + deletesResults.add(deletesResult); + } + + public class DeletesTask implements Callable { + private Index index; + private Deletes deletes; + + public DeletesTask(Deletes deletes, Index index) { + this.deletes = deletes; + this.index = index; + } + + public DeletesResult call() throws Exception { + DeletesResult deletesResult = index.commitDeletes(deletes, AbstractTransaction.this); + newIndexSnapshots.add(index.getLatestIndexSnapshot()); + addDeletesResult(deletesResult); + return deletesResult; + } + } + + public abstract static class Failure extends Exception { + private String string; + + public Failure(String message) { + super(message); + string = message; + } + + public Failure(Throwable throwable) { + super(throwable); + string = ExceptionUtils.getFullStackTrace(throwable); + } + + public String toString() { + return string; + } + } + + public static class IndexFailure extends Failure { + Index index; + + public IndexFailure(Index index, Throwable throwable) { + super(throwable); + this.index = index; + } + } + + public static class TimeoutFailure extends Failure { + public TimeoutFailure(String message) { + super(message); + } + } + + public static class LogFailure extends Failure { + public LogFailure(Throwable throwable) { + super(throwable); + } + } + + public List getNewIndexSnapshots() { + return newIndexSnapshots; + } +} Index: ocean/src/org/apache/lucene/ocean/Batch.java =================================================================== --- ocean/src/org/apache/lucene/ocean/Batch.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/Batch.java (revision 0) @@ -0,0 +1,249 @@ +package org.apache.lucene.ocean; + +import java.io.IOException; +import java.util.Date; + +import org.apache.commons.lang.SerializationUtils; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.ocean.log.RecordData; +import org.apache.lucene.ocean.log.TransactionLog; +import org.apache.lucene.ocean.util.Bytes; +import org.apache.lucene.ocean.util.Constants; +import org.apache.lucene.store.RAMDirectory; + +/** + * Stays in memory + * + */ +public abstract class Batch { + protected Documents documents; + protected RAMDirectory ramDirectory; + protected Analyzer analyzer; + protected long schemaVersion; + protected Deletes deletes; + protected Date timestamp; + protected boolean isClosed = false; + + public Batch() { + timestamp = new Date(); + } + + public boolean hasRAMDirectory() { + return ramDirectory != null; + } + + public RAMDirectory getRamDirectory() { + return ramDirectory; + } + + public long getSchemaVersion() { + return schemaVersion; + } + + public static class MasterBatch extends Batch { + private TransactionSystem transactionSystem; + private Bytes docBytes; + private int docType; + private Bytes otherBytes; + private int otherType; + + public MasterBatch(TransactionSystem transactionSystem) { + this.transactionSystem = transactionSystem; + } + + public RecordData getRecordData() throws IOException { + if (hasDocuments() && !hasDocData()) { + createDocData(); + } + if (hasDeletes() && !hasOtherData()) { + createDeleteData(); + } + return new RecordData(docType, docBytes, otherType, otherBytes); + } + + public boolean hasOtherData() { + return otherBytes != null; + } + + public Bytes getOtherBytes() { + return otherBytes; + } + + public int getOtherType() { + return otherType; + } + + public Bytes getDocBytes() { + return docBytes; + } + + public int getDocType() { + return docType; + } + + public boolean hasDocData() { + return docBytes != null; + } + + public void createDeleteData() throws IOException { + if (hasDeletes()) { + Deletes deletes = getDeletes(); + otherBytes = new Bytes(1024); + SerializationUtils.serialize(deletes, otherBytes.getOutputStream()); + otherType = Constants.DELETES_SERIALIZE_TYPE; + } + } + + public void createDocData() throws IOException { + if (hasRAMDirectory()) { + docBytes = TransactionLog.serialize(getRamDirectory()); + docType = Constants.RAM_DIRECTORY_TYPE; + } else if (hasDocuments()) { + Documents documents = getDocuments(); + docBytes = TransactionLog.serialize(documents); + docType = Constants.DOCUMENTS_TYPE; + } + } + + void setRAMDirectory(RAMDirectory ramDirectory) { + documents = null; + this.ramDirectory = ramDirectory; + } + + void setAnalyzer(Analyzer analyzer) { + this.analyzer = analyzer; + } + + void setDeletes(Deletes deletes) { + if (isClosed) + throw new RuntimeException("batch closed"); + this.deletes = deletes; + } + + void addDocument(Document document) { + if (this.documents == null) + this.documents = new Documents(); + this.documents.add(document); + } + + void addDocuments(Documents documents) { + if (isClosed) + throw new RuntimeException("batch closed"); + if (this.documents == null) + this.documents = new Documents(); + this.documents.addAll(documents); + } + + /** + * public Serializable getSerializableRamDirectory(Long id) { assert + * ramDirectory != null; return new SerializableRamDirectory(id, + * ramDirectory); } + * + * public Serializable getSerializableBatchDeletes(Long id) { return new + * SerializableBatchDeletes(id, deletes); } + * + * public Serializable getSerializableBatchDocuments(Long id) { return new + * SerializableBatchDocuments(id, documents); } + */ + void commit() throws Exception { + transactionSystem.commitBatch(this); + } + } + + public static class SlaveBatch extends Batch { + private Long id; + + public SlaveBatch(Long id, Documents documents, Deletes deletes) { + this.id = id; + this.documents = documents; + this.deletes = deletes; + } + + public SlaveBatch(Long id, RAMDirectory ramDirectory, Deletes deletes) { + this.id = id; + this.ramDirectory = ramDirectory; + this.deletes = deletes; + } + + public Long getId() { + return id; + } + } + + /** + * public static class SerializableRamDirectory extends SerializableBatch + * implements Externalizable { private static final long serialVersionUID = + * 1l; private RAMDirectory ramDirectory; + * + * public SerializableRamDirectory(Long id, RAMDirectory ramDirectory) { + * super(id); this.ramDirectory = ramDirectory; } // TODO: use native version + * not externalizable public void readExternal(ObjectInput objectInput) throws + * IOException, ClassNotFoundException { long objectVersion = + * objectInput.readLong(); ramDirectory = + * RamDirectorySerializer.deserialize(objectInput); } + * + * public void writeExternal(ObjectOutput objectOutput) throws IOException { + * objectOutput.writeLong(serialVersionUID); + * RamDirectorySerializer.serialize(ramDirectory, objectOutput); } + * + * public RAMDirectory getRamDirectory() { return ramDirectory; } } + * + * public static class SerializableBatchDocuments extends SerializableBatch { + * private static final long serialVersionUID = 1l; private Documents + * documents; + * + * public SerializableBatchDocuments(Long id, Documents documents) { + * super(id); this.documents = documents; } + * + * public Documents getDocuments() { return documents; } } + * + * public static class SerializableBatchDeletes extends SerializableBatch { + * private static final long serialVersionUID = 1l; private Deletes deletes; + * + * public SerializableBatchDeletes(Long id, Deletes deletes) { super(id); } + * + * public Deletes getDeletes() { return deletes; } } + * + * public abstract static class SerializableBatch implements Serializable { + * private Long id; + * + * public SerializableBatch(Long id) { this.id = id; } public Long getId() { + * return id; } } + */ + public Analyzer getAnalyzer() { + return analyzer; + } + + public boolean hasDocuments() { + if (documents == null || documents.size() == 0) + return false; + else + return true; + } + + public boolean hasDeletes() { + if (deletes == null || !deletes.hasDeletes()) { + return false; + } else { + return true; + } + } + + // disallow any more additions + public void close() { + isClosed = true; + } + + public Documents getDocuments() { + return documents; + } + + public Deletes getDeletes() { + return deletes; + } + + public Date getTimestamp() { + return timestamp; + } +} Index: ocean/src/org/apache/lucene/ocean/CommitResult.java =================================================================== --- ocean/src/org/apache/lucene/ocean/CommitResult.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/CommitResult.java (revision 0) @@ -0,0 +1,69 @@ +package org.apache.lucene.ocean; + +import java.util.List; + +public class CommitResult { + private Snapshot snapshot; + private List deletesResults; + private Integer numAdded; + private IndexID addedIndexId; + private int numDeleted = 0; + private List documentIds; + + public CommitResult(Snapshot snapshot, List documentIds, List deletesResults, Integer numAdded, IndexID addedIndexId) { + this.snapshot = snapshot; + this.documentIds = documentIds; + this.deletesResults = deletesResults; + this.numAdded = numAdded; + this.addedIndexId = addedIndexId; + if (deletesResults != null) { + for (DeletesResult deletesResult : deletesResults) { + numDeleted += deletesResult.getNumDeleted(); + } + } + } + + public String toString() { + return "numAdded: "+numAdded+" numDeleted: "+numDeleted; + } + + public List getDocumentIds() { + return documentIds; + } + + public Snapshot getSnapshot() { + return snapshot; + } + + public int getNumDeleted() { + return numDeleted; + } + + public int getNumDocChanges() { + int numChanged = 0; + if (numAdded != null) + numChanged += numAdded; + if (deletesResults != null) { + for (DeletesResult deletesResult : deletesResults) { + numChanged += deletesResult.getNumDeleted(); + } + } + return numChanged; + } + + public Long getSnapshotId() { + return snapshot.getSnapshotId(); + } + + public List getDeletesResults() { + return deletesResults; + } + + public Integer getNumAdded() { + return numAdded; + } + + public IndexID getAddedIndexId() { + return addedIndexId; + } +} Index: ocean/src/org/apache/lucene/ocean/database/OceanDatabase.java =================================================================== --- ocean/src/org/apache/lucene/ocean/database/OceanDatabase.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/database/OceanDatabase.java (revision 0) @@ -0,0 +1,80 @@ +package org.apache.lucene.ocean.database; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.lucene.ocean.TransactionSystem; + +/** + * Assigns unique ID for each OceanObject. The OceanObject may be updated in + * place versioned using optimistic concurrency. Every field is stored. + * + */ +public class OceanDatabase { + private Map fieldMap = new HashMap(); + private final TransactionSystem transactionSystem; + + public OceanDatabase(TransactionSystem transactionSystem) { + this.transactionSystem = transactionSystem; + } + + public class FieldInfo { + + } + + public Result perform(List actions) { + transactionSystem.commitTransaction(List documents, Analyzer analyzer, List deleteByTerms, List deleteByQueries); + } + + public static class ActionResult { + public final String actionId; + + public ActionResult(String actionId) { + this.actionId = actionId; + } + } + + public static class Result { + + } + + public static class Insert extends Action { + public final OceanObject object; + + public Insert(String actionId, OceanObject object) { + super(actionId); + this.object = object; + } + } + + public static class Update extends Action { + public final Long id; + public final OceanObject object; + + public Update(String actionId, Long id, OceanObject object) { + super(actionId); + this.id = id; + this.object = object; + } + } + + public static class Delete extends Action { + public final Long id; + + public Delete(String actionId, Long id) { + super(actionId); + this.id = id; + } + } + + public static class Action { + public final String actionId; // returned in Result + + public Action(String actionId) { + this.actionId = actionId; + } + + + } +} Index: ocean/src/org/apache/lucene/ocean/database/OceanObject.java =================================================================== --- ocean/src/org/apache/lucene/ocean/database/OceanObject.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/database/OceanObject.java (revision 0) @@ -0,0 +1,37 @@ +package org.apache.lucene.ocean.database; + +import org.apache.lucene.document.Field; +import org.apache.lucene.document.Field.Index; +import org.apache.lucene.document.Field.Store; +import org.apache.lucene.document.Field.TermVector; + +public class OceanObject { + + public void add(String name, Object value) { + + } + + public void add(OceanField oceanField) { + + } + + public static class OceanField { + public final String name; + public final Object value; + public final Field.Index fieldIndex; + public final Field.Store fieldStore; + public final Field.TermVector fieldTermVector; + + public OceanField(String name, Object value, Index fieldIndex, Store fieldStore, TermVector fieldTermVector) { + this.name = name; + this.value = value; + this.fieldIndex = fieldIndex; + this.fieldStore = fieldStore; + this.fieldTermVector = fieldTermVector; + } + + public OceanField(String name, Object value) { + this(name, value, null, null, null); + } + } +} Index: ocean/src/org/apache/lucene/ocean/Deletes.java =================================================================== --- ocean/src/org/apache/lucene/ocean/Deletes.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/Deletes.java (revision 0) @@ -0,0 +1,129 @@ +package org.apache.lucene.ocean; + +import java.io.IOException; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.lang.SerializationUtils; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.Query; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; + +/** + * Encapsulates deletes for a transaction + * + */ +public class Deletes implements Serializable { + public static final int DOC_IDS = 1; + public static final int TERMS = 2; + private static final long serialVersionUID = 1l; + private List terms = new ArrayList(); + private List deleteByQueries = new ArrayList(); + private long[] docIds; + + /** + * public Deletes(IndexInput input) throws IOException { int docIdsLength = + * input.readVInt(); if (docIdsLength > 0) { docIds = new long[docIdsLength]; + * for (int x=0; x < docIdsLength; x++) { docIds[x] = input.readVLong(); } } + * int termsLength = input.readVInt(); for (int x=0; x < termsLength; x++) { + * String field = input.readString(); String text = input.readString(); + * terms.add(new Term(field, text)); } int dqLength = input.readVInt(); for + * (int x=0; x < dqLength; x++) { int blen = input.readVInt(); byte[] bytes = + * new byte[blen]; input.readBytes(bytes, 0, blen); + * deleteByQueries.add((DeleteByQuery)SerializationUtils.deserialize(bytes)); } } + */ + public Deletes() { + } + + /** + * public void writeTo(IndexOutput output) throws IOException { if (docIds == + * null) { output.writeVInt(0); } else { output.writeVInt(docIds.length); for + * (int x = 0; x < docIds.length; x++) { output.writeVLong(docIds[x]); } } + * output.writeVInt(terms.size()); for (int x = 0; x < terms.size(); x++) { + * output.writeString(terms.get(x).field()); + * output.writeString(terms.get(x).text()); } + * output.writeVInt(deleteByQueries.size()); for (int x = 0; x < + * deleteByQueries.size(); x++) { byte[] bytes = + * SerializationUtils.serialize(deleteByQueries.get(x)); + * output.writeVInt(bytes.length); output.writeBytes(bytes, bytes.length); } } + */ + public void merge(Deletes deletes) { + terms.addAll(deletes.getTerms()); + deleteByQueries.addAll(deletes.getDeleteByQueries()); + } + + public boolean hasDocIds() { + return docIds != null; + } + + void setDocIds(long[] docIds) { + assert this.docIds == null; + this.docIds = docIds; + } + + public void addTerm(Term term) { + terms.add(term); + } + + public void addQuery(Query query) { + deleteByQueries.add(new DeleteByQuery(query)); + } + + public void addDeleteByQuery(DeleteByQuery deleteByQuery) { + deleteByQueries.add(deleteByQuery); + } + + public long[] getDocIds() { + return docIds; + } + + public boolean hasDeletes() { + if (terms.size() > 0 || deleteByQueries.size() > 0 || docIds != null && docIds.length > 0) { + return true; + } else { + return false; + } + } + + public boolean hasDeleteByQueries() { + if (deleteByQueries != null && deleteByQueries.size() > 0) + return true; + return false; + } + + public List getDeleteByQueries() { + return deleteByQueries; + } + + public static class DeleteByQuery implements Serializable { + private Query query; + + public DeleteByQuery(Query query) { + this.query = query; + } + + public Query getQuery() { + return query; + } + } + + public List getQueries() { + List queries = new ArrayList(deleteByQueries.size()); + for (DeleteByQuery deleteByQuery : deleteByQueries) { + queries.add(deleteByQuery.getQuery()); + } + return queries; + } + + public List getTerms() { + return terms; + } + + public boolean hasTerms() { + if (terms != null && terms.size() > 0) + return true; + return false; + } +} Index: ocean/src/org/apache/lucene/ocean/DeletesResult.java =================================================================== --- ocean/src/org/apache/lucene/ocean/DeletesResult.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/DeletesResult.java (revision 0) @@ -0,0 +1,60 @@ +package org.apache.lucene.ocean; + +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; + +public class DeletesResult { + private IndexID indexId; + private List results = new ArrayList(); + private int numDeleted = 0; + private List docIds = new LinkedList(); + private int numAdded = 0; + + public DeletesResult(IndexID indexId) { + this.indexId = indexId; + } + + public List getDocIds() { + return docIds; + } + + public IndexID getIndexId() { + return indexId; + } + + void setNumAdded(int numAdded) { + this.numAdded = numAdded; + } + + public void add(Result result) { + numDeleted += result.getNumDeleted(); + results.add(result); + } + + public int getNumAdded() { + return numAdded; + } + + public int getNumDeleted() { + return numDeleted; + } + + public static class Result { + private Object delete; + private int numDeleted; + + public Result(Object delete, int numDeleted) { + this.delete = delete; + this.numDeleted = numDeleted; + } + + public Object getDelete() { + return delete; + } + + public int getNumDeleted() { + return numDeleted; + } + } +} \ No newline at end of file Index: ocean/src/org/apache/lucene/ocean/DirectoryIndex.java =================================================================== --- ocean/src/org/apache/lucene/ocean/DirectoryIndex.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/DirectoryIndex.java (revision 0) @@ -0,0 +1,410 @@ +package org.apache.lucene.ocean; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.lucene.index.IndexCommit; +import org.apache.lucene.index.IndexDeletionPolicy; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.OceanSegmentReader; +import org.apache.lucene.ocean.Batch.SlaveBatch; +import org.apache.lucene.ocean.util.Constants; +import org.apache.lucene.ocean.util.SortedList; +import org.apache.lucene.ocean.util.Util; +import org.apache.lucene.store.Directory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Abstract class used by RamIndex and DiskIndex. Assumes a + * org.apache.lucene.store.Directory based IndexReader implementation. + * + */ +public abstract class DirectoryIndex extends Index { + final static Logger LOG = LoggerFactory.getLogger(DirectoryIndex.class); + protected final SortedList indexSnapshotMap = new SortedList(); + private int numDeleteTransactionsSinceLastFlush = 0; + protected DirectoryIndexDeletionPolicy indexDeletionPolicy = new DirectoryIndexDeletionPolicy(); + protected IndexReader initialIndexReader; + protected final List checkpointSnapshotIds = new ArrayList(); + protected final List lastAppliedSnapshotIds = new ArrayList(); + private Long maxSnapshotId; + private Long maxDocumentId; + private Long minSnapshotId; + private Long minDocumentId; + + public DirectoryIndex(IndexID id, TransactionSystem system) { + super(id, system); + } + + public void close() throws IOException { + for (DirectoryIndexSnapshot snapshot : indexSnapshotMap.values()) { + snapshot.close(); + } + } + + public DirectoryIndexSnapshot getLatestIndexSnapshot() { + return indexSnapshotMap.lastValue(); + } + + public DirectoryIndexSnapshot getIndexSnapshot(Long snapshotId) { + return indexSnapshotMap.get(snapshotId); + } + + /** + * public IndexSnapshot initialize(Long snapshotId, List + * deletesList, TransactionSystem system) throws Exception, IndexException, + * IOException { IndexReader indexReader = initialIndexReader; if (deletesList == + * null || deletesList.size() == 0) { createNewSnapshot(snapshotId, false, + * indexReader); } else { for (Deletes deletes : deletesList) { + * applyDeletes(true, deletes, null, indexReader); indexReader.flush(); // + * always flush here because it is the during the creation of the index + * createNewSnapshot(slaveBatch.getId(), true, indexReader); } } assert + * snapshotId.equals(indexSnapshotMap.lastKey()); return + * indexSnapshotMap.get(indexSnapshotMap.lastKey()); } + */ + + public IndexSnapshot initialize(Long snapshotId, List deleteOnlySlaveBatches, TransactionSystem system) throws Exception, + IndexException, IOException { + IndexReader indexReader = initialIndexReader; + if (deleteOnlySlaveBatches == null || deleteOnlySlaveBatches.size() == 0) { + createNewSnapshot(snapshotId, true, indexReader); + } else { + int numDeleted = 0; + boolean checkpoint = false; + for (int x = 0; x < deleteOnlySlaveBatches.size(); x++) { + SlaveBatch slaveBatch = deleteOnlySlaveBatches.get(x); + if (slaveBatch.hasDeletes()) { + DeletesResult deletesResult = applyDeletes(true, slaveBatch.getDeletes(), null, indexReader); + numDeleted += deletesResult.getNumDeleted(); + } + if (numDeleted > 0 && x == deleteOnlySlaveBatches.size() - 1) { + checkpoint = true; + indexReader.flush(); + } + createNewSnapshot(slaveBatch.getId(), checkpoint, indexReader); + } + } + // assert snapshotId.equals(indexSnapshotMap.lastKey()); + return indexSnapshotMap.lastValue(); + } + + protected void onCommit() throws Exception { + + } + + private List getSnapshotsByGeneration(long generation) throws IOException { + List snapshots = new ArrayList(); + for (DirectoryIndexSnapshot indexSnapshot : indexSnapshotMap.values()) { + if (indexSnapshot.getIndexReader().getIndexCommit().getGeneration() == generation) { + snapshots.add(indexSnapshot); + } + } + return snapshots; + } + + /** + * Finds reader by version by iterating over snapshots and comparing versions + * + * @param version + * @return + */ + private DirectoryIndexSnapshot getSnapshotByReaderVersion(long version) { + for (DirectoryIndexSnapshot indexSnapshot : indexSnapshotMap.values()) { + if (indexSnapshot.getIndexReaderVersion() == version) { + return indexSnapshot; + } + } + return null; + } + + public class DirectoryIndexDeletionPolicy implements IndexDeletionPolicy { + private IndexCommit lastCommit; + SortedList commitPoints = new SortedList(); // key + + // is + // generation + + // is + // generation + + public void onInit(List commits) throws IOException { + onCommit(commits); + } + + public IndexCommit getLastIndexCommitPoint() { + return lastCommit; + } + + public void onCommit(List commits) throws IOException { + try { + commitPoints.clear(); + for (int x = 0; x < commits.size(); x++) { + IndexCommit indexCommit = (IndexCommit) commits.get(x); + commitPoints.put(indexCommit.getGeneration(), indexCommit); + } + DirectoryIndex.this.onCommit(); + lastCommit = (IndexCommit) commits.get(commits.size() - 1); + for (int x = 0; x < commits.size() - 1; x++) { + IndexCommit indexCommitPoint = (IndexCommit) commits.get(x); + // Multiple snapshots may have the same generation + // so deleting a commitpoint may affect multiple snapshots + long generation = indexCommitPoint.getGeneration(); + List snapshots = getSnapshotsByGeneration(generation); + // if there are no snapshots it needs to be deleted, nothing + // is using it anymore + if (snapshots.size() == 0) { + indexCommitPoint.delete(); + commitPoints.remove(indexCommitPoint.getGeneration()); + } + for (DirectoryIndexSnapshot indexSnapshot : snapshots) { + if (!indexSnapshot.hasRef()) { + // not referenced in Snapshots anymore + indexCommitPoint.delete(); + indexSnapshot.delete(); + } + } + } + } catch (Exception exception) { + throw Util.asIOException(exception); + } + } + } + + public abstract class DirectoryIndexSnapshot extends IndexSnapshot { + protected OceanSegmentReader indexReader; + private int maxDoc; + + public DirectoryIndexSnapshot(Long snapshotId, OceanSegmentReader indexReader) throws IOException { + super(snapshotId); + this.indexReader = indexReader; + maxDoc = indexReader.maxDoc(); + getMinDocumentId(); + getMaxDocumentId(); + } + + public void delete() throws Exception { + // TODO: maybe need to check for snapshot refs here + LOG.info(DirectoryIndex.this.getId() + " deleting snapshotid: " + snapshotId); + long generation = getGeneration(); + List snapshotsGreaterWithGeneration = new ArrayList(); + for (DirectoryIndexSnapshot snapshot : indexSnapshotMap.values()) { + if (snapshot.getGeneration() == generation && snapshot.snapshotId.longValue() > snapshotId) { + snapshotsGreaterWithGeneration.add(snapshot); + } + } + indexReader.close(false); + if (snapshotsGreaterWithGeneration.size() == 0) { + IndexCommit indexCommit = indexDeletionPolicy.commitPoints.get(generation); + if (indexCommit != null) { + LOG.info(DirectoryIndex.this.getId() + " deleting snapshotid: " + snapshotId + " indexCommit: " + indexCommit.getGeneration()); + indexCommit.delete(); + } + } + indexSnapshotMap.remove(snapshotId); + } + + public Long getMinDocumentId() throws IOException { + if (minDocumentId == null) { + String string = getMin(Constants.DOCUMENTID); + if (string == null) + return null; + minDocumentId = Util.longFromEncoded(string); + } + return minDocumentId; + } + + public Long getMinSnapshotId() throws IOException { + if (minSnapshotId == null) { + String string = getMin(Constants.SNAPSHOTID); + if (string == null) + return null; + minSnapshotId = Util.longFromEncoded(string); + } + return minSnapshotId; + } + + public Long getMaxSnapshotId() throws IOException { + if (maxSnapshotId == null) { + String string = getMax(Constants.SNAPSHOTID); + if (string == null) + return null; + maxSnapshotId = Util.longFromEncoded(string); + } + return maxSnapshotId; + } + + public Long getMaxDocumentId() throws IOException { + if (maxDocumentId == null) { + String string = getMax(Constants.DOCUMENTID); + if (string == null) + return null; + maxDocumentId = Util.longFromEncoded(string); + } + return maxDocumentId; + } + + public Long getLastAppliedSnapshotId() { + return getLastId(lastAppliedSnapshotIds, "getLastAppliedSnapshotId"); + } + + private Long getLastId(List ids, String name) { + if (ids.size() == 0) + return null; + return ids.get(ids.size() - 1); + /** + * for (int x = ids.size() - 1; x >= 0; x--) { Long id = ids.get(x); if + * (id.longValue() <= getSnapshotId()) { return id; } } Long lastId = + * ids.get(ids.size() - 1); LOG.error(name + " pos is negative, should not + * be possible for: " + getSnapshotId() + " last valid id: " + lastId); + * return null; + */ + } + + public Long getLastCheckpointId() { + return getLastId(checkpointSnapshotIds, "getLastSnapshotIdDeletesFlushed"); + } + + public long getGeneration() throws IOException { + return indexReader.getIndexCommit().getGeneration(); + } + + public void close() throws IOException { + indexReader.close(false); + indexSnapshotMap.remove(snapshotId); + } + + public int deletedDoc() { + return indexReader.maxDoc() - indexReader.numDocs(); + } + + public int maxDoc() { + return indexReader.maxDoc(); + } + + public IndexReader getIndexReader() { + return indexReader; + } + + public long getIndexReaderVersion() { + return indexReader.getVersion(); + } + + public boolean hasRef() throws Exception { + return getSystem().getSnapshots().hasRefs(snapshotId); + } + } + + public long getSize() throws IOException { + Directory directory = getDirectory(); + return Util.getSize(directory); + } + + /** + * Creates a new snapshot only without performing any changes to the index + * + * @param transaction + * @throws IndexException + * @throws InterruptedException + * @throws IOException + */ + public void commitNothing(Transaction transaction) throws IndexException, InterruptedException, IOException, Exception { + IndexSnapshot latestIndexSnapshot = getLatestIndexSnapshot(); + assert latestIndexSnapshot != null; + //if (latestIndexSnapshot.getSnapshotId().equals(transaction.getPreviousId())) { + //} + transaction.ready(this); + if (transaction.go()) { + Long snapshotId = transaction.getId(); + IndexReader previousIndexReader = latestIndexSnapshot.getIndexReader(); + IndexReader newIndexReader = (IndexReader)previousIndexReader.clone(); + createNewSnapshot(snapshotId, false, newIndexReader); + removeOldSnapshots(indexSnapshotMap); + } + } + + public boolean rollback(Long snapshotId) throws Exception { + LOG.info("rollback " + snapshotId); + DirectoryIndexSnapshot indexSnapshot = indexSnapshotMap.get(snapshotId); + if (indexSnapshot != null) { + indexSnapshot.delete(); + return true; + } + return false; + } + + public DeletesResult commitDeletes(Deletes deletes, Transaction transaction) throws Exception, IndexException, InterruptedException, + IOException { + IndexSnapshot latestIndexSnapshot = getLatestIndexSnapshot(); + assert latestIndexSnapshot != null; + // assert + // latestIndexSnapshot.getSnapshotId().equals(transaction.getPreviousId()); + // if + // (latestIndexSnapshot.getSnapshotId().equals(transaction.getPreviousId())) + // { + // System.out.println("latest: " + latestIndexSnapshot.getSnapshotId() + " + // previous: " + transaction.getPreviousId()); + // } + IndexReader previousIndexReader = latestIndexSnapshot.getIndexReader(); + IndexReader newIndexReader = (IndexReader) previousIndexReader.clone(); + try { + DeletesResult deletesResult = applyDeletes(true, deletes, null, newIndexReader); + transaction.ready(this); + if (transaction.go()) { + // if (deletesResult.getNumDeleted() > 0) { + // if (TransactionSystem.flushIndexReaderAfterDelete()) + // newIndexReader.flush(); + // } + boolean checkpoint = false; + int deleteThreshold = (int) (getSystem().getDeletesFlushThresholdPercent() * newIndexReader.maxDoc()); + if (deleteThreshold > 0 && deletesResult.getNumDeleted() > 0 && numDeleteTransactionsSinceLastFlush > deleteThreshold) { + if (LOG.isInfoEnabled()) + LOG.info("deletes flushed numDeleteTransactionsSinceLastFlush: " + numDeleteTransactionsSinceLastFlush + " threshold: " + + deleteThreshold); + newIndexReader.flush(); + checkpoint = true; + } + Long snapshotId = transaction.getId(); + if (deletesResult.getNumDeleted() > 0) { + numDeleteTransactionsSinceLastFlush++; + lastAppliedSnapshotIds.add(snapshotId); + // if flush was called check to make sure there is a new generation + // for the indexreader + LOG.info("previous reader gen: " + previousIndexReader.getIndexCommit().getGeneration() + " newIndexReader gen: " + + newIndexReader.getIndexCommit().getGeneration()); + } + createNewSnapshot(snapshotId, checkpoint, newIndexReader); + removeOldSnapshots(indexSnapshotMap); + return deletesResult; + } else { + rollback(transaction.getId()); + return null; + } + } catch (Throwable throwable) { + LOG.error("error in " + DirectoryIndex.this.getId(), throwable); + transaction.failed(this, throwable); + // rollback(transaction.getId()); + return null; + } + } + + protected DirectoryIndexSnapshot createNewSnapshot(Long snapshotId, boolean checkpoint, IndexReader newIndexReader) throws IOException { + if (newIndexReader == null) + newIndexReader = this.initialIndexReader; + if (checkpoint) { + assert (newIndexReader.maxDoc() - newIndexReader.numDocs()) >= 0; + numDeleteTransactionsSinceLastFlush = 0; + checkpointSnapshotIds.add(snapshotId); + } + return doCreateNewSnapshot(snapshotId, newIndexReader); + } + + protected abstract DirectoryIndexSnapshot doCreateNewSnapshot(Long snapshotId, IndexReader newIndexReader) throws IOException; + + protected void registerSnapshot(DirectoryIndexSnapshot indexSnapshot) throws IOException { + indexSnapshotMap.put(indexSnapshot.getSnapshotId(), indexSnapshot); + } + + public abstract Directory getDirectory(); +} Index: ocean/src/org/apache/lucene/ocean/DirectoryMap.java =================================================================== --- ocean/src/org/apache/lucene/ocean/DirectoryMap.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/DirectoryMap.java (revision 0) @@ -0,0 +1,19 @@ +package org.apache.lucene.ocean; + +import java.io.IOException; + +import org.apache.lucene.store.Directory; + +public abstract class DirectoryMap { + public abstract Directory create(String name) throws IOException; + + public abstract void delete(String name) throws IOException; + + public abstract Directory get(String name) throws IOException; + + public abstract String[] list() throws IOException; + + public abstract LogDirectory getDirectory(); + + public abstract LogDirectory getLogDirectory(); +} Index: ocean/src/org/apache/lucene/ocean/DiskIndex.java =================================================================== --- ocean/src/org/apache/lucene/ocean/DiskIndex.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/DiskIndex.java (revision 0) @@ -0,0 +1,224 @@ +package org.apache.lucene.ocean; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.logging.Logger; + +import org.apache.lucene.index.IndexCommit; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.OceanSegmentReader; +import org.apache.lucene.index.SerialMergeScheduler; +import org.apache.lucene.ocean.SnapshotInfo.IndexInfo; +import org.apache.lucene.ocean.util.Util; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.store.RAMDirectory; + +/** + * On disk index. Only deletes are allowed to occur to the index. + * There is a unique IndexReader per snapshot. + * + */ +public class DiskIndex extends DirectoryIndex { + public static Logger log = Logger.getLogger(DiskIndex.class.getName()); + private final Directory directory; + //private IndexInfo indexInfo; + + public DiskIndex(IndexID id, Directory directory, TransactionSystem system) throws Exception, IndexException, IOException { + super(id, system); + //assert segmentGeneration != null; + this.directory = directory; + if (directory.fileExists("writing.index")) { + throw new IndexNeverCompletedCopyException("index never completed copying"); + } + if (IndexWriter.isLocked(directory)) { + LOG.info("directory: "+directory+" locked. being unlocked"); + IndexWriter.unlock(directory); + } + initialIndexReader = IndexReader.open(directory, indexDeletionPolicy); + long readerGeneration = initialIndexReader.getIndexCommit().getGeneration(); + //indexInfo = loadIndexInfo(); + //createNewSnapshot(snapshotId, false, initialIndexReader); + } + + // load existing index + public DiskIndex(IndexID id, Directory directory, Long snapshotId, IndexInfo indexInfo, TransactionSystem system) throws Exception, IndexException, IOException { + super(id, system); + if (indexInfo.segmentGeneration == null) { + throw new IndexException(id+" indexInfo.segmentGeneration null"); + } + assert indexInfo.segmentGeneration != null; + this.directory = directory; + this.mergedfromIndexIds.addAll(mergedfromIndexIds); + this.checkpointSnapshotIds.add(indexInfo.checkpointId); + this.lastAppliedSnapshotIds.add(indexInfo.lastAppliedId); + if (directory.fileExists("writing.index")) { + throw new IndexNeverCompletedCopyException("index never completed copying"); + } + if (IndexWriter.isLocked(directory)) { + LOG.info("directory: "+directory+" locked. being unlocked"); + IndexWriter.unlock(directory); + } + initialIndexReader = IndexReader.open(directory, indexDeletionPolicy); + long readerGeneration = initialIndexReader.getIndexCommit().getGeneration(); + assert indexInfo.segmentGeneration.longValue() == readerGeneration; + //indexInfo = loadIndexInfo(); + // TODO: delete generations above the one known in indexinfo + Iterator> commitPointIterator = indexDeletionPolicy.commitPoints.entrySet().iterator(); + while (commitPointIterator.hasNext()) { + Map.Entry entry = commitPointIterator.next(); + IndexCommit indexCommit = entry.getValue(); + if (indexCommit.getGeneration() > indexInfo.getSegmentGeneration().longValue()) { + LOG.info(((FSDirectory)directory).getFile().getName()+" deleting generation: "+indexCommit.getGeneration()); + } + } + createNewSnapshot(snapshotId, false, initialIndexReader); + } + + // merge indexes creating new index + public DiskIndex(IndexID id, Directory directory, List indexSnapshots, TransactionSystem system) throws Exception, IOException { + super(id, system); + this.directory = directory; + // TODO: capture exceptions here and throw exception with indexsnapshots info + Util.touchFile("writing.index", directory); + IndexReader[] indexReaders = getIndexReaders(indexSnapshots); + // create in ram first, is faster than copying to disk due to less hard disk head movement + int diskIndexRAMDirectoryBufferSize = system.getDiskIndexRAMDirectoryBufferSize(); + RAMDirectory ramDirectory = new RAMDirectory(diskIndexRAMDirectoryBufferSize); + IndexWriter indexWriter = new IndexWriter(ramDirectory, false, system.getDefaultAnalyzer(), true); + indexWriter.setMaxBufferedDocs(Integer.MAX_VALUE); + indexWriter.setRAMBufferSizeMB(IndexWriter.DISABLE_AUTO_FLUSH); + indexWriter.setMergeScheduler(new SerialMergeScheduler()); + indexWriter.setUseCompoundFile(true); + indexWriter.addIndexes(indexReaders); + indexWriter.close(); + + Util.copy(ramDirectory, directory); // copy out 1 megabyte at a time + + //indexInfo = new IndexInfo(); + //indexInfo.setMaxDocumentID(maxDocumentId); + //indexInfo.setMaxSnapshotID(maxSnapshotId); + //saveIndexInfo(indexInfo); + directory.deleteFile("writing.index"); + addMergedFromIndexIds(indexSnapshots); + initialIndexReader = IndexReader.open(directory, indexDeletionPolicy); + } + + public Directory getDirectory() { + return directory; + } + /** + private IndexInfo loadIndexInfo() throws Exception { + String xml = Util.getString("indexinfo.xml", directory); + Element element = XMLUtil.parseElement(xml); + return new IndexInfo(element); + } + + private void saveIndexInfo(IndexInfo indexInfo) throws Exception { + Element element = indexInfo.toElement(); + String xml = XMLUtil.outputElement(element); + Util.save(xml, "indexinfo.xml", directory); + } + + public static class IndexInfo implements CElement { + private Long maxSnapshotId; + private Long maxDocumentId; + + public IndexInfo() { + } + + public Long getMaxSnapshotID() { + return maxSnapshotId; + } + + public void setMaxSnapshotID(Long maxSnapshotId) { + this.maxSnapshotId = maxSnapshotId; + } + + public Long getMaxDocumentID() { + return maxDocumentId; + } + + public void setMaxDocumentID(Long maxDocumentId) { + this.maxDocumentId = maxDocumentId; + } + + public IndexInfo(Element element) throws Exception { + maxSnapshotId = XMLUtil.getAttributeLong("maxSnapshotId", element); + maxDocumentId = XMLUtil.getAttributeLong("maxDocumentId", element); + } + + public Element toElement() throws Exception { + Element element = new Element("indexinfo"); + XMLUtil.setAttribute("maxSnapshotId", maxSnapshotId, element); + XMLUtil.setAttribute("maxDocumentId", maxDocumentId, element); + return element; + } + } + **/ + public boolean hasTooManyDeletedDocs(double percent) { + assert percent <= 1.0; + DirectoryIndexSnapshot indexSnapshot = getLatestIndexSnapshot(); + if (indexSnapshot != null) { + IndexReader indexReader = indexSnapshot.getIndexReader(); + int maxDoc = indexReader.maxDoc(); + int deletedDocs = maxDoc - indexReader.numDocs(); + if (deletedDocs > (maxDoc * percent)) + return true; + } + return false; + } + + public class DiskIndexSnapshot extends DirectoryIndexSnapshot { + private Collection indexReaderFileNames; + + public DiskIndexSnapshot(Long snapshotID, OceanSegmentReader indexReader, Collection indexReaderFileNames) throws IOException { + super(snapshotID, indexReader); + this.indexReaderFileNames = indexReaderFileNames; + } + + //public Long getMaxSnapshotId() throws IOException { + // return indexInfo.getMaxSnapshotID(); + //} + + //public Long getMaxDocumentId() throws IOException { + // return indexInfo.getMaxDocumentID(); + //} + + public void delete() throws Exception { + super.delete(); + deleteFiles(); + } + + public boolean hasRef() throws Exception { + return getSystem().getSnapshots().contains(snapshotId); + } + + public void deleteFiles() throws IOException { + } + + public List getFiles() throws Exception { + List files = new ArrayList(); + for (String fileName : indexReaderFileNames) { + files.add(fileName); + } + return files; + } + } + + protected void onCommit() throws Exception { + } + + protected DiskIndexSnapshot doCreateNewSnapshot(Long snapshotId, IndexReader newIndexReader) throws IOException { + IndexCommit indexCommit = newIndexReader.getIndexCommit(); + Collection fileNames = indexCommit.getFileNames(); + DiskIndexSnapshot diskIndexSnapshot = new DiskIndexSnapshot(snapshotId, (OceanSegmentReader)newIndexReader, fileNames); + registerSnapshot(diskIndexSnapshot); + return diskIndexSnapshot; + } +} Index: ocean/src/org/apache/lucene/ocean/Documents.java =================================================================== --- ocean/src/org/apache/lucene/ocean/Documents.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/Documents.java (revision 0) @@ -0,0 +1,35 @@ +package org.apache.lucene.ocean; + +import java.io.Reader; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Fieldable; + +public class Documents extends ArrayList { + + public Documents() {} + + public Documents(Collection documents) { + for (Document document : documents) { + add(document); + } + } + + public boolean hasFieldsWithTokenStreamOrReader() { + for (Document document : this) { + List fields = document.getFields(); + for (int x=0; x < fields.size(); x++) { + Fieldable fieldable = (Fieldable)fields.get(x); + TokenStream tokenStream = fieldable.tokenStreamValue(); + if (tokenStream != null) return true; + Reader reader = fieldable.readerValue(); + if (reader != null) return true; + } + } + return false; + } +} Index: ocean/src/org/apache/lucene/ocean/FSDirectoryMap.java =================================================================== --- ocean/src/org/apache/lucene/ocean/FSDirectoryMap.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/FSDirectoryMap.java (revision 0) @@ -0,0 +1,120 @@ +package org.apache.lucene.ocean; + +import java.io.File; +import java.io.IOException; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.io.FileUtils; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.ocean.util.Util; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.store.Lock; +import org.apache.lucene.store.LockObtainFailedException; +import org.apache.lucene.store.NativeFSLockFactory; + +public class FSDirectoryMap extends DirectoryMap { + public static final String WRITE_LOCK_NAME = "write.lock"; + private Map map = new HashMap(); + private File fileDirectory; + private LogDirectory rootDirectory; + private LogDirectory logDirectory; + private static MessageDigest DIGESTER; + private static final char[] HEX_DIGITS = + {'0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f'}; + private NativeFSLockFactory lockFactory; + + static { + try { + DIGESTER = MessageDigest.getInstance("MD5"); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e.toString(), e); + } + } + + public FSDirectoryMap(File fileDirectory, String logDirectoryName) throws IOException { + this.fileDirectory = fileDirectory; + Util.mkdir(fileDirectory); + + lockFactory = new NativeFSLockFactory(fileDirectory); + //lockFactory.clearLock(WRITE_LOCK_NAME); + lockFactory.setLockPrefix(getLockID()); + Lock lock = lockFactory.makeLock(WRITE_LOCK_NAME); + boolean obtained = lock.obtain(1000*5); + System.out.println("lock obtained: "+obtained); + if (!obtained) throw new LockObtainFailedException("Index locked for write: " + lock); + + rootDirectory = new FSLogDirectory(fileDirectory); + for (File file : fileDirectory.listFiles()) { + if (file.isDirectory() && !file.getName().equals(logDirectoryName)) { + FSDirectory dir = FSDirectory.getDirectory(file); + map.put(file.getName(), dir); + } + } + logDirectory = new FSLogDirectory(new File(fileDirectory, logDirectoryName)); + } + + public String getLockID() { + String dirName; // name to be hashed + try { + dirName = fileDirectory.getCanonicalPath(); + } catch (IOException e) { + throw new RuntimeException(e.toString(), e); + } + + byte digest[]; + synchronized (DIGESTER) { + digest = DIGESTER.digest(dirName.getBytes()); + } + StringBuilder buf = new StringBuilder(); + buf.append("ocean-"); + for (int i = 0; i < digest.length; i++) { + int b = digest[i]; + buf.append(HEX_DIGITS[(b >> 4) & 0xf]); + buf.append(HEX_DIGITS[b & 0xf]); + } + return buf.toString(); + } + + public LogDirectory getLogDirectory() { + return logDirectory; + } + + public LogDirectory getDirectory() { + return rootDirectory; + } + + public Directory create(String name) throws IOException { + Directory directory = FSDirectory.getDirectory(new File(fileDirectory, name)); + directory.setLockFactory(new NativeFSLockFactory(fileDirectory)); + IndexWriter.unlock(directory); + return directory; + } + + public void delete(String name) throws IOException { + FSDirectory directory = get(name); + directory.close(); + File file = directory.getFile(); + FileUtils.deleteDirectory(file); + map.remove(name); + } + + public FSDirectory get(String name) throws IOException { + return map.get(name); + } + + public String[] list() throws IOException { + int x = 0; + String[] array = new String[map.size()]; + for (String string : map.keySet()) { + array[x] = string; + x++; + } + return array; + } + + +} Index: ocean/src/org/apache/lucene/ocean/FSLogDirectory.java =================================================================== --- ocean/src/org/apache/lucene/ocean/FSLogDirectory.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/FSLogDirectory.java (revision 0) @@ -0,0 +1,85 @@ +package org.apache.lucene.ocean; + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.locks.ReentrantLock; + +import org.apache.commons.io.FileUtils; +import org.apache.lucene.ocean.util.OceanRandomAccessFile; +import org.apache.lucene.ocean.util.Util; + +public class FSLogDirectory extends LogDirectory { + private File fileDirectory; + private ReentrantLock outputLock = new ReentrantLock(); + private ReentrantLock inputLock = new ReentrantLock(); + + public FSLogDirectory(File fileDirectory) { + Util.mkdir(fileDirectory); + this.fileDirectory = fileDirectory; + } + + public String[] list() throws IOException { + List list = new ArrayList(); + for (File file : fileDirectory.listFiles()) { + if (!file.isDirectory()) { + list.add(file.getName()); + } + } + return (String[]) list.toArray(new String[0]); + } + + public boolean fileExists(String name) throws IOException { + File file = new File(fileDirectory, name); + return file.exists(); + } + + public long fileModified(String name) throws IOException { + File file = new File(fileDirectory, name); + return file.lastModified(); + } + + public boolean deleteFile(String name) throws IOException { + File file = new File(fileDirectory, name); + if (file.isDirectory()) { + FileUtils.deleteDirectory(file); + return file.exists(); + } else { + boolean deleted = file.delete(); + return deleted; + } + //if (!deleted) { + //throw new IOException("file: "+name+" not deleted"); + //} + } + + public long fileLength(String name) throws IOException { + File file = new File(fileDirectory, name); + return file.length(); + } + + public OceanRandomAccessFile openInput(String name) throws IOException { + inputLock.lock(); + try { + File file = new File(fileDirectory, name); + return new OceanRandomAccessFile(file, "r"); + } finally { + inputLock.unlock(); + } + } + + public OceanRandomAccessFile getOutput(String name, boolean overwrite) throws IOException { + outputLock.lock(); + try { + File file = new File(fileDirectory, name); + if (overwrite) { + file.delete(); + } + return new OceanRandomAccessFile(file, "rw"); + } finally { + outputLock.unlock(); + } + } +} Index: ocean/src/org/apache/lucene/ocean/Index.java =================================================================== --- ocean/src/org/apache/lucene/ocean/Index.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/Index.java (revision 0) @@ -0,0 +1,424 @@ +package org.apache.lucene.ocean; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; + +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.StaleReaderException; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermDocs; +import org.apache.lucene.index.TermEnum; +import org.apache.lucene.ocean.Deletes.DeleteByQuery; +import org.apache.lucene.ocean.util.Constants; +import org.apache.lucene.ocean.util.SortedList; +import org.apache.lucene.ocean.util.Util; +import org.apache.lucene.search.ExtendedFieldCache; +import org.apache.lucene.search.HitCollector; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.Searchable; +import org.apache.lucene.search.ExtendedFieldCache.LongParser; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +// TODO: implement close, and isClosed +public abstract class Index { + final static Logger LOG = LoggerFactory.getLogger(Index.class); + private final IndexID id; + protected boolean isClosed = false; + private final TransactionSystem transactionSystem; + private boolean isReadOnly = false; + private boolean isDeleteOnly = false; + protected List mergedfromIndexIds = new ArrayList(); + + public Index(IndexID id, TransactionSystem transactionSystem) { + this.id = id; + this.transactionSystem = transactionSystem; + } + + public static interface IndexSnapshotSearchable extends Searchable { + public IndexSnapshot getIndexSnapshot(); + } + + public static class IndexSnapshotSearcher extends IndexSearcher implements IndexSnapshotSearchable { + private IndexSnapshot indexSnapshot; + + public IndexSnapshotSearcher(IndexReader indexReader, IndexSnapshot indexSnapshot) { + super(indexReader); + this.indexSnapshot = indexSnapshot; + } + + public IndexSnapshot getIndexSnapshot() { + return indexSnapshot; + } + } + + protected void addMergedFromIndexIds(Collection indexSnapshots) { + for (IndexSnapshot indexSnapshot : indexSnapshots) { + mergedfromIndexIds.add(indexSnapshot.getIndex().getId()); + } + } + + protected void addMergedFromIndexId(IndexSnapshot indexSnapshot) { + mergedfromIndexIds.add(indexSnapshot.getIndex().getId()); + } + + public List getMergedFromIndexIds() { + return mergedfromIndexIds; + } + + /** + * Remove indexsnapshots after N descending + * + * @param snapshotMap + * @throws Exception + */ + // TODO: check main snapshot + public void removeOldSnapshots(SortedList snapshotMap) throws Exception { + ListIterator> iterator = snapshotMap.endListIterator(); + int count = 0; + List toDelete = new ArrayList(); + while (iterator.hasPrevious()) { + Map.Entry entry = iterator.previous(); + if (count > 5) { + IndexSnapshot indexSnapshot = entry.getValue(); + Snapshots snapshots = transactionSystem.getSnapshots(); + Snapshot snapshot = snapshots.get(indexSnapshot.getSnapshotId()); + try { + int refCount = snapshot.refCount(); + if (!snapshots.hasRefs(indexSnapshot.getSnapshotId())) { + toDelete.add(indexSnapshot); + } + } finally { + snapshot.decRef(); + } + // indexSnapshot.delete(); + } + count++; + } + for (IndexSnapshot indexSnapshot : toDelete) { + indexSnapshot.delete(); + } + /** + * Long last = snapshotMap.lastKey(); for (Iterator iterator = + * snapshotMap.keySet().iterator(); iterator.hasNext();) { Long snapshotId = + * iterator.next(); if (!transactionSystem.snapshots.contains(snapshotId) && + * (last != null && !last.equals(snapshotId))) { iterator.remove(); } } + */ + } + + public void close() throws IOException { + } + + public TransactionSystem getSystem() { + return transactionSystem; + } + + public static IndexReader[] getIndexReaders(List indexSnapshots) { + IndexReader[] indexReaders = new IndexReader[indexSnapshots.size()]; + for (int x = 0; x < indexSnapshots.size(); x++) { + indexReaders[x] = indexSnapshots.get(x).getIndexReader(); + } + return indexReaders; + } + + public static Long getMaxSnapshotId(List indexSnapshots) throws Exception { + List snapshotIdList = new ArrayList(indexSnapshots.size()); + for (IndexSnapshot indexSnapshot : indexSnapshots) { + snapshotIdList.add(indexSnapshot.getMaxSnapshotId()); + } + if (snapshotIdList.size() > 0) { + return Collections.max(snapshotIdList); + } else { + return null; + } + } + + public static Long getMaxDocumentId(List indexSnapshots) throws Exception { + List documentIdList = new ArrayList(indexSnapshots.size()); + for (IndexSnapshot indexSnapshot : indexSnapshots) { + documentIdList.add(indexSnapshot.getMaxDocumentId()); + } + if (documentIdList.size() > 0) { + return Collections.max(documentIdList); + } else { + return null; + } + } + + public void setDeleteOnly(boolean isDeleteOnly) { + this.isDeleteOnly = isDeleteOnly; + } + + public boolean isDeleteOnly() { + return isDeleteOnly; + } + + public boolean isReadOnly() { + return isReadOnly; + } + + public void setReadOnly(boolean isReadOnly) { + this.isReadOnly = isReadOnly; + } + + protected DeletesResult applyDeletes(boolean deleteFromReader, Deletes deletes, Collection deletedDocs, IndexReader indexReader) + throws CorruptIndexException, IOException, Exception { + long[] ids = ExtendedFieldCache.EXT_DEFAULT.getLongs(indexReader, Constants.DOCUMENTID, new LongParser() { + public long parseLong(String string) { + return Util.longFromEncoded(string); + } + }); + DeletesResult deletesResult = new DeletesResult(getId()); + if (deletes.hasDeletes()) { + if (deletes.hasDocIds()) { + int docsDeleted = 0; + long[] docIdsArray = deletes.getDocIds(); + for (long id : docIdsArray) { + int doc = Util.getDoc(Constants.DOCUMENTID, id, indexReader); + if (doc >= 0) { + if (deleteFromReader) + indexReader.deleteDocument(doc); + if (deletedDocs != null) + deletedDocs.add(doc); + docsDeleted++; + } + } + deletesResult.add(new DeletesResult.Result(docIdsArray, docsDeleted)); + } else { + List docIds = deletesResult.getDocIds(); + if (deletes.hasTerms()) { + List terms = deletes.getTerms(); + for (Term term : terms) { + int docsDeleted = deleteByTerm(deleteFromReader, term, deletedDocs, docIds, ids, indexReader); + + deletesResult.add(new DeletesResult.Result(term, docsDeleted)); + } + } + if (deletes.hasDeleteByQueries()) { + List deleteByQueries = deletes.getDeleteByQueries(); + for (DeleteByQuery deleteByQuery : deleteByQueries) { + int docsDeleted = deleteByQuery(deleteFromReader, deleteByQuery, deletedDocs, ids, docIds, indexReader); + deletesResult.add(new DeletesResult.Result(deleteByQuery, docsDeleted)); + } + } + } + } + return deletesResult; + } + + protected int deleteByTerm(boolean deleteFromReader, Term term, Collection deletedDocs, List docIds, long[] ids, + IndexReader indexReader) throws IOException { + TermDocs docs = indexReader.termDocs(term); + if (docs == null) + return 0; + int n = 0; + try { + while (docs.next()) { + int doc = docs.doc(); + if (deletedDocs != null) + deletedDocs.add(doc); + Long docId = ids[doc]; + docIds.add(docId); + if (deleteFromReader) + indexReader.deleteDocument(doc); + n++; + } + } finally { + docs.close(); + } + return n; + } + + protected int deleteByQuery(final boolean deleteFromReader, DeleteByQuery deleteByQuery, final Collection deletedDocs, + final long[] ids, final List deletedIds, final IndexReader indexReader) throws IOException { + Query query = deleteByQuery.getQuery(); + IndexSearcher indexSearcher = new IndexSearcher(indexReader); + final int[] numDeleted = new int[1]; + indexSearcher.search(query, new HitCollector() { + public void collect(int doc, float score) { + try { + if (deleteFromReader) + indexReader.deleteDocument(doc); + if (deletedDocs != null) + deletedDocs.add(doc); + Long docId = ids[doc]; + deletedIds.add(docId); + numDeleted[0]++; + } catch (StaleReaderException staleReaderException) { + throw new RuntimeException(staleReaderException); + } catch (IOException ioException) { + throw new RuntimeException(ioException); + } + } + }); + return numDeleted[0]; + } + + public static class IndexException extends Exception { + public IndexException(String message) { + super(message); + } + + public IndexException(String message, Exception exception) { + super(message, exception); + } + } + + public static class IndexNeverCompletedCopyException extends IndexException { + public IndexNeverCompletedCopyException(String message) { + super(message); + } + } + + public abstract boolean rollback(Long snapshotId) throws Exception; + + public abstract IndexSnapshot getIndexSnapshot(Long snapshotID); + + public abstract IndexSnapshot getLatestIndexSnapshot(); + + public abstract class IndexSnapshot { + protected final Long snapshotId; + + public IndexSnapshot(Long snapshotId) { + this.snapshotId = snapshotId; + } + + public IndexSnapshotSearchable getSearcher() { + return new IndexSnapshotSearcher(getIndexReader(), this); + } + + public abstract void delete() throws Exception; + + public abstract Long getMinDocumentId() throws IOException; + + public abstract Long getMinSnapshotId() throws IOException; + + public abstract Long getMaxSnapshotId() throws IOException; + + public abstract Long getMaxDocumentId() throws IOException; + + public abstract int deletedDoc(); + + public abstract int maxDoc(); + + public Long getSnapshotId() { + return snapshotId; + } + + public Index getIndex() { + return Index.this; + } + + /** + * Iterates terms until next field is reached, returns text + * + * @param field + * @return + * @throws Exception + */ + public String getMax(String field) throws IOException { + IndexReader indexReader = getIndexReader(); + TermEnum termEnum = indexReader.terms(new Term(field, "")); + try { + String text = null; + do { + Term term = termEnum.term(); + if (term == null || term.field() != field) + break; + text = term.text(); + } while (termEnum.next()); + return text; + } finally { + termEnum.close(); + } + } + + public String getMin(String field) throws IOException { + IndexReader indexReader = getIndexReader(); + TermEnum termEnum = indexReader.terms(new Term(field, "")); + try { + do { + Term term = termEnum.term(); + if (term == null || term.field() != field) + break; + return term.text(); + } while (termEnum.next()); + return null; + } finally { + termEnum.close(); + } + } + + public abstract IndexReader getIndexReader(); + } + + public static class MergedDocMap { + private Map oldMap; // maps old doc to new doc + private RI[] merged; // maps new doc to old doc and reader + + public MergedDocMap(List indexSnapshots) { + int newMaxDoc = 0; + for (IndexSnapshot indexSnapshot : indexSnapshots) { + newMaxDoc += indexSnapshot.getIndexReader().numDocs(); + } + oldMap = new HashMap(indexSnapshots.size()); + RI[] merged = new RI[newMaxDoc]; + int pos = 0; + for (IndexSnapshot indexSnapshot : indexSnapshots) { + IndexReader indexReader = indexSnapshot.getIndexReader(); + int maxDoc = indexReader.maxDoc(); + int[] old = new int[maxDoc]; + for (int x = 0; x < maxDoc; x++) { + if (indexReader.hasDeletions() && indexReader.isDeleted(x)) { + merged[pos] = null; + old[x] = -1; + } else { + merged[pos] = new RI(x, indexSnapshot); + old[x] = pos; + pos++; + } + } + oldMap.put(indexSnapshot, old); + } + } + + public static class RI { + public int doc; + public IndexSnapshot oldIndexSnapshot; + + public RI(int doc, IndexSnapshot oldIndexSnapshot) { + this.doc = doc; + this.oldIndexSnapshot = oldIndexSnapshot; + } + } + + public Map getOldMap() { + return oldMap; + } + + public RI[] getMerged() { + return merged; + } + } + + public boolean isClosed() { + return isClosed; + } + + public IndexID getId() { + return id; + } + + public abstract void commitNothing(Transaction transaction) throws IndexException, InterruptedException, IOException, Exception; + + public abstract DeletesResult commitDeletes(Deletes deletes, Transaction transaction) throws Exception, IndexException, + InterruptedException, IOException; +} Index: ocean/src/org/apache/lucene/ocean/IndexCreator.java =================================================================== --- ocean/src/org/apache/lucene/ocean/IndexCreator.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/IndexCreator.java (revision 0) @@ -0,0 +1,188 @@ +package org.apache.lucene.ocean; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.SerialMergeScheduler; +import org.apache.lucene.ocean.log.RawLogFile; +import org.apache.lucene.store.Directory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Allows creation of an index using multiple threads by feeding documents into + * a BlockingQueue. + * + */ +// TODO: after create called make object unusable +public class IndexCreator { + final static Logger LOG = LoggerFactory.getLogger(IndexCreator.class); + //private Directory directory; + private Long maxSize; + private int threads; + private ExecutorService threadPool; + private IndexWriter indexWriter; + private boolean isFinished; + private List> futures = new ArrayList>(); + //private Analyzer analyzer; + private BlockingQueue queue; + private int numAdded = 0; + private String name; + + public IndexCreator(String name, Directory directory, long maxSize, int threads, Analyzer analyzer, ExecutorService threadPool) throws IOException { + //this.directory = directory; + this.name = name; + this.maxSize = maxSize; + this.threads = threads; + this.threadPool = threadPool; + isFinished = false; + indexWriter = new IndexWriter(directory, false, analyzer, true); + indexWriter.setUseCompoundFile(false); + indexWriter.setMergeScheduler(new SerialMergeScheduler()); + indexWriter.setMaxBufferedDocs(Integer.MAX_VALUE); + indexWriter.setRAMBufferSizeMB(IndexWriter.DISABLE_AUTO_FLUSH); + } + + public IndexCreator(IndexWriter indexWriter, int threads, ExecutorService threadPool) throws IOException { + this.indexWriter = indexWriter; + this.threads = threads; + this.threadPool = threadPool; + isFinished = false; + } + + public void close() throws Exception { + indexWriter.close(); + setFinished(true); + } + + public int getNumAdded() { + return numAdded; + } + + public static class Add { + private final Document document; + private final Analyzer analyzer; + + // private RAMDirectory ramDirectory; + + // public Add(RAMDirectory ramDirectory) { + // this.ramDirectory = ramDirectory; + // } + + public Add(Document document) { + this.document = document; + this.analyzer = null; + } + + public Add(Document document, Analyzer analyzer) { + this.document = document; + this.analyzer = analyzer; + } + + public Analyzer getAnalyzer() { + return analyzer; + } + + // public RAMDirectory getRamDirectory() { + // return ramDirectory; + // } + + public Document getDocument() { + return document; + } + } + + public void start(BlockingQueue queue) throws Exception { + this.queue = queue; + + // set impossibly high to never be + // triggered, setting both to + // DISABLE_AUTO_FLUSH causes an + // exception + // List> callables = new + // ArrayList>(threads); + for (int x = 0; x < threads; x++) { + // callables.add(new Task(queue)); + futures.add(threadPool.submit(new Task())); + } + // futures = threadPool.invokeAll(callables); + + } + + public void create(boolean optimize) throws Exception { + while (queue.peek() != null) { + Thread.sleep(5); + } + setFinished(true); + try { + for (Future future : futures) { + if (future.isDone()) { + try { + future.get(); + } catch (ExecutionException executionException) { + Throwable cause = executionException.getCause(); + if (cause instanceof Exception) + throw (Exception) cause; + else + throw new Exception(cause); + } + } + Thread.sleep(10); + } + if (optimize) indexWriter.optimize(); // should not be necessary + indexWriter.commit(); + numAdded = indexWriter.numDocs(); + } finally { + indexWriter.close(); + } + } + + public void setFinished(boolean isFinished) { + this.isFinished = isFinished; + } + + private boolean isFinished() { + if (isFinished) + return true; + if (indexWriter.ramSizeInBytes() >= maxSize) { + isFinished = true; + } + return isFinished; + } + + public class Task implements Callable { + //private BlockingQueue queue; + + //public Task(BlockingQueue queue) { + // this.queue = queue; + //} + + public Object call() throws Exception { + while (!isFinished()) { + Add add = queue.poll(); + if (add != null) { + Document document = add.getDocument(); + Analyzer analyzer = add.getAnalyzer(); + if (analyzer == null) { + indexWriter.addDocument(document); + } else { + indexWriter.addDocument(document, analyzer); + } + } else { + Thread.sleep(5); + } + } + return null; + } + } +} Index: ocean/src/org/apache/lucene/ocean/Indexes.java =================================================================== --- ocean/src/org/apache/lucene/ocean/Indexes.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/Indexes.java (revision 0) @@ -0,0 +1,38 @@ +package org.apache.lucene.ocean; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.lucene.ocean.util.Util; + +public class Indexes { + private Map indexMap = new HashMap(); + + public IndexID getMaxId(String type) { + List list = new ArrayList(); + for (IndexID indexId : indexMap.keySet()) { + if (indexId.type.equals(type)) { + list.add(indexId); + } + } + if (list.size() == 0) return null; + return Util.max(list); + } + + public List getIndexes() { + return new ArrayList(indexMap.values()); + } + + public Indexes() { + } + + public Index get(IndexID indexId) { + return indexMap.get(indexId); + } + + public void add(Index index) { + indexMap.put(index.getId(), index); + } +} Index: ocean/src/org/apache/lucene/ocean/IndexID.java =================================================================== --- ocean/src/org/apache/lucene/ocean/IndexID.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/IndexID.java (revision 0) @@ -0,0 +1,52 @@ +package org.apache.lucene.ocean; + +import org.apache.commons.lang.builder.CompareToBuilder; + +public class IndexID implements Comparable { + public final Long id; + public final String type; + + public IndexID(Long id, String type) { + this.id = id; + this.type = type; + } + + public String toString() { + return type+":"+id; + } + + public int compareTo(IndexID other) { + return new CompareToBuilder().append(id, other.id).append(other.type, type).toComparison(); + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((id == null) ? 0 : id.hashCode()); + result = prime * result + ((type == null) ? 0 : type.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + final IndexID other = (IndexID) obj; + if (id == null) { + if (other.id != null) + return false; + } else if (!id.equals(other.id)) + return false; + if (type == null) { + if (other.type != null) + return false; + } else if (!type.equals(other.type)) + return false; + return true; + } +} Index: ocean/src/org/apache/lucene/ocean/LargeBatchTransaction.java =================================================================== --- ocean/src/org/apache/lucene/ocean/LargeBatchTransaction.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/LargeBatchTransaction.java (revision 0) @@ -0,0 +1,36 @@ +package org.apache.lucene.ocean; + +public class LargeBatchTransaction extends Transaction { + private Long id; + private Snapshot previousSnapshot; + + public LargeBatchTransaction(Long id, Snapshot previousSnapshot) { + this.id = id; + this.previousSnapshot = previousSnapshot; + } + + public Snapshot getPreviousSnapshot() { + return previousSnapshot; + } + + // for now doesn't need to do anything + public CommitResult call() throws Exception { + return null; + } + + public boolean go() { + return true; + } + + public void ready(Index index) { + + } + + public Long getId() { + return id; + } + + public void failed(Index index, Throwable throwable) { + + } +} Index: ocean/src/org/apache/lucene/ocean/log/LogFile.java =================================================================== --- ocean/src/org/apache/lucene/ocean/log/LogFile.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/log/LogFile.java (revision 0) @@ -0,0 +1,253 @@ +package org.apache.lucene.ocean.log; + +import java.io.EOFException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +import org.apache.lucene.ocean.LogDirectory; +import org.apache.lucene.ocean.log.RawLogFile.CRCException; +import org.apache.lucene.ocean.log.RawLogFile.FileStreamRecord; +import org.apache.lucene.ocean.log.RawLogFile.StreamData; +import org.apache.lucene.ocean.log.RawLogFile.StreamRecord; +import org.apache.lucene.ocean.util.OceanRandomAccessFile; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Log file. Contains all record headers for loading of the actual record. + * + */ +public class LogFile implements Comparable { + final static Logger LOG = LoggerFactory.getLogger(LogFile.class); + private List recordHeaders; // sorted by id + private ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); + private RawLogFile rawLogFile; + private Long id; + + LogFile(Long id) { + this.id = id; + } + + public LogFile(Long id, String file, LogDirectory logDirectory) throws IOException { + this.id = id; + rawLogFile = new RawLogFile(file, logDirectory); + recordHeaders = rawLogFile.loadRecordHeaders(); + if (recordHeaders.size() > 0) { + // check to make sure last record was completely written + // by checking the crc32 + RecordHeader last = recordHeaders.get(recordHeaders.size()-1); + try { + checkCrc(last.id); + } catch (CRCException crcException) { + handleLastHeaderFailure(); + } catch (EOFException eofException) { + handleLastHeaderFailure(); + } + } + } + + private void handleLastHeaderFailure() { + // last one did not make it, remove it + RecordHeader last = recordHeaders.get(recordHeaders.size()-1); + LOG.info("removing last record "+last.id+" as it did not make it"); + recordHeaders.remove(recordHeaders.size()-1); + // reset to position after previous known header + rawLogFile.resetWritePosition(recordHeaders); + } + + public String getName() { + return rawLogFile.getName(); + } + + public int getNumRecords() { + return recordHeaders.size(); + } + + public boolean delete() throws IOException { + close(); + return rawLogFile.delete(); + } + + public void close() throws IOException { + rawLogFile.close(); + } + + public long getPreviousId(long id) { + long pos = getPosition(id); + if (pos == 0) return -1; + return pos-1; + } + + public int compareTo(LogFile other) { + return (id < other.id ? -1 : (id == other.id ? 0 : 1)); + } + + private void add(RecordHeader recordHeader) { + recordHeaders.add(recordHeader); + } + + public long size() throws IOException { + return rawLogFile.size(); + } + + private int getPosition(long id) { + RecordHeader h = new RecordHeader(); + h.id = id; + int pos = Collections.binarySearch(recordHeaders, h); + if (pos < 0) + pos = -1 - pos; + return pos; + } + + public void checkCrc(Long id) throws IOException { + RecordIterator iterator = getRecordIterator(id); + if (iterator.hasNext()) { + Record record = iterator.next(); + StreamRecord streamRecord = record.getStreamRecord(); + StreamData docData = streamRecord.getDocuments(); + if (docData != null) docData.verifyCrc(); + StreamData otherData = streamRecord.getOther(); + if (otherData != null) otherData.verifyCrc(); + } else { + throw new IOException("id: "+id+" does not exist"); + } + } + + public RecordHeader get(long id) { + if (recordHeaders.size() > 0) { + int pos = getPosition(id); + if (pos >= recordHeaders.size()) return null; + return recordHeaders.get(pos); + } else return null; + } + /** + public boolean delete(long id) throws IOException { + lock.writeLock().lock(); + try { + RecordHeader recordHeader = get(id); + boolean deleted = rawLogFile.delete(recordHeader); + if (deleted) { + int pos = getPosition(id); + recordHeaders.remove(pos); + } + return deleted; + } finally { + lock.writeLock().unlock(); + } + } + **/ + public long getId() { + return id; + } + + public long getMinId() { + if (recordHeaders.size() == 0) + return -1; + else + return recordHeaders.get(0).id; + } + + public boolean containsId(Long id) { + if (recordHeaders.size() == 0) return false; + return id <= getMaxId() && id >= getMinId(); + } + + public Long getMaxId() { + if (recordHeaders.size() == 0) return null; + return recordHeaders.get(recordHeaders.size() - 1).id; + } + + public List getRecordHeaders(long from, int num) { + int pos = getPosition(from); + int count = 0; + List recordHeaders = new ArrayList(num); + while (count < num) { + RecordHeader recordHeader = recordHeaders.get(pos); + recordHeaders.add(recordHeader); + count++; + } + return recordHeaders; + } + + public void commit(RecordHeader recordHeader) { + // TODO: commit + } + + public RecordHeader writeRecord(Long id, RecordData recordData) throws IOException { + lock.writeLock().lock(); + try { + RecordHeader recordHeader = rawLogFile.write(id, recordData.docType, recordData.docBytes, recordData.otherType, recordData.otherBytes); + add(recordHeader); + return recordHeader; + } finally { + lock.writeLock().unlock(); + } + } + + public RecordIterator getRecordIterator(Long snapshotId) throws IOException { + OceanRandomAccessFile input = rawLogFile.openInput(); + return new RecordIterator(snapshotId, input); + } + + public class RecordIterator { + private OceanRandomAccessFile input; + int pos; + + public RecordIterator(Long snapshotId, OceanRandomAccessFile input) throws IOException { + this.input = input; + if (snapshotId == null) { + pos = 0; + } else { + pos = getPosition(snapshotId); + } + } + + public Record next() throws IOException { + RecordHeader recordHeader = recordHeaders.get(pos); + if (LOG.isDebugEnabled()) LOG.debug("recordHeader: "+recordHeader); + pos++; + return getRecord(recordHeader); + } + + private Record getRecord(RecordHeader recordHeader) throws IOException { + FileStreamRecord fileStreamRecord = rawLogFile.readRecordData(recordHeader, input); + Record record = new Record(recordHeader.id, recordHeader, fileStreamRecord); + return record; + } + + public boolean hasNext() { + return pos < recordHeaders.size(); + } + + public void close() throws IOException { + input.close(); + } + } + + public static class Record { + private long id; + private StreamRecord streamRecord; + private RecordHeader recordHeader; + + public Record(long id, RecordHeader recordHeader, StreamRecord streamRecord) { + this.id = id; + this.recordHeader = recordHeader; + this.streamRecord = streamRecord; + } + + public RecordHeader getRecordHeader() { + return recordHeader; + } + + public Long getId() { + return id; + } + + public StreamRecord getStreamRecord() { + return streamRecord; + } + } +} Index: ocean/src/org/apache/lucene/ocean/log/LogFileManager.java =================================================================== --- ocean/src/org/apache/lucene/ocean/log/LogFileManager.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/log/LogFileManager.java (revision 0) @@ -0,0 +1,331 @@ +package org.apache.lucene.ocean.log; + +import java.io.IOException; +import java.text.DecimalFormat; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +import org.apache.lucene.ocean.LogDirectory; +import org.apache.lucene.ocean.log.LogFile.Record; +import org.apache.lucene.ocean.util.Bytes; +import org.apache.lucene.ocean.util.LongSequence; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Manages the log files. There is one active log file at a time that updates + * are written to. When the active log file reaches the MAX_FILE_SIZE a new + * active log file is created. + * + */ +// TODO: delete log files that are no longer needed +public class LogFileManager { + final static Logger LOG = LoggerFactory.getLogger(LogFileManager.class); + private List logFiles = new ArrayList(); + private LongSequence logIdSequence = new LongSequence(1, 1); + //private ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); + private ReentrantReadWriteLock logFilesLock = new ReentrantReadWriteLock(); + private ScheduledExecutorService logFileCheckTimer; + private LogDirectory logDirectory; + private long maxFileSize; + private long logFileSizeCheckDelayMillis; + + // TODO: load existing max snapshot id + public LogFileManager(long maxFileSize, long logFileSizeCheckDelayMillis, LogDirectory logDirectory) throws IOException { + this.logDirectory = logDirectory; + this.maxFileSize = maxFileSize; + this.logFileSizeCheckDelayMillis = logFileSizeCheckDelayMillis; + String[] list = logDirectory.list(); + List logFileNames = new ArrayList(list.length); + for (String file : list) { + long id = getLogFileNumber(file); + LogFile logFile = new LogFile(id, file, logDirectory); + if (logFiles.size() > 0) { + if (logFile.getNumRecords() == 0 || logFile.size() == 0) { + boolean deleted = logFile.delete(); + LOG.info("deleted: "+deleted+" logFile: "+logFile.getName()+" numrecords: "+logFile.getNumRecords()); + // do not add log file that is empty + // TODO: need to handle log file that has only 1 broken record as it is empty as well + continue; + } + } + logFiles.add(logFile); + logFileNames.add(file); + } + Collections.sort(logFiles); + if (logFiles.size() > 0) { + long last = logFiles.get(logFiles.size() - 1).getId(); + long next = last + 1; + logIdSequence.set(next); + } + LOG.info("initial log files: "+logFileNames); + logFileCheckTimer = Executors.newSingleThreadScheduledExecutor(); + logFileCheckTimer.scheduleWithFixedDelay(new LogFileSizeCheck(), 1000, logFileSizeCheckDelayMillis, TimeUnit.MILLISECONDS); + } + + /** + * Any log file with a maxSnapshotId less than minIndexSnapshotId can be + * deleted. This is because it means the index would no longer recover + * anything from the transaction log. + * + * @param minSnapshotId + */ + public void deleteOldLogFiles(Long minIndexSnapshotId) throws IOException { + List deletes = new ArrayList(); + logFilesLock.writeLock().lock(); + try { + Iterator iterator = logFiles.iterator(); + while (iterator.hasNext()) { + LogFile logFile = iterator.next(); + Long maxId = logFile.getMaxId(); + if (maxId != null) { + if (minIndexSnapshotId.longValue() > maxId.longValue()) { + if (LOG.isDebugEnabled()) LOG.info("minIndexSnapshotId:"+minIndexSnapshotId.longValue()+" maxLogFileId: "+maxId); + deletes.add(logFile); + iterator.remove(); + } + } + } + } finally { + logFilesLock.writeLock().unlock(); + } + for (LogFile deleteLogFile : deletes) { + boolean deleted = deleteLogFile.delete(); + if (LOG.isInfoEnabled()) LOG.info(deleteLogFile.getName()+" deleted: "+deleted); + } + } + + public int getNumRecords() { + logFilesLock.readLock().lock(); + try { + int num = 0; + for (LogFile logFile : logFiles) { + num += logFile.getNumRecords(); + } + return num; + } finally { + logFilesLock.readLock().unlock(); + } + } + + public void close() throws IOException { + for (LogFile logFile : logFiles) { + logFile.close(); + } + } + + private int getPosition(long id) { + LogFile lf = new LogFile(id); + int pos = Collections.binarySearch(logFiles, lf); + if (pos < 0) + pos = -1 - pos; + return pos; + } + + public Long getMinSnapshotId() { + if (logFiles.size() > 0) { + LogFile logFile = logFiles.get(0); + return logFile.getMinId(); + } else { + return null; + } + } + + public Long getMaxSnapshotId() { + if (logFiles.size() > 0) { + LogFile logFile = logFiles.get(logFiles.size() - 1); + return logFile.getMaxId(); + } else { + return null; + } + } + + public Long getMaxId() { + if (logFiles.size() == 0) + return null; + return logFiles.get(logFiles.size() - 1).getMaxId(); + } + + private LogFile getLast() { + if (logFiles.size() == 0) + return null; + return logFiles.get(logFiles.size() - 1); + } + + public class LogFileSizeCheck implements Runnable { + public void run() { + try { + LogFile logFile = getLast(); + if (logFile != null) { + if (logFile.size() >= maxFileSize) { + LogFile newLogFile = createNewLogFile(); + logFiles.add(newLogFile); + } + } + } catch (IOException ioException) { + LOG.error("", ioException); + } + } + } + /** + public boolean delete(Long id) throws Exception { + LogFile logFile = getLogFileContaining(id); + if (logFile == null) + throw new Exception("unknown id: " + id); + return logFile.delete(id); + } + **/ + public Long getPreviousId(Long id) { + LogFile logFile = getLogFileContaining(id); + if (logFile == null) + return null; + int pos = logFiles.indexOf(logFile); + Long previousId = logFile.getPreviousId(id); + if (previousId == null && pos > 0) { + logFile = logFiles.get(pos - 1); + previousId = logFile.getPreviousId(id); + } + return previousId; + } + + public long getMinId() { + if (logFiles.size() > 0) { + return logFiles.get(0).getMinId(); + } + return -1; + } + + public LogFile createNewLogFile() throws IOException { + long id = logIdSequence.getAndIncrement(); + String fileName = createLogFileName(id); + if (logDirectory.fileExists(fileName) && logDirectory.fileLength(fileName) > 0) { + throw new IOException(fileName+" exists"); + } + LOG.info("creating new log file: "+fileName); + LogFile logFile = new LogFile(id, fileName, logDirectory); + logFilesLock.writeLock().lock(); + try { + logFiles.add(logFile); + return logFile; + } finally { + logFilesLock.writeLock().unlock(); + } + } + + private static String createLogFileName(long value) { + DecimalFormat decimalFormat = (DecimalFormat) DecimalFormat.getInstance(); + decimalFormat.applyPattern("00000000"); + return "log" + decimalFormat.format(value) + ".bin"; + } + + private static long getLogFileNumber(String fileName) { + String numberString = fileName.substring(3, fileName.lastIndexOf('.')); + return Long.parseLong(numberString); + } + + private LogFile getLogFileContaining(Long id) { + logFilesLock.readLock().lock(); + try { + for (LogFile logFile : logFiles) { + if (logFile.containsId(id)) { + return logFile; + } + } + return null; + } finally { + logFilesLock.readLock().unlock(); + } + } + + public boolean contains(Long id) { + logFilesLock.readLock().lock(); + try { + for (LogFile logFile : logFiles) { + if (logFile.containsId(id)) { + return true; + } + } + return false; + } finally { + logFilesLock.readLock().unlock(); + } + } + + public RecordIterator getRecordIterator(Long snapshotId) throws IOException { + return new RecordIterator(snapshotId); + } + + public class RecordIterator { + private Iterator logFileIterator; + private LogFile.RecordIterator currentRecordIterator; + + public RecordIterator(Long snapshotId) throws IOException { + logFileIterator = logFiles.iterator(); + while (logFileIterator.hasNext()) { + LogFile logFile = logFileIterator.next(); + if (snapshotId == null || logFile.containsId(snapshotId)) { + currentRecordIterator = logFile.getRecordIterator(snapshotId); + } + } + } + + public void close() throws IOException { + if (currentRecordIterator != null) + currentRecordIterator.close(); + } + + public Record next() throws IOException { + if (currentRecordIterator != null) { + return currentRecordIterator.next(); + } else { + return null; + } + } + + public boolean hasNext() throws IOException { + if (currentRecordIterator == null) + return false; + if (currentRecordIterator.hasNext()) { + return true; + } else { + currentRecordIterator.close(); + if (logFileIterator.hasNext()) { + LogFile logFile = logFileIterator.next(); + currentRecordIterator = logFile.getRecordIterator(null); + return hasNext(); + } + } + return false; + } + } + + public void commit(RecordHeader recordHeader) { + // TODO: implement commit + } + + public RecordHeader writeRecord(Long id, RecordData recordData) throws IOException { + LogFile logFile = getCurrentLogFile(); + return logFile.writeRecord(id, recordData); + } + + public LogFile getCurrentLogFile() throws IOException { + logFilesLock.readLock().lock(); + try { + LogFile logFile = getLast(); + if (logFile == null) { + logFilesLock.readLock().unlock(); + logFile = createNewLogFile(); + logFilesLock.readLock().lock(); + } + return logFile; + } finally { + logFilesLock.readLock().unlock(); + } + } +} Index: ocean/src/org/apache/lucene/ocean/log/RawLogFile.java =================================================================== --- ocean/src/org/apache/lucene/ocean/log/RawLogFile.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/log/RawLogFile.java (revision 0) @@ -0,0 +1,445 @@ +package org.apache.lucene.ocean.log; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.ByteArrayInputStream; +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.locks.ReentrantLock; +import java.util.zip.CRC32; +import java.util.zip.CheckedInputStream; +import java.util.zip.CheckedOutputStream; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.io.output.ByteArrayOutputStream; +import org.apache.commons.io.output.CountingOutputStream; +import org.apache.lucene.ocean.LogDirectory; +import org.apache.lucene.ocean.util.Bytes; +import org.apache.lucene.ocean.util.OceanRandomAccessFile; +import org.apache.lucene.ocean.util.VDataInputStream; +import org.apache.lucene.ocean.util.VDataOutputStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Performs raw file access including serializing records to and from the + * underlying file + * + */ +// TODO: add version and some header info to file +public class RawLogFile { + final static Logger LOG = LoggerFactory.getLogger(RawLogFile.class); + public static final byte DELETED = 101; + public static final byte OK = 9; + // public static final byte[] HEADER = new byte[] { 'O', 'c', 'N' }; + private long currentWritePosition = 0; + private ReentrantLock writeLock = new ReentrantLock(); + private String file; + private OceanRandomAccessFile output; + private LogDirectory logDirectory; + private byte[] crcBuffer = new byte[1024]; + + public RawLogFile(String file, LogDirectory logDirectory) throws IOException { + this.file = file; + this.logDirectory = logDirectory; + output = logDirectory.getOutput(file, false); + // TODO: need better way to make sure we're starting at the end of the file + currentWritePosition = logDirectory.fileLength(file); + LOG.info("starting currentWritePosition: " + currentWritePosition); + } + + public void resetWritePosition(List recordHeaders) { + writeLock.lock(); + try { + if (recordHeaders.size() > 0) { + RecordHeader lastHeader = recordHeaders.get(recordHeaders.size() - 1); + currentWritePosition = lastHeader.headerPosition + lastHeader.headersLength + lastHeader.docsLength + lastHeader.otherLength; + } else { + currentWritePosition = 0; // TODO: change when file header added + } + } finally { + writeLock.unlock(); + } + } + + public String getName() { + return file; + } + + public boolean delete() throws IOException { + return logDirectory.deleteFile(file); + } + + public OceanRandomAccessFile openInput() throws IOException { + return logDirectory.openInput(file); + } + + public long size() throws IOException { + return logDirectory.fileLength(file); + } + + public static interface StreamRecord { + public StreamData getDocuments(); + + public StreamData getOther(); + } + + /** + * Throws an error if the crcs do not match and the read is at the end of the + * stream. + * + */ + public static class CRCInputStream extends CheckedInputStream { + private long matchCrc; + CRC32 crc = new CRC32(); + + public CRCInputStream(InputStream input, long matchCrc) { + super(input, new CRC32()); + this.matchCrc = matchCrc; + } + + public long skip(long n) throws IOException { + byte[] buf = new byte[512]; + long total = 0; + while (total < n) { + long len = n - total; + len = read(buf, 0, len < buf.length ? (int) len : buf.length); + if (len == -1) { + return total; + } + total += len; + } + return total; + } + + public int read() throws IOException { + int b = in.read(); + if (b != -1) { + crc.update(b); + } + if (b == -1) + checkCrc(); + return b; + } + + private void checkCrc() throws IOException { + long inputCrc = crc.getValue(); + if (matchCrc != inputCrc) { + throw new IOException("crc does not match"); + } + } + + public int read(byte[] buf, int off, int len) throws IOException { + len = in.read(buf, off, len); + if (len != -1) { + crc.update(buf, off, len); + } + if (len == -1) + checkCrc(); + return len; + } + } + + public static class CRCException extends IOException { + public CRCException(String msg) { + super(msg); + } + } + + public static interface StreamData { + public CRCInputStream getInputStream() throws IOException; + + public void verifyCrc() throws IOException, CRCException; + + public int getLength(); + } + + public static class FileStreamRecord implements StreamRecord { + private RecordHeader recordHeader; + private OceanRandomAccessFile input; + + public FileStreamRecord(RecordHeader recordHeader, OceanRandomAccessFile input) { + this.recordHeader = recordHeader; + this.input = input; + } + + public FileStreamData getDocuments() { + if (recordHeader.docsLength == 0) + return null; + long position = recordHeader.getDocsPosition(); + return new FileStreamData(recordHeader.docsLength, position, recordHeader.docsCrc32, input); + } + + public FileStreamData getOther() { + if (recordHeader.otherLength == 0) + return null; + long position = recordHeader.getOtherPosition(); + return new FileStreamData(recordHeader.otherLength, position, recordHeader.otherCrc32, input); + } + } + + public static class FileStreamData implements StreamData { + private int length; + private long position; + private OceanRandomAccessFile input; + private long crc32; + + public FileStreamData(int length, long position, long crc32, OceanRandomAccessFile input) { + this.length = length; + this.position = position; + this.crc32 = crc32; + this.input = input; + } + + public long getCrc() { + return crc32; + } + + public void verifyCrc() throws IOException, CRCException { + InputStream input = getInputStream(); + int numRead; + byte[] buffer = new byte[1024]; + while ((numRead = input.read(buffer)) >= 0) { + } + } + + public CRCInputStream getInputStream() throws IOException { + int bufferSize = 1024 * 8; + if (bufferSize > length) { + bufferSize = length; + } + return new CRCInputStream(new BufferedInputStream(input.getInputStream(position, length), bufferSize), crc32); + } + + /** + * public byte[] getBytes() throws IOException { input.seek(position); + * byte[] bytes = new byte[length]; input.readFully(bytes); return bytes; } + */ + public int getLength() { + return length; + } + } + + public FileStreamRecord readRecordData(RecordHeader recordHeader, OceanRandomAccessFile input) { + return new FileStreamRecord(recordHeader, input); + } + + List loadRecordHeaders() throws IOException { + OceanRandomAccessFile input = openInput(); + if (input.length() > 0) { + LoadRecordHeaders loadRecordHeaders = new LoadRecordHeaders(input); + return loadRecordHeaders.getRecordHeaders(); + } else { + return new ArrayList(); + } + } + + /** + * public boolean delete(RecordHeader recordHeader) throws IOException { long + * lastPosition = -1; writeLock.lock(); try { lastPosition = + * output.getFilePointer(); output.seek(recordHeader.headerPosition); byte[] + * header = new byte[3]; output.readFully(header, 0, header.length); if + * (!Arrays.equals(HEADER, header)) { throw new IOException("no header"); } + * byte first = (byte) output.readByte(); if (first == OK) { int + * nextHeaderPosition = output.readInt(); long id = output.readLong(); assert + * id == recordHeader.id; output.seek(recordHeader.headerPosition + 4); + * output.writeByte(DELETED); output.getFD().sync(); return true; } else { // + * already deleted return false; } } finally { if (lastPosition != -1) + * output.seek(lastPosition); writeLock.unlock(); } } + */ + /** + * public class LoadRecordHeaders { private long currentReadPosition = 0; + * private List recordHeaders = new ArrayList(); + * + * public LoadRecordHeaders(RandomAccessFile input) throws IOException { try { + * while (true) { input.seek(currentReadPosition); RecordHeader recordHeader = + * new RecordHeader(); byte[] header = new byte[3]; + * recordHeader.headerPosition = (int) input.getFilePointer(); + * input.readFully(header, 0, header.length); if (!Arrays.equals(HEADER, + * header)) { // TODO: scan to next header if there is one LOG.error("header + * incorrect: "+new String(header)+" at pos: "+currentReadPosition); while + * (true) { recordHeader.headerPosition = (int) input.getFilePointer(); + * input.readFully(header, 0, header.length); if (Arrays.equals(HEADER, + * header)) { LOG.error("found header at position: + * "+recordHeader.headerPosition); break; } } // continue; } byte status = + * input.readByte(); recordHeader.id = input.readLong(); if (status == + * DELETED) { // TODO: skip it if (LOG.isDebugEnabled()) LOG.debug("record + * "+recordHeader.id+" status deleted"); } recordHeader.docsLength = + * input.readInt(); recordHeader.otherLength = input.readInt(); + * recordHeader.docsPosition = input.getFilePointer(); if (status != DELETED) + * recordHeaders.add(recordHeader); currentReadPosition = + * input.getFilePointer() + recordHeader.docsLength + + * recordHeader.otherLength; } } catch (EOFException eofException) { // at end + * of file } finally { if (input != null) input.close(); } } + * + * public List getRecordHeaders() { return recordHeaders; } } + */ + + public class LoadRecordHeaders { + private long currentReadPosition = 0; + private List recordHeaders = new ArrayList(); + private byte[] buffer = new byte[64]; + + public LoadRecordHeaders(OceanRandomAccessFile input) throws IOException { + try { + while (true) { + RecordHeader recordHeader = new RecordHeader(); + input.seek(currentReadPosition); + recordHeader.headerPosition = (int) input.getFilePointer(); + int numRead = input.read(buffer); + if (numRead == -1) + break; + ByteArrayInputStream byteInput = new ByteArrayInputStream(buffer); + VDataInputStream firstVInput = new VDataInputStream(byteInput); + long headerCrc32 = firstVInput.readVLong(); + int headersSize = firstVInput.readVInt(); + int initialSize = buffer.length - byteInput.available(); + // start reading from the buffer after the headercrc32 value + // TODO: can reuse CRC32 object + CheckedInputStream checkedInputStream = new CheckedInputStream(byteInput, new CRC32()); + VDataInputStream vInput = new VDataInputStream(checkedInputStream); + byte status = vInput.readByte(); + recordHeader.headersLength = headersSize + initialSize; + recordHeader.id = vInput.readVLong(); + recordHeader.docsLength = vInput.readVInt(); + recordHeader.docType = vInput.readVInt(); + recordHeader.docsCrc32 = vInput.readVLong(); + recordHeader.otherLength = vInput.readVInt(); + recordHeader.otherType = vInput.readVInt(); + recordHeader.otherCrc32 = vInput.readVLong(); + + CRC32 crc32 = (CRC32) checkedInputStream.getChecksum(); + long crcValue = crc32.getValue(); + if (headerCrc32 != crcValue) { + throw new IOException("header crc: " + headerCrc32 + " crc: " + crc32.getValue()); + } + if (status == DELETED) { + // TODO: skip it + if (LOG.isDebugEnabled()) + LOG.debug("record " + recordHeader.id + " status deleted"); + } + if (status != DELETED) + recordHeaders.add(recordHeader); + currentReadPosition = recordHeader.headerPosition + recordHeader.headersLength + recordHeader.docsLength + + recordHeader.otherLength; + } + } catch (EOFException eofException) { + // at end of file + } finally { + if (input != null) + input.close(); + } + } + + public List getRecordHeaders() { + return recordHeaders; + } + } + + private static long getCrc32(byte[] bytes) { + if (bytes == null) + return -1; + CRC32 crc32 = new CRC32(); + crc32.update(bytes); + return crc32.getValue(); + } + + private long getCrc32(Bytes bytes) throws IOException { + if (bytes == null) + return -1; + CRC32 crc32 = new CRC32(); + InputStream input = bytes.getInputStream(); + int numRead; + while ((numRead = input.read(crcBuffer)) >= 0) { + crc32.update(crcBuffer, 0, numRead); + } + return crc32.getValue(); + } + + public RecordHeader write(Long id, int docType, Bytes docBytes, int otherType, Bytes otherBytes) throws IOException { + writeLock.lock(); + try { + int documentsLength = 0; + if (docBytes != null) + documentsLength = docBytes.length(); + int otherLength = 0; + if (otherBytes != null) + otherLength = otherBytes.length(); + RecordHeader recordHeader = new RecordHeader(); + recordHeader.id = id; + recordHeader.headerPosition = (int) currentWritePosition; + recordHeader.docsLength = documentsLength; + recordHeader.docType = docType; + recordHeader.docsCrc32 = getCrc32(docBytes); + recordHeader.otherLength = otherLength; + recordHeader.otherType = otherType; + recordHeader.otherCrc32 = getCrc32(otherBytes); + + output.seek(currentWritePosition); + ByteArrayOutputStream byteOutput = new ByteArrayOutputStream(64); // can + // be + // reused + CheckedOutputStream checkedOutputStream = new CheckedOutputStream(byteOutput, new CRC32()); + VDataOutputStream dataOutputStream = new VDataOutputStream(checkedOutputStream); + dataOutputStream.writeByte(OK); + // TODO: can do previous difference compression on these + dataOutputStream.writeVLong(id); + dataOutputStream.writeVInt(recordHeader.docsLength); + dataOutputStream.writeVInt(recordHeader.docType); + dataOutputStream.writeVLong(recordHeader.docsCrc32); + dataOutputStream.writeVInt(recordHeader.otherLength); + dataOutputStream.writeVInt(recordHeader.otherType); + dataOutputStream.writeVLong(recordHeader.otherCrc32); + dataOutputStream.flush(); + checkedOutputStream.flush(); + CRC32 crc32 = (CRC32) checkedOutputStream.getChecksum(); + long crcValue = crc32.getValue(); + + int headersSize = byteOutput.size(); + + OutputStream fileOutputStream = output.getOutputStream(currentWritePosition); + BufferedOutputStream bufferedFileOutput = new BufferedOutputStream(fileOutputStream); + CountingOutputStream countingOutput = new CountingOutputStream(bufferedFileOutput); + VDataOutputStream vOutput = new VDataOutputStream(countingOutput); + vOutput.writeVLong(crcValue); + vOutput.writeVInt(headersSize); + int sizeOfInitial = countingOutput.getCount(); + recordHeader.headersLength = headersSize + sizeOfInitial; + + byteOutput.writeTo(vOutput); + if (docBytes != null) + docBytes.writeTo(vOutput); + if (otherBytes != null) + otherBytes.writeTo(vOutput); + vOutput.flush(); + bufferedFileOutput.flush(); + fileOutputStream.flush(); + /** + * output.write(HEADER); output.writeByte(OK); output.writeLong(id); + * output.writeInt(recordHeader.docsLength); + * output.writeInt(recordHeader.otherLength); + */ + /** + * recordHeader.docsPosition = output.getFilePointer(); if (docBytes != + * null) output.write(docBytes); if (otherBytes != null) + * output.write(otherBytes); int totalLength = recordHeader.docsLength + + * recordHeader.otherLength+3+1+8+4+4; System.out.println("write record + * length: "+totalLength); + */ + output.getFD().sync(); + LOG.info("currentWritePosition: " + currentWritePosition+" length: "+recordHeader.getLength()+" docs length: "+recordHeader.docsLength); + currentWritePosition = output.getFilePointer(); + return recordHeader; + } finally { + writeLock.unlock(); + } + } + + public void close() throws IOException { + output.close(); + } +} Index: ocean/src/org/apache/lucene/ocean/log/RecordData.java =================================================================== --- ocean/src/org/apache/lucene/ocean/log/RecordData.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/log/RecordData.java (revision 0) @@ -0,0 +1,17 @@ +package org.apache.lucene.ocean.log; + +import org.apache.lucene.ocean.util.Bytes; + +public class RecordData { + public final int docType; + public final Bytes docBytes; + public final int otherType; + public final Bytes otherBytes; + + public RecordData(int docType, Bytes docBytes, int otherType, Bytes otherBytes) { + this.docType = docType; + this.docBytes = docBytes; + this.otherType = otherType; + this.otherBytes = otherBytes; + } +} Index: ocean/src/org/apache/lucene/ocean/log/RecordHeader.java =================================================================== --- ocean/src/org/apache/lucene/ocean/log/RecordHeader.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/log/RecordHeader.java (revision 0) @@ -0,0 +1,40 @@ +package org.apache.lucene.ocean.log; + +import org.apache.commons.lang.builder.ReflectionToStringBuilder; + +/** + * Log record header + * + */ +public class RecordHeader implements Comparable { + public long id; + public int headerPosition; + public int headersLength; + public int docsLength; + public int docType; + public int otherLength; + public int otherType; + //public int docsPosition; + public long docsCrc32; + public long otherCrc32; + + public int getLength() { + return headersLength + docsLength + otherLength; + } + + public long getDocsPosition() { + return headerPosition + headersLength; + } + + public long getOtherPosition() { + return headerPosition + headersLength + docsLength; + } + + public int compareTo(RecordHeader other) { + return (id < other.id ? -1 : (id == other.id ? 0 : 1)); + } + + public String toString() { + return ReflectionToStringBuilder.toString(this); + } +} \ No newline at end of file Index: ocean/src/org/apache/lucene/ocean/log/TransactionLog.java =================================================================== --- ocean/src/org/apache/lucene/ocean/log/TransactionLog.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/log/TransactionLog.java (revision 0) @@ -0,0 +1,201 @@ +package org.apache.lucene.ocean.log; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.Serializable; +import java.util.concurrent.locks.ReentrantLock; + +import org.apache.commons.lang.SerializationUtils; +import org.apache.lucene.ocean.Deletes; +import org.apache.lucene.ocean.Documents; +import org.apache.lucene.ocean.LogDirectory; +import org.apache.lucene.ocean.Batch.MasterBatch; +import org.apache.lucene.ocean.Batch.SlaveBatch; +import org.apache.lucene.ocean.log.LogFile.Record; +import org.apache.lucene.ocean.log.LogFileManager.RecordIterator; +import org.apache.lucene.ocean.log.RawLogFile.StreamData; +import org.apache.lucene.ocean.util.Bytes; +import org.apache.lucene.ocean.util.Constants; +import org.apache.lucene.ocean.util.LongSequence; +import org.apache.lucene.ocean.util.RAMDirectorySerializer; +import org.apache.lucene.ocean.util.Util; +import org.apache.lucene.store.RAMDirectory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Serializes transactions known internally as batches to an underlying log + * file. Provides an iterator over the batches. + * + */ +public class TransactionLog { + final static Logger LOG = LoggerFactory.getLogger(TransactionLog.class); + // private ByteBufferPool byteBufferPool = new ByteBufferPool(50 * 1024, 5, + // 5); // not used right now + LogFileManager logFileManager; + private ReentrantLock writeLock = new ReentrantLock(); + private LogDirectory logDirectory; + private LongSequence snapshotIdSequence; + + public TransactionLog(long maxFileSize, LogDirectory logDirectory) throws IOException { + this.logDirectory = logDirectory; + logFileManager = new LogFileManager(maxFileSize, 1000 * 30, logDirectory); + Long maxId = logFileManager.getMaxId(); + if (maxId == null) + maxId = new Long(0); + snapshotIdSequence = new LongSequence(maxId + 1, 1); + } + + /** + * Any log file with a maxSnapshotId less than minIndexSnapshotId can be + * deleted. This is because it means the index would no longer recover + * anything from the transaction log. + * + * @param minSnapshotId + */ + public void deleteOldLogFiles(Long minIndexSnapshotId) throws IOException { + logFileManager.deleteOldLogFiles(minIndexSnapshotId); + } + + public void close() throws IOException { + logFileManager.close(); + } + + public Object deserialize(int type, InputStream input) throws IOException { + if (type == Constants.RAM_DIRECTORY_TYPE) { + return RAMDirectorySerializer.deserialize(input); + } else if (type == Constants.DOCUMENTS_TYPE) { + return SerializationUtils.deserialize(input); + } else { + throw new RuntimeException("unknown object"); + } + } + + public static Bytes serialize(Serializable object) throws IOException { + if (object instanceof RAMDirectory) { + RAMDirectory ramDirectory = (RAMDirectory) object; + int size = (int) Util.getSize(ramDirectory); + int bufferSize = 8 * 1024; + if (bufferSize > size) { + bufferSize = size + 1024; + } + Bytes bytes = new Bytes(bufferSize); + OutputStream output = bytes.getOutputStream(); + RAMDirectorySerializer.serialize(ramDirectory, output); + return bytes; + } else if (object instanceof Documents) { + Bytes bytes = new Bytes(8 * 1024); + OutputStream output = bytes.getOutputStream(); + SerializationUtils.serialize(object, output); + return bytes; + } else { + throw new RuntimeException("unknown object"); + } + } + /** + public RecordData getRecordData(MasterBatch masterBatch) throws IOException { + Bytes docBytes = null; + Bytes otherBytes = null; + int docType = -1; + int otherType = -1; + if (masterBatch.hasRAMDirectory()) { + // docBytes = SerializationUtils.serialize(masterBatch.getRamDirectory()); + docBytes = serialize(masterBatch.getRamDirectory()); + docType = Constants.RAM_DIRECTORY_TYPE; + } else if (masterBatch.hasDocuments()) { + Documents documents = masterBatch.getDocuments(); + docBytes = serialize(documents); + docType = Constants.DOCUMENTS_TYPE; + } + if (masterBatch.hasDeletes()) { + Deletes deletes = masterBatch.getDeletes(); + otherBytes = new Bytes(1024); + SerializationUtils.serialize(deletes, otherBytes.getOutputStream()); + docType = Constants.DELETES_SERIALIZE_TYPE; + } + return new RecordData(docType, docBytes, otherType, otherBytes); + } + **/ + public void writeMasterBatch(final Long id, MasterBatch masterBatch) throws Exception { + RecordData recordData = masterBatch.getRecordData(); + writeLock.lock(); + try { + logFileManager.writeRecord(id, recordData); + } finally { + writeLock.unlock(); + } + } + + public int getNumRecords() { + return logFileManager.getNumRecords(); + } + + public Long getNextId() { + return snapshotIdSequence.getAndIncrement(); + } + + public Long getMinId() { + return logFileManager.getMinId(); + } + + public Long getMaxId() { + return logFileManager.getMaxId(); + } + + public Long getPreviousId(long id) { + return logFileManager.getPreviousId(id); + } + + public SlaveBatchIterator getSlaveBatchIterator(Long snapshotId) throws Exception { + return new SlaveBatchIterator(snapshotId); + } + + public class SlaveBatchIterator { + RecordIterator recordIterator; + + public SlaveBatchIterator(Long snapshotId) throws IOException { + recordIterator = logFileManager.getRecordIterator(snapshotId); + } + + public boolean hasNext() throws IOException { + return recordIterator.hasNext(); + } + + public SlaveBatch next(boolean loadDocuments, boolean loadOther) throws Exception { + Record record = recordIterator.next(); + RecordHeader recordHeader = record.getRecordHeader(); + Documents documents = null; + RAMDirectory ramDirectory = null; + Deletes deletes = null; + if (loadDocuments) { + StreamData docData = record.getStreamRecord().getDocuments(); + if (docData != null) { + InputStream docInput = docData.getInputStream(); + Object object = deserialize(recordHeader.docType, docInput); + if (object instanceof RAMDirectory) { + ramDirectory = (RAMDirectory) object; + } else { + documents = (Documents) object; + } + } + } + if (loadOther) { + StreamData otherData = record.getStreamRecord().getOther(); + if (otherData != null) { + InputStream otherInput = otherData.getInputStream(); + deletes = (Deletes) SerializationUtils.deserialize(otherInput); + } + } + if (ramDirectory != null) { + return new SlaveBatch(record.getId(), ramDirectory, deletes); + } else { + return new SlaveBatch(record.getId(), documents, deletes); + } + } + + public void close() throws IOException { + recordIterator.close(); + } + } +} Index: ocean/src/org/apache/lucene/ocean/LogDirectory.java =================================================================== --- ocean/src/org/apache/lucene/ocean/LogDirectory.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/LogDirectory.java (revision 0) @@ -0,0 +1,21 @@ +package org.apache.lucene.ocean; + +import java.io.IOException; + +import org.apache.lucene.ocean.util.OceanRandomAccessFile; + +public abstract class LogDirectory { + public abstract String[] list() throws IOException; + + public abstract boolean fileExists(String name) throws IOException; + + public abstract long fileModified(String name) throws IOException; + + public abstract boolean deleteFile(String name) throws IOException; + + public abstract long fileLength(String name) throws IOException; + + public abstract OceanRandomAccessFile openInput(String name) throws IOException; + + public abstract OceanRandomAccessFile getOutput(String name, boolean overwrite) throws IOException; +} Index: ocean/src/org/apache/lucene/ocean/MemoryIndexThreadLocal.java =================================================================== --- ocean/src/org/apache/lucene/ocean/MemoryIndexThreadLocal.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/MemoryIndexThreadLocal.java (revision 0) @@ -0,0 +1,36 @@ +package org.apache.lucene.ocean; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + +public class MemoryIndexThreadLocal { + private Map map = new HashMap(); + private Map threadMap = new HashMap(); + private TransactionSystem transactionSystem; + + public MemoryIndexThreadLocal(TransactionSystem transactionSystem) { + this.transactionSystem = transactionSystem; + } + + public WriteableMemoryIndex get() throws Exception { + Thread thread = Thread.currentThread(); + WriteableMemoryIndex index = threadMap.get(thread); + if (index == null) { + index = transactionSystem.newWriteableMemoryIndex(); + index.createIndexSnapshot(new Long(1)); + map.put(index.getId(), index); + threadMap.put(thread, index); + } + return index; + } + + public void remove(IndexID indexId) { + map.remove(indexId); + threadMap.remove(indexId); + } + + public Collection getAll() { + return map.values(); + } +} Index: ocean/src/org/apache/lucene/ocean/MultiThreadSearcherPolicy.java =================================================================== --- ocean/src/org/apache/lucene/ocean/MultiThreadSearcherPolicy.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/MultiThreadSearcherPolicy.java (revision 0) @@ -0,0 +1,25 @@ +package org.apache.lucene.ocean; + +public class MultiThreadSearcherPolicy extends SearcherPolicy { + private final int minThreads; + private final int maxThreads; + private final int queueSize; + + public MultiThreadSearcherPolicy(int minThreads, int maxThreads, int queueSize) { + this.minThreads = minThreads; + this.maxThreads = maxThreads; + this.queueSize = queueSize; + } + + public int getQueueSize() { + return queueSize; + } + + public int getMinThreads() { + return minThreads; + } + + public int getMaxThreads() { + return maxThreads; + } +} Index: ocean/src/org/apache/lucene/ocean/MultiThreadTransaction.java =================================================================== --- ocean/src/org/apache/lucene/ocean/MultiThreadTransaction.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/MultiThreadTransaction.java (revision 0) @@ -0,0 +1,11 @@ +package org.apache.lucene.ocean; + +/** + * Coordinates a multithreaded transaction commit between multiple indexes. + * Utilizes java.util.concurrent.CountDownLatch for synchronization between the + * indexes as each index operation is performed in it's own thread. + * + */ +public class MultiThreadTransaction { + +} Index: ocean/src/org/apache/lucene/ocean/MultiWriteableMemoryIndex.java =================================================================== --- ocean/src/org/apache/lucene/ocean/MultiWriteableMemoryIndex.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/MultiWriteableMemoryIndex.java (revision 0) @@ -0,0 +1,9 @@ +package org.apache.lucene.ocean; + +/** + * For use with MemoryIndexThreadLocal + * + */ +public class MultiWriteableMemoryIndex { + +} Index: ocean/src/org/apache/lucene/ocean/OceanConsole.java =================================================================== --- ocean/src/org/apache/lucene/ocean/OceanConsole.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/OceanConsole.java (revision 0) @@ -0,0 +1,11 @@ +package org.apache.lucene.ocean; + +/** + * Display indexes + * Display index snapshots + * Run merge + * + */ +public class OceanConsole { + +} Index: ocean/src/org/apache/lucene/ocean/OceanInstantiatedIndexReader.java =================================================================== --- ocean/src/org/apache/lucene/ocean/OceanInstantiatedIndexReader.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/OceanInstantiatedIndexReader.java (revision 0) @@ -0,0 +1,84 @@ +package org.apache.lucene.ocean; + +import java.io.IOException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.lucene.store.instantiated.InstantiatedIndex; +import org.apache.lucene.store.instantiated.InstantiatedIndexReader; + +/** + * Simulates a multiple version IndexReader with InstantiatedIndexReader by + * having documents over the set maxDoc be deleted. + * + */ +public class OceanInstantiatedIndexReader extends InstantiatedIndexReader implements Cloneable { + private HashSet deletedDocs; + + public OceanInstantiatedIndexReader(InstantiatedIndex index) { + super(index); + this.deletedDocs = new HashSet(); + } + + public OceanInstantiatedIndexReader(InstantiatedIndex index, HashSet deletedDocs) { + super(index); + this.deletedDocs = deletedDocs; + } + + protected void doDelete(int docNum) throws IOException { + deletedDocs.add(docNum); + } + + public synchronized Object clone() { + return new OceanInstantiatedIndexReader(getIndex(), (HashSet)deletedDocs.clone()); + } + /** + protected void doSetNorm(int doc, String field, byte value) throws IOException { + if (updatedNormsByFieldNameAndDocumentNumber == null) { + updatedNormsByFieldNameAndDocumentNumber = new HashMap>(normsByFieldNameAndDocumentNumber.size()); + } + List list = updatedNormsByFieldNameAndDocumentNumber.get(field); + if (list == null) { + list = new LinkedList(); + updatedNormsByFieldNameAndDocumentNumber.put(field, list); + } + list.add(new NormUpdate(doc, value)); + } + + public byte[] norms(String field) throws IOException { + byte[] norms = normsByFieldNameAndDocumentNumber.get(field); + if (updatedNormsByFieldNameAndDocumentNumber != null) { + norms = norms.clone(); + List updated = updatedNormsByFieldNameAndDocumentNumber.get(field); + if (updated != null) { + for (NormUpdate normUpdate : updated) { + norms[normUpdate.doc] = normUpdate.value; + } + } + } + return norms; + } + + public void norms(String field, byte[] bytes, int offset) throws IOException { + byte[] norms = normsByFieldNameAndDocumentNumber.get(field); + System.arraycopy(norms, offset, bytes, 0, norms.length); + } + **/ + public int numDocs() { + return maxDoc() - deletedDocs.size(); + } + + public boolean isDeleted(int n) { + if (deletedDocs != null && deletedDocs.contains(n)) + return true; + return false; + } + + public boolean hasDeletions() { + return deletedDocs != null && deletedDocs.size() > 0; + } +} Index: ocean/src/org/apache/lucene/ocean/OceanSearcher.java =================================================================== --- ocean/src/org/apache/lucene/ocean/OceanSearcher.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/OceanSearcher.java (revision 0) @@ -0,0 +1,67 @@ +package org.apache.lucene.ocean; + +import java.io.IOException; +import java.util.HashSet; +import java.util.Set; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.FieldSelector; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.search.MultiSearcher; + +public class OceanSearcher extends MultiSearcher { + Snapshot snapshot; + + public OceanSearcher(Snapshot snapshot) throws IOException { + super(snapshot.getSearchables()); + this.snapshot = snapshot; + } + + boolean verifyDocsIdsUnique() throws IOException { + IndexReader reader = snapshot.getIndexReader(); + int maxDoc = reader.maxDoc(); + Set set = new HashSet(maxDoc); + for (int x=0; x < maxDoc; x++) { + if (reader.isDeleted(x)) continue; + Document document = reader.document(x); + Long id = new Long(document.get("id")); + if (set.contains(id)) { + return false; + } else { + set.add(id); + } + } + return true; + } + + /** + * Searcher must be closed to decref the snapshot + */ + public void close() { + snapshot.decRef(); + } + + public Document doc(int n) throws CorruptIndexException, IOException { + int i = subSearcher(n); // find searcher index + Document document = getSearchables()[i].doc(n - getStarts()[i]); // dispatch to searcher + //SnapshotSearcher snapshotSearcher = (SnapshotSearcher)searchables[i]; + //IndexID indexId = snapshotSearcher.getIndexSnapshot().getIndex().getId(); + //document.add(new Field(Constants.INDEXID, indexId.toString(), Field.Store.NO, Field.Index.NO)); + return document; + } + + // inherit javadoc + public Document doc(int n, FieldSelector fieldSelector) throws CorruptIndexException, IOException { + int i = subSearcher(n); // find searcher index + Document document = getSearchables()[i].doc(n - getStarts()[i], fieldSelector); // dispatch to searcher + //SnapshotSearcher snapshotSearcher = (SnapshotSearcher)searchables[i]; + //IndexID indexId = snapshotSearcher.getIndexSnapshot().getIndex().getId(); + //document.add(new Field(Constants.INDEXID, indexId.toString(), Field.Store.NO, Field.Index.NO)); + return document; + } + + public Snapshot getSnapshot() { + return snapshot; + } +} Index: ocean/src/org/apache/lucene/ocean/RamIndex.java =================================================================== --- ocean/src/org/apache/lucene/ocean/RamIndex.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/RamIndex.java (revision 0) @@ -0,0 +1,111 @@ +package org.apache.lucene.ocean; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.KeepOnlyLastCommitDeletionPolicy; +import org.apache.lucene.index.OceanSegmentReader; +import org.apache.lucene.index.SerialMergeScheduler; +import org.apache.lucene.ocean.WriteableMemoryIndex.MemoryIndexSnapshot; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.RAMDirectory; + +public class RamIndex extends DirectoryIndex { + private RAMDirectory ramDirectory; + private Long maxSnapshotId; + private Long maxDocumentId; + + public RamIndex(IndexID id, List indexSnapshots, TransactionSystem system) throws Exception { + super(id, system); + ramDirectory = new RAMDirectory(); + IndexReader[] indexReaders = getIndexReaders(indexSnapshots); + RAMDirectory ramDirectory = new RAMDirectory(); + IndexWriter indexWriter = new IndexWriter(ramDirectory, false, system.getDefaultAnalyzer(), true); + indexWriter.setMaxBufferedDocs(Integer.MAX_VALUE); + indexWriter.setRAMBufferSizeMB(0.0f); + indexWriter.setMergeScheduler(new SerialMergeScheduler()); + indexWriter.setUseCompoundFile(false); + indexWriter.addIndexes(indexReaders); + indexWriter.close(); + maxSnapshotId = getMaxSnapshotId(indexSnapshots); + maxDocumentId = getMaxDocumentId(indexSnapshots); + addMergedFromIndexIds(indexSnapshots); + } + + // TODO: add timestamp so ramindex can be removed from indices + public RamIndex(IndexID id, Long snapshotId, List deletesList, RAMDirectory ramDirectory, TransactionSystem system) throws Exception { + super(id, system); + this.ramDirectory = ramDirectory; + initialIndexReader = IndexReader.open(ramDirectory, indexDeletionPolicy); + if (deletesList != null) { + for (Deletes deletes : deletesList) { + applyDeletes(true, deletes, null, initialIndexReader); + } + initialIndexReader.flush(); + } + createNewSnapshot(snapshotId, true, initialIndexReader); + } + + // converts memoryIndexSnapshot into ramindexsnapshot + public RamIndex(IndexID id, MemoryIndexSnapshot memoryIndexSnapshot) throws Exception, IOException { + super(id, memoryIndexSnapshot.getIndex().getSystem()); + this.maxSnapshotId = memoryIndexSnapshot.getMaxSnapshotId(); + this.maxDocumentId = memoryIndexSnapshot.getMaxDocumentId(); + ramDirectory = new RAMDirectory(); + Analyzer defaultAnalyzer = memoryIndexSnapshot.getIndex().getSystem().getDefaultAnalyzer(); + IndexWriter indexWriter = new IndexWriter(ramDirectory, false, defaultAnalyzer, true, new KeepOnlyLastCommitDeletionPolicy()); + indexWriter.setMaxBufferedDocs(Integer.MAX_VALUE); + indexWriter.setRAMBufferSizeMB(IndexWriter.DISABLE_AUTO_FLUSH); + indexWriter.setMergeScheduler(new SerialMergeScheduler()); + indexWriter.setUseCompoundFile(false); + indexWriter.addIndexes(new IndexReader[] {memoryIndexSnapshot.getIndexReader()}); + indexWriter.close(); + initialIndexReader = IndexReader.open(ramDirectory, indexDeletionPolicy); + addMergedFromIndexId(memoryIndexSnapshot); + List indexSnapshots = new ArrayList(1); + indexSnapshots.add(memoryIndexSnapshot); + createNewSnapshot(memoryIndexSnapshot.getSnapshotId(), true, initialIndexReader); + } + + public RamIndexSnapshot commitIndex(Transaction transaction) throws IndexException, InterruptedException, IOException { + try { + transaction.ready(this); + if (transaction.go()) { + Long snapshotId = transaction.getId(); + RamIndexSnapshot indexSnapshot = (RamIndexSnapshot)createNewSnapshot(snapshotId, true, initialIndexReader); + return indexSnapshot; + } else { + // if commit fails this snapshot and ramindex won't make it + return null; + } + } catch (Throwable throwable) { + LOG.error("", throwable); + transaction.failed(this, throwable); + return null; + } + } + + protected RamIndexSnapshot doCreateNewSnapshot(Long snapshotId, IndexReader newIndexReader) throws IOException { + RamIndexSnapshot ramIndexSnapshot = new RamIndexSnapshot(snapshotId, (OceanSegmentReader)newIndexReader); + registerSnapshot(ramIndexSnapshot); + return ramIndexSnapshot; + } + + public class RamIndexSnapshot extends DirectoryIndexSnapshot { + public RamIndexSnapshot(Long snapshotId, OceanSegmentReader indexReader) throws IOException { + super(snapshotId, indexReader); + } + + public String toString() { + return "RamIndexSnapshot index: "+RamIndex.this.getId()+" snapshotid: "+snapshotId+" maxDoc: "+indexReader.maxDoc(); + } + } + + public Directory getDirectory() { + return ramDirectory; + } +} Index: ocean/src/org/apache/lucene/ocean/SearcherPolicy.java =================================================================== --- ocean/src/org/apache/lucene/ocean/SearcherPolicy.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/SearcherPolicy.java (revision 0) @@ -0,0 +1,5 @@ +package org.apache.lucene.ocean; + +public class SearcherPolicy { + +} Index: ocean/src/org/apache/lucene/ocean/SingleThreadSearcherPolicy.java =================================================================== --- ocean/src/org/apache/lucene/ocean/SingleThreadSearcherPolicy.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/SingleThreadSearcherPolicy.java (revision 0) @@ -0,0 +1,5 @@ +package org.apache.lucene.ocean; + +public class SingleThreadSearcherPolicy extends SearcherPolicy { + +} Index: ocean/src/org/apache/lucene/ocean/SingleThreadTransaction.java =================================================================== --- ocean/src/org/apache/lucene/ocean/SingleThreadTransaction.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/SingleThreadTransaction.java (revision 0) @@ -0,0 +1,39 @@ +package org.apache.lucene.ocean; + +import java.util.List; + +import org.apache.lucene.analysis.Analyzer; + +public class SingleThreadTransaction extends AbstractTransaction { + + public SingleThreadTransaction(Long id, List documentIds, Snapshot previousSnapshot, Batch batch, TransactionSystem system) throws Exception { + super(id, documentIds, previousSnapshot, batch, system); + } + + public CommitResult call() throws Exception { + if (batch.hasDeletes()) { + for (Index index : previousSnapshot.getDeleteOnlyIndices()) { + new DeletesTask(batch.getDeletes(), index).call(); + } + } else { + for (Index index : previousSnapshot.getDeleteOnlyIndices()) { + index.commitNothing(this); + //new DeletesTask(batch.getDeletes(), index).call(); + } + } + WriteableMemoryIndex writeableIndex = (WriteableMemoryIndex) previousSnapshot.getWriteableSnapshot().getIndex(); + if (batch.hasRAMDirectory()) { + new AddRamIndexDocumentsTask(batch.getRamDirectory(), writeableIndex).call(); + } else if (batch.hasDocuments()) { + Documents documents = batch.getDocuments(); + Analyzer analyzer = batch.getAnalyzer(); + new AddWriteableMemoryDocumentsTask(documents, analyzer, batch.getDeletes(), writeableIndex).call(); + } else if (batch.hasDeletes()) { + new DeletesTask(batch.getDeletes(), writeableIndex).call(); + } else { + writeableIndex.commitNothing(this); + } + handleMasterBatch(); + return createCommitResult(); + } +} Index: ocean/src/org/apache/lucene/ocean/Snapshot.java =================================================================== --- ocean/src/org/apache/lucene/ocean/Snapshot.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/Snapshot.java (revision 0) @@ -0,0 +1,415 @@ +package org.apache.lucene.ocean; + +import java.io.IOException; +import java.math.BigDecimal; +import java.text.DecimalFormat; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.MultiReader; +import org.apache.lucene.ocean.DirectoryIndex.DirectoryIndexSnapshot; +import org.apache.lucene.ocean.DiskIndex.DiskIndexSnapshot; +import org.apache.lucene.ocean.Index.IndexSnapshot; +import org.apache.lucene.ocean.Index.IndexSnapshotSearchable; +import org.apache.lucene.ocean.RamIndex.RamIndexSnapshot; +import org.apache.lucene.ocean.SnapshotInfo.IndexInfo; +import org.apache.lucene.ocean.WriteableMemoryIndex.MemoryIndexSnapshot; +import org.apache.lucene.ocean.util.SortedList; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Searchable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class Snapshot implements Comparable { + final static Logger LOG = LoggerFactory.getLogger(Snapshot.class); + private BigDecimal id; + private SortedList indexSnapshotMap; + private MemoryIndexSnapshot writeableSnapshot; + private IndexReader indexReader; + private IndexSnapshotSearchable[] searchables; + private final int[] starts; + private int maxDoc; + private TransactionSystem system; + private final long timestamp; + private int refs = 0; + + public Snapshot(BigDecimal id, MemoryIndexSnapshot writeableSnapshot, Collection indexSnapshots, TransactionSystem system, + long timestamp) throws IOException { + this.id = id; + this.writeableSnapshot = writeableSnapshot; + this.timestamp = timestamp; + indexSnapshotMap = new SortedList(); + for (IndexSnapshot indexSnapshot : indexSnapshots) { + assert indexSnapshot != null; + indexSnapshotMap.put(indexSnapshot.getIndex().getId(), indexSnapshot); + } + indexSnapshotMap.put(writeableSnapshot.getIndex().getId(), writeableSnapshot); + IndexReader[] readerArray = getReaderArray(indexSnapshotMap.values()); + int i = 0; + //searchables = new IndexSnapshotSearchable[indexSnapshotMap.size()]; + List searchablesList = new ArrayList(indexSnapshotMap.size()); + for (IndexSnapshot indexSnapshot : indexSnapshotMap.values()) { + IndexSnapshotSearchable searcher = indexSnapshot.getSearcher(); + if (searcher == null) { + LOG.error("searcher null"); + } + if (searcher != null) searchablesList.add(searcher); + } + searchables = (IndexSnapshotSearchable[])searchablesList.toArray(new IndexSnapshotSearchable[0]); + indexReader = new MultiReader(readerArray); + maxDoc = indexReader.maxDoc(); + starts = makeStarts(); + } + + public Snapshot(Long snapshotId, int minorVersion, MemoryIndexSnapshot writeableSnapshot, List indexSnapshots, + TransactionSystem system, long timestamp) throws IOException { + this(toId(snapshotId, minorVersion), writeableSnapshot, indexSnapshots, system, timestamp); + } + + public String getIdString() { + return formatId(id); + } + + public IndexSnapshotSearchable[] getSearchables() { + return searchables; + } + + public boolean isMinor() { + return getMinorVersion() > 0; + } + + public String toString() { + return "snapshot: "+formatId(id); + } + + public TransactionSystem getSystem() { + return system; + } + + public synchronized int refCount() { + return refs; + } + + public synchronized boolean hasRefs() { + return refs > 0; + } + + public synchronized void decRef() { + refs--; + } + + synchronized void incRef() { + refs++; + } + + private static boolean hasDuplicates(List allIndexSnapshots) { + Set set = new HashSet(); + for (IndexSnapshot indexSnapshot : allIndexSnapshots) { + IndexID indexId = indexSnapshot.getIndex().getId(); + if (set.contains(indexId)) { + return true; + } else { + set.add(indexId); + } + } + return false; + } + + private IndexReader[] getReaderArray(Collection indexSnapshots) { + IndexReader[] readerArray = new IndexReader[indexSnapshots.size()]; + int x = 0; + for (IndexSnapshot indexSnapshot : indexSnapshots) { + indexSnapshotMap.put(indexSnapshot.getIndex().getId(), indexSnapshot); + readerArray[x] = indexSnapshot.getIndexReader(); + x++; + } + return readerArray; + } + + public int numDocs() { + return indexReader.numDocs(); + } + + private int[] makeStarts() { + IndexSnapshot[] indexSnapshotsArray = indexSnapshotMap.values().toArray(new IndexSnapshot[0]); + // build starts array + int[] starts = new int[indexSnapshotsArray.length + 1]; + for (int i = 0; i < indexSnapshotsArray.length; i++) { + starts[i] = maxDoc; + maxDoc += indexSnapshotsArray[i].maxDoc(); // compute maxDocs + } + starts[indexSnapshotsArray.length] = maxDoc; + return starts; + } + + public long getTimestamp() { + return timestamp; + } + + public int compareTo(Snapshot other) { + return id.compareTo(other.id); + } + + //public Searcher getSearcher() throws IOException { + // MultiSearcher multiSearcher = new MultiSearcher(getSearchers()); + // return multiSearcher; + //} + + + /** + public Searcher[] getSearchers() { + IndexSnapshot[] indexSnapshots = (IndexSnapshot[]) indexSnapshotMap.values().toArray(new IndexSnapshot[0]); + Searcher[] searchers = new Searcher[indexSnapshotMap.size()]; + for (int x = 0; x < indexSnapshots.length; x++) { + searchers[x] = new SnapshotSearcher(indexSnapshots[x].getIndexReader(), indexSnapshots[x]); + } + return searchers; + } + **/ + public IndexReader getIndexReader() { + return indexReader; + } + + public int maxDoc() { + return maxDoc; + } + + public int[] getStarts() { + return starts; + } + + public List getRamIndexSnapshots() { + List ramIndexSnapshots = new ArrayList(); + for (IndexSnapshot indexSnapshot : indexSnapshotMap.values()) { + if (indexSnapshot instanceof RamIndexSnapshot) { + ramIndexSnapshots.add((RamIndexSnapshot) indexSnapshot); + } + } + return ramIndexSnapshots; + } + + public IndexReader[] getIndexReaders() { + IndexReader[] indexReaders = new IndexReader[indexSnapshotMap.size()]; + int i = 0; + for (IndexSnapshot indexSnapshot : indexSnapshotMap.values()) { + indexReaders[i] = indexSnapshot.getIndexReader(); + i++; + } + return indexReaders; + } + + public int getMaxDoc() { + int maxDoc = 0; + for (IndexSnapshot indexSnapshot : indexSnapshotMap.values()) { + maxDoc += indexSnapshot.getIndexReader().maxDoc(); + } + return maxDoc; + } + + public int getMinorVersion() { + return getMinorVersion(id); + } + + public static void main(String[] args) { + BigDecimal id = toId(210l, 1); + String string = formatId(id); + System.out.println(string); + } + + public static BigDecimal toId(Long snapshotId, int minorVersion) { + StringBuilder builder = new StringBuilder(); + builder.append(snapshotId); + builder.append("."); + if (10 > minorVersion) + builder.append("0"); + builder.append(minorVersion); + BigDecimal value = new BigDecimal(builder.toString()); + return value; + } + + public static int getMinorVersion(BigDecimal value) { + value = value.subtract(new BigDecimal(value.longValue())); + BigDecimal decimal = value.scaleByPowerOfTen(2); + return decimal.intValue(); + } + + public static List getSnapshotIds(Collection indexSnapshots) { + List ids = new ArrayList(); + for (IndexSnapshot indexSnapshot : indexSnapshots) { + ids.add(indexSnapshot.getSnapshotId()); + } + return ids; + } + /** + public static boolean snapshotIdsMatch(Collection indexSnapshots) { + Long current = null; + for (IndexSnapshot indexSnapshot : indexSnapshots) { + if (current == null) { + current = indexSnapshot.getSnapshotId(); + } else if (!current.equals(indexSnapshot.getSnapshotId())) { + return false; + } + } + return true; + } + **/ + public SnapshotInfo getSnapshotInfo() throws IOException { + int deletedDocs = indexReader.maxDoc() - indexReader.numDocs(); + SnapshotInfo snapshotInfo = new SnapshotInfo(id, indexReader.maxDoc(), indexReader.numDocs(), deletedDocs, timestamp); + for (IndexSnapshot indexSnapshot : indexSnapshotMap.values()) { + Index index = indexSnapshot.getIndex(); + String type = null; + Long segmentGeneration = null; + Long deleteFlushId = null; + Long lastAppliedId = null; + List mergedFrom = index.getMergedFromIndexIds(); + if (index instanceof DiskIndex) { + segmentGeneration = indexSnapshot.getIndexReader().getIndexCommit().getGeneration(); + type = "disk"; + } else if (index instanceof WriteableMemoryIndex) + type = "memory"; + else if (index instanceof RamIndex) + type = "ram"; + if (indexSnapshot instanceof DirectoryIndexSnapshot) { + DirectoryIndexSnapshot directoryIndexSnapshot = (DirectoryIndexSnapshot)indexSnapshot; + deleteFlushId = directoryIndexSnapshot.getLastCheckpointId(); + lastAppliedId = directoryIndexSnapshot.getLastAppliedSnapshotId(); + } + IndexInfo indexInfo = new IndexInfo(indexSnapshot.getSnapshotId(), index.getId().id, segmentGeneration, type, indexSnapshot.maxDoc(), indexSnapshot.getIndexReader().numDocs(), indexSnapshot.deletedDoc(), indexSnapshot.getMinDocumentId(), indexSnapshot.getMaxDocumentId(), indexSnapshot.getMinSnapshotId(), indexSnapshot.getMaxSnapshotId(), deleteFlushId, lastAppliedId, mergedFrom); + snapshotInfo.add(indexInfo); + } + return snapshotInfo; + } + + public static String formatId(BigDecimal id) { + DecimalFormat format = new DecimalFormat("##0.00"); + return format.format(id); + } + + public static String getFileName(BigDecimal id) { + String string = formatId(id); + String replaced = string.replace('.', '_'); + return "snapshot_" + replaced + ".xml"; + } + + /** + * Create minor snapshot (meaning a merged snapshot with no real index + * changes) reusing the existing writeableSnapshot. + * + * @param removeIndexIds + * @param newIndexSnapshot + * @return + * @throws IOException + */ + public Snapshot createMinor(Collection removeIndexIds, IndexSnapshot newIndexSnapshot) throws IOException { + return createMinor(removeIndexIds, writeableSnapshot, newIndexSnapshot); + } + + public Snapshot createMinor(Collection removeIndexIds) throws IOException { + int minorVersion = getMinorVersion(); + Long snapshotId = getSnapshotId(); + int newMinorVersion = minorVersion + 1; + HashMap mapCopy = new HashMap(indexSnapshotMap); + for (IndexID indexid : removeIndexIds) { + IndexSnapshot removed = mapCopy.remove(indexid); + if (removed != null) { + System.out.println("removed indexid: "+indexid+" from snapshot: "+snapshotId+"."+newMinorVersion); + } + } + // make sure to put in the writeableSnapshot + mapCopy.put(writeableSnapshot.getIndex().getId(), writeableSnapshot); + System.out.println("mapCopy.values(): "+mapCopy.values()); + Snapshot newSnapshot = new Snapshot(snapshotId, newMinorVersion, writeableSnapshot, new ArrayList(mapCopy.values()), system, System + .currentTimeMillis()); + return newSnapshot; + } + + public Snapshot createMinor(Collection removeIndexIds, MemoryIndexSnapshot writeableSnapshot, IndexSnapshot newIndexSnapshot) + throws IOException { + int minorVersion = getMinorVersion(); + Long snapshotId = getSnapshotId(); + int newMinorVersion = minorVersion + 1; + HashMap mapCopy = new HashMap(indexSnapshotMap); + for (IndexID indexid : removeIndexIds) { + IndexSnapshot removed = mapCopy.remove(indexid); + if (removed != null) { + LOG.info("removed indexid: "+indexid+" from snapshot: "+snapshotId+"."+newMinorVersion); + } + } + IndexID newIndexId = newIndexSnapshot.getIndex().getId(); + assert !mapCopy.containsKey(newIndexId); + mapCopy.put(newIndexId, newIndexSnapshot); + mapCopy.put(writeableSnapshot.getIndex().getId(), writeableSnapshot); + + LOG.info("snapshotId: " + snapshotId + " newMinorVersion: " + newMinorVersion+" newIndexId: "+newIndexId); + // BigDecimal newId = toId(snapshotId, minorVersion); + Snapshot newSnapshot = new Snapshot(snapshotId, newMinorVersion, writeableSnapshot, new ArrayList(mapCopy.values()), system, System + .currentTimeMillis()); + return newSnapshot; + } + + public List getDiskIndexes() { + List diskIndexes = new ArrayList(); + for (IndexSnapshot indexSnapshot : indexSnapshotMap.values()) { + if (indexSnapshot instanceof DiskIndexSnapshot) { + diskIndexes.add((DiskIndex)indexSnapshot.getIndex()); + } + } + return diskIndexes; + } + + public List getDeleteOnlyIndices() { + HashMap mapCopy = new HashMap(indexSnapshotMap); + mapCopy.remove(writeableSnapshot.getIndex().getId()); + List indices = new ArrayList(); + for (IndexSnapshot indexSnapshot : mapCopy.values()) { + indices.add(indexSnapshot.getIndex()); + } + return indices; + } + + public MemoryIndexSnapshot getWriteableSnapshot() { + return writeableSnapshot; + } + + public boolean containsIndex(long indexid) { + return indexSnapshotMap.containsKey(indexid); + } + + public List getDeleteOnlyIndexSnapshots() { + List indexSnapshots = new ArrayList(); + for (IndexSnapshot indexSnapshot : indexSnapshotMap.values()) { + if (indexSnapshot != writeableSnapshot) { + indexSnapshots.add(indexSnapshot); + } + } + return indexSnapshots; + } + + public List getDiskIndexSnapshots() { + List diskIndexSnapshots = new ArrayList(); + for (IndexSnapshot indexSnapshot : indexSnapshotMap.values()) { + if (indexSnapshot instanceof DiskIndexSnapshot) { + diskIndexSnapshots.add((DiskIndexSnapshot) indexSnapshot); + } + } + return diskIndexSnapshots; + } + + public List getIndexSnapshots() { + return new ArrayList(indexSnapshotMap.values()); + } + + public Long getSnapshotId() { + return id.longValue(); + } + + public BigDecimal getId() { + return id; + } +} Index: ocean/src/org/apache/lucene/ocean/SnapshotInfo.java =================================================================== --- ocean/src/org/apache/lucene/ocean/SnapshotInfo.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/SnapshotInfo.java (revision 0) @@ -0,0 +1,240 @@ +package org.apache.lucene.ocean; + +import java.math.BigDecimal; +import java.text.DateFormat; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Date; +import java.util.List; +import java.util.Map; + +import org.apache.commons.lang.StringUtils; +import org.apache.lucene.ocean.util.CElement; +import org.apache.lucene.ocean.util.SortedList; +import org.apache.lucene.ocean.util.XMLUtil; +import org.jdom.Element; + +public class SnapshotInfo implements CElement, Comparable { + private BigDecimal id; + private SortedList indexInfos; + private int numDocs; + private int maxDoc; + private int deletedDocs; + private long timestamp; + + public SnapshotInfo(BigDecimal id, int maxDoc, int numDocs, int deletedDocs, long timestamp) { + assert id != null; + this.id = id; + this.maxDoc = maxDoc; + this.numDocs = numDocs; + this.deletedDocs = deletedDocs; + this.timestamp = timestamp; + indexInfos = new SortedList(); + } + + public IndexInfo getIndexInfo(IndexID id) { + return indexInfos.get(id); + } + + public int compareTo(SnapshotInfo other) { + return id.compareTo(other.id); + } + + public void add(IndexInfo indexInfo) { + indexInfos.put(indexInfo.getIndexID(), indexInfo); + } + + public SnapshotInfo(Element element) { + indexInfos = new SortedList(); + id = new BigDecimal(element.getAttributeValue("id")); + for (Element indexElement : XMLUtil.getChildren("index", element)) { + IndexInfo indexInfo = new IndexInfo(indexElement); + indexInfos.put(indexInfo.getIndexID(), indexInfo); + } + } + + public Long getSnapshotId() { + return id.longValue(); + } + + public BigDecimal getId() { + return id; + } + + public Collection getIndexInfos() { + return indexInfos.values(); + } + + public SnapshotInfo(BigDecimal id, Map indexInfos) { + this.id = id; + this.indexInfos = new SortedList(indexInfos); + } + + public static class IndexInfo implements CElement { + public Long snapshotId; + public Long id; + public Long segmentGeneration; + public String type; + public Integer maxDoc; + public Integer deletedDoc; + public Integer numDocs; + public Long minDocumentId; + public Long maxDocumentId; + public Long minSnapshotId; + public Long maxSnapshotId; + public Long checkpointId; + public Long lastAppliedId; // last transaction that made changes to this snapshot/index + public List mergedFrom; + + public IndexInfo() { + } + + public IndexInfo(Long snapshotId, Long id, Long segmentGeneration, String type, int maxDoc, int numDocs, int deletedDoc, Long minDocumentId, Long maxDocumentId, Long minSnapshotId, Long maxSnapshotId, Long checkpointId, Long lastAppliedId, List mergedFrom) { + this.snapshotId = snapshotId; + this.id = id; + this.segmentGeneration = segmentGeneration; + this.type = type; + this.maxDoc = maxDoc; + this.numDocs = numDocs; + this.deletedDoc = deletedDoc; + this.minDocumentId = minDocumentId; + this.maxDocumentId = maxDocumentId; + this.minSnapshotId = minSnapshotId; + this.maxSnapshotId = maxSnapshotId; + this.checkpointId = checkpointId; + this.lastAppliedId = lastAppliedId; + this.mergedFrom = mergedFrom; + } + + public List getMergedFrom() { + return mergedFrom; + } + + public Long getLastAppliedId() { + return lastAppliedId; + } + + public IndexID getIndexID() { + return new IndexID(id, type); + } + + public IndexInfo(Element element) { + snapshotId = XMLUtil.getAttributeLong("snapshotid", element); + id = XMLUtil.getAttributeLong("id", element); + segmentGeneration = XMLUtil.getAttributeLong("segmentGeneration", element); + type = XMLUtil.getAttributeString("type", element); + maxDoc = XMLUtil.getAttributeInteger("maxDoc", element); + numDocs = XMLUtil.getAttributeInteger("numDocs", element); + deletedDoc = XMLUtil.getAttributeInteger("deletedDoc", element); + minDocumentId = XMLUtil.getAttributeLong("minDocumentId", element); + minSnapshotId = XMLUtil.getAttributeLong("minSnapshotId", element); + maxDocumentId = XMLUtil.getAttributeLong("maxDocumentId", element); + maxSnapshotId = XMLUtil.getAttributeLong("maxSnapshotId", element); + checkpointId = XMLUtil.getAttributeLong("checkpointId", element); + lastAppliedId = XMLUtil.getAttributeLong("lastAppliedId", element); + String mergedFromString = XMLUtil.getAttributeString("mergedFrom", element); + if (StringUtils.isNotBlank(mergedFromString)) { + String[] strings = StringUtils.split(mergedFromString, ","); + mergedFrom = new ArrayList(strings.length); + for (String string : strings) { + String[] array = StringUtils.split(string, ":"); + String type = array[0]; + Long id = new Long(array[1]); + mergedFrom.add(new IndexID(id, type)); + } + } + } + + public Integer getNumDocs() { + return numDocs; + } + + public Long getCheckpointId() { + return checkpointId; + } + + public Long getSnapshotId() { + return snapshotId; + } + + public Long getMinSnapshotId() { + return minSnapshotId; + } + + public Long getMinDocumentId() { + return minDocumentId; + } + + public Long getMaxSnapshotId() { + return maxSnapshotId; + } + + public Long getMaxDocumentId() { + return maxDocumentId; + } + + public Integer getDeletedDoc() { + return deletedDoc; + } + + public Long getSegmentGeneration() { + return segmentGeneration; + } + + public Integer getMaxDoc() { + return maxDoc; + } + + public Long getId() { + return id; + } + + public String getType() { + return type; + } + + public Element toElement() { + Element element = new Element("index"); + XMLUtil.setAttribute("snapshotid", snapshotId, element); + XMLUtil.setAttribute("type", type, element); + XMLUtil.setAttribute("id", id, element); + XMLUtil.setAttribute("segmentGeneration", segmentGeneration, element); + XMLUtil.setAttribute("maxDoc", maxDoc, element); + XMLUtil.setAttribute("numDocs", numDocs, element); + XMLUtil.setAttribute("deletedDoc", deletedDoc, element); + XMLUtil.setAttribute("minDocumentId", minDocumentId, element); + XMLUtil.setAttribute("maxDocumentId", maxDocumentId, element); + XMLUtil.setAttribute("minSnapshotId", minSnapshotId, element); + XMLUtil.setAttribute("maxSnapshotId", maxSnapshotId, element); + XMLUtil.setAttribute("checkpointId", checkpointId, element); + XMLUtil.setAttribute("lastAppliedId", lastAppliedId, element); + if (mergedFrom != null && mergedFrom.size() > 0) { + String mergedFromString = StringUtils.join(mergedFrom, ","); + XMLUtil.setAttribute("mergedFrom", mergedFromString, element); + } + return element; + } + } + /** + public void writeTo(RandomAccessIO output) throws Exception { + Element element = toElement(); + String xml = XMLUtil.outputElement(element); + byte[] bytes = xml.getBytes("UTF-8"); + output.write(bytes); + } + **/ + public Element toElement() { + Element element = new Element("snapshot"); + XMLUtil.setAttribute("id", id, element); + XMLUtil.setAttribute("numDocs", numDocs, element); + XMLUtil.setAttribute("maxDoc", maxDoc, element); + XMLUtil.setAttribute("deletedDocs", deletedDocs, element); + DateFormat dateFormat = DateFormat.getDateTimeInstance(); + String timeString = dateFormat.format(new Date(timestamp)); + XMLUtil.setAttribute("timestamp", timeString, element); + for (IndexInfo indexInfo : indexInfos.values()) { + element.addContent(indexInfo.toElement()); + } + return element; + } +} \ No newline at end of file Index: ocean/src/org/apache/lucene/ocean/snapshotlog/SnapshotLogFile.java =================================================================== --- ocean/src/org/apache/lucene/ocean/snapshotlog/SnapshotLogFile.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/snapshotlog/SnapshotLogFile.java (revision 0) @@ -0,0 +1,174 @@ +package org.apache.lucene.ocean.snapshotlog; + +import java.io.EOFException; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.locks.ReentrantLock; + +import org.apache.lucene.ocean.LogDirectory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Log file for snapshot infos. It is human readable using strings. + * + */ +public class SnapshotLogFile { + final static Logger LOG = LoggerFactory.getLogger(SnapshotLogFile.class); + private long currentWritePosition = 0; + private RandomAccessFile output; + private String file; + private LogDirectory logDirectory; + private List infoHeaders = new ArrayList(); // sorted + // descending + // index + private Long id; + private int size = 0; + private ReentrantLock writeLock = new ReentrantLock(); + + + public SnapshotLogFile(Long id, String file, LogDirectory logDirectory, int numHeadersOnHand) throws IOException { + this.id = id; + this.file = file; + this.logDirectory = logDirectory; + output = logDirectory.getOutput(file, false); + currentWritePosition = logDirectory.fileLength(file); + loadLastInfos(numHeadersOnHand); + } + + public int getSize() { + return size; + } + + public void close() throws IOException { + output.close(); + } + + public void delete() throws IOException { + output.close(); + boolean deleted = logDirectory.deleteFile(file); + System.out.println("file: "+file+" deleted: "+deleted); + } + + public long length() throws IOException { + return output.length(); + } + + public Long getId() { + return id; + } + + public static class InfoHeader { + public final Long logId; + public final long position; + public final int length; + public final int index; + + public InfoHeader(Long logId, int index, long position, int length) { + this.logId = logId; + this.index = index; + this.position = position; + this.length = length; + } + } + + public InfoHeader write(String string) throws IOException { + writeLock.lock(); + try { + int index = 0; + if (infoHeaders.size() > 0) { + index = infoHeaders.get(0).index + 1; + } + byte[] bytes = string.getBytes("UTF-8"); + output.seek(currentWritePosition); + output.writeBytes(Integer.toString(bytes.length)); + output.write('\n'); + long position = output.getFilePointer(); + output.write(bytes); + output.getFD().sync(); + currentWritePosition = output.getFilePointer(); + InfoHeader infoHeader = new InfoHeader(id, index, position, bytes.length); + infoHeaders.add(0, infoHeader); + size++; + return infoHeader; + } finally { + writeLock.unlock(); + } + } + + public String loadData(InfoHeader infoHeader) throws IOException { + RandomAccessFile randomAccessFile = logDirectory.openInput(file); + try { + randomAccessFile.seek(infoHeader.position); + byte[] bytes = new byte[infoHeader.length]; + randomAccessFile.readFully(bytes); + return new String(bytes, "UTF-8"); + } finally { + randomAccessFile.close(); + } + } + + public List getLastInfos(int n) { + List list = new ArrayList(n); + Iterator iterator = infoHeaders.iterator(); + for (int x = 0; x < n; x++) { + if (iterator.hasNext()) { + list.add(iterator.next()); + } else + break; + } + return list; + } + + private String readLine(RandomAccessFile randomAccessFile) throws IOException { + StringBuilder builder = new StringBuilder(5); + while (true) { + byte b = randomAccessFile.readByte(); + if (b == '\n') { + break; + } + builder.append((char)b); + } + return builder.toString(); + } + + /** + * Loads only the last n snapshotinfos + * + * @param n + * number of last infos to load + * @throws IOException + */ + public void loadLastInfos(int n) throws IOException { + int index = 0; + RandomAccessFile randomAccessFile = logDirectory.openInput(file); + try { + while (true) { + try { + //String lengthLine = randomAccessFile.readLine(); + String lengthLine = readLine(randomAccessFile); + int length = Integer.parseInt(lengthLine); + long position = randomAccessFile.getFilePointer(); + InfoHeader infoHeader = new InfoHeader(id, index, position, length); + if (infoHeaders.size() > n) { + // remove the last one + infoHeaders.remove(infoHeaders.size() - 1); + } + // add the newest to the beginning + infoHeaders.add(0, infoHeader); + index++; + size++; + // skip to the next info + randomAccessFile.seek(randomAccessFile.getFilePointer() + length); + } catch (EOFException eofException) { + break; + } + } + } finally { + randomAccessFile.close(); + } + } +} Index: ocean/src/org/apache/lucene/ocean/snapshotlog/SnapshotLogManager.java =================================================================== --- ocean/src/org/apache/lucene/ocean/snapshotlog/SnapshotLogManager.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/snapshotlog/SnapshotLogManager.java (revision 0) @@ -0,0 +1,153 @@ +package org.apache.lucene.ocean.snapshotlog; + +import java.io.IOException; +import java.text.DecimalFormat; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.locks.ReentrantLock; + +import org.apache.lucene.ocean.LogDirectory; +import org.apache.lucene.ocean.SnapshotInfo; +import org.apache.lucene.ocean.snapshotlog.SnapshotLogFile.InfoHeader; +import org.apache.lucene.ocean.util.CElement; +import org.apache.lucene.ocean.util.SortedList; +import org.apache.lucene.ocean.util.XMLUtil; +import org.jdom.Element; + +/** + * Writes to a files that contain snapshotinfos + * + */ +public class SnapshotLogManager { + private SortedList logFiles = new SortedList(); + private int numHeadersOnHand; + private int maxSize; + private LogDirectory logDirectory; + private ReentrantLock writeLock = new ReentrantLock(); + + public SnapshotLogManager(int maxSize, int numHeadersOnHand, LogDirectory logDirectory) throws IOException { + if (numHeadersOnHand > maxSize) throw new IOException("maxSize must be greater than numHeadersOnHand"); + this.maxSize = maxSize; + this.numHeadersOnHand = numHeadersOnHand; + this.logDirectory = logDirectory; + for (String name : logDirectory.list()) { + if (name.startsWith("snapshot")) { + Long id = getIdFromName(name); + SnapshotLogFile snapshotLogFile = new SnapshotLogFile(id, name, logDirectory, numHeadersOnHand); + logFiles.put(id, snapshotLogFile); + } + } + if (logFiles.size() == 0) { + createNewLogFile(); + } + } + + public void close() throws IOException { + for (SnapshotLogFile logFile : logFiles.values()) { + logFile.close(); + } + } + + public int numLogFiles() { + return logFiles.size(); + } + + public static String getNameFromId(Long id) { + DecimalFormat decimalFormat = (DecimalFormat) DecimalFormat.getInstance(); + decimalFormat.applyPattern("00000000"); + return "snapshot" + decimalFormat.format(id) + ".log"; + } + + public static Long getIdFromName(String string) { + int dot = string.lastIndexOf('.'); + String substring = string.substring("snapshot".length(), dot); + return new Long(substring); + } + + public List getLastInfos(int n) throws IOException { + if (n > numHeadersOnHand) + throw new IOException("n greater than " + numHeadersOnHand); + List infos = getCurrent().getLastInfos(n); + List returnInfos = new ArrayList(infos); + // add more from previous log file if there is one + if (n > infos.size() && logFiles.size() > 1) { + List otherInfos = logFiles.get(logFiles.size() - 2).getLastInfos(n - infos.size()); + returnInfos.addAll(otherInfos); + } + return returnInfos; + } + + public SnapshotLogFile getLogFile(Long id) { + return logFiles.get(id); + } + + private SnapshotLogFile getCurrent() { + return logFiles.lastValue(); + } + + private void deleteOldLogFile() throws IOException { + if (logFiles.size() > 1) { + SnapshotLogFile current = logFiles.lastValue(); + List currentHeaders = current.getLastInfos(numHeadersOnHand); + // if current log file has enough headers + if (currentHeaders.size() >= numHeadersOnHand) { + // can delete old files + List deleteIds = new ArrayList(); + for (int x = 0; x < logFiles.size() - 1; x++) { + SnapshotLogFile logFile = logFiles.get(x); + logFile.delete(); + deleteIds.add(logFile.getId()); + } + for (Long id : deleteIds) { + logFiles.remove(id); + } + } + } + } + + private void createNewLogFile() throws IOException { + Long id = null; + if (logFiles.size() == 0) { + id = new Long(1); + } else { + id = logFiles.lastKey().longValue() + 1; + } + String name = getNameFromId(id); + SnapshotLogFile snapshotLogFile = new SnapshotLogFile(id, name, logDirectory, numHeadersOnHand); + logFiles.put(id, snapshotLogFile); + } + + public InfoHeader write(CElement object) throws Exception { + Element element = object.toElement(); + String xml = XMLUtil.outputElementOmitDeclaration(element); + return write(xml); + } + + public InfoHeader write(String string) throws IOException { + writeLock.lock(); + try { + SnapshotLogFile snapshotLogFile = getCurrent(); + InfoHeader infoHeader = snapshotLogFile.write(string); + deleteOldLogFile(); + if (snapshotLogFile.getSize() >= maxSize) { + createNewLogFile(); + } + return infoHeader; + } finally { + writeLock.unlock(); + } + } + + public SnapshotInfo loadMaxSnapshotInfo() throws Exception { + List infos = getLastInfos(5); + if (infos.size() == 0) return null; + String xml = loadData(infos.get(0)); + Element element = XMLUtil.parseElement(xml); + return new SnapshotInfo(element); + } + + public String loadData(InfoHeader infoHeader) throws IOException { + SnapshotLogFile snapshotLogFile = getLogFile(infoHeader.logId); + return snapshotLogFile.loadData(infoHeader); + } +} Index: ocean/src/org/apache/lucene/ocean/Snapshots.java =================================================================== --- ocean/src/org/apache/lucene/ocean/Snapshots.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/Snapshots.java (revision 0) @@ -0,0 +1,190 @@ +package org.apache.lucene.ocean; + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import org.apache.lucene.ocean.snapshotlog.SnapshotLogManager; +import org.apache.lucene.ocean.util.SortedList; +import org.apache.lucene.ocean.util.Util; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class Snapshots { + final static Logger LOG = LoggerFactory.getLogger(Snapshots.class); + private SortedList list = new SortedList(); + private TransactionSystem system; + + public Snapshots(TransactionSystem system) { + this.system = system; + } + + public synchronized boolean hasRefs(Long snapshotId) { + Snapshot snapshot = get(snapshotId, false); + if (snapshot == null) + return false; + return snapshot.hasRefs(); + } + + public synchronized void remove(int max, long durationMillis) { + if (list.size() > max) { + long now = System.currentTimeMillis(); + int numToCheck = list.size() - max; + Iterator iterator = list.values().iterator(); + for (int x = 0; x < numToCheck; x++) { + Snapshot snapshot = iterator.next(); + if (!snapshot.hasRefs() && (snapshot.getTimestamp() + durationMillis) > now) { + iterator.remove(); + } + } + } + } + + /** + * Loads the ids from the file names rather than loading each xml file. + * + * @param directory + * @return + * @throws Exception + * + * public static List loadSnapshotInfoIds(LogDirectory directory) + * throws Exception { List list = new ArrayList(); + * for (String file : directory.list()) { if (directory.fileLength(file) > 0) { + * String str = "snapshot_"; if (file.startsWith(str)) { String main = + * file.substring(str.length(), file.lastIndexOf('.')); String[] split = + * StringUtils.split(main, "_"); if (split.length > 1) { String replace = + * main.replace('_', '.'); //System.out.println("replace: "+replace); + * list.add(new BigDecimal(replace)); } else { Long snapshotId = new + * Long(split[0]); list.add(new BigDecimal(snapshotId)); } } } } + * Collections.sort(list); return list; } + * + * public static List loadSnapshotInfos(LogDirectory directory) + * throws Exception { List snapshotInfos = new ArrayList(); + * for (String file : directory.list()) { if (directory.fileLength(file) > 0) { + * String str = "snapshot_"; if (file.startsWith(str)) { String main = + * file.substring(str.length(), file.lastIndexOf('.')); String[] split = + * StringUtils.split(main, "_"); Long snapshotId = new Long(split[0]); Integer + * version = new Integer(0); if (split.length > 1) version = new + * Integer(split[1]); String xml = Util.getString(file, directory); Element + * element = XMLUtil.parseElement(xml); snapshotInfos.add(new + * SnapshotInfo(element)); // sorted.add(new BigDecimal(snapshotId + "." + + * version)); } } } Collections.sort(snapshotInfos); return snapshotInfos; } + * + * public static SnapshotInfo loadMaxSnapshotInfo(LogDirectory directory) + * throws Exception { List ids = loadSnapshotInfoIds(directory); + * if (ids.size() == 0) return null; BigDecimal maxId = Util.max(ids); String + * fileName = Snapshot.getFileName(maxId); String xml = + * Util.getString(fileName, directory); Element element = + * XMLUtil.parseElement(xml); return new SnapshotInfo(element); } + */ + private synchronized Snapshot get(long snapshotId, boolean incref) { + List snapshots = getForSnapshot(snapshotId, false); + Snapshot snapshot = Util.max(snapshots); + snapshot.incRef(); + return snapshot; + } + + public synchronized Snapshot get(long snapshotId) { + return get(snapshotId, true); + } + + private synchronized List getForSnapshot(long snapshotId, boolean incref) { + List inrange = new ArrayList(); + Iterator> iterator = list.iterator(new BigDecimal(snapshotId), true, new BigDecimal(snapshotId+1), false); + while (iterator.hasNext()) { + Snapshot snapshot = iterator.next().getValue(); + inrange.add(snapshot); + } + return inrange; + } + + /** + public synchronized List getForSnapshot(long snapshotId) { + List inrange = new ArrayList(); + for (Snapshot snapshot : list.values()) { + long l = snapshot.getId().toBigInteger().longValue(); + if (l == snapshotId) { + inrange.add(snapshot); + } + } + return inrange; + } + **/ + public synchronized boolean contains(BigDecimal id) { + return list.containsKey(id); + } + + public synchronized boolean contains(Long snapshotId) { + return get(snapshotId, false) != null; + } + + /** + * public synchronized boolean containsIndex(long indexid) { for (Snapshot + * snapshot : list) { if (snapshot.containsIndex(indexid)) return true; } + * return false; } + */ + /** + private void remove(Snapshot snapshot) throws IOException { + Iterator iterator = list.values().iterator(); + while (iterator.hasNext()) { + Snapshot s = iterator.next(); + if (s.getId().equals(snapshot.getId())) { + iterator.remove(); + String file = Snapshot.getFileName(snapshot.getId()); + system.directoryMap.getDirectory().deleteFile(file); + } + } + } + **/ + public synchronized Snapshot getLatestSnapshot() { + return getLatestSnapshot(true); + } + + private synchronized Snapshot getLatestSnapshot(boolean incref) { + if (list.size() == 0) + return null; + Snapshot snapshot = list.get(list.size() - 1); + if (incref) snapshot.incRef(); + return snapshot; + } + + synchronized void add(Snapshot snapshot, boolean createFile) throws Exception { + // verify on minor version snapshot numdocs is the same + if (snapshot.isMinor()) { + // assert snapshot.numDocs() == getLatestSnapshot().numDocs(); + } + if (createFile) { + addCreateFile(snapshot); + } else { + list.put(snapshot.getId(), snapshot); + } + LOG.info("snapshot: " + snapshot.toString() + " added"); + } + + private void addCreateFile(Snapshot snapshot) throws Exception { + BigDecimal id = snapshot.getId(); + SnapshotInfo snapshotInfo = snapshot.getSnapshotInfo(); + SnapshotLogManager snapshotLogManager = system.getSnapshotLogManager(); + snapshotLogManager.write(snapshotInfo); + list.put(id, snapshot); + } + + /** + * private void addCreateFile(Snapshot snapshot) throws Exception { BigDecimal + * id = snapshot.getId(); SnapshotInfo snapshotInfo = + * snapshot.getSnapshotInfo(); String fileName = Snapshot.getFileName(id); + * LogDirectory directory = system.directoryMap.getDirectory(); if + * (directory.fileExists(fileName)) { throw new IOException("fileName: " + + * fileName + " already exists"); } RandomAccessFile output = + * directory.getOutput(fileName, true); Element element = + * snapshotInfo.toElement(); String xml = + * XMLUtil.outputElementOmitDeclaration(element); byte[] bytes = + * xml.getBytes("UTF-8"); list.add(snapshot); output.write(bytes, 0, + * bytes.length); output.close(); // remove previous versions for this id // + * SortedMap headMap = snapshotMap.headMap(id); // for + * (Snapshot removeSnapshot : headMap.values()) { // remove(removeSnapshot); // } + * Collections.sort(list); } + */ +} \ No newline at end of file Index: ocean/src/org/apache/lucene/ocean/Transaction.java =================================================================== --- ocean/src/org/apache/lucene/ocean/Transaction.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/Transaction.java (revision 0) @@ -0,0 +1,17 @@ +package org.apache.lucene.ocean; + +import java.util.concurrent.Callable; + +public abstract class Transaction implements Callable { + public abstract boolean go(); + + //abstract void addDocsAdded(int value); + + public abstract void ready(Index index); + + public abstract Long getId(); + + public abstract void failed(Index index, Throwable throwable); + + public abstract Snapshot getPreviousSnapshot(); +} Index: ocean/src/org/apache/lucene/ocean/TransactionSystem.java =================================================================== --- ocean/src/org/apache/lucene/ocean/TransactionSystem.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/TransactionSystem.java (revision 0) @@ -0,0 +1,1061 @@ +package org.apache.lucene.ocean; + +import java.io.IOException; +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantLock; + +import org.apache.commons.lang.StringUtils; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.OceanSegmentReader; +import org.apache.lucene.index.SerialMergeScheduler; +import org.apache.lucene.index.Term; +import org.apache.lucene.ocean.Batch.MasterBatch; +import org.apache.lucene.ocean.Batch.SlaveBatch; +import org.apache.lucene.ocean.DiskIndex.DiskIndexSnapshot; +import org.apache.lucene.ocean.Index.IndexException; +import org.apache.lucene.ocean.Index.IndexSnapshot; +import org.apache.lucene.ocean.IndexCreator.Add; +import org.apache.lucene.ocean.RamIndex.RamIndexSnapshot; +import org.apache.lucene.ocean.SnapshotInfo.IndexInfo; +import org.apache.lucene.ocean.WriteableMemoryIndex.MemoryIndexSnapshot; +import org.apache.lucene.ocean.log.TransactionLog; +import org.apache.lucene.ocean.log.TransactionLog.SlaveBatchIterator; +import org.apache.lucene.ocean.snapshotlog.SnapshotLogManager; +import org.apache.lucene.ocean.util.Bytes; +import org.apache.lucene.ocean.util.Constants; +import org.apache.lucene.ocean.util.LongSequence; +import org.apache.lucene.ocean.util.SortedList; +import org.apache.lucene.ocean.util.Timeout; +import org.apache.lucene.ocean.util.Util; +import org.apache.lucene.search.OceanMultiThreadSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.RAMDirectory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Main class for search transaction system. + * + * Indexes on disk are immutable, they can only be deleted from or merged + * periodically. Merges occur in the background. There is always one active + * WriteableMemoryIndex that new documents are written to. + * + * A snapshot corresponds to a transaction. Each transaction creates a new + * snapshot. Snapshot ids have both major and minor version represented as a + * decimal. The major represents the transaction. The minor increments with + * index merges. Transaction data is known as a batch. There is a MasterBatch + * and SlaveBatch. A MasterBatch is created in the initial update call to + * TransactionSystem such as addDocument. All update calls eventually go through + * commitBatch(Batch batch). A SlaveBatch is what is loaded from the + * transactionlog during a recovery. + * + * IndexWriter like methods such as addDocument, updateDocument are provided. + * The commitTransaction method provides complete transaction access. + * + * A _documentid field is added to each document. This is an internal number for + * tracking a document and allows the transaction log system to be recovered + * properly. During recovery a delete will use the _documentid rather than the + * actual query or term to insure the exact documents are deleted at the point + * in time the transaction occurred. + * + * + */ +// TODO: need test case of maybeMergeDiskIndices +// TODO: custom efficient document serializer +// TODO: not sure how to handle Document fields with a TokenStream +// TODO: make transaction timeout a batch parameter +// TODO: make multithreaded transactions optional +// TODO: test disk indexes with too many deletes merging +// TODO: remove lang and io library dependencies +// TODO: write test cases for LogFileManager +// TODO: build recovery to load based on last deletes id +// TODO: allow optional queryparser to be defined for serializing deletes in a +// more space efficient manner +// TODO: add optional LRU size cache to transaction log for new records +// TODO: remove snapshot id from document, rely on snapshotinfo +// TODO: add writeableindex to master batch +// TODO: create wrapper for TransactionSystem enabling optimistic concurrency +// and support for dynamic fields +// TODO: test RawLogFile.FileStreamData.getInputStream crc check +// TODO: add commitnothing +// TODO: logfilemanager if logfile is empty then delete and use previous log +// file +// TODO: create file deleter that tries to delete files as windows has problems +// with this +// TODO: create how to use Ocean section in wiki +public class TransactionSystem { + final static Logger LOG = LoggerFactory.getLogger(TransactionSystem.class); + public static final int DEFAULT_MEMORY_INDEX_MAX_DOCS = 50; + public static final int DEFAULT_MAYBE_MERGE_DOC_CHANGES = 2000; + public static final int DEFAULT_MAX_RAM_INDEXES_SIZE = 1024 * 1024 * 30; + public static final float DEFAULT_MERGE_DISK_DELETED_PERCENT = 0.3f; + public static final float DEFAULT_DELETES_FLUSH_THRESHOLD_PERCENT = 0.3f; + public static final long DEFAULT_MAYBE_MERGES_TIMER_INTERVAL = 60 * 1000; + public static final long DEFAULT_LOG_FILE_DELETE_TIMER_INTERVAL = 100 * 1000; + private ThreadPoolExecutor commitThreadPool; + private ThreadPoolExecutor mergeThreadPool; + private TransactionLog transactionLog; + private Indexes indexes = new Indexes(); + private ReentrantLock commitLock = new ReentrantLock(); + Snapshots snapshots; + private ReentrantLock mergeIndexesLock = new ReentrantLock(); + private int docChangesSinceLastMerge = 0; + private Analyzer defaultAnalyzer; + private int serverNumber = 0; + private LongSequence documentSequence; + private LongSequence diskIndexSequence; + private LongSequence ramIndexSequence; + private int memoryIndexMaxDocs = DEFAULT_MEMORY_INDEX_MAX_DOCS; + private int maybeMergeDocChanges = DEFAULT_MAYBE_MERGE_DOC_CHANGES; + private int maxRamIndexesSize = DEFAULT_MAX_RAM_INDEXES_SIZE; + private int maxDocsIndexes = -1; + private int maxSnapshots = 5; + private float mergeDiskDeletedPercent = DEFAULT_MERGE_DISK_DELETED_PERCENT; + private long snapshotExpiration = 20 * 1000; + private float deletesFlushThresholdPercent = DEFAULT_DELETES_FLUSH_THRESHOLD_PERCENT; + private long maybeMergesTimerInterval = DEFAULT_MAYBE_MERGES_TIMER_INTERVAL; + private long logFileDeleteTimerInterval = DEFAULT_LOG_FILE_DELETE_TIMER_INTERVAL; + private int diskIndexRAMDirectoryBufferSize = 1024 * 512; + DirectoryMap directoryMap; + private ArrayBlockingQueue mergeQueue; + private SearcherPolicy searcherPolicy; + private ExecutorService searchThreadPool; + private ArrayBlockingQueue searchQueue; + private SortedList largeBatches = new SortedList(); + private ReentrantLock largeBatchLock = new ReentrantLock(); + static { + System.setProperty("org.apache.lucene.SegmentReader.class", OceanSegmentReader.class.getName()); + } + private long maybeMergesTimestamp = -1; + private ScheduledExecutorService maybeMergesExecutorService; + private ScheduledExecutorService logFileDeleteExecutorService; + private SnapshotLogManager snapshotLogManager; + + public TransactionSystem(TransactionLog transactionLog, Analyzer defaultAnalyzer, DirectoryMap directoryMap) throws Exception { + this(transactionLog, defaultAnalyzer, directoryMap, DEFAULT_MAYBE_MERGE_DOC_CHANGES, -1, DEFAULT_MEMORY_INDEX_MAX_DOCS, + DEFAULT_MERGE_DISK_DELETED_PERCENT, new SingleThreadSearcherPolicy()); + } + + public TransactionSystem(TransactionLog transactionLog, Analyzer defaultAnalyzer, DirectoryMap directoryMap, int maybeMergeDocChanges, + int maxDocsIndexes, int memoryIndexMaxDocs, float mergeDiskDeletedPercent, SearcherPolicy searcherPolicy) throws Exception { + this.transactionLog = transactionLog; + this.defaultAnalyzer = defaultAnalyzer; + this.directoryMap = directoryMap; + this.maybeMergeDocChanges = maybeMergeDocChanges; + this.maxDocsIndexes = maxDocsIndexes; + this.memoryIndexMaxDocs = memoryIndexMaxDocs; + this.mergeDiskDeletedPercent = mergeDiskDeletedPercent; + this.searcherPolicy = searcherPolicy; + if (searcherPolicy instanceof MultiThreadSearcherPolicy) { + MultiThreadSearcherPolicy multiThreadSearcherPolicy = (MultiThreadSearcherPolicy) searcherPolicy; + searchQueue = new ArrayBlockingQueue(multiThreadSearcherPolicy.getQueueSize()); + searchThreadPool = new ThreadPoolExecutor(multiThreadSearcherPolicy.getMinThreads(), multiThreadSearcherPolicy.getMaxThreads(), + 1000 * 60, TimeUnit.MILLISECONDS, searchQueue); + } + mergeQueue = new ArrayBlockingQueue(2); + mergeThreadPool = new ThreadPoolExecutor(1, 1, 1000 * 60, TimeUnit.MILLISECONDS, mergeQueue); + commitThreadPool = new ThreadPoolExecutor(1, 10, 1000 * 60, TimeUnit.MILLISECONDS, new LinkedBlockingQueue()); + snapshotLogManager = new SnapshotLogManager(100, 10, directoryMap.getDirectory()); + snapshots = new Snapshots(this); + if (LOG.isInfoEnabled()) + LOG.info("TransactionSystem"); + load(); + maybeMergesExecutorService = (ScheduledExecutorService) Executors.newSingleThreadScheduledExecutor(); + maybeMergesExecutorService.scheduleWithFixedDelay(new MaybeMergesTimer(), maybeMergesTimerInterval, maybeMergesTimerInterval, + TimeUnit.MILLISECONDS); + logFileDeleteExecutorService = (ScheduledExecutorService) Executors.newSingleThreadScheduledExecutor(); + logFileDeleteExecutorService.scheduleWithFixedDelay(new LogFileDeleteTimer(), logFileDeleteTimerInterval, logFileDeleteTimerInterval, + TimeUnit.MILLISECONDS); + } + + public int getDiskIndexRAMDirectoryBufferSize() { + return diskIndexRAMDirectoryBufferSize; + } + + public SnapshotLogManager getSnapshotLogManager() { + return snapshotLogManager; + } + + // TODO: can use deleteFlushId instead of getMinSnapshotId + public class LogFileDeleteTimer implements Runnable { + public void run() { + Snapshot snapshot = snapshots.getLatestSnapshot(); + try { + List minSnapshotIds = new ArrayList(); + List diskIndexSnapshots = snapshot.getDiskIndexSnapshots(); + for (DiskIndexSnapshot diskIndexSnapshot : diskIndexSnapshots) { + Long minSnapshotId = diskIndexSnapshot.getMinSnapshotId(); + if (minSnapshotId != null) { + minSnapshotIds.add(minSnapshotId); + } + } + Long minIndexSnapshotId = Util.min(minSnapshotIds); + transactionLog.deleteOldLogFiles(minIndexSnapshotId); + } catch (IOException ioException) { + LOG.error("", ioException); + } finally { + snapshot.decRef(); + } + } + } + + /** + * If the last merge timestamp is greater than maybeMergesTimerInterval then + * MaybeMergeIndices is submitted to the mergeThreadPool + * + */ + public class MaybeMergesTimer implements Runnable { + public void run() { + if (mergeIndexesLock.isLocked()) + return; + if ((System.currentTimeMillis() - maybeMergesTimestamp) > maybeMergesTimerInterval) { + if (LOG.isInfoEnabled()) + LOG.info("submitting MaybeMergeIndices from timer"); + mergeThreadPool.submit(new MaybeMergeIndexes()); + } + } + } + + public float getDeletesFlushThresholdPercent() { + return deletesFlushThresholdPercent; + } + + public IndexID getIndexId(Long documentId) throws IOException { + Snapshot snapshot = snapshots.getLatestSnapshot(); + try { + for (IndexSnapshot indexSnapshot : snapshot.getIndexSnapshots()) { + IndexReader indexReader = indexSnapshot.getIndexReader(); + int freq = indexReader.docFreq(new Term(Constants.DOCUMENTID, Util.longToEncoded(documentId))); + if (freq > 0) { + return indexSnapshot.getIndex().getId(); + } + } + return null; + } finally { + snapshot.decRef(); + } + } + + /** + * Number of docs that have to change, adds + deletes, before + * MaybeMergeIndexes is called + * + * @param maybeMergeDocChanges + */ + public void setMaybeMergeDocChanges(int maybeMergeDocChanges) { + this.maybeMergeDocChanges = maybeMergeDocChanges; + } + + public void setMaxDocsIndexes(int maxDocsIndexes) { + this.maxDocsIndexes = maxDocsIndexes; + } + + public void close() throws IOException { + if (LOG.isInfoEnabled()) + LOG.info("close"); + maybeMergesExecutorService.shutdown(); + mergeThreadPool.shutdown(); + commitThreadPool.shutdown(); + snapshotLogManager.close(); + transactionLog.close(); + for (Index index : indexes.getIndexes()) { + index.close(); + } + } + + public Document getDocument(Term term) throws IOException { + Snapshot snapshot = snapshots.getLatestSnapshot(); + try { + IndexReader reader = snapshot.getIndexReader(); + int doc = Util.getTermDoc(term, reader); + if (doc >= 0) + return reader.document(doc); + else + return null; + } finally { + snapshot.decRef(); + } + } + + public OceanSearcher getSearcher() throws IOException { + Snapshot snapshot = snapshots.getLatestSnapshot(); + if (searcherPolicy instanceof SingleThreadSearcherPolicy) { + return new OceanSearcher(snapshot); + } else { + return new OceanMultiThreadSearcher(snapshot, searchThreadPool); + } + } + + public CommitResult deleteDocument(Query query) throws Exception { + List deleteByQueries = new ArrayList(1); + deleteByQueries.add(query); + return commitTransaction(null, null, null, deleteByQueries); + } + + public CommitResult deleteDocument(Term term) throws Exception { + List dterms = new ArrayList(1); + dterms.add(term); + return commitTransaction(null, null, dterms, null); + } + + public CommitResult updateDocument(Term term, Document document) throws Exception { + return updateDocument(term, document, defaultAnalyzer); + } + + public CommitResult updateDocument(Term term, Document document, Analyzer analyzer) throws Exception { + List list = new ArrayList(1); + list.add(document); + List dterms = new ArrayList(1); + dterms.add(term); + return commitTransaction(list, analyzer, dterms, null); + } + + public CommitResult addDocument(Document document) throws Exception { + return addDocument(document, defaultAnalyzer); + } + + public CommitResult addDocument(Document document, Analyzer analyzer) throws Exception { + List list = new ArrayList(1); + list.add(document); + return commitTransaction(list, analyzer, null, null); + } + + // TODO: possibly allow Iterator or BlockingQueue + public CommitResult commitTransaction(List documents, Analyzer analyzer, List deleteByTerms, List deleteByQueries) + throws Exception { + if (analyzer == null) + analyzer = getDefaultAnalyzer(); + MasterBatch masterBatch = new MasterBatch(this); + if (documents != null) + masterBatch.addDocuments(new Documents(documents)); + masterBatch.setAnalyzer(analyzer); + Deletes deletes = new Deletes(); + if (deleteByTerms != null) { + for (Term deleteTerm : deleteByTerms) { + deletes.addTerm(deleteTerm); + } + } + if (deleteByQueries != null) { + for (Query query : deleteByQueries) { + deletes.addQuery(query); + } + } + if (deletes.hasDeletes()) + masterBatch.setDeletes(deletes); + return commitBatch(masterBatch); + } + + public void merge(boolean doWait) { + if (doWait) + new MaybeMergeIndexes().run(); + else + mergeThreadPool.submit(new MaybeMergeIndexes()); + } + + public Analyzer getDefaultAnalyzer() { + return defaultAnalyzer; + } + + public long getNextRamIndexId() { + return ramIndexSequence.getAndIncrement(); + } + + public long getNextDiskIndexId() { + return diskIndexSequence.getAndIncrement(); + } + + public TransactionLog getTransactionLog() { + return transactionLog; + } + + public ExecutorService getCommitThreadPool() { + return commitThreadPool; + } + + public void load() throws Exception { + BigDecimal id; + Long snapshotId; + List indexSnapshots = null; + SnapshotInfo snapshotInfo = snapshotLogManager.loadMaxSnapshotInfo(); + // Snapshots.loadMaxSnapshotInfo(directoryMap.getDirectory()); + if (LOG.isDebugEnabled()) + LOG.debug("snapshotInfo: " + snapshotInfo); + long timestamp = System.currentTimeMillis(); + if (snapshotInfo != null) { + id = snapshotInfo.getId(); + snapshotId = snapshotInfo.getSnapshotId(); + if (snapshotId.longValue() != transactionLog.getMaxId().longValue()) { + LOG.error("snapshotId: " + snapshotId + " transactionLog.getMaxId: " + transactionLog.getMaxId()); + assert snapshotId == transactionLog.getMaxId(); + } + loadDiskIndexes(snapshotInfo, indexes); + IndexID diskMaxId = indexes.getMaxId("disk"); + if (diskMaxId != null) + diskIndexSequence = new LongSequence(diskMaxId.id.longValue() + 1, 1); + else + diskIndexSequence = new LongSequence(1, 1); + ramIndexSequence = new LongSequence(1, 1); + indexSnapshots = new ArrayList(); + List snapshotIds = new LinkedList(); + // TODO: what if index directory is deleted and it is still referenced + for (IndexInfo indexInfo : snapshotInfo.getIndexInfos()) { + if (indexInfo.getType().equals("disk")) { + DiskIndex diskIndex = (DiskIndex) indexes.get(indexInfo.getIndexID()); + if (diskIndex != null) { + IndexSnapshot indexSnapshot = diskIndex.getIndexSnapshot(snapshotInfo.getSnapshotId()); + indexSnapshots.add(indexSnapshot); + Long indexSnapshotId = indexSnapshot.getMaxSnapshotId(); + if (indexSnapshotId == null) { + throw new Exception(indexInfo.getIndexID() + " does not have a max snapshot id"); + } + snapshotIds.add(indexSnapshotId); + } + } + } + Long maxDiskIndexSnapshotId = Util.max(snapshotIds); + Long fromSnapshotId = null; + System.out.println("maxDiskIndexSnapshotId: " + maxDiskIndexSnapshotId); + if (maxDiskIndexSnapshotId != null) { + fromSnapshotId = new Long(maxDiskIndexSnapshotId.longValue() + 1); + } + List ramIndexSnapshots = runTransactionsNotInIndex(fromSnapshotId); + System.out.println("ramIndexSnapshots: " + ramIndexSnapshots); + // TODO: verify all snapshots have same id + indexSnapshots.addAll(ramIndexSnapshots); + List documentIds = new ArrayList(indexSnapshots.size()); + for (IndexSnapshot indexSnapshot : indexSnapshots) { + documentIds.add(indexSnapshot.getMaxDocumentId()); + } + Long maxDocumentId = Util.max(documentIds); + if (maxDocumentId != null) { + Long documentSequenceId = Util.getNextServerSequence(maxDocumentId, serverNumber); + documentSequence = new LongSequence(documentSequenceId, 100); + } else { + documentSequence = new LongSequence(serverNumber, 100); + } + } else { + snapshotId = new Long(0); + id = new BigDecimal(snapshotId.toString()); + documentSequence = new LongSequence(serverNumber, 100); + diskIndexSequence = new LongSequence(1, 1); + ramIndexSequence = new LongSequence(1, 1); + } + WriteableMemoryIndex writeableMemoryIndex = newWriteableMemoryIndex(); + MemoryIndexSnapshot writeableSnapshot = writeableMemoryIndex.createIndexSnapshot(snapshotId); + if (indexSnapshots == null) { + indexSnapshots = new ArrayList(); + indexSnapshots.add(writeableSnapshot); + } + Snapshot snapshot = new Snapshot(id, writeableSnapshot, indexSnapshots, this, timestamp); + snapshots.add(snapshot, false); + new MaybeMergeIndexes().run(); + } + + /** + * Delete snapshotinfo if no longer referenced in Snapshots + * + * @throws Exception + * + * private void deleteUnreferencedSnapshots() throws Exception { + * snapshots.remove(maxSnapshots, snapshotExpiration); LogDirectory directory = + * directoryMap.getDirectory(); List ids = + * Snapshots.loadSnapshotInfoIds(directory); for (BigDecimal id : ids) { if + * (!snapshots.contains(id)) { // not referenced, delete it String fileName = + * Snapshot.getFileName(id); //System.out.println("deleteFile: " + fileName + " + * id: " + Snapshot.formatId(id)); try { directory.deleteFile(fileName); //if + * (LOG.isDebugEnabled()) // LOG.debug("deleteFile: " + fileName); } catch + * (Exception exception) { LOG.error(exception.getMessage()); } } } } + */ + public Indexes getIndexes() { + return indexes; + } + + public Snapshots getSnapshots() { + return snapshots; + } + + /** + * Runs the transactions from the transaction log that are not already in + * Lucene indexes + * + * @param startSnapshotId + * @return loaded ram snapshots + * @throws Exception + * @throws CategoryException + * @throws IOException + */ + private List runTransactionsNotInIndex(Long startSnapshotId) throws Exception, IOException { + LOG.info("startSnapshotId: " + startSnapshotId); + SlaveBatchIterator iterator = transactionLog.getSlaveBatchIterator(startSnapshotId); + if (!iterator.hasNext()) + return new ArrayList(); + try { + long indexIdNum = ramIndexSequence.getAndIncrement(); + IndexID indexId = new IndexID(indexIdNum, "ram"); + RAMDirectory ramDirectory = new RAMDirectory(); + ExecutorService threadPool = getCommitThreadPool(); + IndexCreator indexCreator = new IndexCreator("runTransactionsNotInIndex", ramDirectory, Long.MAX_VALUE, 4, defaultAnalyzer, + threadPool); + BlockingQueue addQueue = new ArrayBlockingQueue(4000, true); + List deletesList = new ArrayList(); // deletes are + // recorded and run + // against all of the + // snapshots at the + // end + indexCreator.start(addQueue); + List ramDirectories = new ArrayList(); + int docCount = 0; + while (iterator.hasNext()) { + SlaveBatch slaveBatch = iterator.next(true, true); + Analyzer analyzer = slaveBatch.getAnalyzer(); + if (slaveBatch.hasDocuments()) { + Documents documents = slaveBatch.getDocuments(); + for (Document document : documents) { + addQueue.add(new IndexCreator.Add(document)); + docCount++; + } + } else if (slaveBatch.hasRAMDirectory()) { + ramDirectories.add(slaveBatch.getRamDirectory()); + } + if (slaveBatch.hasDeletes()) { + deletesList.add(slaveBatch.getDeletes()); + } + } + LOG.info("docCount: " + docCount); + // if zero means all the transactions were deletes + if (docCount == 0) { + indexCreator.close(); + return new ArrayList(); + } + indexCreator.create(false); + ramDirectories.add(ramDirectory); + Long snapshotId = transactionLog.getMaxId(); + + RAMDirectory totalRAMDirectory = new RAMDirectory(); + IndexWriter indexWriter = new IndexWriter(totalRAMDirectory, false, getDefaultAnalyzer(), true); + indexWriter.setMergeScheduler(new SerialMergeScheduler()); + indexWriter.setUseCompoundFile(false); + indexWriter.addIndexes((RAMDirectory[]) ramDirectories.toArray(new RAMDirectory[0])); + indexWriter.close(); + RamIndex ramIndex = new RamIndex(indexId, snapshotId, deletesList, totalRAMDirectory, this); + indexes.add(ramIndex); + RamIndexSnapshot indexSnapshot = (RamIndexSnapshot) ramIndex.getIndexSnapshot(snapshotId); + assert indexSnapshot != null; + List indexSnapshots = new ArrayList(); + indexSnapshots.add(indexSnapshot); + + /** + * // alternative, create individual snapshots List + * indexSnapshots = new ArrayList(ramDirectories.size()); + * for (RAMDirectory rd : ramDirectories) { RamIndex ramIndex = new + * RamIndex(indexId, snapshotId, deletesList, rd, this); + * indexes.add(ramIndex); RamIndexSnapshot indexSnapshot = + * (RamIndexSnapshot) ramIndex.getIndexSnapshot(snapshotId); assert + * indexSnapshot != null; indexSnapshots.add(indexSnapshot); } + */ + return indexSnapshots; + } finally { + if (iterator != null) + iterator.close(); + } + } + + private void loadDiskIndexes(SnapshotInfo snapshotInfo, Indexes indices) throws Exception, IOException { + for (String name : directoryMap.list()) { + try { + if (name.endsWith("_index")) { + String idString = StringUtils.split(name, "_")[0]; + Directory directory = directoryMap.get(name); + Long indexIdNum = new Long(idString); + IndexID indexId = new IndexID(indexIdNum, "disk"); + try { + IndexInfo indexInfo = snapshotInfo.getIndexInfo(indexId); + if (indexInfo != null) { + Long snapshotId = snapshotInfo.getSnapshotId(); + DiskIndex diskIndex = new DiskIndex(indexId, directory, snapshotId, indexInfo, this); + indices.add(diskIndex); + } else { + LOG.info("index no longer referenced deleting: " + name); + // directoryMap.delete(name); + } + } catch (IndexException indexException) { + LOG.error("index not ready, deleting: " + name, indexException); + // directoryMap.delete(name); + } catch (IOException ioException) { + LOG.error("index not ready, deleting: " + name, ioException); + // directoryMap.delete(name); + } + } + } catch (Exception exception) { + LOG.error("", exception); + // if exception simply skip over the index + } + } + } + + public MasterBatch createMasterBatch() throws Exception { + return new MasterBatch(this); + } + + public class MaybeMergeIndexes implements Runnable { + public MaybeMergeIndexes() { + } + + public void run() { + if (LOG.isDebugEnabled()) + LOG.debug("MaybeMergeIndexes"); + mergeIndexesLock.lock(); + try { + docChangesSinceLastMerge = 0; + maybeMergeWriteable(); + maybeMergeRamIndexes(); + maybeMergeDiskIndexes(); + maybeMergesTimestamp = System.currentTimeMillis(); + } catch (Throwable throwable) { + LOG.error("", throwable); + } finally { + mergeIndexesLock.unlock(); + } + } + + /** + * If the existing ram indexes are above maxRamIndexesSize, then they are + * merged and a new disk index is created from them. Or if the number of + * documents exceeds maxDocsIndexes. + * + * @param snapshot + * @throws Exception + */ + private void maybeMergeRamIndexes() throws Throwable { + Snapshot snapshot = snapshots.getLatestSnapshot(); + try { + long size = 0; + int numDocs = 0; + List ramIndexSnapshots = snapshot.getRamIndexSnapshots(); + for (RamIndexSnapshot ramIndexSnapshot : ramIndexSnapshots) { + RamIndex ramIndex = (RamIndex) ramIndexSnapshot.getIndex(); + size += ramIndex.getSize(); + numDocs += ramIndexSnapshot.getIndexReader().maxDoc(); + } + // if merging based on number of docs + if (maxDocsIndexes > 0 && numDocs > maxDocsIndexes) { + if (LOG.isDebugEnabled()) + LOG.debug("executeMerge because numDocs: " + numDocs + " more than maxDocsIndexes: " + maxDocsIndexes); + executeMerge(ramIndexSnapshots, snapshot); + } else if (size > maxRamIndexesSize) { + // merging based on size of ram indexes + executeMerge(ramIndexSnapshots, snapshot); + } + } finally { + snapshot.decRef(); + } + } + + // TODO: needs to limit the size of the resulting diskindex + private void maybeMergeDiskIndexes() throws Throwable { + Snapshot snapshot = snapshots.getLatestSnapshot(); + try { + Long snapshotId = snapshot.getSnapshotId(); + List indexSnapshotsToMerge = new ArrayList(); + for (DiskIndex diskIndex : snapshot.getDiskIndexes()) { + DiskIndexSnapshot indexSnapshot = (DiskIndexSnapshot) diskIndex.getIndexSnapshot(snapshotId); + if (diskIndex.hasTooManyDeletedDocs(mergeDiskDeletedPercent)) { + indexSnapshotsToMerge.add(indexSnapshot); + } + } + if (indexSnapshotsToMerge.size() > 0) { + executeMerge(indexSnapshotsToMerge, snapshot); + } + } finally { + snapshot.decRef(); + } + } + + /** + * converts current memorywriteableindex to a ramindex + * + * @param snapshot + * @throws Exception + */ + private void maybeMergeWriteable() throws Exception { + // have to do everything in commit lock otherwise updates will + // change the writeableIndexSnapshot. The negative case + // however is fast + commitLock.lock(); + try { + Snapshot snapshot = snapshots.getLatestSnapshot(); + try { + MemoryIndexSnapshot writeableIndexSnapshot = snapshot.getWriteableSnapshot(); + int numDocs = writeableIndexSnapshot.getIndexReader().numDocs(); + if (writeableIndexSnapshot.maxDoc() >= memoryIndexMaxDocs) { + if (LOG.isInfoEnabled()) + LOG.info("merge writeable"); + long indexIdNum = ramIndexSequence.getAndIncrement(); + IndexID indexId = new IndexID(indexIdNum, "ram"); + RamIndex ramIndex = new RamIndex(indexId, writeableIndexSnapshot); + indexes.add(ramIndex); + IndexSnapshot ramIndexSnapshot = ramIndex.getLatestIndexSnapshot(); + assert ramIndexSnapshot.getIndexReader().numDocs() == numDocs; + Snapshot currentSnapshot = snapshots.getLatestSnapshot(); + try { + List removeIndexIds = new ArrayList(); + removeIndexIds.add(writeableIndexSnapshot.getIndex().getId()); + + // create new WriteableMemoryIndex for the new snapshot because + // the + // one that was there + // has been converted to a RamIndex + WriteableMemoryIndex newWriteableMemoryIndex = newWriteableMemoryIndex(); + MemoryIndexSnapshot newMemoryIndexSnapshot = newWriteableMemoryIndex.createIndexSnapshot(snapshot.getSnapshotId()); + Snapshot newSnapshot = currentSnapshot.createMinor(removeIndexIds, newMemoryIndexSnapshot, ramIndex.getLatestIndexSnapshot()); + snapshots.add(newSnapshot, true); + if (LOG.isInfoEnabled()) + LOG.info("merge writeable completed"); + } finally { + currentSnapshot.decRef(); + } + } + } finally { + snapshot.decRef(); + } + } finally { + commitLock.unlock(); + } + } + + /** + * Takes snapshots and makes a DiskIndex. + * + * @param indexSnapshots + * @param snapshot + * @throws Exception + */ + private void executeMerge(List incoming, Snapshot snapshot) throws Throwable { + try { + List oldIndexSnapshots = new ArrayList(incoming); + Set removeIndexIds = new HashSet(); + // Snapshot currentSnapshot = snapshots.getLatestSnapshot(); + // if snapshot consists only of deletions skip it + Iterator iterator = oldIndexSnapshots.iterator(); + while (iterator.hasNext()) { + IndexSnapshot indexSnapshot = iterator.next(); + if (indexSnapshot.deletedDoc() == indexSnapshot.maxDoc()) { + removeIndexIds.add(indexSnapshot.getIndex().getId()); + iterator.remove(); + } + } + System.out.println("executeMerge removeIndexIds all deleted: " + removeIndexIds); + // if there are removeIndexIds now, it means + if (removeIndexIds.size() > 0 && removeIndexIds.size() == oldIndexSnapshots.size()) { + commitLock.lock(); + try { + Snapshot currentSnapshot = snapshots.getLatestSnapshot(); + try { + Snapshot newSnapshot = currentSnapshot.createMinor(removeIndexIds); + snapshots.add(newSnapshot, true); + } finally { + currentSnapshot.decRef(); + } + } finally { + commitLock.unlock(); + } + return; + } + if (oldIndexSnapshots.size() == 0) + return; + Long snapshotId = snapshot.getSnapshotId(); + Long indexIdNum = diskIndexSequence.getAndIncrement(); + IndexID indexId = new IndexID(indexIdNum, "disk"); + Directory directory = directoryMap.create(indexIdNum + "_index"); + // initial creation happens outside of commitlock because it is the most + // time consuming + // deletes that could have occured since the DiskIndex was created + // happen inside the commitlock as deletes have minimal performance + // impact + DiskIndex newDiskIndex = new DiskIndex(indexId, directory, oldIndexSnapshots, TransactionSystem.this); + // TODO: ram directory snapshots need to merge somehow with the new + // snapshot + indexes.add(newDiskIndex); + commitLock.lock(); + try { + Snapshot currentSnapshot = snapshots.getLatestSnapshot(); + try { + List deleteOnlySlaveBatches = new ArrayList(); + Long latestSnapshotId = currentSnapshot.getSnapshotId(); + if (latestSnapshotId.longValue() > snapshotId.longValue()) { + SlaveBatchIterator batchIterator = transactionLog.getSlaveBatchIterator(snapshotId.longValue()+1); + while (batchIterator.hasNext()) { + SlaveBatch slaveBatch = batchIterator.next(false, true); + if (slaveBatch.deletes != null) { + if (slaveBatch.deletes.getDocIds() == null) { + assert slaveBatch.deletes.getDocIds() != null; + } + } + deleteOnlySlaveBatches.add(slaveBatch); + } + } + IndexSnapshot newIndexSnapshot = newDiskIndex.initialize(latestSnapshotId, deleteOnlySlaveBatches, TransactionSystem.this); + for (IndexSnapshot indexSnapshot : oldIndexSnapshots) { + Index index = indexSnapshot.getIndex(); + removeIndexIds.add(index.getId()); + } + StringBuilder builder = new StringBuilder(); + Iterator indexSnapshotsIterator = oldIndexSnapshots.iterator(); + while (indexSnapshotsIterator.hasNext()) { + IndexSnapshot indexSnapshot = indexSnapshotsIterator.next(); + builder.append(indexSnapshot.getIndex().getId().toString()); + if (indexSnapshotsIterator.hasNext()) { + builder.append(","); + } + } + builder.append(" ").append(" indexes written to disk index: ").append(indexId.toString()); + LOG.info(builder.toString()); + LOG.info("newIndexSnapshot: " + newIndexSnapshot.getClass().getName()); + Snapshot newSnapshot = currentSnapshot.createMinor(removeIndexIds, newIndexSnapshot); + System.out.println("snapshot: " + newSnapshot.getId() + " disk num: " + newSnapshot.getDiskIndexSnapshots().size()); + // assert newSnapshot.getDiskIndexSnapshots().size() != 0; + snapshots.add(newSnapshot, true); + } finally { + currentSnapshot.decRef(); + } + } finally { + commitLock.unlock(); + } + } catch (Throwable throwable) { + throw throwable; + } + } + } + + /** + * Allows large indexes to be created and committed without putting the + * documents in the transaction log. Deletes from the LargeBatch are placed in + * the transaction log however. + * + * @param threads + * @param ramBufferSize + * @return + * @throws Exception + */ + // TODO: not sure about recovery as eventually index will be merged, could be + // ok + public LargeBatch createLargeBatch(int threads, float ramBufferSize, boolean optimize) throws Exception { + if (threads > commitThreadPool.getMaximumPoolSize()) { + commitThreadPool.setMaximumPoolSize(threads); + } + Long indexIdNum = diskIndexSequence.getAndIncrement(); + IndexID indexId = new IndexID(indexIdNum, "disk"); + LargeBatch largeBatch = new LargeBatch(indexId, threads, ramBufferSize, optimize); + largeBatchLock.lock(); + try { + largeBatches.add(indexId, largeBatch); + } finally { + largeBatchLock.unlock(); + } + return largeBatch; + } + + public class LargeBatch { + IndexID indexId; + ArrayBlockingQueue documentQueue; + IndexCreator indexCreator; + long timestamp = System.currentTimeMillis(); + Deletes deletes = new Deletes(); + boolean optimize; + IndexWriter indexWriter; + Directory directory; + + public LargeBatch(IndexID indexId, int threads, float ramBufferSize, boolean optimize) throws Exception { + this.indexId = indexId; + this.optimize = optimize; + directory = directoryMap.create(indexId.id + "_index"); + indexWriter = new IndexWriter(directory, false, getDefaultAnalyzer(), true); + indexWriter.setUseCompoundFile(false); + indexWriter.setMergeScheduler(new SerialMergeScheduler()); + indexWriter.setMaxBufferedDocs(Integer.MAX_VALUE); + indexWriter.setRAMBufferSizeMB(ramBufferSize); + indexCreator = new IndexCreator(indexWriter, threads, commitThreadPool); + documentQueue = new ArrayBlockingQueue(2000); + indexCreator.start(documentQueue); + } + + public void addDocument(Document document, Analyzer analyzer) { + documentQueue.add(new IndexCreator.Add(document, analyzer)); + timestamp = Timeout.TIMER_THREAD.getTime(); + } + + public void deleteDocuments(Term term) { + deletes.addTerm(term); + } + + public void deleteDocuments(Query query) { + deletes.addQuery(query); + } + + public CommitResult commit() throws Exception { + // wait until queue is empty + while (documentQueue.peek() != null) + Thread.sleep(5); + if (deletes.hasDeleteByQueries()) { + Query[] queries = deletes.getQueries().toArray(new Query[0]); + indexWriter.deleteDocuments(queries); + } + if (deletes.hasTerms()) { + Term[] terms = deletes.getTerms().toArray(new Term[0]); + indexWriter.deleteDocuments(terms); + } + indexCreator.create(optimize); + int numAdded = indexCreator.getNumAdded(); + // write to transaction log and create new snapshot + // also perform deletes on indexsnapshots + DiskIndex diskIndex = new DiskIndex(indexId, directory, TransactionSystem.this); + commitLock.lock(); + try { + Snapshot currentSnapshot = snapshots.getLatestSnapshot(); + try { + Long snapshotId = transactionLog.getNextId(); + LargeBatchTransaction transaction = new LargeBatchTransaction(snapshotId, currentSnapshot); + List indexSnapshots = currentSnapshot.getIndexSnapshots(); + // apply deletes to the other + List deletesResults = new ArrayList(indexSnapshots.size()); + for (IndexSnapshot indexSnapshot : indexSnapshots) { + Index index = indexSnapshot.getIndex(); + DeletesResult deletesResult = index.commitDeletes(deletes, transaction); + deletesResults.add(deletesResult); + } + List newIndexSnapshots = new ArrayList(indexSnapshots); + DiskIndexSnapshot newIndexSnapshot = (DiskIndexSnapshot) diskIndex.createNewSnapshot(snapshotId, true, null); + newIndexSnapshots.add(newIndexSnapshot); + Snapshot newSnapshot = new Snapshot(snapshotId, 0, currentSnapshot.getWriteableSnapshot(), newIndexSnapshots, + TransactionSystem.this, System.currentTimeMillis()); + snapshots.add(newSnapshot, true); + return new CommitResult(newSnapshot, null, deletesResults, numAdded, indexId); + } finally { + currentSnapshot.decRef(); + } + } finally { + commitLock.unlock(); + } + } + } + + /** + * Commits a batch to the transaction log + * + * @param batch + * @return CommitResult + * @throws Exception + * @throws IOException + */ + CommitResult commitBatch(Batch batch) throws Exception, IOException { + batch.close(); + commitLock.lock(); + try { + Long snapshotId = null; + List documentIds = null; + if (batch instanceof SlaveBatch) { + SlaveBatch slaveBatch = (SlaveBatch) batch; + snapshotId = slaveBatch.getId(); + } else { + MasterBatch masterBatch = (MasterBatch) batch; + snapshotId = transactionLog.getNextId(); + if (batch.hasDocuments()) { + Documents documents = batch.getDocuments(); + documentIds = new ArrayList(documents.size()); + for (Document document : documents) { + Long documentId = documentSequence.getAndIncrement(); + documentIds.add(documentId); + Util.setValue(Constants.DOCUMENTID, documentId, document); + Util.setValue(Constants.SNAPSHOTID, snapshotId, document); + } + if (documents.size() >= memoryIndexMaxDocs || documents.hasFieldsWithTokenStreamOrReader()) { + RAMDirectory ramDirectory = createRamDirectory(documents, batch.getAnalyzer()); + masterBatch.setRAMDirectory(ramDirectory); + } + // create here before the commitlock + masterBatch.createDocData(); + } + } + Snapshot currentSnapshot = snapshots.getLatestSnapshot(); + try { + MemoryIndexSnapshot writeableIndexSnapshot = currentSnapshot.getWriteableSnapshot(); + WriteableMemoryIndex writeableMemoryIndex = (WriteableMemoryIndex) writeableIndexSnapshot.getIndex(); + List nonWriteableIndices = currentSnapshot.getDeleteOnlyIndices(); + Transaction transaction = null; + CommitResult commitResult = null; + try { + // Long previousId = transactionLog.getPreviousId(snapshotId); + transaction = new SingleThreadTransaction(snapshotId, documentIds, currentSnapshot, batch, this); + commitResult = transaction.call(); + } catch (Exception exception) { + LOG.error("transaction failed"); + throw new Exception("transaction failed", exception); + } + Snapshot newSnapshot = commitResult.getSnapshot(); + snapshots.add(newSnapshot, true); + docChangesSinceLastMerge += commitResult.getNumDocChanges(); + int writeableMaxDoc = writeableMemoryIndex.getLatestIndexSnapshot().getIndexReader().maxDoc(); + if (docChangesSinceLastMerge > maybeMergeDocChanges || writeableMaxDoc >= memoryIndexMaxDocs) { + // System.out.println("docChangesSinceLastMerge: " + + // docChangesSinceLastMerge + " maybeMergeDocChanges: " + + // maybeMergeDocChanges); + // System.out.println("writeableMaxDoc: " + writeableMaxDoc + " + // memoryIndexMaxDocs: " + memoryIndexMaxDocs); + // only submit if nothing is currently executing or pending + if (mergeThreadPool.getActiveCount() == 0) + merge(false); + // mergeThreadPool.submit(new MaybeMergeIndexes()); + } + // deleteUnreferencedSnapshots(); + return commitResult; + } finally { + currentSnapshot.decRef(); + } + } finally { + commitLock.unlock(); + } + } + + // TODO: if documents is really small then just use one thread + RAMDirectory createRamDirectory(Documents documents, Analyzer analyzer) throws Exception { + RAMDirectory ramDirectory = new RAMDirectory(); + ExecutorService threadPool = getCommitThreadPool(); + IndexCreator indexCreator = new IndexCreator("createRamDirectory", ramDirectory, Long.MAX_VALUE, 4, analyzer, threadPool); + BlockingQueue addQueue = new ArrayBlockingQueue(1000); + indexCreator.start(addQueue); + for (Document document : documents) { + boolean worked = addQueue.offer(new IndexCreator.Add(document)); + if (!worked) { + LOG.warn("addQueue worked: " + worked); + } + } + indexCreator.create(false); + return ramDirectory; + } + + WriteableMemoryIndex newWriteableMemoryIndex() throws Exception { + Long indexIdNum = ramIndexSequence.getAndIncrement(); + IndexID indexId = new IndexID(indexIdNum, "memory"); + WriteableMemoryIndex writeableMemoryIndex = new WriteableMemoryIndex(indexId, this); + indexes.add(writeableMemoryIndex); + return writeableMemoryIndex; + } +} Index: ocean/src/org/apache/lucene/ocean/util/ByteArrayIndexInput.java =================================================================== --- ocean/src/org/apache/lucene/ocean/util/ByteArrayIndexInput.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/util/ByteArrayIndexInput.java (revision 0) @@ -0,0 +1,41 @@ +package org.apache.lucene.ocean.util; + +import java.io.IOException; + +import org.apache.lucene.store.IndexInput; + +public class ByteArrayIndexInput extends IndexInput { + private byte[] bytes; + protected int pos = 0; + + public ByteArrayIndexInput(byte[] bytes) { + this.bytes = bytes; + } + + public byte readByte() throws IOException { + return bytes[pos++]; + } + + public void readBytes(byte[] b, int offset, int len) throws IOException { + if (pos + len > bytes.length) { + len = bytes.length - pos; + } + System.arraycopy(bytes, pos, b, offset, len); + pos += len; + } + + public void close() throws IOException { + } + + public long getFilePointer() { + return pos; + } + + public void seek(long pos) throws IOException { + this.pos = (int)pos; + } + + public long length() { + return bytes.length; + } +} Index: ocean/src/org/apache/lucene/ocean/util/ByteArrayIndexOutput.java =================================================================== --- ocean/src/org/apache/lucene/ocean/util/ByteArrayIndexOutput.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/util/ByteArrayIndexOutput.java (revision 0) @@ -0,0 +1,51 @@ +package org.apache.lucene.ocean.util; + +import java.io.IOException; + +import org.apache.commons.io.output.ByteArrayOutputStream; +import org.apache.lucene.store.IndexOutput; + +public class ByteArrayIndexOutput extends IndexOutput { + private ByteArrayOutputStream byteArrayOutputStream; + private int pos = 0; + + public ByteArrayIndexOutput() { + byteArrayOutputStream = new ByteArrayOutputStream(1024); + } + + public ByteArrayIndexOutput(ByteArrayOutputStream byteArrayOutputStream) { + this.byteArrayOutputStream = byteArrayOutputStream; + } + + public byte[] toByteArray() { + return byteArrayOutputStream.toByteArray(); + } + + public long length() throws IOException { + return byteArrayOutputStream.size(); + } + + public void writeByte(byte b) throws IOException { + byteArrayOutputStream.write(b); + pos++; + } + + public void writeBytes(byte[] b, int offset, int length) throws IOException { + byteArrayOutputStream.write(b, offset, length); + pos += length; + } + + public void flush() throws IOException { + } + + public void close() throws IOException { + } + + public long getFilePointer() { + return (int)pos; + } + + public void seek(long pos) throws IOException { + throw new UnsupportedOperationException(""); + } +} Index: ocean/src/org/apache/lucene/ocean/util/Bytes.java =================================================================== --- ocean/src/org/apache/lucene/ocean/util/Bytes.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/util/Bytes.java (revision 0) @@ -0,0 +1,277 @@ +package org.apache.lucene.ocean.util; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.List; + +/** + * Byte buffers maintained in a list. Can obtain OutputStream and InputStream + * for writing and reading to buffers. + * + */ +public class Bytes { + private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; + private List buffers = new ArrayList(); + private List bufferPool = new ArrayList(); + private int bufferSize; + private int count = 0; + private ByteArrayOutputStream output; + + public Bytes(int bufferSize) { + this.bufferSize = bufferSize; + } + + public void writeTo(OutputStream out) throws IOException { + int remaining = count; + for (int i = 0; i < buffers.size(); i++) { + byte[] buf = getBuffer(i); + int c = Math.min(buf.length, remaining); + out.write(buf, 0, c); + remaining -= c; + if (remaining == 0) { + break; + } + } + } + + public void reset() { + count = 0; + output = null; + bufferPool.addAll(buffers); + buffers.clear(); + } + + public int length() { + return count; + } + + private byte[] getBuffer(int index) { + return (byte[]) buffers.get(index); + } + + /** + * public class ByteArrayInputStream extends InputStream { private int pos = + * 0; private int bufferPos = 0; private byte[] buffer; private int bufferLen; + * private int bufferIdx = 0; + * + * public ByteArrayInputStream() { if (buffers.size() > 0) { buffer = + * buffers.get(bufferIdx); } } + * + * public int read(byte[] b, int offset, int len) throws IOException { if (pos >= + * count) return -1; int total = 0; while (len > 0) { if (bufferPos >= + * buffer.length) { if (!nextBuffer()) { return total; } } int remainInBuffer = + * buffer.length - bufferPos; int bytesToCopy = len < remainInBuffer ? len : + * remainInBuffer; System.arraycopy(buffer, bufferPos, b, offset, + * bytesToCopy); offset += bytesToCopy; len -= bytesToCopy; bufferPos += + * bytesToCopy; total += bytesToCopy; pos += bytesToCopy; } return total; } + * + * public int read() { if (buffers.size() == 0) return -1; if (pos >= count) + * return -1; if (bufferPos >= buffer.length) { if (!nextBuffer()) return -1; } + * pos++; return buffer[bufferPos++]; } + * + * private boolean nextBuffer() { bufferIdx++; if (bufferIdx >= + * buffers.size()-1) { return false; } buffer = buffers.get(bufferIdx); return + * true; } } + */ + + public class ByteArrayInputStream extends InputStream { + private byte[] currentBuffer; + private int currentBufferIndex; + private int bufferPosition; + private long bufferStart; + private int bufferLength; + + public ByteArrayInputStream() { + currentBufferIndex = -1; + currentBuffer = null; + } + + public long getPosition() { + return currentBufferIndex < 0 ? 0 : bufferStart + bufferPosition; + } + + public int read() throws IOException { + if (getPosition() >= count) + return -1; + if (bufferPosition >= bufferLength) { + currentBufferIndex++; + switchCurrentBuffer(); + } + return currentBuffer[bufferPosition++]; + } + + public int read(byte[] b, int offset, int len) throws IOException { + int pos = (int) getPosition(); + if (pos >= count) + return -1; + int total = 0; + while (len > 0) { + if (bufferPosition >= bufferLength) { + currentBufferIndex++; + if (!switchCurrentBuffer()) { + return total; + } + } + int remainInBuffer = bufferLength - bufferPosition; + int bytesToCopy = len < remainInBuffer ? len : remainInBuffer; + System.arraycopy(currentBuffer, bufferPosition, b, offset, bytesToCopy); + offset += bytesToCopy; + len -= bytesToCopy; + bufferPosition += bytesToCopy; + total += bytesToCopy; + } + return total; + } + + private final boolean switchCurrentBuffer() throws IOException { + if (currentBufferIndex >= buffers.size()) { // end of file reached, no + // more + // buffers left + return false; + } else { + currentBuffer = buffers.get(currentBufferIndex); + bufferPosition = 0; + bufferStart = (long) bufferSize * (long) currentBufferIndex; + long buflen = count - bufferStart; + bufferLength = buflen > bufferSize ? bufferSize : (int) buflen; + return true; + } + } + } + + public class ByteArrayOutputStream extends OutputStream { + private byte[] currentBuffer; + private int currentBufferIndex; + + private int bufferPosition; + private long bufferStart; + private int bufferLength; + + public ByteArrayOutputStream() throws IOException { + seek(0); + } + + public void seek(long pos) throws IOException { + // set the file length in case we seek back + // and flush() has not been called yet + // setFileLength(); + if (pos < bufferStart || pos >= bufferStart + bufferLength) { + currentBufferIndex = (int) (pos / bufferSize); + switchCurrentBuffer(); + } + bufferPosition = (int) (pos % bufferSize); + } + + public void write(int b) throws IOException { + if (bufferPosition == bufferLength) { + currentBufferIndex++; + switchCurrentBuffer(); + } + currentBuffer[bufferPosition++] = (byte) b; + count++; + } + + public void write(byte[] b, int offset, int len) throws IOException { + while (len > 0) { + if (bufferPosition == bufferLength) { + currentBufferIndex++; + switchCurrentBuffer(); + } + + int remainInBuffer = currentBuffer.length - bufferPosition; + int bytesToCopy = len < remainInBuffer ? len : remainInBuffer; + System.arraycopy(b, offset, currentBuffer, bufferPosition, bytesToCopy); + offset += bytesToCopy; + len -= bytesToCopy; + bufferPosition += bytesToCopy; + count += bytesToCopy; + } + } + + private final void switchCurrentBuffer() throws IOException { + if (currentBufferIndex == buffers.size()) { + if (bufferPool.size() > 0) { + currentBuffer = bufferPool.get(0); + bufferPool.remove(0); + } else { + currentBuffer = new byte[bufferSize]; + } + buffers.add(currentBuffer); + } else { + currentBuffer = (byte[]) buffers.get(currentBufferIndex); + } + bufferPosition = 0; + bufferStart = (long) bufferSize * (long) currentBufferIndex; + bufferLength = currentBuffer.length; + } + } + + /** + * public class ByteArrayOutputStream extends OutputStream { private int + * currentBufferIndex; private int pos = 0; private byte[] currentBuffer; + * + * public ByteArrayOutputStream() { if (buffers.size() > 0) { + * currentBufferIndex = 0; currentBuffer = getBuffer(currentBufferIndex); } + * needNewBuffer(); } + * + * private void needNewBuffer() { if (currentBufferIndex < buffers.size() - 1) { // + * Recycling old buffer pos += currentBuffer.length; + * + * currentBufferIndex++; currentBuffer = getBuffer(currentBufferIndex); + * numValidBuffers++; } else { // Creating new buffer if (currentBuffer == + * null) { pos = 0; } else { pos += currentBuffer.length; } + * + * currentBufferIndex++; currentBuffer = new byte[bufferSize]; + * buffers.add(currentBuffer); numValidBuffers++; } } + * + * public void write(byte[] b, int off, int len) { if ((off < 0) || (off > + * b.length) || (len < 0) || ((off + len) > b.length) || ((off + len) < 0)) { + * throw new IndexOutOfBoundsException(); } else if (len == 0) { return; } int + * newcount = count + len; int remaining = len; int inBufferPos = count - pos; + * while (remaining > 0) { int part = Math.min(remaining, currentBuffer.length - + * inBufferPos); System.arraycopy(b, off + len - remaining, currentBuffer, + * inBufferPos, part); remaining -= part; if (remaining > 0) { + * needNewBuffer(); inBufferPos = 0; } } count = newcount; } + * + * public void write(int b) { int inBufferPos = count - pos; if (inBufferPos == + * currentBuffer.length) { needNewBuffer(); inBufferPos = 0; } + * currentBuffer[inBufferPos] = (byte) b; count++; } + * + * public int size() { return count; } + * + * public void close() throws IOException { // nop } + * + * public void reset() { count = 0; pos = 0; currentBufferIndex = 0; + * currentBuffer = getBuffer(currentBufferIndex); } + * + * public void writeTo(OutputStream out) throws IOException { int remaining = + * count; for (int i = 0; i < buffers.size(); i++) { byte[] buf = + * getBuffer(i); int c = Math.min(buf.length, remaining); out.write(buf, 0, + * c); remaining -= c; if (remaining == 0) { break; } } } + * + * /** Gets the curent contents of this byte stream as a byte array. The + * result is independent of this stream. + * + * @return the current contents of this output stream, as a byte array + * @see java.io.ByteArrayOutputStream#toByteArray() + * + * public byte[] toByteArray() { int remaining = count; if (remaining == 0) { + * return EMPTY_BYTE_ARRAY; } byte newbuf[] = new byte[remaining]; int pos = + * 0; for (int i = 0; i < buffers.size(); i++) { byte[] buf = getBuffer(i); + * int c = Math.min(buf.length, remaining); System.arraycopy(buf, 0, newbuf, + * pos, c); pos += c; remaining -= c; if (remaining == 0) { break; } } return + * newbuf; } } + */ + public OutputStream getOutputStream() throws IOException { + if (output == null) { + output = new ByteArrayOutputStream(); + } + return output; + } + + public InputStream getInputStream() { + return new ByteArrayInputStream(); + } +} Index: ocean/src/org/apache/lucene/ocean/util/BytesPool.java =================================================================== --- ocean/src/org/apache/lucene/ocean/util/BytesPool.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/util/BytesPool.java (revision 0) @@ -0,0 +1,129 @@ +package org.apache.lucene.ocean.util; + +import java.util.Arrays; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantLock; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class BytesPool { + final static Logger LOG = LoggerFactory.getLogger(BytesPool.class); + public static final long DEFAULT_EVICT_TIME = 1000*60; + private final ConcurrentHashMap map = new ConcurrentHashMap(); + private final ScheduledExecutorService timer; + private final long evictTime; + private static BytesPool instance; + + public synchronized static BytesPool getInstance() { + if (instance == null) { + instance = new BytesPool(DEFAULT_EVICT_TIME); + } + return instance; + } + + public BytesPool(long evictTime) { + this.evictTime = evictTime; + timer = Executors.newSingleThreadScheduledExecutor(); + timer.scheduleWithFixedDelay(new Timer(), evictTime, evictTime, TimeUnit.MILLISECONDS); + } + + public class Timer implements Runnable { + public void run() { + for (ByteArrays byteArrays : map.values()) { + byteArrays.evict(); + } + } + } + + public static class Entry { + public final byte[] bytes; + public long accessTime; + + public Entry(byte[] bytes) { + this.bytes = bytes; + } + } + + public class ByteArrays { + public final int length; + private LinkedList queue = new LinkedList(); + private ReentrantLock lock = new ReentrantLock(); + + public ByteArrays(int length) { + this.length = length; + } + + public void evict() { + lock.lock(); + try { + long now = Timeout.TIMER_THREAD.getTime(); + Iterator iterator = queue.iterator(); + while (iterator.hasNext()) { + Entry entry = iterator.next(); + if ((now - entry.accessTime) > evictTime) { + iterator.remove(); + LOG.info("removed entry length: "+length); + } + } + } finally { + lock.lock(); + } + } + + public Entry add(byte[] bytes) { + lock.lock(); + try { + Entry entry = new Entry(new byte[length]); + queue.add(entry); + return entry; + } finally { + lock.lock(); + } + } + + public byte[] get() { + lock.lock(); + try { + Entry entry = null; + if (queue.size() > 0) { + entry = queue.removeLast(); + } + if (entry == null) { + return new byte[length]; + //entry = new Entry(new byte[length]);//add(new byte[length]); + } + entry.accessTime = Timeout.TIMER_THREAD.getTime(); + return entry.bytes; + } finally { + lock.lock(); + } + } + } + + public byte[] getBytes(int length) { + ByteArrays byteArrays = getByteArrays(length); + return byteArrays.get(); + } + + private ByteArrays getByteArrays(int length) { + ByteArrays byteArrays = map.get(length); + if (byteArrays == null) { + byteArrays = new ByteArrays(length); + ByteArrays absentByteArrays = map.putIfAbsent(length, byteArrays); + if (absentByteArrays != null) byteArrays = absentByteArrays; + } + return byteArrays; + } + + public void returnBytes(byte[] bytes) { + Arrays.fill(bytes, (byte)0); + ByteArrays byteArrays = getByteArrays(bytes.length); + byteArrays.add(bytes); + } +} Index: ocean/src/org/apache/lucene/ocean/util/CElement.java =================================================================== --- ocean/src/org/apache/lucene/ocean/util/CElement.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/util/CElement.java (revision 0) @@ -0,0 +1,11 @@ +package org.apache.lucene.ocean.util; + +import org.jdom.Element; + +/** + * + * @author jasonr + */ +public interface CElement { + public Element toElement() throws Exception; +} Index: ocean/src/org/apache/lucene/ocean/util/Constants.java =================================================================== --- ocean/src/org/apache/lucene/ocean/util/Constants.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/util/Constants.java (revision 0) @@ -0,0 +1,13 @@ +package org.apache.lucene.ocean.util; + +public interface Constants { + //public static final String ID = "_id".intern(); + public static final String INDEXID = "_indexid".intern(); + public static final String DOCUMENTID = "_documentid".intern(); + public static final String SNAPSHOTID = "_snapshotid".intern(); + + public static final int DOCUMENTS_TYPE = 1; + public static final int RAM_DIRECTORY_TYPE = 2; + + public static final int DELETES_SERIALIZE_TYPE = 1; +} Index: ocean/src/org/apache/lucene/ocean/util/DocumentSerializer.java =================================================================== --- ocean/src/org/apache/lucene/ocean/util/DocumentSerializer.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/util/DocumentSerializer.java (revision 0) @@ -0,0 +1,138 @@ +package org.apache.lucene.ocean.util; + +import java.io.IOException; +import java.util.List; + +import org.apache.commons.lang.SerializationUtils; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.Token; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.Fieldable; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.SmallFloat; + +public class DocumentSerializer { + final static int INDEXED = 0x00000001; + final static int TOKENIZED = 0x00000002; + final static int STORED = 0x00000004; + final static int BINARY = 0x00000008; + final static int COMPRESSED = 0x00000010; + final static int OMIT_NORMS = 0x00000020; + final static int STORE_TERMVECTORS = 0x00000040; + final static int STORE_TERMPOSITIONS = 0x00000080; + final static int STORE_TERMOFFSETS = 0x00000100; + final static int STRING = 0x00000200; + final static int TOKENSTREAM = 0x00000400; + final static int TOKENS = 0x00000800; + + // final static int REQUIRED = 0x00001000; + + public static Document toDocument(IndexInput input) throws IOException { + int numFields = input.readVInt(); + Document document = new Document(); + for (int x=0; x < numFields; x++) { + Field field = null; + Field.Store store = Field.Store.NO; + Field.Index index = Field.Index.NO; + String name = input.readString(); + int type = input.readVInt(); + if ((type & STORED) != 0) { + store = Field.Store.YES; + } else if ((type & COMPRESSED) != 0) { + store = Field.Store.COMPRESS; + } + if ((type & TOKENIZED) != 0) { + index = Field.Index.TOKENIZED; + } else if ((type & INDEXED) != 0) { + index = Field.Index.UN_TOKENIZED; + } else if ((type & INDEXED) != 0) { + index = Field.Index.NO_NORMS; + } + float boost = SmallFloat.byte315ToFloat(input.readByte()); + if ((type & STRING) != 0) { + String string = input.readString(); + field = new Field(name, string, store, index); + } else if ((type & BINARY) != 0) { + int length = input.readVInt(); + byte[] binary = new byte[length]; + input.readBytes(binary, 0, length); + field = new Field(name, binary, store); + } + if (field != null) { + document.add(field); + } + } + return document; + } + + // TODO: implement serializing tokens + public static void writeDocument(Document document, IndexOutput output, boolean includeTokens) throws IOException { + List fields = document.getFields(); + output.writeVInt(fields.size()); + for (Fieldable field : fields) { + output.writeString(field.name()); + int type = 0; + if (field.isBinary()) { + type |= BINARY; + } else if (field.isIndexed()) { + type |= INDEXED; + } else if (field.getOmitNorms()) { + type |= OMIT_NORMS; + } else if (field.isCompressed()) { + type |= COMPRESSED; + } else if (field.isStored()) { + type |= STORED; + } else if (field.isTermVectorStored()) { + type |= STORE_TERMVECTORS; + } else if (field.isStorePositionWithTermVector()) { + type |= STORE_TERMPOSITIONS; + } else if (field.isStoreOffsetWithTermVector()) { + type |= STORE_TERMOFFSETS; + } else if (field.isTokenized()) { + type |= TOKENIZED; + } else if (field.tokenStreamValue() != null) { + type |= TOKENSTREAM; + } else if (field.stringValue() != null) { + type |= STRING; + } else if (includeTokens) { + type |= TOKENS; + } + output.writeVInt(type); + output.writeByte(SmallFloat.floatToByte315(field.getBoost())); // boost + if (field.stringValue() != null) { + String string = field.stringValue(); + output.writeString(string); + } else if (field.binaryValue() != null) { + byte[] bytes = field.binaryValue(); + output.writeVInt(bytes.length); + output.writeBytes(bytes, bytes.length); + } + // if (includeTokens && field.isTokenized()) { + // TokenStream tokenStream = analyzer.tokenStream(field.name(), reader); + // } + } + } + + public static void writeTokens(TokenStream tokenStream) throws IOException { + Token token; + while ((token = tokenStream.next()) != null) { + + } + } + + public static void write(Document document, IndexOutput output) throws IOException { + byte[] bytes = SerializationUtils.serialize(document); + output.writeVInt(bytes.length); + output.writeBytes(bytes, bytes.length); + } + + public static Document read(IndexInput input) throws IOException { + int length = input.readVInt(); + byte[] bytes = new byte[length]; + input.readBytes(bytes, 0, length); + return (Document) SerializationUtils.deserialize(bytes); + } +} Index: ocean/src/org/apache/lucene/ocean/util/LimitedFileInputStream.java =================================================================== --- ocean/src/org/apache/lucene/ocean/util/LimitedFileInputStream.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/util/LimitedFileInputStream.java (revision 0) @@ -0,0 +1,131 @@ +package org.apache.lucene.ocean.util; + +import java.io.IOException; +import java.io.InputStream; +import java.io.RandomAccessFile; + +public class LimitedFileInputStream extends InputStream { + private RandomAccessFile randomAccessFile; + /** + * The maximum number of bytes that can be read from the stream. Subsequent + * read operations will return -1. + */ + private long contentLength; + + /** The current position */ + private long pos = 0; + + /** True if the stream is closed. */ + private boolean closed = false; + + public LimitedFileInputStream(RandomAccessFile randomAccessFile, int contentLength) { + this.randomAccessFile = randomAccessFile; + this.contentLength = contentLength; + } + + public void close() throws IOException { + } + + /** + * Read the next byte from the stream + * + * @return The next byte or -1 if the end of stream has been reached. + * @throws IOException + * If an IO problem occurs + * @see java.io.InputStream#read() + */ + public int read() throws IOException { + if (closed) { + throw new IOException("Attempted read from closed stream."); + } + + if (pos >= contentLength) { + return -1; + } + pos++; + return randomAccessFile.read(); + } + + /** + * Does standard {@link InputStream#read(byte[], int, int)} behavior, but also + * notifies the watcher when the contents have been consumed. + * + * @param b + * The byte array to fill. + * @param off + * Start filling at this position. + * @param len + * The number of bytes to attempt to read. + * @return The number of bytes read, or -1 if the end of content has been + * reached. + * + * @throws java.io.IOException + * Should an error occur on the wrapped stream. + */ + public int read(byte[] b, int off, int len) throws IOException { + if (closed) { + throw new IOException("Attempted read from closed stream."); + } + + if (pos >= contentLength) { + return -1; + } + + if (pos + len > contentLength) { + len = (int) (contentLength - pos); + } + int count = randomAccessFile.read(b, off, len); + pos += count; + return count; + } + + /** + * Read more bytes from the stream. + * + * @param b + * The byte array to put the new data in. + * @return The number of bytes read into the buffer. + * @throws IOException + * If an IO problem occurs + * @see java.io.InputStream#read(byte[]) + */ + public int read(byte[] b) throws IOException { + return read(b, 0, b.length); + } + + /** + * Skips and discards a number of bytes from the input stream. + * + * @param n + * The number of bytes to skip. + * @return The actual number of bytes skipped. <= 0 if no bytes are skipped. + * @throws IOException + * If an error occurs while skipping bytes. + * @see InputStream#skip(long) + */ + public long skip(long n) throws IOException { + // make sure we don't skip more bytes than are + // still available + long length = Math.min(n, contentLength - pos); + // skip and keep track of the bytes actually skipped + long fp = length + randomAccessFile.getFilePointer(); + randomAccessFile.seek(fp); + // only add the skipped bytes to the current position + // if bytes were actually skipped + if (length > 0) { + pos += length; + } + return length; + } + + public int available() throws IOException { + if (this.closed) { + return 0; + } + int avail = (int) (randomAccessFile.length() - randomAccessFile.getFilePointer()); + if (this.pos + avail > this.contentLength) { + avail = (int) (this.contentLength - this.pos); + } + return avail; + } +} Index: ocean/src/org/apache/lucene/ocean/util/LongSequence.java =================================================================== --- ocean/src/org/apache/lucene/ocean/util/LongSequence.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/util/LongSequence.java (revision 0) @@ -0,0 +1,43 @@ +package org.apache.lucene.ocean.util; + +import java.util.concurrent.locks.ReentrantLock; + +public class LongSequence { + private long value; + private int increment; + private ReentrantLock lock = new ReentrantLock(); + + public LongSequence(long value, int increment) { + this.value = value; + this.increment = increment; + } + + public long get() { + lock.lock(); + try { + return value; + } finally { + lock.unlock(); + } + } + + public void set(long i) { + lock.lock(); + try { + value = i; + } finally { + lock.unlock(); + } + } + + public long getAndIncrement() { + lock.lock(); + try { + long v = value; + value += increment; + return v; + } finally { + lock.unlock(); + } + } +} Index: ocean/src/org/apache/lucene/ocean/util/OceanRandomAccessFile.java =================================================================== --- ocean/src/org/apache/lucene/ocean/util/OceanRandomAccessFile.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/util/OceanRandomAccessFile.java (revision 0) @@ -0,0 +1,111 @@ +package org.apache.lucene.ocean.util; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.RandomAccessFile; + +/** + * adds writeVInt, writeVLong, readVInt, readVLong, getInputStream(long + * position, int length) + * + */ +public class OceanRandomAccessFile extends RandomAccessFile { + private String mode; + private boolean isClosed = false; + + public OceanRandomAccessFile(File file, String mode) throws IOException { + super(file, mode); + this.mode = mode; + } + + public void close() throws IOException { + super.close(); + isClosed = true; + } + + public static class FileOutputStream extends OutputStream { + OceanRandomAccessFile randomAccessFile; + + public FileOutputStream(OceanRandomAccessFile randomAccessFile) { + this.randomAccessFile = randomAccessFile; + } + + public void write(byte[] b, int off, int len) throws IOException { + randomAccessFile.write(b, off, len); + } + + public void write(int b) throws IOException { + randomAccessFile.write(b); + } + } + + public static class FileInputStream extends InputStream { + OceanRandomAccessFile randomAccessFile; + + public FileInputStream(OceanRandomAccessFile randomAccessFile) { + this.randomAccessFile = randomAccessFile; + } + + public int read(byte[] b, int off, int len) throws IOException { + return randomAccessFile.read(b, off, len); + } + + public int read() throws IOException { + return randomAccessFile.read(); + } + } + + public OutputStream getOutputStream(long position) throws IOException { + seek(position); + return new FileOutputStream(this); + } + + public InputStream getInputStream(long position) throws IOException { + seek(position); + return new FileInputStream(this); + } + + public InputStream getInputStream(long position, int length) throws IOException { + seek(position); + return new LimitedFileInputStream(this, length); + } + + public long readVLong() throws IOException { + byte b = readByte(); + long i = b & 0x7F; + for (int shift = 7; (b & 0x80) != 0; shift += 7) { + b = readByte(); + i |= (b & 0x7FL) << shift; + } + return i; + } + + public int readVInt() throws IOException { + byte b = readByte(); + int i = b & 0x7F; + for (int shift = 7; (b & 0x80) != 0; shift += 7) { + b = readByte(); + i |= (b & 0x7F) << shift; + } + return i; + } + + public void writeVLong(long i) throws IOException { + while ((i & ~0x7F) != 0) { + writeByte((byte) ((i & 0x7f) | 0x80)); + i >>>= 7; + } + writeByte((byte) i); + } + + public void writeVInt(int i) throws IOException { + while ((i & ~0x7F) != 0) { + writeByte((byte) ((i & 0x7f) | 0x80)); + i >>>= 7; + } + writeByte((byte) i); + } +} Index: ocean/src/org/apache/lucene/ocean/util/RAMDirectorySerializer.java =================================================================== --- ocean/src/org/apache/lucene/ocean/util/RAMDirectorySerializer.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/util/RAMDirectorySerializer.java (revision 0) @@ -0,0 +1,67 @@ +package org.apache.lucene.ocean.util; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.store.RAMFile; +import org.apache.lucene.store.RAMInputStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class RAMDirectorySerializer { + final static Logger LOG = LoggerFactory.getLogger(RAMDirectorySerializer.class); + + public static RAMDirectory deserialize(InputStream inputStream) throws IOException { + VDataInputStream input = new VDataInputStream(inputStream); + byte[] buffer = new byte[1024]; + RAMDirectory ramDirectory = new RAMDirectory(); + int numFiles = input.readVInt(); + for (int x=0; x < numFiles; x++) { + String file = input.readString(); + int length = input.readVInt(); + IndexOutput ramOutput = ramDirectory.createOutput(file); + long readCount = 0; + while (readCount < length) { + int toRead = readCount + buffer.length > length ? (int)(length - readCount) : buffer.length; + input.read(buffer, 0, toRead); + ramOutput.writeBytes(buffer, toRead); + readCount += toRead; + } + ramOutput.close(); + } + return ramDirectory; + } + + public static void serialize(RAMDirectory ramDirectory, OutputStream out) throws IOException { + //ByteArrayIndexOutput output = new ByteArrayIndexOutput(); + VDataOutputStream output = new VDataOutputStream(out); + byte[] buffer = new byte[1024]; + String[] files = ramDirectory.list(); + output.writeVInt(files.length); + for (String file : files) { + output.writeString(file); + int length = (int)ramDirectory.fileLength(file); + output.writeVInt(length); + //LOG.info("file: "+file+" length: "+length); + RAMInputStream input = (RAMInputStream)ramDirectory.openInput(file); + RAMFile ramFile = input.getRAMFile(); + Util.copyRamFile(ramFile, output); + + /** + + long readCount = 0; + while (readCount < length) { + int toRead = readCount + buffer.length > length ? (int)(length - readCount) : buffer.length; + input.readBytes(buffer, 0, toRead); + output.write(buffer); + readCount += toRead; + } + **/ + //RAMOutputStream ramOutput = (RAMOutputStream)ramDirectory.createOutput(file); + // ramOutput.writeTo(output); + } + } +} Index: ocean/src/org/apache/lucene/ocean/util/SortedList.java =================================================================== --- ocean/src/org/apache/lucene/ocean/util/SortedList.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/util/SortedList.java (revision 0) @@ -0,0 +1,397 @@ +package org.apache.lucene.ocean.util; + +import java.util.AbstractCollection; +import java.util.AbstractSet; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.lang.ObjectUtils; + +public class SortedList,V> implements Map { + private ArrayList> entries; + private transient Set> entrySet = null; + transient volatile Set keySet = null; + transient volatile Collection values = null; + + public SortedList() { + entries = new ArrayList>(); + } + + public SortedList(int size) { + entries = new ArrayList>(size); + } + + public SortedList(Map map) { + this(map.size()); + putAll(map); + } + + public String toString() { + return entries.toString(); + } + + public Iterator> iterator(K fromKey, boolean fromInclusive, K toKey, boolean toInclusive) { + int fromPos = 0; + if (fromKey != null) { + fromPos = Collections.binarySearch(entries, new SortedEntry(fromKey)); + if (fromPos >= 0) { + if (!fromInclusive) + fromPos++; + } else if (fromPos < 0) { + fromPos = -1 - fromPos; + } + } + int toPos = entries.size()-1; + if (toKey != null) { + toPos = Collections.binarySearch(entries, new SortedEntry(toKey)); + if (toPos >= 0) { + if (!toInclusive) + toPos--; + } else if (toPos < 0) { + toPos = -1 - toPos; + toPos--; + } + } + Iterator> iterator = entries.subList(fromPos, toPos+1).iterator(); + return new EntryIterator(iterator); + } + + public ListIterator> endListIterator() { + return entries.listIterator(entries.size() - 1); + } + + public ListIterator> listIterator() { + return entries.listIterator(); + } + + public ListIterator> listIterator(int index) { + return entries.listIterator(index); + } + + private Map.Entry removeMapping(Map.Entry toRemove) { + int pos = Collections.binarySearch(entries, new SortedEntry((K) toRemove.getKey())); + if (pos >= 0) { + SortedEntry entry = entries.get(pos); + if (entry.value.equals(toRemove.getValue())) { + entries.remove(pos); + return entry; + } + } + return null; + } + + public static class SortedEntry,V> implements Map.Entry, Comparable> { + private K key; + private V value; + + private SortedEntry(K key, V value) { + this.key = key; + this.value = value; + } + + public String toString() { + String keyStr = null; + if (key == null) keyStr = ""; + else keyStr = key.toString(); + String valueStr = null; + if (value == null) valueStr = ""; + else valueStr = value.toString(); + return keyStr+"->"+value; + } + + public K setKey(K key) { + K oldKey = this.key; + this.key = key; + return oldKey; + } + + public V setValue(V value) { + V oldValue = this.value; + this.value = value; + return oldValue; + } + + public K getKey() { + return key; + } + + public V getValue() { + return value; + } + + private SortedEntry(K key) { + this.key = key; + } + + public int compareTo(SortedEntry other) { + return key.compareTo(other.key); + } + } + + public void putAll(Map m) { + entries.ensureCapacity(m.size()); + for (Iterator> i = m.entrySet().iterator(); i.hasNext();) { + Map.Entry e = i.next(); + put(e.getKey(), e.getValue()); + } + } + + /** + * public Set keySet() { Set set = new HashSet(entries.size()); + * for (SortedEntry entry : entries) { set.add(entry.key); } return set; } + */ + // public Set> entrySet() { + // Set> set = new HashSet>(entries); + // return set; + // } + public boolean isEmpty() { + return entries.size() == 0; + } + + public void clear() { + entries.clear(); + } + + public int size() { + return entries.size(); + } + + private int getPos(K key) { + int pos = Collections.binarySearch(entries, new SortedEntry(key)); + if (pos < 0) + pos = -1 - pos; + return pos; + } + + public boolean containsValue(Object value) { + for (SortedEntry entry : entries) { + boolean b = ObjectUtils.equals(value, entry.value); + if (b) + return true; + } + return false; + } + + public boolean containsKey(Object key) { + int pos = Collections.binarySearch(entries, new SortedEntry((K) key)); + return pos >= 0; + } + + public K lastKey() { + if (entries.size() == 0) + return null; + return entries.get(entries.size() - 1).key; + } + + public V lastValue() { + if (entries.size() == 0) + return null; + return entries.get(entries.size() - 1).value; + } + + public V put(K key, V value) { + V oldValue = null; + int pos = Collections.binarySearch(entries, new SortedEntry(key)); + if (pos >= 0) { + oldValue = entries.get(pos).value; + entries.set(pos, new SortedEntry(key, value)); + return oldValue; + } + if (pos < 0) + pos = -1 - pos; + entries.add(pos, new SortedEntry(key, value)); + return null; + } + + private SortedEntry getEntry(Object key) { + int pos = Collections.binarySearch(entries, new SortedEntry((K) key)); + if (pos >= 0) { + return entries.get(pos); + } else { + return null; + } + } + + public V get(int index) { + return entries.get(index).value; + } + + public V get(Object key) { + int pos = Collections.binarySearch(entries, new SortedEntry((K) key)); + if (pos >= 0) { + return entries.get(pos).value; + } else { + return null; + } + } + + public V remove(Object key) { + assert key != null; + int pos = Collections.binarySearch(entries, new SortedEntry((K) key)); + if (pos >= 0) { + SortedEntry entry = entries.get(pos); + entries.remove(pos); + return entry.value; + } + return null; + } + + public void add(K key, V value) { + int pos = getPos(key); + entries.add(pos, new SortedEntry(key, value)); + } + + private abstract class SortedListIterator implements Iterator { + private Iterator> iterator; + + public SortedListIterator() { + iterator = entries.iterator(); + } + + public SortedListIterator(Iterator> iterator) { + this.iterator = iterator; + } + + protected Map.Entry nextEntry() { + return iterator.next(); + } + + public void remove() { + iterator.remove(); + } + + public boolean hasNext() { + return iterator.hasNext(); + } + } + + private class ValueIterator extends SortedListIterator { + public V next() { + return nextEntry().getValue(); + } + } + + private class KeyIterator extends SortedListIterator { + public K next() { + return nextEntry().getKey(); + } + } + + private class EntryIterator extends SortedListIterator> { + private EntryIterator() {} + + private EntryIterator(Iterator> iterator) { + super(iterator); + } + + public Map.Entry next() { + return nextEntry(); + } + } + + // Subclass overrides these to alter behavior of views' iterator() method + Iterator newKeyIterator() { + return new KeyIterator(); + } + + Iterator newValueIterator() { + return new ValueIterator(); + } + + Iterator> newEntryIterator() { + return new EntryIterator(); + } + + public Set keySet() { + Set ks = keySet; + return (ks != null ? ks : (keySet = new KeySet())); + } + + private class KeySet extends AbstractSet { + public Iterator iterator() { + return newKeyIterator(); + } + + public int size() { + return entries.size(); + } + + public boolean contains(Object o) { + return containsKey(o); + } + + public boolean remove(Object o) { + return SortedList.this.remove(o) != null; + } + + public void clear() { + SortedList.this.clear(); + } + } + + public Collection values() { + Collection vs = values; + return (vs != null ? vs : (values = new Values())); + } + + private class Values extends AbstractCollection { + public Iterator iterator() { + return newValueIterator(); + } + + public int size() { + return SortedList.this.size(); + } + + public boolean contains(Object o) { + return containsValue(o); + } + + public void clear() { + SortedList.this.clear(); + } + } + + public Set> entrySet() { + Set> es = entrySet; + return (es != null ? es : (entrySet = (Set>) (Set) new EntrySet())); + } + + private class EntrySet extends AbstractSet/* > */{ + public Iterator/* > */iterator() { + return newEntryIterator(); + } + + public boolean contains(Object o) { + if (!(o instanceof Map.Entry)) + return false; + Map.Entry e = (Map.Entry) o; + SortedEntry candidate = getEntry(e.getKey()); + return candidate != null && candidate.equals(e); + } + + public boolean remove(Object o) { + return removeMapping((Map.Entry) o) != null; + } + + public int size() { + return SortedList.this.size(); + } + + public void clear() { + SortedList.this.clear(); + } + } + + /** + * public Collection values() { List values = new ArrayList(entries.size()); + * for (SortedEntry entry : entries) { values.add(entry.value); } return + * values; } + */ +} Index: ocean/src/org/apache/lucene/ocean/util/Timeout.java =================================================================== --- ocean/src/org/apache/lucene/ocean/util/Timeout.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/util/Timeout.java (revision 0) @@ -0,0 +1,138 @@ +package org.apache.lucene.ocean.util; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.commons.lang.builder.ReflectionToStringBuilder; + +public class Timeout { + public static final int CANCELLED = 10; + public static final int OK = 1; + public static final int NO_TIMEOUT = 20; + public AtomicInteger status; + public long startTime; + public long timeout; + public Throwable throwable; + public List listeners = new ArrayList(5); + public final static TimerThread TIMER_THREAD = new TimerThread(); + + public String toString() { + return ReflectionToStringBuilder.toString(this); + } + + public Timeout() { + this.status = new AtomicInteger(NO_TIMEOUT); + } + + public long getElapsed() { + return TIMER_THREAD.getTime() - startTime; + } + + public static Timeout FOREVER() { + return new Timeout(Long.MAX_VALUE); + } + + public Timeout(long timeout) { + this.timeout = timeout; + this.status = new AtomicInteger(OK); + startTime = TIMER_THREAD.getTime(); + } + + public void notifyListenersCancelled() { + for (Listener listener : listeners) + listener.cancelled(); + } + + public static class Listener { + public void cancelled() { + } + } + + public void addListener(Listener listener) { + listeners.add(listener); + } + + public long getTimeLeft() { + return (timeout + startTime) - System.currentTimeMillis(); + } + + public void cancel() { + cancel(null); + } + + public void cancel(Throwable throwable) { + if (status.get() != CANCELLED) { + this.throwable = throwable; + status.set(CANCELLED); + } + notifyListenersCancelled(); + } + + public boolean notOK() { + return !isOK(); + } + + public boolean isOK() { + if (status.get() == NO_TIMEOUT) + return true; + if (status.get() == CANCELLED) + return false; + long dif = TIMER_THREAD.getTime() - startTime; + boolean ok = timeout > dif; + if (!ok) + status.set(CANCELLED); + return ok; + } + + public static class TimeoutException extends Exception { + private Timeout timeout; + + public TimeoutException(Timeout timeout) { + this.timeout = timeout; + } + + public TimeoutException(String message, Timeout timeout) { + super(message); + this.timeout = timeout; + } + } + + public static class TimeoutRuntimeException extends RuntimeException { + private Timeout timeout; + + public TimeoutRuntimeException(String message, Timeout timeout) { + super(message); + this.timeout = timeout; + } + } + + public static class TimerThread extends Thread { + private final int resolution; + private volatile long time; + + public TimerThread() { + this(1); + } + + public TimerThread(final int resolution) { + this.resolution = resolution; + this.setDaemon(true); + } + + public void run() { + for (;;) { + time = System.currentTimeMillis(); + try { + Thread.sleep(resolution); + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + } + + public long getTime() { + return time; + } + } +} Index: ocean/src/org/apache/lucene/ocean/util/TokenStreamInput.java =================================================================== --- ocean/src/org/apache/lucene/ocean/util/TokenStreamInput.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/util/TokenStreamInput.java (revision 0) @@ -0,0 +1,160 @@ +package org.apache.lucene.ocean.util; + +import java.io.IOException; +import java.io.Reader; +import java.util.HashMap; +import java.util.Map; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.Token; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.UnicodeUtil; + +/** + * + * + */ +// TODO: store strings in hash and index by number +public class TokenStreamInput { + private IndexInput originalInput; + private int numDocs; + private FieldInfos[] fieldInfos; + private static ContextThreadLocal contextThreadLocal = new ContextThreadLocal(); + + public TokenStreamInput(IndexInput originalInput) throws IOException { + this.originalInput = originalInput; + numDocs = originalInput.readVInt(); + readFieldIndexes(); + } + + public Analyzer getAnalyzer(int doc) { + Context context = getContext(); + context.analyzerImpl.doc = doc; + return context.analyzerImpl; + } + + public class AnalyzerImpl extends Analyzer { + int doc; + + public TokenStream tokenStream(String fieldName, Reader reader) { + try { + return getTokenStream(doc, fieldName); + } catch (IOException ioException) { + throw new RuntimeException(ioException); + } + } + } + + public static class ContextThreadLocal extends ThreadLocal { + protected Context initialValue() { + return new Context(); + } + } + + public Context getContext() { + Context context = contextThreadLocal.get(); + if (context.input == null) { + context.input = (IndexInput) originalInput.clone(); + } + if (context.analyzerImpl == null) { + context.analyzerImpl = new AnalyzerImpl(); + } + return context; + } + + public static class Context { + private TokenStreamImpl tokenStreamImpl = new TokenStreamImpl(); + private UnicodeUtil.UTF16Result text = new UnicodeUtil.UTF16Result(); + private UnicodeUtil.UTF8Result bytes = new UnicodeUtil.UTF8Result(); + private IndexInput input; + private AnalyzerImpl analyzerImpl; + } + + public TokenStream getTokenStream(int doc, String field) throws IOException { + Context context = getContext(); + FieldInfo fieldInfo = fieldInfos[doc].get(field); + int position = fieldInfo.position; + context.input.seek(position); + TokenStreamImpl tokenStreamImpl = context.tokenStreamImpl; + tokenStreamImpl.setNum(fieldInfo.num); + return tokenStreamImpl; + } + + public static class TokenStreamImpl extends TokenStream { + int position; + int num; + int i; + private IndexInput input; + private Context context; + + public TokenStreamImpl() { + + } + + public void setNum(int num) { + this.num = num; + i = 0; + } + + public Token next() throws IOException { + return next(new Token()); + } + + public Token next(Token token) throws IOException { + if (i == num) + return null; + int length = input.readVInt(); + context.bytes.setLength(length); + input.readBytes(context.bytes.result, 0, length); + UnicodeUtil.UTF8toUTF16(context.bytes.result, 0, length, context.text); + token.setTermBuffer(context.text.result, 0, context.text.length); + token.setStartOffset(input.readVInt()); + token.setEndOffset(input.readVInt()); + i++; + return token; + } + } + + public static class FieldInfo { + public int position; + public int num; + } + + public static class FieldInfos { + private Map map; + + public FieldInfos() { + + } + + public FieldInfo get(String field) { + return map.get(field); + } + } + + private void readFieldIndexes() throws IOException { + int numFields = originalInput.readVInt(); + Map map = new HashMap(numFields); + Map otherMap = new HashMap(numFields); + for (int x = 0; x < numFields; x++) { + String name = originalInput.readString(); + map.put(name, x); + otherMap.put(x, name); + } + fieldInfos = new FieldInfos[numDocs]; + for (int x = 0; x < fieldInfos.length; x++) { + fieldInfos[x] = new FieldInfos(); + int fieldInfoNum = originalInput.readVInt(); + fieldInfos[x].map = new HashMap(); + for (int y = 0; y < fieldInfoNum; y++) { + int stringNum = originalInput.readVInt(); + FieldInfo fieldInfo = new FieldInfo(); + fieldInfo.position = originalInput.readVInt(); + fieldInfo.num = originalInput.readVInt(); + String name = otherMap.get(stringNum); + fieldInfos[x].map.put(name, fieldInfo); + } + } + } +} Index: ocean/src/org/apache/lucene/ocean/util/TokenStreamOutput.java =================================================================== --- ocean/src/org/apache/lucene/ocean/util/TokenStreamOutput.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/util/TokenStreamOutput.java (revision 0) @@ -0,0 +1,150 @@ +package org.apache.lucene.ocean.util; + +import java.io.IOException; +import java.io.Reader; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.Token; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Fieldable; +import org.apache.lucene.index.Payload; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.UnicodeUtil; + +// incomplete +public class TokenStreamOutput { + private Analyzer analyzer; + private Token localToken = new Token(); + int position; + int length; + int offset; + int offsetEnd; + final int maxFieldLength; + private IndexOutput output; + private UnicodeUtil.UTF8Result utf8Result = new UnicodeUtil.UTF8Result(); + private int fieldIndexPosition; + private Map fieldIndexes; + + public TokenStreamOutput(List documents, Analyzer analyzer, IndexOutput output, int maxFieldLength) throws IOException { + this.maxFieldLength = maxFieldLength; + output.writeVInt(documents.size()); + for (Document document : documents) { + writeDocument(document); + } + } + + public void writeFieldIndexes(IndexOutput output) throws IOException { + int doc = -1; + //output.writeVInt(i); + //for (FieldIndex fieldIndex : fieldIndexes) { + // if (doc != fieldIndex.doc) output.writeVInt(fieldIndex.doc); + + //} + } + + public static class FieldIndex { + public int doc; + public int position; + } + + public void writeDocument(Document document) throws IOException { + List fields = document.getFields(); + List indexFields = new ArrayList(fields.size()); + for (Fieldable field : fields) { + if (field.isIndexed()) { + indexFields.add(field); + } + } + output.writeVInt(indexFields.size()); + for (Fieldable field : indexFields) { + writeField(field); + } + } + + public void writeField(Fieldable field) throws IOException { + long filePointer = output.getFilePointer(); + + if (length > 0) + position += analyzer.getPositionIncrementGap(field.name()); + //output.writeString(field.name()); + if (!field.isTokenized()) { // un-tokenized field + String stringValue = field.stringValue(); + final int valueLength = stringValue.length(); + Token token = localToken; + token.clear(); + char[] termBuffer = token.termBuffer(); + if (termBuffer.length < valueLength) + termBuffer = token.resizeTermBuffer(valueLength); + stringValue.getChars(0, valueLength, termBuffer, 0); + token.setTermLength(valueLength); + token.setStartOffset(offset); + token.setEndOffset(offset + stringValue.length()); + addPosition(token); + offset += stringValue.length(); + length++; + } else { + final TokenStream stream; + final TokenStream streamValue = field.tokenStreamValue(); + if (streamValue != null) + stream = streamValue; + else { + final Reader reader; // find or make Reader + final Reader readerValue = field.readerValue(); + if (readerValue != null) + reader = readerValue; + else { + String stringValue = field.stringValue(); + if (stringValue == null) + throw new IllegalArgumentException("field must have either TokenStream, String or Reader value"); + reader = new StringReader(stringValue); + } + stream = analyzer.reusableTokenStream(field.name(), reader); + } + stream.reset(); + try { + offsetEnd = offset - 1; + for (;;) { + Token token = stream.next(localToken); + if (token == null) + break; + position += (token.getPositionIncrement() - 1); + addPosition(token); + if (++length >= maxFieldLength) { + // if (threadState.docWriter.infoStream != null) + // threadState.docWriter.infoStream.println("maxFieldLength " + // +maxFieldLength+ " reached for field " + fieldInfo.name + ", + // ignoring following tokens"); + break; + } + } + offset = offsetEnd + 1; + } finally { + stream.close(); + } + } + } + + private void addPosition(Token token) throws IOException { + final char[] tokenText = token.termBuffer(); + final int tokenTextLen = token.termLength(); + UnicodeUtil.UTF16toUTF8(tokenText, 0, tokenTextLen, utf8Result); + output.writeVInt(tokenTextLen); + output.writeBytes(utf8Result.result, 0, utf8Result.length); + output.writeVInt(token.startOffset()); + output.writeVInt(token.endOffset()); + Payload payload = token.getPayload(); + if (payload == null) { + output.writeVInt(position<<1); + } else if (payload != null) { + output.writeVInt((position<<1)|1); + int payloadLength = payload.getData().length; + output.writeVInt(payloadLength); + output.writeBytes(payload.getData(), payload.getOffset(), payloadLength); + } + } +} Index: ocean/src/org/apache/lucene/ocean/util/Util.java =================================================================== --- ocean/src/org/apache/lucene/ocean/util/Util.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/util/Util.java (revision 0) @@ -0,0 +1,506 @@ +package org.apache.lucene.ocean.util; + +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.RandomAccessFile; +import java.io.Reader; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.net.MalformedURLException; +import java.net.URL; +import java.text.DateFormat; +import java.text.DecimalFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.SortedMap; +import java.util.SortedSet; +import java.util.TimeZone; +import java.util.TreeSet; +import java.util.regex.Pattern; + +import javax.naming.Context; +import javax.naming.InitialContext; +import javax.naming.NameNotFoundException; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermDocs; +import org.apache.lucene.ocean.LogDirectory; +import org.apache.lucene.search.ExtendedFieldCache; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.store.RAMFile; +import org.apache.lucene.store.RAMInputStream; + +public class Util { + public static TimeZone UTC = TimeZone.getTimeZone("UTC"); + public final static Pattern splitPattern = Pattern.compile(",| "); + private static ThreadLocalDateFormat dateFormatThreadLocal = new ThreadLocalDateFormat(); + + public static String formatSnapshotId(BigDecimal id) { + DecimalFormat format = new DecimalFormat("##0.00"); + return format.format(id); + } + + public static > T min(Collection collection) { + if (collection.size() == 0) + return null; + return Collections.min(collection); + } + + /** + * Handles size 0 collections returns null + */ + public static > T max(Collection collection) { + if (collection.size() == 0) + return null; + return Collections.max(collection); + } + + public static void touchFile(String file, Directory directory) throws IOException { + IndexOutput output = directory.createOutput(file); + output.close(); + } + + public static String getString(String file, LogDirectory directory) throws IOException { + try { + RandomAccessFile input = directory.openInput(file); + if (input.length() == 0) + return null; + byte[] bytes = new byte[(int) input.length()]; + input.read(bytes); + return new String(bytes, "UTF-8"); + } catch (Throwable ioException) { + IOException newIOException = new IOException("file: " + file); + newIOException.initCause(ioException); + throw newIOException; + } + } + + public static void save(String string, String file, LogDirectory directory) throws IOException { + byte[] bytes = string.getBytes("UTF-8"); + RandomAccessFile output = directory.getOutput(file, true); + output.write(bytes); + output.getFD().sync(); + output.close(); + } + + public static String getString(String file, Directory directory) throws IOException { + IndexInput input = directory.openInput(file); + byte[] bytes = new byte[(int) input.length()]; + input.readBytes(bytes, 0, bytes.length); + return new String(bytes, "UTF-8"); + } + + public static void save(String string, String file, Directory directory) throws IOException { + byte[] bytes = string.getBytes("UTF-8"); + IndexOutput output = directory.createOutput(file); + output.writeBytes(bytes, bytes.length); + output.flush(); + output.close(); + } + + public static void copy(IndexInput input, IndexOutput output, byte[] buffer) throws IOException { + long len = input.length(); + long readCount = 0; + while (readCount < len) { + int toRead = readCount + buffer.length > len ? (int) (len - readCount) : buffer.length; + input.readBytes(buffer, 0, toRead); + output.writeBytes(buffer, toRead); + readCount += toRead; + } + } + + public static V getLastValue(SortedMap map) { + if (map.size() == 0) + return null; + return map.get(map.lastKey()); + } + + public static void setValue(String name, long value, Document document) { + String encoded = longToEncoded(value); + document.add(new Field(name, encoded, Field.Store.YES, Field.Index.UN_TOKENIZED)); + } + + public static boolean mkdir(File dir) { + if (!dir.exists()) { + return dir.mkdirs(); + } + return false; + } + + public static String longToEncoded(long value) { + return long2sortableStr(value); + } + + public static long longFromEncoded(String string) { + return SortableStr2long(string, 0, 5); + } + + public static String long2sortableStr(long val) { + char[] arr = new char[5]; + long2sortableStr(val, arr, 0); + return new String(arr, 0, 5); + } + + // uses binary representation of an int to build a string of + // chars that will sort correctly. Only char ranges + // less than 0xd800 will be used to avoid UCS-16 surrogates. + // we can use the lowest 15 bits of a char, (or a mask of 0x7fff) + public static int long2sortableStr(long val, char[] out, int offset) { + val += Long.MIN_VALUE; + out[offset++] = (char) (val >>> 60); + out[offset++] = (char) (val >>> 45 & 0x7fff); + out[offset++] = (char) (val >>> 30 & 0x7fff); + out[offset++] = (char) (val >>> 15 & 0x7fff); + out[offset] = (char) (val & 0x7fff); + return 5; + } + + public static long SortableStr2long(String sval, int offset, int len) { + long val = (long) (sval.charAt(offset++)) << 60; + val |= ((long) sval.charAt(offset++)) << 45; + val |= ((long) sval.charAt(offset++)) << 30; + val |= sval.charAt(offset++) << 15; + val |= sval.charAt(offset); + val -= Long.MIN_VALUE; + return val; + } + + public static int getDoc(String fieldName, long value, IndexReader indexReader) throws IOException { + String encoded = longToEncoded(value); + return getTermDoc(new Term(fieldName, encoded), indexReader); + } + + public static int getTermDoc(Term term, IndexReader indexReader) throws IOException { + TermDocs docs = indexReader.termDocs(term); + try { + if (docs.next()) { + return docs.doc(); + } + } finally { + docs.close(); + } + return -1; + } + + public static List getTermDocs(Term term, IndexReader indexReader) throws IOException { + List list = new ArrayList(); + TermDocs docs = indexReader.termDocs(term); + try { + while (docs.next()) { + list.add(docs.doc()); + } + } finally { + docs.close(); + } + return list; + } + + public static long getSize(Directory directory) throws IOException { + long total = 0; + for (String file : directory.list()) { + total += directory.fileLength(file); + } + return total; + } + + public static void copyRamFile(RAMFile ramFile, OutputStream os) throws IOException { + int numBuffers = ramFile.numBuffers(); + long length = ramFile.getLength(); + int bytesLength; + int numWritten = 0; + for (int x=0; x < numBuffers; x++) { + byte[] buffer = ramFile.getBuffer(x); + if (buffer.length + numWritten > length) + bytesLength = (int)(length - numWritten); + else + bytesLength = buffer.length; + os.write(buffer, 0, bytesLength); + } + } + + public static void copy(RAMDirectory src, Directory dest) throws IOException { + final String[] files = src.list(); + + if (files == null) + throw new IOException("cannot read directory " + src + ": list() returned null"); + + for (int i = 0; i < files.length; i++) { + IndexOutput os = null; + RAMInputStream is = null; + try { + // create file in dest directory + os = dest.createOutput(files[i]); + // read current file + is = (RAMInputStream)src.openInput(files[i]); + RAMFile ramFile = is.getRAMFile(); + int numBuffers = ramFile.numBuffers(); + long length = ramFile.getLength(); + int bytesLength; + int numWritten = 0; + for (int x=0; x < numBuffers; x++) { + byte[] buffer = ramFile.getBuffer(x); + if (buffer.length + numWritten > length) + bytesLength = (int)(length - numWritten); + else + bytesLength = buffer.length; + os.writeBytes(buffer, bytesLength); + } + } finally { + // graceful cleanup + try { + if (os != null) + os.close(); + } finally { + if (is != null) + is.close(); + } + } + } + } + + public static void copy(InputStream is, RandomAccessFile ras, byte[] buf) throws IOException { + int numRead; + while ((numRead = is.read(buf)) >= 0) { + ras.write(buf, 0, numRead); + } + } + + public static File getDirectory(File root, String path) { + File file = new File(root, path); + if (!file.exists()) { + file.mkdirs(); + } + return file; + } + + public static DateFormat getThreadLocalDateFormat() { + return dateFormatThreadLocal.get(); + } + + public static String formatDate(Date date) { + DateFormat dateFormat = getThreadLocalDateFormat(); + return dateFormat.format(date); + } + + public static Date parseDate(String string) throws ParseException { + DateFormat dateFormat = getThreadLocalDateFormat(); + return dateFormat.parse(string); + } + + private static class ThreadLocalDateFormat extends ThreadLocal { + DateFormat proto; + + public ThreadLocalDateFormat() { + super(); + SimpleDateFormat tmp = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.US); + tmp.setTimeZone(UTC); + proto = tmp; + } + + protected DateFormat initialValue() { + return (DateFormat) proto.clone(); + } + } + + public static long[] getFieldCacheLong(String field, IndexReader indexReader) throws IOException { + return ExtendedFieldCache.EXT_DEFAULT.getLongs(indexReader, field); + } + + public static boolean isTrue(String string) { + return StringUtils.equalsIgnoreCase("true", string); + } + + public static Object construct(Object parameter, Class clazz) throws Exception { + Constructor constructor = clazz.getDeclaredConstructor(parameter.getClass()); + try { + return constructor.newInstance(parameter); + } catch (InvocationTargetException invocationTargetException) { + Throwable cause = invocationTargetException.getCause(); + if (cause instanceof Exception) + throw (Exception) cause; + else + throw invocationTargetException; + } + } + + public static Long parseLong(String value) { + if (StringUtils.isBlank(value)) { + return null; + } + try { + return new Long(value); + } catch (Throwable ex) { + return null; + } + } + + public static URL parseURL(String string) throws MalformedURLException { + if (StringUtils.isBlank(string)) { + return null; + } + return new URL(string); + } + + public static List readLines(Reader reader) throws Exception { + return IOUtils.readLines(reader); + } + + public static IOException asIOException(String message, Throwable throwable) throws IOException { + IOException ioException = new IOException(message); + ioException.initCause(throwable); + return ioException; + } + + public static IOException asIOException(Throwable throwable) { + if (throwable instanceof IOException) { + return (IOException) throwable; + } + IOException ioException = new IOException(throwable.getMessage()); + ioException.initCause(throwable); + return ioException; + } + + public static void copy(InputStream is, OutputStream os, byte[] buf) throws IOException { + int numRead; + while ((numRead = is.read(buf)) >= 0) { + os.write(buf, 0, numRead); + } + } + + public static int[] toIntArray(List list) { + int size = list.size(); + int[] array = new int[size]; + int x = 0; + for (int i : list) { + array[x] = i; + x++; + } + return array; + } + + public static List loadUrls(File file) throws IOException { + List lines = IOUtils.readLines(new FileReader(file)); + List urls = new ArrayList(); + for (String line : lines) { + urls.add(new URL(line)); + } + return urls; + } + + /** + * public static HttpParameters toHttpParameters(HttpServletRequest request) { + * try { if (StringUtils.equalsIgnoreCase("post", request.getMethod())) { + * HttpParameters parameters = new HttpParameters(); URL url = new + * URL(request.getRequestURL().toString()); CGIParser cgiParser = new + * CGIParser(url.toString(), "UTF-8"); for (String name : + * cgiParser.getParameterNameList()) { for (String value : + * cgiParser.getParameterValues(name)) { parameters.add(name, value); } } + * return parameters; } } catch (Exception exception) { throw new + * RuntimeException(exception); } if (StringUtils.equalsIgnoreCase("get", + * request.getMethod())) { HttpParameters parameters = new HttpParameters(); + * Enumeration paramEnum = request.getParameterNames(); while + * (paramEnum.hasMoreElements()) { String name = (String) + * paramEnum.nextElement(); String[] array = request.getParameterValues(name); + * if (array != null && array.length > 0) { for (String value : array) { + * parameters.add(name, value); } } } return parameters; } throw new + * RuntimeException("unknown http method " + request.getMethod()); } + */ + public static Long getNextServerSequence(Long value, int serverNumber) { + if (value == null) { + return new Long(serverNumber); + } + Long i = null; + if (value > 99) { + String string = value.toString(); + String substring = string.substring(0, string.length() - 2); + i = new Long(substring + "00"); + } else { + i = new Long(0); + } + long v = i + serverNumber; + return v + 100; + } + + public static int getServerNumber(BigInteger id) { + String string = id.toString(); + String substring = string.substring(string.length() - 2, string.length()); + return Integer.parseInt(substring); + } + + public static SortedSet splitToSortedSet(String string) { + if (StringUtils.isBlank(string)) { + return null; + } + String[] array = splitPattern.split(string.trim(), 0); + TreeSet sortedSet = new TreeSet(); + for (int x = 0; x < array.length; x++) { + sortedSet.add(array[x]); + } + return sortedSet; + } + + public static File getAppServerHome() { + return new File(System.getProperty("catalina.home")); + } + + public static File getHomeDirectory(String name, File defaultDirectory) throws Exception { + String value = System.getenv(name); + if (value != null) + return new File(value); + value = System.getProperty(name); + if (value != null) + return new File(value); + Context context = (Context) new InitialContext().lookup("java:comp/env"); + try { + String string = (String) context.lookup(name); + if (StringUtils.isNotBlank(value)) + return new File(value); + } catch (NameNotFoundException nameNotFoundException) { + } + defaultDirectory.mkdirs(); + return defaultDirectory; + } + + public static Object getFirst(List list) { + Iterator iterator = list.iterator(); + if (iterator.hasNext()) { + return iterator.next(); + } else { + return null; + } + } + + public static Map toMapExcept(org.jdom.Element element, String exceptName) { + Map map = new HashMap(); + for (Object object : element.getAttributes()) { + org.jdom.Attribute attribute = (org.jdom.Attribute) object; + String name = attribute.getName(); + if (!StringUtils.equals(name, exceptName)) { + map.put(name, attribute.getValue()); + } + } + return map; + } +} Index: ocean/src/org/apache/lucene/ocean/util/VDataInputStream.java =================================================================== --- ocean/src/org/apache/lucene/ocean/util/VDataInputStream.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/util/VDataInputStream.java (revision 0) @@ -0,0 +1,41 @@ +package org.apache.lucene.ocean.util; + +import java.io.DataInputStream; +import java.io.IOException; +import java.io.InputStream; + +public class VDataInputStream extends DataInputStream { + private byte[] bytes; // used by readString() + + public VDataInputStream(InputStream input) { + super(input); + } + + public String readString() throws IOException { + int length = readVInt(); + if (bytes == null || length > bytes.length) + bytes = new byte[(int) (length*1.25)]; + readFully(bytes, 0, length); + return new String(bytes, 0, length, "UTF-8"); + } + + public long readVLong() throws IOException { + byte b = readByte(); + long i = b & 0x7F; + for (int shift = 7; (b & 0x80) != 0; shift += 7) { + b = readByte(); + i |= (b & 0x7FL) << shift; + } + return i; + } + + public int readVInt() throws IOException { + byte b = readByte(); + int i = b & 0x7F; + for (int shift = 7; (b & 0x80) != 0; shift += 7) { + b = readByte(); + i |= (b & 0x7F) << shift; + } + return i; + } +} Index: ocean/src/org/apache/lucene/ocean/util/VDataOutputStream.java =================================================================== --- ocean/src/org/apache/lucene/ocean/util/VDataOutputStream.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/util/VDataOutputStream.java (revision 0) @@ -0,0 +1,37 @@ +package org.apache.lucene.ocean.util; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.OutputStream; + +import org.apache.lucene.util.UnicodeUtil; + +public class VDataOutputStream extends DataOutputStream { + private UnicodeUtil.UTF8Result utf8Result = new UnicodeUtil.UTF8Result(); + + public VDataOutputStream(OutputStream output) { + super(output); + } + + public void writeString(String s) throws IOException { + UnicodeUtil.UTF16toUTF8(s, 0, s.length(), utf8Result); + writeVInt(utf8Result.length); + write(utf8Result.result, 0, utf8Result.length); + } + + public void writeVLong(long i) throws IOException { + while ((i & ~0x7F) != 0) { + writeByte((byte)((i & 0x7f) | 0x80)); + i >>>= 7; + } + writeByte((byte)i); + } + + public void writeVInt(int i) throws IOException { + while ((i & ~0x7F) != 0) { + writeByte((byte)((i & 0x7f) | 0x80)); + i >>>= 7; + } + writeByte((byte)i); + } +} Index: ocean/src/org/apache/lucene/ocean/util/XMLUtil.java =================================================================== --- ocean/src/org/apache/lucene/ocean/util/XMLUtil.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/util/XMLUtil.java (revision 0) @@ -0,0 +1,444 @@ +package org.apache.lucene.ocean.util; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringReader; +import java.io.Writer; +import java.lang.reflect.Field; +import java.math.BigDecimal; +import java.net.URL; +import java.text.ParseException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Date; +import java.util.List; +import java.util.logging.Logger; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang.BooleanUtils; +import org.apache.commons.lang.StringUtils; +import org.jdom.Attribute; +import org.jdom.Document; +import org.jdom.Element; +import org.jdom.Namespace; +import org.jdom.Verifier; +import org.jdom.input.SAXBuilder; +import org.jdom.output.Format; +import org.jdom.output.XMLOutputter; +import org.xml.sax.EntityResolver; +import org.xml.sax.InputSource; + +/** + * + * @author Jason Rutherglen + */ +public class XMLUtil { + public static Logger log = Logger.getLogger(XMLUtil.class.getName()); + + public XMLUtil() { + } + + public static Collection getAttributes(Element element) { + return (Collection)element.getAttributes(); + } + + public static String printAndDetach(Element element) throws Exception { + List list = element.cloneContent(); + Element root = null; + if (list.size() == 1) { + root = list.get(0); + } else { + root = new Element("root"); + root.addContent(list); + } + return outputElement(root); + } + + public static Object getProperValue(Field field, String string) throws Exception { + if (!field.getType().isAssignableFrom(String.class)) { + if (field.getType().isAssignableFrom(URL.class)) { + return new URL(string); + } else if (field.getType().isAssignableFrom(Date.class)) { + return Util.parseDate(string); + } else if (field.getType().isAssignableFrom(File.class)) { + return new File(string); + } else if (field.getType().isAssignableFrom(Long.class)) { + return new Long(string); + } else if (field.getType().isAssignableFrom(Double.class)) { + return new Double(string); + } + throw new Exception("unknown type " + field.getType().getName()); + } else + return string; + } + + public static void addAll(List children, Element parent) { + for (Element element : children) { + parent.addContent(element); + } + } + + public static void printList(String rootName, List list, PrintWriter writer) throws Exception { + Format format = Format.getPrettyFormat(); + format.setLineSeparator("\n"); + XMLOutputter outputter = new XMLOutputter(format); + Element root = new Element(rootName); + for (Object object : list) { + if (object instanceof CElement) { + CElement cElement = (CElement) object; + root.addContent(cElement.toElement()); + } + } + Document document = new Document(); + document.addContent(root); + outputter.output(document, writer); + } + + public static List getChildren(String xml) throws Exception { + return getChildren(parseElement(xml)); + } + + public static List getChildren(String name, Element element) { + List elements = (List) element.getChildren(name); + if (element == null) + return new ArrayList(0); + else + return elements; + } + + public static List getChildren(Element element) { + List elements = (List) element.getChildren(); + if (element == null) + return new ArrayList(0); + else + return elements; + } + + public static Element getFirstChild(Element element) { + return (Element) Util.getFirst(element.getChildren()); + } + + public static void setAttribute(String name, Object value, Element element) { + if (value == null) { + return; + } + if (value instanceof Date) { + value = Util.formatDate((Date) value); + } + element.setAttribute(name, value.toString()); + } + + public static java.net.URL getChildURL(String name, Element element) { + String childText = element.getChildText(name); + try { + return new java.net.URL(childText); + } catch (Throwable th) { + } + return null; + } + + public static Element getChild(String name, Element root) { + return root.getChild(name); + } + + public static java.math.BigInteger getChildBigInteger(String name, Element element) { + String childText = element.getChildText(name); + try { + return new java.math.BigInteger(childText); + } catch (Throwable throwable) { + } + return null; + } + + public static Long getChildLong(String name, Element element) throws NumberFormatException { + String childText = element.getChildText(name); + return new Long(childText); + } + + public static String getAttributeString(String name, Element element) { + String text = element.getAttributeValue(name); + return text; + } + + public static Integer getAttributeInteger(String name, Element element) throws NumberFormatException { + String text = element.getAttributeValue(name); + if (text == null || text.equals("")) { + return null; + } + return new Integer(text); + } + + public static Float getAttributeFloat(String name, Element element) throws NumberFormatException { + String text = element.getAttributeValue(name); + if (text == null || text.equals("")) { + return null; + } + return new Float(text); + } + + public static BigDecimal getAttributeBigDecimal(String name, Element element) throws NumberFormatException { + String text = element.getAttributeValue(name); + if (text == null || text.equals("")) { + return null; + } + return new BigDecimal(text); + } + + public static Long getAttributeLong(String name, Element element) throws NumberFormatException { + String text = element.getAttributeValue(name); + if (text == null || text.equals("")) { + return null; + } + return new Long(text); + } + + public static Date getAttributeDate(String name, Element element) throws ParseException { + String text = element.getAttributeValue(name); + if (text == null || text.equals("")) { + return null; + } + return Util.parseDate(text); + } + + public static Boolean getChildBoolean(String name, Element element) { + String text = element.getChildText(name); + if (StringUtils.isBlank(text)) { + return null; + } + return BooleanUtils.toBooleanObject(text); + } + + public static boolean getAttributeBooleanPrimitive(String name, Element element) { + String text = element.getAttributeValue(name); + if (StringUtils.isBlank(text)) { + return false; + } + return BooleanUtils.toBooleanObject(text); + } + + public static Boolean getAttributeBoolean(String name, Element element) { + String text = element.getAttributeValue(name); + if (StringUtils.isBlank(text)) { + return null; + } + return BooleanUtils.toBooleanObject(text); + } + + public static Date getChildDate(String name, Element element) throws ParseException { + String text = element.getChildText(name); + if (text == null || text.equals("")) { + return null; + } + return parseDate(text); + } + + public static Date parseDate(String dateStr) throws ParseException { + if (org.apache.commons.lang.StringUtils.isEmpty(dateStr)) { + return null; + } + return Util.parseDate(dateStr); + } + + public static String formatDate(Date date) { + if (date == null) { + return ""; + } + return Util.formatDate(date); + } + + public static String getChildText(String name, Element element) { + return element.getChildText(name); + } + + public static Integer getChildInteger(String name, Element element) { + try { + String text = element.getChildText(name); + return new Integer(text); + } catch (Throwable ex) { + } + return null; + } + + public static Double getChildDouble(String name, Element element) throws NumberFormatException { + try { + String text = element.getChildText(name); + return new Double(text); + } catch (Throwable ex) { + } + return null; + } + + public static void outputElement(Element element, Writer writer) throws Exception { + XMLOutputter xmlOut = new XMLOutputter(Format.getPrettyFormat()); + xmlOut.output(new Document(element), writer); + } + + public static String outputElement(Element element) throws Exception { + XMLOutputter xmlOut = new XMLOutputter(Format.getPrettyFormat()); + String xmlString = xmlOut.outputString(new Document(element)); + return xmlString; + } + + public static String outputElementMinimal(Element element) throws Exception { + Format format = Format.getCompactFormat(); + format.setOmitDeclaration(true); + XMLOutputter xmlOut = new XMLOutputter(format); + String xmlString = xmlOut.outputString(new Document(element)); + return xmlString; + } + + public static String outputElementOmitDeclaration(Element element) { + Format format = Format.getPrettyFormat(); + format.setOmitDeclaration(true); + XMLOutputter xmlOut = new XMLOutputter(format); + String xmlString = xmlOut.outputString(new Document(element)); + return xmlString; + } + + public static String outputDocument(Document document) throws Exception { + XMLOutputter xmlOut = new XMLOutputter(Format.getPrettyFormat()); + String xmlString = xmlOut.outputString(document); + return xmlString; + } + + public static String removeInvalidXMLChars(String value) { + StringBuffer buffer = new StringBuffer(); + char[] array = value.toCharArray(); + for (int x = 0; x < array.length; x++) { + if (Verifier.isXMLCharacter(array[x])) { + buffer.append(array[x]); + } + } + return buffer.toString(); + } + + public static Element createTextElement(String name, Object object, Element parentElement) { + if (object == null) { + return null; + } + + String text = null; + if (object instanceof Date) { + Date date = (Date) object; + + text = formatDate(date); + } else { + text = object.toString(); + } + Element element = new Element(name); + // XMLOutputter outputter = new XMLOutputter(); + // text = outputter.escapeElementEntities(text); + // text = removeInvalidXMLChars(text); + element.setText(text); + + parentElement.addContent(element); + + return element; + } + + public static Element createTextElement(String name, Object object, Namespace namespace, Element parentElement) { + if (object == null) { + return null; + } + + String text = null; + if (object instanceof File) { + text = ((File) object).getAbsolutePath(); + } else { + text = object.toString(); + } + Element element = new Element(name, namespace); + XMLOutputter outputter = new XMLOutputter(); + text = outputter.escapeElementEntities(text); + text = removeInvalidXMLChars(text); + element.setText(text); + + parentElement.addContent(element); + + return element; + } + + public static void saveXML(Element element, File file) throws IOException { + Document document = new Document(); + document.addContent(element); + saveXML(document, file); + } + + public static void saveXML(Document document, File file) throws IOException { + File parentDir = file.getParentFile(); + if (!parentDir.exists()) { + parentDir.mkdirs(); + } + Format format = Format.getPrettyFormat(); + format.setLineSeparator("\n"); + XMLOutputter outputter = new XMLOutputter(format); + String channelXMLStr = outputter.outputString(document); + FileUtils.writeStringToFile(file, channelXMLStr, "UTF-8"); + } + + /** + * public static XmlPullParser parseDocumentSAX(InputStream input) throws + * Exception { XmlPullParserFactory factory = + * XmlPullParserFactory.newInstance("org.xmlpull.mxp1.MXParserFactory", null); + * factory.setNamespaceAware(false); factory.setValidating(false); + * XmlPullParser xpp = factory.newPullParser(); xpp.setInput(input, "UTF-8"); + * return xpp; } + */ + public static File getChildFile(String name, Element element) throws Exception { + String path = element.getChildTextTrim(name); + if (StringUtils.isBlank(path)) { + return null; + } + return new File(path); + } + + public static Document parseDocument(File file) throws XMLException, IOException { + String xml = FileUtils.readFileToString(file, "UTF-8"); + return parseDocument(xml); + } + + public static Element parseElement(File file) throws XMLException, IOException { + if (!file.exists()) { + return null; + } + String xml = FileUtils.readFileToString(file, "UTF-8"); + return parseElement(xml); + } + + public static Element parseElement(String xml) throws XMLException { + Document document = parseDocument(xml); + return document.getRootElement(); + } + + public static class XMLException extends Exception { + public XMLException(String message, Throwable throwable) { + super(message, throwable); + } + } + + public static Document parseDocument(String xml) throws XMLException { + if (StringUtils.isBlank(xml)) + throw new IllegalArgumentException("xml blank"); + EntityResolver RESOLVER = new EmptyEntityResolver(); + SAXBuilder saxBuilder = new SAXBuilder(false); + saxBuilder.setEntityResolver(RESOLVER); + try { + Document document = saxBuilder.build(new StringReader(xml)); + return document; + } catch (Exception exception) { + throw new XMLException(xml, exception); + } + } + + public static class EmptyEntityResolver implements EntityResolver { + public InputSource resolveEntity(String publicId, String systemId) { + InputSource EMPTY_INPUTSOURCE = new InputSource(new ByteArrayInputStream(new byte[0])); + return EMPTY_INPUTSOURCE; + // if (systemId != null && systemId.endsWith(".dtd")) return + // EMPTY_INPUTSOURCE; + // return null; + } + } +} Index: ocean/src/org/apache/lucene/ocean/WriteableMemoryIndex.java =================================================================== --- ocean/src/org/apache/lucene/ocean/WriteableMemoryIndex.java (revision 0) +++ ocean/src/org/apache/lucene/ocean/WriteableMemoryIndex.java (revision 0) @@ -0,0 +1,329 @@ +package org.apache.lucene.ocean; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.MultiReader; +import org.apache.lucene.ocean.Index.IndexSnapshot; +import org.apache.lucene.ocean.Index.IndexSnapshotSearchable; +import org.apache.lucene.ocean.util.Constants; +import org.apache.lucene.ocean.util.SortedList; +import org.apache.lucene.ocean.util.Util; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MultiSearcher; +import org.apache.lucene.search.Searchable; +import org.apache.lucene.store.instantiated.InstantiatedIndex; +import org.apache.lucene.store.instantiated.InstantiatedIndexWriter; + +/** + * Keeps a list of OceanMemoryIndex around + * + */ +public class WriteableMemoryIndex extends Index { + private SortedList snapshotMap = new SortedList(); + + public WriteableMemoryIndex(IndexID id, TransactionSystem system) { + super(id, system); + } + + public MemoryIndexSnapshot createIndexSnapshot(Long id) throws IOException { + return createIndexSnapshot(id, null, null); + } + + public MemoryIndexSnapshot createIndexSnapshot(Long id, OceanInstantiatedIndexReader[] readers, IndexReader indexReader) throws IOException { + List readersList = null; + if (readers != null) { + readersList = new ArrayList(readers.length+1); + for (OceanInstantiatedIndexReader reader : readers) + readersList.add(reader); + } else + readersList = new ArrayList(1); + OceanInstantiatedIndexReader oceanInstantiatedIndexReader = null; + if (indexReader == null) + oceanInstantiatedIndexReader = new OceanInstantiatedIndexReader(new InstantiatedIndex()); + else + oceanInstantiatedIndexReader = new OceanInstantiatedIndexReader(new InstantiatedIndex(indexReader)); + readersList.add(oceanInstantiatedIndexReader); + MemoryIndexSnapshot memoryIndexSnapshot = new MemoryIndexSnapshot(id, (OceanInstantiatedIndexReader[])readersList.toArray(new OceanInstantiatedIndexReader[0])); + snapshotMap.put(id, memoryIndexSnapshot); + return memoryIndexSnapshot; + } + + public MemoryIndexSnapshot getIndexSnapshot(Long snapshotId) { + return snapshotMap.get(snapshotId); + } + + public void commitNothing(Transaction transaction) throws IndexException, InterruptedException, IOException, Exception { + MemoryIndexSnapshot latestIndexSnapshot = getLatestIndexSnapshot(); + assert latestIndexSnapshot != null; + // if + // (latestIndexSnapshot.getSnapshotId().equals(transaction.getPreviousId())) + // { + // } + transaction.ready(this); + if (transaction.go()) { + Long snapshotId = transaction.getId(); + // IndexReader previousIndexReader = latestIndexSnapshot.getIndexReader(); + // MemoryIndexSnapshot(snapshotId, latestIndexSnapshot.indexReaders); + //MemoryIndexSnapshot newIndexSnapshot = latestIndexSnapshot.newIndexSnapshot(snapshotId); + OceanInstantiatedIndexReader[] clonedReaders = latestIndexSnapshot.cloneReaders(false); + createIndexSnapshot(transaction.getId(), clonedReaders); + removeOldSnapshots(snapshotMap); + } + } + + public DeletesResult commitChanges(Documents documents, Deletes deletes, Analyzer analyzer, Transaction transaction) + throws InterruptedException, Exception, IOException { + try { + if (isClosed()) { + throw new IOException("index is closed"); + } + if (isReadOnly() && documents != null && documents.size() > 0) { + throw new IOException("index not accepting new documents"); + } + DeletesResult deletesResult = new DeletesResult(getId()); + MemoryIndexSnapshot previousIndexSnapshot = getLatestIndexSnapshot(); + + OceanInstantiatedIndexReader[] clonedReaders = previousIndexSnapshot.cloneReaders(true); + MultiReader clonedMultiReader = new MultiReader(clonedReaders); + + if (deletes != null && deletes.hasDeletes()) { + deletesResult = applyDeletes(true, deletes, null, clonedMultiReader); + } + MemoryIndexSnapshot newIndexSnapshot = null; + if (documents != null) { + InstantiatedIndex newIndex = createInstantiatedIndex(documents, analyzer); + newIndexSnapshot = createIndexSnapshot(transaction.getId(), clonedReaders, new OceanInstantiatedIndexReader(newIndex)); + deletesResult.setNumAdded(documents.size()); + //transaction.addDocsAdded(documents.size()); + } else { + newIndexSnapshot = createIndexSnapshot(transaction.getId(), clonedReaders, null); + } + transaction.ready(this); + if (transaction.go()) { + Long snapshotId = transaction.getId(); + //snapshotMap.put(transaction.getId(), newIndexSnapshot); + removeOldSnapshots(snapshotMap); + return deletesResult; + } else { + snapshotMap.remove(transaction.getId()); + return null; + } + } catch (Throwable throwable) { + transaction.failed(this, throwable); + if (throwable instanceof Exception) { + throw (Exception) throwable; + } else { + throw new Exception(throwable); + } + } + // return null; + } + + public void createIndexSnapshot(Long id, OceanInstantiatedIndexReader[] readers) { + + } + + public InstantiatedIndex createInstantiatedIndex(Documents documents, Analyzer analyzer) throws IOException { + InstantiatedIndex index = new InstantiatedIndex(); + InstantiatedIndexWriter writer = new InstantiatedIndexWriter(index); + for (Document document : documents) { + writer.addDocument(document, analyzer); + } + writer.commit(); + return index; + } + + public DeletesResult commitDeletes(Deletes deletes, Transaction transaction) throws Exception, InterruptedException, IOException { + return commitChanges(null, deletes, null, transaction); + } + + public static class IndexSnapshotMultiSearcher extends MultiSearcher implements IndexSnapshotSearchable { + private IndexSnapshot indexSnapshot; + + public IndexSnapshotMultiSearcher(IndexSnapshot indexSnapshot, Searchable[] searchables) throws IOException { + super(searchables); + this.indexSnapshot = indexSnapshot; + } + + public IndexSnapshot getIndexSnapshot() { + return indexSnapshot; + } + } + + public class MemoryIndexSnapshot extends IndexSnapshot { + //private List indexReaders = new ArrayList(); + private OceanInstantiatedIndexReader[] subReaders; + private MultiReader indexReader; + private Integer deletedDoc; + private Long maxSnapshotId; + private Long maxDocumentId; + private Long minSnapshotId; + private Long minDocumentId; + private IndexSnapshotMultiSearcher searcher; + private int maxDoc = 0; + + public MemoryIndexSnapshot(Long snapshotId, OceanInstantiatedIndexReader[] subReaders) throws IOException { + super(snapshotId); + this.subReaders = subReaders; + // indexReaders.iterator(); + /** + Iterator iterator = indexReaders.iterator(); + while (iterator.hasNext()) { + OceanInstantiatedIndexReader reader = iterator.next(); + int maxDoc = reader.maxDoc(); + if (maxDoc == 0) { + iterator.remove(); + } + this.maxDoc += maxDoc; + // throw new RuntimeException("maxDoc: 0"); + // } + } + **/ + } + + public OceanInstantiatedIndexReader[] cloneReaders(boolean removeMaxDocZero) { + List cloneList = new ArrayList(); + for (OceanInstantiatedIndexReader reader : subReaders) { + if (removeMaxDocZero) { + if (reader.maxDoc() >= 0) + cloneList.add(reader); + } else + cloneList.add(reader); + } + return cloneList.toArray(new OceanInstantiatedIndexReader[0]); + } + + public IndexSnapshotSearchable getSearcher() { + if (searcher == null) { + try { + if (subReaders.length > 0) { + Searchable[] searchables = new Searchable[subReaders.length]; + for (int x=0; x < searchables.length; x++) { + searchables[x] = new IndexSearcher(subReaders[x]); + } + searcher = new IndexSnapshotMultiSearcher(this, searchables); + } + } catch (IOException ioException) { + throw new RuntimeException(ioException); + } + } + return searcher; + } + + public void delete() throws Exception { + snapshotMap.remove(snapshotId); + } + + public Long getMinDocumentId() throws IOException { + if (minDocumentId == null) { + String string = getMin(Constants.DOCUMENTID); + if (string == null) + return null; + minDocumentId = Util.longFromEncoded(string); + } + return minDocumentId; + } + + public Long getMinSnapshotId() throws IOException { + if (minSnapshotId == null) { + String string = getMin(Constants.SNAPSHOTID); + if (string == null) + return null; + minSnapshotId = Util.longFromEncoded(string); + } + return minSnapshotId; + } + + public Long getMaxSnapshotId() throws IOException { + if (maxSnapshotId == null) { + String string = getMax(Constants.SNAPSHOTID); + if (string == null) + return null; + maxSnapshotId = Util.longFromEncoded(string); + } + return maxSnapshotId; + } + + public Long getMaxDocumentId() throws IOException { + if (maxDocumentId == null) { + String string = getMax(Constants.DOCUMENTID); + if (string == null) + return null; + maxDocumentId = Util.longFromEncoded(string); + } + return maxDocumentId; + } + + //public MemoryIndexSnapshot newIndexSnapshot(Long id) throws IOException { + // return newIndexSnapshot(id, (InstantiatedIndex) null); + //} + + //public MemoryIndexSnapshot newIndexSnapshot(Long id, IndexReader indexReader) throws IOException { + // return newIndexSnapshot(id, new InstantiatedIndex(indexReader)); + //} + + //public MemoryIndexSnapshot newIndexSnapshot(Long id, InstantiatedIndex instantiatedIndex) throws IOException { + //List newReaders = new ArrayList(indexReaders.size() + 1); + //for (OceanInstantiatedIndexReader reader : indexReaders) { + // newReaders.add((OceanInstantiatedIndexReader) reader.clone()); + //} + //if (instantiatedIndex != null) + // newReaders.add(new OceanInstantiatedIndexReader(instantiatedIndex)); + //MemoryIndexSnapshot newMemoryIndexSnapshot = new MemoryIndexSnapshot(id); + //for (OceanInstantiatedIndexReader reader : indexReaders) { + // if (reader.maxDoc() > 0) newMemoryIndexSnapshot.add((OceanInstantiatedIndexReader) reader.clone()); + //} + // newMemoryIndexSnapshot.add(new OceanInstantiatedIndexReader(instantiatedIndex)); + // WriteableMemoryIndex.this.snapshotMap.put(id, newMemoryIndexSnapshot); + // return newMemoryIndexSnapshot; + //} + + // void add(IndexReader indexReader) throws IOException { + // InstantiatedIndex newIndex = new InstantiatedIndex(indexReader); + // add(newIndex); + // } + + // private void add(InstantiatedIndex newIndex) { + // OceanInstantiatedIndexReader newReader = new + // OceanInstantiatedIndexReader(newIndex); + // indexReaders.add(newReader); + // /} + + public int deletedDoc() { + if (deletedDoc == null) { + deletedDoc = getIndexReader().maxDoc() - getIndexReader().numDocs(); + } + return deletedDoc; + } + + public int maxDoc() { + return getIndexReader().maxDoc(); + } + + public IndexReader getIndexReader() { + if (indexReader == null) { + List list = new ArrayList(subReaders.length); + for (OceanInstantiatedIndexReader reader : subReaders) { + if (reader.maxDoc() > 0) + list.add(reader); + } + indexReader = new MultiReader((IndexReader[])list.toArray(new IndexReader[0])); + } + return indexReader; + } + } + + public MemoryIndexSnapshot getLatestIndexSnapshot() { + return snapshotMap.lastValue(); + } + + public boolean rollback(Long snapshotId) { + return snapshotMap.remove(snapshotId) != null; + } +} Index: ocean/src/org/apache/lucene/search/OceanMultiThreadSearcher.java =================================================================== --- ocean/src/org/apache/lucene/search/OceanMultiThreadSearcher.java (revision 0) +++ ocean/src/org/apache/lucene/search/OceanMultiThreadSearcher.java (revision 0) @@ -0,0 +1,254 @@ +package org.apache.lucene.search; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.locks.ReentrantLock; + +import org.apache.lucene.ocean.OceanSearcher; +import org.apache.lucene.ocean.Snapshot; +import org.apache.lucene.ocean.util.Util; +import org.apache.lucene.util.PriorityQueue; + +public class OceanMultiThreadSearcher extends OceanSearcher { + private Snapshot snapshot; + private ExecutorService searchThreadPool; + private Searchable[] searchables; + private int[] starts; + + public OceanMultiThreadSearcher(Snapshot snapshot, ExecutorService searchThreadPool) throws IOException { + super(snapshot); + searchables = snapshot.getSearchables(); + starts = snapshot.getStarts(); + this.snapshot = snapshot; + this.searchThreadPool = searchThreadPool; + } + + public TopDocs search(Weight weight, Filter filter, int nDocs) throws IOException { + HitQueue hq = new HitQueue(nDocs); + int totalHits = 0; + MultiSearcherThread[] msta = new MultiSearcherThread[searchables.length]; + ReentrantLock lock = new ReentrantLock(); + List futures = new ArrayList(searchables.length); + for (int i = 0; i < searchables.length; i++) { // search each searcher + msta[i] = new MultiSearcherThread(searchables[i], weight, filter, nDocs, hq, i, starts, lock); + futures.add(searchThreadPool.submit(msta[i])); + } + processFutures(futures); + for (int i = 0; i < searchables.length; i++) { + IOException ioe = msta[i].getIOException(); + if (ioe == null) { + totalHits += msta[i].hits(); + } else { + // if one search produced an IOException, rethrow it + throw ioe; + } + } + ScoreDoc[] scoreDocs = new ScoreDoc[hq.size()]; + for (int i = hq.size() - 1; i >= 0; i--) + // put docs in array + scoreDocs[i] = (ScoreDoc) hq.pop(); + float maxScore = (totalHits == 0) ? Float.NEGATIVE_INFINITY : scoreDocs[0].score; + return new TopDocs(totalHits, scoreDocs, maxScore); + } + + private void processFutures(List futures) throws IOException { + for (Future future : futures) { + try { + future.get(); + } catch (ExecutionException executionException) { + if (executionException.getCause() instanceof IOException) { + throw (IOException)executionException.getCause(); + } + throw Util.asIOException(executionException.getCause()); + } catch (InterruptedException interruptedException) { + throw Util.asIOException(interruptedException); + } + } + } + + /** + * A search implementation allowing sorting which spans a new thread for each + * Searchable, waits for each search to complete and merges the results back + * together. + */ + public TopFieldDocs search(Weight weight, Filter filter, int nDocs, Sort sort) throws IOException { + // don't specify the fields - we'll wait to do this until we get results + FieldDocSortedHitQueue hq = new FieldDocSortedHitQueue(null, nDocs); + int totalHits = 0; + MultiSearcherThread[] msta = new MultiSearcherThread[searchables.length]; + ReentrantLock lock = new ReentrantLock(); + List futures = new ArrayList(searchables.length); + for (int i = 0; i < searchables.length; i++) { // search each searcher + // Assume not too many searchables and cost of creating a thread is by far + // inferior to a search + msta[i] = new MultiSearcherThread(searchables[i], weight, filter, nDocs, hq, sort, i, starts, lock); + futures.add(searchThreadPool.submit(msta[i])); + } + processFutures(futures); + float maxScore = Float.NEGATIVE_INFINITY; + for (int i = 0; i < searchables.length; i++) { + IOException ioe = msta[i].getIOException(); + if (ioe == null) { + totalHits += msta[i].hits(); + maxScore = Math.max(maxScore, msta[i].getMaxScore()); + } else { + // if one search produced an IOException, rethrow it + throw ioe; + } + } + ScoreDoc[] scoreDocs = new ScoreDoc[hq.size()]; + for (int i = hq.size() - 1; i >= 0; i--) + // put docs in array + scoreDocs[i] = (ScoreDoc) hq.pop(); + + return new TopFieldDocs(totalHits, scoreDocs, hq.getFields(), maxScore); + } + + class MultiSearcherThread implements Callable { + private Searchable searchable; + private Weight weight; + private Filter filter; + private int nDocs; + private TopDocs docs; + private int i; + private PriorityQueue hq; + private int[] starts; + private IOException ioe; + private Sort sort; + private ReentrantLock lock; + + public MultiSearcherThread(Searchable searchable, Weight weight, Filter filter, int nDocs, HitQueue hq, int i, int[] starts, ReentrantLock lock) { + this.searchable = searchable; + this.weight = weight; + this.filter = filter; + this.nDocs = nDocs; + this.hq = hq; + this.i = i; + this.starts = starts; + this.lock = lock; + } + + public MultiSearcherThread(Searchable searchable, Weight weight, Filter filter, int nDocs, FieldDocSortedHitQueue hq, Sort sort, int i, + int[] starts, ReentrantLock lock) { + this.searchable = searchable; + this.weight = weight; + this.filter = filter; + this.nDocs = nDocs; + this.hq = hq; + this.i = i; + this.starts = starts; + this.sort = sort; + this.lock = lock; + } + + public Object call() throws Exception { + try { + docs = (sort == null) ? searchable.search(weight, filter, nDocs) : searchable.search(weight, filter, nDocs, sort); + } + // Store the IOException for later use by the caller of this thread + catch (IOException ioe) { + this.ioe = ioe; + } + if (ioe == null) { + // if we are sorting by fields, we need to tell the field sorted hit + // queue + // the actual type of fields, in case the original list contained AUTO. + // if the searchable returns null for fields, we'll have problems. + if (sort != null) { + ((FieldDocSortedHitQueue) hq).setFields(((TopFieldDocs) docs).fields); + } + ScoreDoc[] scoreDocs = docs.scoreDocs; + for (int j = 0; j < scoreDocs.length; j++) { // merge scoreDocs into hq + ScoreDoc scoreDoc = scoreDocs[j]; + scoreDoc.doc += starts[i]; // convert doc + // it would be so nice if we had a thread-safe insert + lock.lock(); + try { + if (!hq.insert(scoreDoc)) + break; + } finally { + lock.unlock();// no more scores > minScore + } + } + } + return null; + } + + public int hits() { + return docs.totalHits; + } + + public float getMaxScore() { + return docs.getMaxScore(); + } + + public IOException getIOException() { + return ioe; + } + } + + /** + * Lower-level search API. + * + *

+ * {@link HitCollector#collect(int,float)} is called for every non-zero + * scoring document. + * + *

+ * Applications should only use this if they need all of the matching + * documents. The high-level search API ({@link Searcher#search(Query)}) is + * usually more efficient, as it skips non-high-scoring hits. + * + * @param weight + * to match documents + * @param filter + * if non-null, a bitset used to eliminate some documents + * @param results + * to receive hits + * + */ + public void search(Weight weight, Filter filter, final HitCollector results) throws IOException { + List futures = new ArrayList(searchables.length); + for (int i = 0; i < searchables.length; i++) { + final int start = starts[i]; + HitCollectorThread hitCollectorThread = new HitCollectorThread(start, searchables[i], weight, filter, results); + futures.add(searchThreadPool.submit(hitCollectorThread)); + //searchables[i].search(weight, filter, new HitCollector() { + // public void collect(int doc, float score) { + // results.collect(doc + start, score); + // } + //}); + } + processFutures(futures); + } + + private class HitCollectorThread implements Callable { + private Searchable searchable; + private Weight weight; + private Filter filter; + private HitCollector hitCollector; + private int start; + + public HitCollectorThread(int start, Searchable searchable, Weight weight, Filter filter, HitCollector hitCollector) { + this.start = start; + this.searchable = searchable; + this.weight = weight; + this.filter = filter; + this.hitCollector = hitCollector; + } + + public Object call() throws Exception { + searchable.search(weight, filter, new HitCollector() { + public void collect(int doc, float score) { + hitCollector.collect(doc + start, score); + } + }); + return null; + } + } +} Index: ocean/test/FileReadTest.java =================================================================== --- ocean/test/FileReadTest.java (revision 0) +++ ocean/test/FileReadTest.java (revision 0) @@ -0,0 +1,317 @@ +import java.io.RandomAccessFile; +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.nio.channels.Channel; +import java.nio.channels.WritableByteChannel; +import java.nio.ByteBuffer; +import java.lang.reflect.Constructor; +import java.util.Random; +import java.util.LinkedList; +import java.util.ArrayList; + +/** + * @author yonik + * @version $Id$ + */ +public class FileReadTest { + + static int poolsize=2; + + public static void main(String[] args) throws Exception { + int argpos=0; + final String filename = args[argpos++]; + final String implementation = args[argpos++]; + final boolean serial = args[argpos++].toLowerCase().equals("true"); + final int nThreads = Integer.parseInt(args[argpos++]); + final int iterations = Integer.parseInt(args[argpos++]); + final int bufsize=argpos=nPages) pg-=nPages; + long pos = (serial ? pg : rn) * bufsize; + try { + val += reader.read(pos); + } catch (IOException e) { + e.printStackTrace(); + throw new RuntimeException(e); + } + } + } + synchronized (answer) { answer[0]+=val; } + times[seed] = System.currentTimeMillis() - start; + } + }; + } + + for (int i=0;i>1]; + } + }; + } +} + + +class PooledPread extends MyFile { + FileChannel[] all; + int[] useCount; + public PooledPread(RandomAccessFile f, String filename) throws Exception { + super(f); + all = new FileChannel[FileReadTest.poolsize]; + all[0]=f.getChannel(); + for (int i=1; i 1) { + synchronized(PooledPread.this) { + int minCount = useCount[0]; + for (int i=1; i 1) { + synchronized(PooledPread.this) { + useCount[idx]--; + } + } + + return b[0]+b[len>>1]; + } + }; + } +} + + +class ChannelFile extends MyFile { + final FileChannel channel; + public ChannelFile(RandomAccessFile f, String filename) { + super(f); + channel = f.getChannel(); + } + + MyReader getReader() { + return new MyReader() { + byte[] b = new byte[BUFFER_SIZE]; + ByteBuffer bb = ByteBuffer.wrap(b); + + public int read(long pos) throws IOException { + bb.clear(); + int len; + synchronized(channel) { + if (filepos != pos) channel.position(pos); + len = channel.read(bb); + filepos = pos + len; + } + return b[0]+b[len>>1]; + } + }; + } +} + + +class ChannelPread extends MyFile { + final FileChannel channel; + public ChannelPread(RandomAccessFile f, String filename) { + super(f); + channel = f.getChannel(); + } + + MyReader getReader() { + return new MyReader() { + byte[] b = new byte[BUFFER_SIZE]; + ByteBuffer bb = ByteBuffer.wrap(b); + + public int read(long pos) throws IOException { + bb.clear(); + int len = channel.read(bb, pos); + return b[0]+b[len>>1]; + } + }; + } +} + + +class ChannelPreadDirect extends MyFile { + final FileChannel channel; + public ChannelPreadDirect(RandomAccessFile f, String filename) { + super(f); + channel = f.getChannel(); + } + + MyReader getReader() { + return new MyReader() { + ByteBuffer bb = ByteBuffer.allocateDirect(BUFFER_SIZE); + + public int read(long pos) throws IOException { + bb.clear(); + int len = channel.read(bb, pos); + return bb.get(0)+bb.get(len>>1); + } + }; + } +} + + +class ChannelTransfer extends MyFile { + final FileChannel channel; + public ChannelTransfer(RandomAccessFile f, String filename) { + super(f); + channel = f.getChannel(); + } + + MyReader getReader() { + return new MyReader() { + final byte[] b = new byte[BUFFER_SIZE]; + int posInBuffer=0; + final ByteBuffer bb = ByteBuffer.wrap(b); + + final WritableByteChannel sink = new WritableByteChannel() { + public int write(ByteBuffer src) throws IOException { + int remaining = src.remaining(); + src.get(b,posInBuffer,remaining); + // may be called multiple times, so we need to keep track + posInBuffer += remaining; + return remaining; + } + public boolean isOpen() {return true;} + public void close() throws IOException {} + }; + + public int read(long pos) throws IOException { + posInBuffer=0; + int len = (int)channel.transferTo(pos, BUFFER_SIZE, sink); + return b[0]+b[len>>1]; + } + }; + } +} + Index: ocean/test/org/apache/lucene/ocean/log/TestRawLogFile.java =================================================================== --- ocean/test/org/apache/lucene/ocean/log/TestRawLogFile.java (revision 0) +++ ocean/test/org/apache/lucene/ocean/log/TestRawLogFile.java (revision 0) @@ -0,0 +1,46 @@ +package org.apache.lucene.ocean.log; + +import java.io.File; +import java.util.List; + +import junit.framework.TestSuite; +import junit.textui.TestRunner; + +import org.apache.commons.io.FileUtils; +import org.apache.lucene.ocean.FSLogDirectory; +import org.apache.lucene.ocean.util.Bytes; +import org.apache.lucene.util.LuceneTestCase; + +public class TestRawLogFile extends LuceneTestCase { + public static void main(String args[]) { + TestRunner.run(new TestSuite(TestRawLogFile.class)); + } + + public void testMain() throws Exception { + File file = new File("g:\\testrawlogfile"); + FileUtils.cleanDirectory(file); + FSLogDirectory logDirectory = new FSLogDirectory(file); + RawLogFile rawLogFile = new RawLogFile("test", logDirectory); + Bytes docBytes = new Bytes(1024); + Bytes otherBytes = new Bytes(1024); + for (int x=0; x < 10; x++) { + docBytes.reset(); + otherBytes.reset(); + docBytes.getOutputStream().write((x+" docs").getBytes()); + otherBytes.getOutputStream().write((x+" other").getBytes()); + rawLogFile.write((long)x, 1, docBytes, 2, otherBytes); + //rawLogFile.write((long)x, 1, , 2, (x+" other").getBytes()); + } + List headers = rawLogFile.loadRecordHeaders(); + assertEquals(10, headers.size()); + for (int x=0; x < headers.size(); x++) { + RecordHeader header = headers.get(x); + assertEquals(x, header.id); + } + rawLogFile.close(); + } + + private static byte[] toBytes(String string) throws Exception { + return string.getBytes("UTF-8"); + } +} Index: ocean/test/org/apache/lucene/ocean/snapshotlog/TestSnapshotLog.java =================================================================== --- ocean/test/org/apache/lucene/ocean/snapshotlog/TestSnapshotLog.java (revision 0) +++ ocean/test/org/apache/lucene/ocean/snapshotlog/TestSnapshotLog.java (revision 0) @@ -0,0 +1,93 @@ +package org.apache.lucene.ocean.snapshotlog; + +import java.io.File; +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +import junit.framework.TestSuite; +import junit.textui.TestRunner; + +import org.apache.lucene.ocean.FSLogDirectory; +import org.apache.lucene.ocean.IndexID; +import org.apache.lucene.ocean.SnapshotInfo; +import org.apache.lucene.ocean.SnapshotInfo.IndexInfo; +import org.apache.lucene.ocean.snapshotlog.SnapshotLogFile.InfoHeader; +import org.apache.lucene.ocean.util.XMLUtil; +import org.apache.lucene.util.LuceneTestCase; +import org.jdom.Element; + +public class TestSnapshotLog extends LuceneTestCase { + Random random = new Random(System.currentTimeMillis()); + + public static void main(String args[]) { + TestRunner.run(new TestSuite(TestSnapshotLog.class)); + } + + private long nextLong() { + return (long) random.nextInt(1000); + } + + private int nextInt() { + return random.nextInt(1000); + } + + public void testSnapshotLogManager() throws Exception { + FSLogDirectory logDirectory = new FSLogDirectory(new File("g:\\testsnapshotlogmanager")); + SnapshotLogManager snapshotLogManager = new SnapshotLogManager(25, 10, logDirectory); + for (int x = 0; x < 40; x++) { + SnapshotInfo snapshotInfo = createSnapshotInfo(x); + Element element = snapshotInfo.toElement(); + String xml = XMLUtil.outputElementOmitDeclaration(element); + snapshotLogManager.write(xml); + } + System.out.println("numLogFiles: "+snapshotLogManager.numLogFiles()); + } + + private SnapshotInfo createSnapshotInfo(int x) { + SnapshotInfo snapshotInfo = new SnapshotInfo(new BigDecimal(x+".02"), 25, 10, 15, System.currentTimeMillis()); + //snapshotInfo.add(new IndexInfo(nextLong(), nextLong(), nextLong(), "disk", nextInt(), nextInt(), nextInt(), nextLong(), nextLong(), + // nextLong(), nextLong(), nextLong(), nextLong())); + IndexInfo indexInfo = new IndexInfo(); + indexInfo.snapshotId = nextLong(); + indexInfo.id = nextLong(); + indexInfo.segmentGeneration = nextLong(); + indexInfo.type = "disk"; + indexInfo.maxDoc = nextInt(); + indexInfo.numDocs = nextInt(); + indexInfo.deletedDoc = nextInt(); + indexInfo.minDocumentId = nextLong(); + indexInfo.maxDocumentId = nextLong(); + indexInfo.minSnapshotId = nextLong(); + indexInfo.maxSnapshotId = nextLong(); + indexInfo.checkpointId = nextLong(); + indexInfo.lastAppliedId = nextLong(); + List mergedFrom = new ArrayList(); + mergedFrom.add(new IndexID(nextLong(), "disk")); + mergedFrom.add(new IndexID(nextLong(), "disk")); + mergedFrom.add(new IndexID(nextLong(), "disk")); + indexInfo.mergedFrom = mergedFrom; + return snapshotInfo; + } + + public void testSnapshotLogFile() throws Exception { + FSLogDirectory logDirectory = new FSLogDirectory(new File("g:\\testsnapshotlogfile")); + SnapshotLogFile snapshotLogFile = new SnapshotLogFile(1l, "snapshot0000001.log", logDirectory, 20); + snapshotLogFile.loadLastInfos(20); + int size = snapshotLogFile.getSize(); + System.out.println("size: "+size); + for (int x = 0; x < 20; x++) { + SnapshotInfo snapshotInfo = createSnapshotInfo(x); + Element element = snapshotInfo.toElement(); + String xml = XMLUtil.outputElementOmitDeclaration(element); + snapshotLogFile.write(xml); + } + List infos = snapshotLogFile.getLastInfos(5); + for (InfoHeader info : infos) { + String xml = snapshotLogFile.loadData(info); + System.out.print(xml); + } + snapshotLogFile.close(); + } +} Index: ocean/test/org/apache/lucene/ocean/TestByteBufferPool.java =================================================================== --- ocean/test/org/apache/lucene/ocean/TestByteBufferPool.java (revision 0) +++ ocean/test/org/apache/lucene/ocean/TestByteBufferPool.java (revision 0) @@ -0,0 +1,11 @@ +package org.apache.lucene.ocean; + +/** + * Write to pool, read from pool make sure results match. + * + */ +public class TestByteBufferPool { + public void testByteBufferPool() { + + } +} Index: ocean/test/org/apache/lucene/ocean/TestIndexCreator.java =================================================================== --- ocean/test/org/apache/lucene/ocean/TestIndexCreator.java (revision 0) +++ ocean/test/org/apache/lucene/ocean/TestIndexCreator.java (revision 0) @@ -0,0 +1,73 @@ +package org.apache.lucene.ocean; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import junit.framework.TestSuite; +import junit.textui.TestRunner; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.WhitespaceAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.util.LuceneTestCase; + +public class TestIndexCreator extends LuceneTestCase { + + /** Main for running test case by itself. */ + public static void main(String args[]) { + TestRunner.run(new TestSuite(TestIndexCreator.class)); + } + + public void testMain() throws Exception { + Directory directory = new RAMDirectory(); + Analyzer analyzer = new WhitespaceAnalyzer(); + ExecutorService threadPool = Executors.newFixedThreadPool(8); + IndexCreator indexCreator = new IndexCreator("test", directory, 1 * 1024 * 1024, 4, analyzer, threadPool); + BlockingQueue queue = new ArrayBlockingQueue(4000, true); + indexCreator.start(queue); // start threads consuming + String[] docs = { "a b c d e", "a b c d e a b c d e", "a b c d e f g h i j", "a c e", "e c a", "a c e a c e", "a c e a b c" }; + for (int x = 0; x < 10; x++) { + for (int j = 0; j < docs.length; j++) { + Document d = new Document(); + d.add(new Field("contents", x+" "+docs[j], Field.Store.YES, Field.Index.TOKENIZED)); + queue.add(new IndexCreator.Add(d)); + } + } + while ( queue.peek() != null) { + Thread.sleep(5); + } + System.out.println("before create"); + indexCreator.create(false); + System.out.println("after create"); + IndexReader indexReader = IndexReader.open(directory); + IndexSearcher searcher = new IndexSearcher(indexReader); + Query query = new MatchAllDocsQuery(); + ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs; + + StringWriter sw = new StringWriter(); + PrintWriter out = new PrintWriter(sw, true); + out.println(hits.length + " total results"); + for (int i = 0; i < hits.length; i++) { + Document d = searcher.doc(hits[i].doc); + out.println(i + " " + hits[i].score + // + " " + DateField.stringToDate(d.get("modified")) + + " " + d.get("contents")); + } + sw.close(); + out.close(); + String singleFileOutput = sw.getBuffer().toString(); + System.out.println(singleFileOutput); + } +} Index: ocean/test/org/apache/lucene/ocean/TestLogRecords.java =================================================================== --- ocean/test/org/apache/lucene/ocean/TestLogRecords.java (revision 0) +++ ocean/test/org/apache/lucene/ocean/TestLogRecords.java (revision 0) @@ -0,0 +1,71 @@ +package org.apache.lucene.ocean; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; + +import junit.framework.TestSuite; +import junit.textui.TestRunner; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.apache.lucene.ocean.log.LogFile; +import org.apache.lucene.ocean.log.RecordData; +import org.apache.lucene.ocean.log.LogFile.Record; +import org.apache.lucene.ocean.log.LogFile.RecordIterator; +import org.apache.lucene.ocean.util.Bytes; +import org.apache.lucene.util.LuceneTestCase; + +public class TestLogRecords extends LuceneTestCase { + public static void main(String args[]) { + TestRunner.run(new TestSuite(TestLogRecords.class)); + } + + public void testLogRecords() throws IOException { + List docList = new ArrayList(); + List otherList = new ArrayList(); + + File file = new File("g:\\testocean"); + FileUtils.cleanDirectory(file); + + FSDirectoryMap directoryMap = new FSDirectoryMap(file, "log"); + LogFile logFile = new LogFile(1l, "test", directoryMap.getLogDirectory()); + Bytes docBytes = new Bytes(5); + Bytes otherBytes = new Bytes(5); + for (int x=0; x < 5; x++) { + docBytes.reset(); + otherBytes.reset(); + String docStr = "docs "+x+" sdfadfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfsdafsdfsdaf"; + docList.add(docStr); + docBytes.getOutputStream().write(docStr.getBytes("UTF-8")); + //System.out.println("docs.length: "+docBytes.length); + String otherStr = "other "+x+" sdfadfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfsdafsdfsdaf"; + otherBytes.getOutputStream().write(otherStr.getBytes("UTF-8")); + otherList.add(otherStr); + RecordData recordData = new RecordData(1, docBytes, 2, otherBytes); + logFile.writeRecord(new Long(x), recordData); + } + System.out.println("min: "+logFile.getMinId()); + System.out.println("max: "+logFile.getMaxId()); + logFile.close(); + logFile = new LogFile(1l, "test", directoryMap.getLogDirectory()); + + // TODO: add test for match + RecordIterator iterator = logFile.getRecordIterator(null); + while (iterator.hasNext()) { + Record record = iterator.next(); + + System.out.println(record.getRecordHeader().toString()); + + InputStream docsInput = record.getStreamRecord().getDocuments().getInputStream(); + String docs = IOUtils.toString(docsInput, "UTF-8"); + InputStream otherInput = record.getStreamRecord().getOther().getInputStream(); + String other = IOUtils.toString(otherInput, "UTF-8"); + System.out.println("docs: "+docs); + System.out.println("other: "+other); + } + iterator.close(); + } +} Index: ocean/test/org/apache/lucene/ocean/TestSearch.java =================================================================== --- ocean/test/org/apache/lucene/ocean/TestSearch.java (revision 0) +++ ocean/test/org/apache/lucene/ocean/TestSearch.java (revision 0) @@ -0,0 +1,246 @@ +package org.apache.lucene.ocean; + +import java.io.File; +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringReader; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.GregorianCalendar; +import java.util.HashSet; +import java.util.List; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.atomic.AtomicLong; + +import junit.framework.TestSuite; +import junit.textui.TestRunner; + +import org.apache.commons.lang.StringUtils; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.SimpleAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.Term; +import org.apache.lucene.ocean.log.TransactionLog; +import org.apache.lucene.ocean.util.Constants; +import org.apache.lucene.ocean.util.Util; +import org.apache.lucene.queryParser.QueryParser; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.util.LuceneTestCase; + +public class TestSearch extends LuceneTestCase { + List deletedDocIds = new ArrayList(); + static Random random = new Random(System.currentTimeMillis()); + int docsAdded = 0; + static String directoryString; + + static { + directoryString = System.getProperty("ocean.test.dir"); + if (StringUtils.isBlank(directoryString)) { + directoryString = System.getProperty("java.io.tmpdir", "tmp"); + } + System.out.println("directory: "+directoryString); + } + + /** Main for running test case by itself. */ + public static void main(String args[]) { + TestRunner.run(new TestSuite(TestSearch.class)); + } + + public static TransactionSystem setupTransactionSystem() throws Exception { + FSDirectoryMap directoryMap = new FSDirectoryMap(new File(directoryString), "log"); + // RAMDirectoryMap directoryMap = new RAMDirectoryMap(); + LogDirectory logDirectory = directoryMap.getLogDirectory(); + TransactionLog transactionLog = new TransactionLog(1024*10, logDirectory); + System.out.println("transactionLog num: " + transactionLog.getNumRecords() + " min: " + transactionLog.getMinId() + " max: " + + transactionLog.getMaxId()); + //SearcherPolicy searcherPolicy = new MultiThreadSearcherPolicy(2, 4, 25); + SearcherPolicy searcherPolicy = new SingleThreadSearcherPolicy(); + return new TransactionSystem(transactionLog, new SimpleAnalyzer(), directoryMap, 20, 5, 10, 0.3f, searcherPolicy); + } + + public void testSearch() throws Exception { + TransactionSystem system = setupTransactionSystem(); + + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw, true); + doTestSearch(pw, true, 2, system); + pw.close(); + sw.close(); + String multiFileOutput = sw.getBuffer().toString(); + + String singleFileOutput = sw.getBuffer().toString(); + System.out.println(multiFileOutput); + + doDeleteRandomDocuments(system, deletedDocIds); + verifyDocumentsDeleted(system); + + //verifyNumDocs(system); + } + + private void verifyNumDocs(TransactionSystem system) throws IOException { + OceanSearcher searcher = system.getSearcher(); + int docs = docsAdded - deletedDocIds.size(); + int numDocs = searcher.getSnapshot().numDocs(); + System.out.println("numDocs: "+numDocs+" should be docs: "+docs); + assertEquals(docs, numDocs); + searcher.close(); + } + + private void verifyDocumentsDeleted(TransactionSystem system) throws Exception { + OceanSearcher searcher = system.getSearcher(); + if (searcher.getSnapshot().getId().compareTo(system.getSnapshots().getLatestSnapshot().getId()) != 0) { + System.out.println("snapshot ids don't match"); + } + for (Long docId : deletedDocIds) { + String encodedId = Util.longToEncoded(docId); + Term term = new Term(Constants.DOCUMENTID, encodedId); + int totalHits = searcher.search(new TermQuery(term), null, 1000).totalHits; + if (totalHits > 0) { + IndexID indexId = system.getIndexId(docId); + System.out.println("documentId: "+docId+" from index: "+indexId+" not deleted"); + } + assertEquals(totalHits, 0); + } + searcher.close(); + } + + public static void doDeleteRandomDocuments(TransactionSystem system, List deletedDocIds) throws Exception { + OceanSearcher searcher = system.getSearcher(); + ScoreDoc[] hits = searcher.search(new MatchAllDocsQuery(), null, 5000).scoreDocs; + int count = 0; + for (ScoreDoc scoreDoc : hits) { + Document document = searcher.doc(scoreDoc.doc); + String docEncoded = document.get(Constants.DOCUMENTID); + Long documentId = Util.longFromEncoded(docEncoded); + if (random.nextBoolean() || count == 0) { + System.out.println("deleting: "+documentId); + CommitResult result = system.deleteDocument(new Term(Constants.DOCUMENTID, docEncoded)); + int numDeleted = result.getNumDeleted(); + if (numDeleted != 1) { + IndexID indexId = system.getIndexId(documentId); + String docIndexId = document.get(Constants.INDEXID); + System.out.println("docIndexId: "+docIndexId+" documentId: "+documentId+" from index: "+indexId+" not deleted maxDoc: "+searcher.maxDoc()); + printDocs(new Term(Constants.DOCUMENTID, docEncoded), system); + } + assertEquals(1, numDeleted); + deletedDocIds.add(documentId); + } + count++; + } + System.out.println("deletedDocIds: "+deletedDocIds); + searcher.close(); + } + + public static void printDocs(Term term, TransactionSystem system) throws IOException { + OceanSearcher searcher = system.getSearcher(); + ScoreDoc[] hits = searcher.search(new TermQuery(term), null, 5000).scoreDocs; + System.out.println("printDocs hits.length: "+hits.length); + int count = 0; + for (ScoreDoc scoreDoc : hits) { + Document document = searcher.doc(scoreDoc.doc); + System.out.println(document.toString()); + } + searcher.close(); + } + + public static Set createDocs(int numTransactions, int numDocs, AtomicLong idSequence, boolean useStringReader, TransactionSystem system) throws Exception { + Set ids = new HashSet(); + for (int x=0; x < numTransactions; x++) { + Set cids = createDocs(numDocs, idSequence, useStringReader, system); + ids.addAll(cids); + } + return ids; + } + + public static Set createDocs(int numDocs, AtomicLong idSequence, boolean useStringReader, TransactionSystem system) throws Exception { + String[] docs = { "a b c d e", "a b c d e a b c d e", "a b c d e f g h i j", "a c e", "e c a", "a c e a c e", "a c e a b c" }; + Set ids = new HashSet(); + List documents = new ArrayList(); + //for (int x = 0; x < rounds; x++) { + boolean run = true; + while (run) { + for (int j = 0; j < docs.length; j++) { + if (ids.size() >= numDocs) { + run = false; + break; + } + Document d = new Document(); + long id = idSequence.getAndIncrement(); + ids.add(id); + d.add(new Field("id", Long.toString(id), Field.Store.YES, Field.Index.UN_TOKENIZED)); + if (useStringReader) + d.add(new Field("contents", new StringReader(docs[j]))); + else + d.add(new Field("contents", docs[j], Field.Store.YES, Field.Index.TOKENIZED)); + d.add(new Field("value", Integer.toString(random.nextInt()), Field.Store.YES, Field.Index.UN_TOKENIZED)); + documents.add(d); + //docsAdded.incrementAndGet(); + } + } + system.commitTransaction(documents, null, null, null); + return ids; + //CommitResult commitResult = system.addDocument(d); + //documentIds.addAll(commitResult.getDocumentIds()); + //return documentIds; + } + + private void doTestSearch(PrintWriter out, boolean addDocs, int rounds, TransactionSystem system) throws Exception { + Analyzer analyzer = new SimpleAnalyzer(); + + if (addDocs) { + String[] docs = { "a b c d e", "a b c d e a b c d e", "a b c d e f g h i j", "a c e", "e c a", "a c e a c e", "a c e a b c" }; + for (int x = 0; x < rounds; x++) { + for (int j = 0; j < docs.length; j++) { + Document d = new Document(); + //d.add(new Field("contents", docs[j], Field.Store.YES, Field.Index.TOKENIZED)); + d.add(new Field("contents", new StringReader(docs[j]))); + system.addDocument(d); + docsAdded++; + } + } + } + + OceanSearcher searcher = system.getSearcher(); + + out.println("snapshot id: " + Util.formatSnapshotId(searcher.getSnapshot().getId())); + + String[] queries = { "a b", "\"a b\"", "\"a b c\"", "a c", "\"a c\"", "\"a c e\"", "*:*" }; + ScoreDoc[] hits = null; + + QueryParser parser = new QueryParser("contents", analyzer); + parser.setPhraseSlop(4); + for (int j = 0; j < queries.length; j++) { + Query query = parser.parse(queries[j]); + out.println("Query: " + query.toString("contents")); + + // DateFilter filter = + // new DateFilter("modified", Time(1997,0,1), Time(1998,0,1)); + // DateFilter filter = DateFilter.Before("modified", Time(1997,00,01)); + // System.out.println(filter); + + hits = searcher.search(query, null, 1000).scoreDocs; + + out.println(hits.length + " total results"); + for (int i = 0; i < hits.length && i < 10; i++) { + Document d = searcher.doc(hits[i].doc); + String docEncoded = d.get(Constants.DOCUMENTID); + out.println(i + " " + hits[i].score + // + " " + DateField.stringToDate(d.get("modified")) + + " " + d.get("contents") + " " + Util.longFromEncoded(docEncoded)); + } + } + searcher.close(); + //system.close(); + } + + static long Time(int year, int month, int day) { + GregorianCalendar calendar = new GregorianCalendar(); + calendar.set(year, month, day); + return calendar.getTime().getTime(); + } +} Index: ocean/test/org/apache/lucene/ocean/TestSystem.java =================================================================== --- ocean/test/org/apache/lucene/ocean/TestSystem.java (revision 0) +++ ocean/test/org/apache/lucene/ocean/TestSystem.java (revision 0) @@ -0,0 +1,177 @@ +package org.apache.lucene.ocean; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +import junit.framework.TestSuite; +import junit.textui.TestRunner; + +import org.apache.commons.lang.StringUtils; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.Term; +import org.apache.lucene.ocean.util.Constants; +import org.apache.lucene.ocean.util.Util; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.util.LuceneTestCase; + +public class TestSystem extends LuceneTestCase { + Random random = new Random(System.currentTimeMillis()); + + public static class Update { + public String field; + public String value; + public Long id; + + public Update(String field, String value, Long id) { + this.field = field; + this.value = value; + this.id = id; + } + } + + public void testSystem() throws Exception { + AtomicInteger docsAdded = new AtomicInteger(0); + AtomicInteger docsDeleted = new AtomicInteger(0); + + List deletedDocIds = new ArrayList(); + List updates = new ArrayList(); + AtomicLong idSequence = new AtomicLong(1); + + TransactionSystem system = TestSearch.setupTransactionSystem(); + Set ids = new HashSet(); + // TestSearch.createDocs(5, docsAdded, system); + // do 50 transactions creating 1 document + ids.addAll(TestSearch.createDocs(50, 1, idSequence, false, system)); + // do 1 transaction creating 50 documents + // this should cause the system to save the documents + // to the transaction log as an encoded ramdirectory index instead of + // serialized objects + ids.addAll(TestSearch.createDocs(1, 100, idSequence, false, system)); + OceanSearcher searcher = system.getSearcher(); + Snapshot snapshot = searcher.getSnapshot(); + System.out.println("searcher snapshot id: " + snapshot.getIdString() + " snapshot.getMaxDoc: " + snapshot.getMaxDoc()); + assertEquals(150, searcher.maxDoc()); + + searcher.verifyDocsIdsUnique(); + + docsAdded.set(ids.size()); + + updateDocuments(20, ids, updates, system); + verifyUpdatedDocuments(updates, system); + + snapshot = system.getSnapshots().getLatestSnapshot(); + try { + assertEquals(snapshot.getIndexReader().numDocs(), ids.size()); + } finally { + snapshot.decRef(); + } + + Set deleted = new HashSet(); + deleteRandomDocuments(docsDeleted, deleted, ids, system); + system.merge(true); + verifyDocsDeleted(deleted, system); + searcher.close(); + } + + public void verifyDocsDeleted(Set deleted, TransactionSystem system) throws IOException { + Snapshot snapshot = system.getSnapshots().getLatestSnapshot(); + try { + IndexReader reader = snapshot.getIndexReader(); + for (Long id : deleted) { + int doc = Util.getTermDoc(new Term("id", id.toString()), reader); + if (doc >= 0) { + boolean isDeleted = reader.isDeleted(doc); + assertTrue(!isDeleted); + } + assertEquals(-1, doc); + } + } finally { + snapshot.decRef(); + } + } + + public void deleteRandomDocuments(AtomicInteger docsDeleted, Set deleted, Set ids, TransactionSystem system) throws Exception { + OceanSearcher searcher = system.getSearcher(); + ScoreDoc[] hits = searcher.search(new MatchAllDocsQuery(), null, 5000).scoreDocs; + List deletes = new ArrayList(); + for (ScoreDoc scoreDoc : hits) { + if (random.nextBoolean()) { + if (!searcher.getSnapshot().getIndexReader().isDeleted(scoreDoc.doc)) { + Document document = searcher.doc(scoreDoc.doc); + String docIdString = document.get(Constants.DOCUMENTID); + CommitResult result = system.deleteDocument(new Term(Constants.DOCUMENTID, docIdString)); + assertEquals(1, result.getNumDeleted()); + Long id = new Long(document.get("id")); + ids.remove(id); + deleted.add(id); + docsDeleted.incrementAndGet(); + } + } + } + searcher.close(); + } + + public void verifyUpdatedDocuments(List updates, TransactionSystem system) throws Exception { + for (Update update : updates) { + Document document = system.getDocument(new Term("id", Long.toString(update.id))); + String newValue = document.get(update.field); + if (!StringUtils.equals(newValue, update.value)) { + System.out.println("id: " + update.id + " field: " + update.field + " value: " + update.value + " newValue: " + newValue); + assertEquals(newValue, update.value); + } + } + } + + public void updateDocuments(int num, Set ids, List updates, TransactionSystem system) throws Exception { + List idList = new ArrayList(ids); + for (int x = 0; x < num; x++) { + updateRandomDocument(idList, updates, system); + } + } + + public void updateRandomDocument(List ids, List updates, TransactionSystem system) throws Exception { + Term idTerm = getRandomIdTerm(ids); + System.out.println("update idterm: " + idTerm); + Document document = system.getDocument(idTerm); + String newStringValue = Integer.toString(random.nextInt()); + OceanSearcher searcher = system.getSearcher(); + if (!StringUtils.equals(newStringValue, document.get("value"))) { + document.removeField("value"); + document.add(new Field("value", newStringValue, Field.Store.YES, Field.Index.UN_TOKENIZED)); + CommitResult commitResult = system.updateDocument(idTerm, document); + if (1 != commitResult.getNumDeleted()) { + boolean docIdsUnique = searcher.verifyDocsIdsUnique(); + // dumpSnapshot(searcher); + assertEquals(1, commitResult.getNumDeleted()); + } + assertEquals(1, commitResult.getNumAdded().intValue()); + } + searcher.close(); + } + + public Term getRandomIdTerm(List ids) { + return new Term("id", Long.toString(getRandomId(ids))); + } + + public Long getRandomId(List ids) { + int i = random.nextInt(ids.size()); + return ids.get(i); + } + + /** Main for running test case by itself. */ + public static void main(String args[]) throws Exception { + //InputStream input = ClassLoader.getSystemResourceAsStream("java/util/Vector.class"); + //byte[] bytes = IOUtils.toByteArray(input); + //System.out.println("bytes.length: "+bytes.length); + TestRunner.run(new TestSuite(TestSystem.class)); + } +} Index: ocean/test/org/apache/lucene/ocean/TestUpdates.java =================================================================== --- ocean/test/org/apache/lucene/ocean/TestUpdates.java (revision 0) +++ ocean/test/org/apache/lucene/ocean/TestUpdates.java (revision 0) @@ -0,0 +1,42 @@ +package org.apache.lucene.ocean; + +import junit.framework.TestSuite; +import junit.textui.TestRunner; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.util.English; +import org.apache.lucene.util.LuceneTestCase; + +/** + * Add documents, delete them, verify they are deleted. Update documents verify + * they have been updated. + * + */ +public class TestUpdates extends LuceneTestCase { + TransactionSystem transactionSystem; + + public static void main(String args[]) { + TestRunner.run(new TestSuite(TestUpdates.class)); + } + + public void testUpdateDocuments() throws Exception { + transactionSystem = TestSearch.setupTransactionSystem(); + // add documents + int count = 2; + for (int i = 0; i < 200; i++) { + Document d = new Document(); + d.add(new Field("id", Integer.toString(i), Field.Store.YES, Field.Index.UN_TOKENIZED)); + d.add(new Field("contents", English.intToEnglish(i + 10 * count), Field.Store.NO, Field.Index.TOKENIZED)); + transactionSystem.addDocument(d); + //transactionSystem.updateDocument(new Term("id", Integer.toString(i)), d); + } + OceanSearcher searcher = transactionSystem.getSearcher(); + int maxDoc = searcher.maxDoc(); + System.out.println("maxDoc: "+maxDoc); + assertEquals(maxDoc, 200); + + + transactionSystem.close(); + } +} Index: ocean/test/org/apache/lucene/ocean/util/TestBytes.java =================================================================== --- ocean/test/org/apache/lucene/ocean/util/TestBytes.java (revision 0) +++ ocean/test/org/apache/lucene/ocean/util/TestBytes.java (revision 0) @@ -0,0 +1,17 @@ +package org.apache.lucene.ocean.util; + +import junit.framework.TestSuite; +import junit.textui.TestRunner; + +import org.apache.lucene.util.LuceneTestCase; + +public class TestBytes extends LuceneTestCase { + public static void main(String args[]) { + TestRunner.run(new TestSuite(TestBytes.class)); + } + + public void testMain() throws Exception { + + } + +} Index: ocean/test/org/apache/lucene/ocean/util/TestSortedList.java =================================================================== --- ocean/test/org/apache/lucene/ocean/util/TestSortedList.java (revision 0) +++ ocean/test/org/apache/lucene/ocean/util/TestSortedList.java (revision 0) @@ -0,0 +1,41 @@ +package org.apache.lucene.ocean.util; + +import java.util.Iterator; +import java.util.Map; + +import junit.framework.TestSuite; +import junit.textui.TestRunner; + +import org.apache.lucene.util.LuceneTestCase; + +public class TestSortedList extends LuceneTestCase { + public static void main(String args[]) { + TestRunner.run(new TestSuite(TestSortedList.class)); + } + + public void testRangeIterator() throws Exception { + SortedList list = new SortedList(); + for (int x=0; x < 10; x++) { + if (x != 4 && x != 8) list.put(x, x); + } + System.out.println(list.toString()); + System.out.println(iteratorToString(list.iterator(3, true, null, false))); + System.out.println(iteratorToString(list.iterator(3, true, 7, false))); + System.out.println(iteratorToString(list.iterator(3, true, 7, true))); + System.out.println(iteratorToString(list.iterator(null, true, 7, true))); + System.out.println(iteratorToString(list.iterator(3, false, 7, true))); + System.out.println(iteratorToString(list.iterator(4, true, 8, true))); + } + + public String iteratorToString(Iterator> iterator) { + StringBuilder b = new StringBuilder(); + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + b.append(entry.toString()); + if (iterator.hasNext()) { + b.append(", "); + } + } + return b.toString(); + } +}