Index: lucene/src/java/org/apache/lucene/util/BytesRef.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 Subsystem: com.intellij.openapi.diff.impl.patch.BaseRevisionTextPatchEP <+>package org.apache.lucene.util;\n\n/**\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements. See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport java.util.Comparator;\n\n/** Represents byte[], as a slice (offset + length) into an\n * existing byte[]. The {@link #bytes} member should never be null;\n * use {@link #EMPTY_BYTES} if necessary.\n *\n * @lucene.experimental */\npublic final class BytesRef implements Comparable,Cloneable {\n /** An empty byte array for convenience */\n public static final byte[] EMPTY_BYTES = new byte[0]; \n\n /** The contents of the BytesRef. Should never be {@code null}. */\n public byte[] bytes;\n\n /** Offset of first valid byte. */\n public int offset;\n\n /** Length of used bytes. */\n public int length;\n\n /** Create a BytesRef with {@link #EMPTY_BYTES} */\n public BytesRef() {\n this(EMPTY_BYTES);\n }\n\n /** This instance will directly reference bytes w/o making a copy.\n * bytes should not be null.\n */\n public BytesRef(byte[] bytes, int offset, int length) {\n assert bytes != null;\n this.bytes = bytes;\n this.offset = offset;\n this.length = length;\n }\n\n /** This instance will directly reference bytes w/o making a copy.\n * bytes should not be null */\n public BytesRef(byte[] bytes) {\n this(bytes, 0, bytes.length);\n }\n\n /** \n * Create a BytesRef pointing to a new array of size capacity.\n * Offset and length will both be zero.\n */\n public BytesRef(int capacity) {\n this.bytes = new byte[capacity];\n }\n\n /**\n * Initialize the byte[] from the UTF8 bytes\n * for the provided String. \n * \n * @param text This must be well-formed\n * unicode text, with no unpaired surrogates.\n */\n public BytesRef(CharSequence text) {\n this();\n copyChars(text);\n }\n\n /**\n * Copies the UTF8 bytes for this string.\n * \n * @param text Must be well-formed unicode text, with no\n * unpaired surrogates or invalid UTF16 code units.\n */\n // TODO broken if offset != 0\n public void copyChars(CharSequence text) {\n UnicodeUtil.UTF16toUTF8(text, 0, text.length(), this);\n }\n \n /**\n * Expert: compares the bytes against another BytesRef,\n * returning true if the bytes are equal.\n * \n * @param other Another BytesRef, should not be null.\n * @lucene.internal\n */\n public boolean bytesEquals(BytesRef other) {\n assert other != null;\n if (length == other.length) {\n int otherUpto = other.offset;\n final byte[] otherBytes = other.bytes;\n final int end = offset + length;\n for(int upto=offset;uptoIt is defined as:\n * 
\n   *  int hash = 0;\n   *  for (int i = offset; i < offset + length; i++) {\n   *    hash = 31*hash + bytes[i];\n   *  }\n   *
\n */\n @Override\n public int hashCode() {\n int hash = 0;\n final int end = offset + length;\n for(int i=offset;i offset) {\n sb.append(' ');\n }\n sb.append(Integer.toHexString(bytes[i]&0xff));\n }\n sb.append(']');\n return sb.toString();\n }\n\n /**\n * Copies the bytes from the given {@link BytesRef}\n *

\n * NOTE: if this would exceed the array size, this method creates a \n * new reference array.\n */\n public void copyBytes(BytesRef other) {\n if (bytes.length < other.length) {\n bytes = new byte[other.length];\n offset = 0;\n }\n System.arraycopy(other.bytes, other.offset, bytes, offset, other.length);\n length = other.length;\n }\n\n /**\n * Appends the bytes from the given {@link BytesRef}\n *

\n * NOTE: if this would exceed the array size, this method creates a \n * new reference array.\n */\n public void append(BytesRef other) {\n int newLen = length + other.length;\n if (bytes.length < newLen) {\n byte[] newBytes = new byte[newLen];\n System.arraycopy(bytes, offset, newBytes, 0, length);\n offset = 0;\n bytes = newBytes;\n }\n System.arraycopy(other.bytes, other.offset, bytes, length+offset, other.length);\n length = newLen;\n }\n\n // TODO: stupid if existing offset is non-zero.\n /** @lucene.internal */\n public void grow(int newLength) {\n bytes = ArrayUtil.grow(bytes, newLength);\n }\n\n /** Unsigned byte order comparison */\n public int compareTo(BytesRef other) {\n return utf8SortedAsUnicodeSortOrder.compare(this, other);\n }\n \n private final static Comparator utf8SortedAsUnicodeSortOrder = new UTF8SortedAsUnicodeComparator();\n\n public static Comparator getUTF8SortedAsUnicodeComparator() {\n return utf8SortedAsUnicodeSortOrder;\n }\n\n private static class UTF8SortedAsUnicodeComparator implements Comparator {\n // Only singleton\n private UTF8SortedAsUnicodeComparator() {};\n\n public int compare(BytesRef a, BytesRef b) {\n final byte[] aBytes = a.bytes;\n int aUpto = a.offset;\n final byte[] bBytes = b.bytes;\n int bUpto = b.offset;\n \n final int aStop;\n if (a.length < b.length) {\n aStop = aUpto + a.length;\n } else {\n aStop = aUpto + b.length;\n }\n\n while(aUpto < aStop) {\n int aByte = aBytes[aUpto++] & 0xff;\n int bByte = bBytes[bUpto++] & 0xff;\n\n int diff = aByte - bByte;\n if (diff != 0) {\n return diff;\n }\n }\n\n // One is a prefix of the other, or, they are equal:\n return a.length - b.length;\n } \n }\n\n /** @deprecated */\n @Deprecated\n private final static Comparator utf8SortedAsUTF16SortOrder = new UTF8SortedAsUTF16Comparator();\n\n /** @deprecated This comparator is only a transition mechanism */\n @Deprecated\n public static Comparator getUTF8SortedAsUTF16Comparator() {\n return utf8SortedAsUTF16SortOrder;\n }\n\n /** @deprecated */\n @Deprecated\n private static class UTF8SortedAsUTF16Comparator implements Comparator {\n // Only singleton\n private UTF8SortedAsUTF16Comparator() {};\n\n public int compare(BytesRef a, BytesRef b) {\n\n final byte[] aBytes = a.bytes;\n int aUpto = a.offset;\n final byte[] bBytes = b.bytes;\n int bUpto = b.offset;\n \n final int aStop;\n if (a.length < b.length) {\n aStop = aUpto + a.length;\n } else {\n aStop = aUpto + b.length;\n }\n\n while(aUpto < aStop) {\n int aByte = aBytes[aUpto++] & 0xff;\n int bByte = bBytes[bUpto++] & 0xff;\n\n if (aByte != bByte) {\n\n // See http://icu-project.org/docs/papers/utf16_code_point_order.html#utf-8-in-utf-16-order\n\n // We know the terms are not equal, but, we may\n // have to carefully fixup the bytes at the\n // difference to match UTF16's sort order:\n \n // NOTE: instead of moving supplementary code points (0xee and 0xef) to the unused 0xfe and 0xff, \n // we move them to the unused 0xfc and 0xfd [reserved for future 6-byte character sequences]\n // this reserves 0xff for preflex's term reordering (surrogate dance), and if unicode grows such\n // that 6-byte sequences are needed we have much bigger problems anyway.\n if (aByte >= 0xee && bByte >= 0xee) {\n if ((aByte & 0xfe) == 0xee) {\n aByte += 0xe;\n }\n if ((bByte&0xfe) == 0xee) {\n bByte += 0xe;\n }\n }\n return aByte - bByte;\n }\n }\n\n // One is a prefix of the other, or, they are equal:\n return a.length - b.length;\n }\n }\n \n /**\n * Creates a new BytesRef that points to a copy of the bytes from \n * other\n *

\n * The returned BytesRef will have a length of other.length\n * and an offset of zero.\n */\n public static BytesRef deepCopyOf(BytesRef other) {\n BytesRef copy = new BytesRef();\n copy.copyBytes(other);\n return copy;\n }\n}\n =================================================================== --- lucene/src/java/org/apache/lucene/util/BytesRef.java (revision 7b227fb77bf61c6ab942243af2c7cf82802f4672) +++ lucene/src/java/org/apache/lucene/util/BytesRef.java (revision ) @@ -17,32 +17,51 @@ * limitations under the License. */ +import sun.misc.Unsafe; + +import java.lang.reflect.Field; +import java.nio.ByteOrder; +import java.security.AccessController; +import java.security.PrivilegedAction; import java.util.Comparator; -/** Represents byte[], as a slice (offset + length) into an +/** + * Represents byte[], as a slice (offset + length) into an - * existing byte[]. The {@link #bytes} member should never be null; + * existing byte[]. The {@link #bytes} member should never be null; - * use {@link #EMPTY_BYTES} if necessary. + * use {@link #EMPTY_BYTES} if necessary. * - * @lucene.experimental */ + * @lucene.experimental + */ -public final class BytesRef implements Comparable,Cloneable { +public final class BytesRef implements Comparable, Cloneable { - /** An empty byte array for convenience */ + /** + * An empty byte array for convenience + */ - public static final byte[] EMPTY_BYTES = new byte[0]; + public static final byte[] EMPTY_BYTES = new byte[0]; - /** The contents of the BytesRef. Should never be {@code null}. */ + /** + * The contents of the BytesRef. Should never be {@code null}. + */ public byte[] bytes; - /** Offset of first valid byte. */ + /** + * Offset of first valid byte. + */ public int offset; - /** Length of used bytes. */ + /** + * Length of used bytes. + */ public int length; - /** Create a BytesRef with {@link #EMPTY_BYTES} */ + /** + * Create a BytesRef with {@link #EMPTY_BYTES} + */ public BytesRef() { this(EMPTY_BYTES); } - /** This instance will directly reference bytes w/o making a copy. + /** + * This instance will directly reference bytes w/o making a copy. * bytes should not be null. */ public BytesRef(byte[] bytes, int offset, int length) { @@ -52,13 +71,15 @@ this.length = length; } - /** This instance will directly reference bytes w/o making a copy. - * bytes should not be null */ + /** + * This instance will directly reference bytes w/o making a copy. + * bytes should not be null + */ public BytesRef(byte[] bytes) { this(bytes, 0, bytes.length); } - /** + /** * Create a BytesRef pointing to a new array of size capacity. * Offset and length will both be zero. */ @@ -68,10 +89,10 @@ /** * Initialize the byte[] from the UTF8 bytes - * for the provided String. - * + * for the provided String. + * * @param text This must be well-formed - * unicode text, with no unpaired surrogates. + * unicode text, with no unpaired surrogates. */ public BytesRef(CharSequence text) { this(); @@ -80,19 +101,19 @@ /** * Copies the UTF8 bytes for this string. - * + * * @param text Must be well-formed unicode text, with no - * unpaired surrogates or invalid UTF16 code units. + * unpaired surrogates or invalid UTF16 code units. */ // TODO broken if offset != 0 public void copyChars(CharSequence text) { UnicodeUtil.UTF16toUTF8(text, 0, text.length(), this); } - + /** * Expert: compares the bytes against another BytesRef, * returning true if the bytes are equal. - * + * * @param other Another BytesRef, should not be null. * @lucene.internal */ @@ -102,7 +123,7 @@ int otherUpto = other.offset; final byte[] otherBytes = other.bytes; final int end = offset + length; - for(int upto=offset;uptoIt is defined as: * 

    *  int hash = 0;
@@ -132,7 +154,7 @@
   public int hashCode() {
     int hash = 0;
     final int end = offset + length;
-    for(int i=offset;i offset) {
         sb.append(' ');
       }
-      sb.append(Integer.toHexString(bytes[i]&0xff));
+      sb.append(Integer.toHexString(bytes[i] & 0xff));
     }
     sb.append(']');
     return sb.toString();
@@ -175,8 +201,8 @@
 
   /**
    * Copies the bytes from the given {@link BytesRef}
-   * 

+   * 

-   * NOTE: if this would exceed the array size, this method creates a 
+   * NOTE: if this would exceed the array size, this method creates a
    * new reference array.
    */
   public void copyBytes(BytesRef other) {
@@ -190,8 +216,8 @@
 
   /**
    * Appends the bytes from the given {@link BytesRef}
-   * 

+   * 

-   * NOTE: if this would exceed the array size, this method creates a 
+   * NOTE: if this would exceed the array size, this method creates a
    * new reference array.
    */
   public void append(BytesRef other) {
@@ -202,82 +228,64 @@
       offset = 0;
       bytes = newBytes;
     }
-    System.arraycopy(other.bytes, other.offset, bytes, length+offset, other.length);
+    System.arraycopy(other.bytes, other.offset, bytes, length + offset, other.length);
     length = newLen;
   }
 
   // TODO: stupid if existing offset is non-zero.
-  /** @lucene.internal */
+
+  /**
+   * @lucene.internal
+   */
   public void grow(int newLength) {
     bytes = ArrayUtil.grow(bytes, newLength);
   }
 
-  /** Unsigned byte order comparison */
+  /**
+   * Unsigned byte order comparison
+   */
   public int compareTo(BytesRef other) {
     return utf8SortedAsUnicodeSortOrder.compare(this, other);
   }
-  
+
-  private final static Comparator utf8SortedAsUnicodeSortOrder = new UTF8SortedAsUnicodeComparator();
+  private final static Comparator utf8SortedAsUnicodeSortOrder = LexicographicalComparatorHolder.BEST_COMPARATOR;
 
   public static Comparator getUTF8SortedAsUnicodeComparator() {
     return utf8SortedAsUnicodeSortOrder;
   }
 
-  private static class UTF8SortedAsUnicodeComparator implements Comparator {
-    // Only singleton
-    private UTF8SortedAsUnicodeComparator() {};
-
-    public int compare(BytesRef a, BytesRef b) {
-      final byte[] aBytes = a.bytes;
-      int aUpto = a.offset;
-      final byte[] bBytes = b.bytes;
-      int bUpto = b.offset;
-      
-      final int aStop;
-      if (a.length < b.length) {
-        aStop = aUpto + a.length;
-      } else {
-        aStop = aUpto + b.length;
-      }
-
-      while(aUpto < aStop) {
-        int aByte = aBytes[aUpto++] & 0xff;
-        int bByte = bBytes[bUpto++] & 0xff;
-
-        int diff = aByte - bByte;
-        if (diff != 0) {
-          return diff;
-        }
-      }
-
-      // One is a prefix of the other, or, they are equal:
-      return a.length - b.length;
-    }    
-  }
-
-  /** @deprecated */
+  /**
+   * @deprecated
+   */
   @Deprecated
   private final static Comparator utf8SortedAsUTF16SortOrder = new UTF8SortedAsUTF16Comparator();
 
-  /** @deprecated This comparator is only a transition mechanism */
+  /**
+   * @deprecated This comparator is only a transition mechanism
+   */
   @Deprecated
   public static Comparator getUTF8SortedAsUTF16Comparator() {
     return utf8SortedAsUTF16SortOrder;
   }
 
-  /** @deprecated */
+  /**
+   * @deprecated
+   */
   @Deprecated
   private static class UTF8SortedAsUTF16Comparator implements Comparator {
     // Only singleton
-    private UTF8SortedAsUTF16Comparator() {};
+    private UTF8SortedAsUTF16Comparator() {
+    }
 
+    ;
+
     public int compare(BytesRef a, BytesRef b) {
 
       final byte[] aBytes = a.bytes;
       int aUpto = a.offset;
       final byte[] bBytes = b.bytes;
       int bUpto = b.offset;
-      
+
       final int aStop;
       if (a.length < b.length) {
         aStop = aUpto + a.length;
@@ -285,7 +293,7 @@
         aStop = aUpto + b.length;
       }
 
-      while(aUpto < aStop) {
+      while (aUpto < aStop) {
         int aByte = aBytes[aUpto++] & 0xff;
         int bByte = bBytes[bUpto++] & 0xff;
 
@@ -296,7 +304,7 @@
           // We know the terms are not equal, but, we may
           // have to carefully fixup the bytes at the
           // difference to match UTF16's sort order:
-          
+
           // NOTE: instead of moving supplementary code points (0xee and 0xef) to the unused 0xfe and 0xff, 
           // we move them to the unused 0xfc and 0xfd [reserved for future 6-byte character sequences]
           // this reserves 0xff for preflex's term reordering (surrogate dance), and if unicode grows such
@@ -305,7 +313,7 @@
             if ((aByte & 0xfe) == 0xee) {
               aByte += 0xe;
             }
-            if ((bByte&0xfe) == 0xee) {
+            if ((bByte & 0xfe) == 0xee) {
               bByte += 0xe;
             }
           }
@@ -317,11 +325,11 @@
       return a.length - b.length;
     }
   }
-  
+
   /**
-   * Creates a new BytesRef that points to a copy of the bytes from 
+   * Creates a new BytesRef that points to a copy of the bytes from
    * other
-   * 

+   * 

    * The returned BytesRef will have a length of other.length
    * and an offset of zero.
    */
@@ -330,4 +338,207 @@
     copy.copyBytes(other);
     return copy;
   }
+
+
+  static Comparator lexicographicalComparatorJavaImpl() {
+    return LexicographicalComparatorHolder.PureJavaComparator.INSTANCE;
+  }
+
+  /**
+   * Provides a lexicographical comparator implementation; either a Java
+   * implementation or a faster implementation based on {@link sun.misc.Unsafe}.
+   * 

+   * 
Uses reflection to gracefully fall back to the Java implementation if
+   * {@code Unsafe} isn't available.
+   */
+  static class LexicographicalComparatorHolder {
+    static final String UNSAFE_COMPARATOR_NAME =
+        LexicographicalComparatorHolder.class.getName() + "$UnsafeComparator";
+
+    static final Comparator BEST_COMPARATOR = getBestComparator();
+
+    enum UnsafeComparator implements Comparator {
+      INSTANCE;
+
+      static final boolean littleEndian =
+          ByteOrder.nativeOrder().equals(ByteOrder.LITTLE_ENDIAN);
+
+      /*
+       * The following static final fields exist for performance reasons.
+       *
+       * In UnsignedBytesBenchmark, accessing the following objects via static
+       * final fields is the fastest (more than twice as fast as the Java
+       * implementation, vs ~1.5x with non-final static fields, on x86_32)
+       * under the Hotspot server compiler. The reason is obviously that the
+       * non-final fields need to be reloaded inside the loop.
+       *
+       * And, no, defining (final or not) local variables out of the loop still
+       * isn't as good because the null check on the theUnsafe object remains
+       * inside the loop and BYTE_ARRAY_BASE_OFFSET doesn't get
+       * constant-folded.
+       *
+       * The compiler can treat static final fields as compile-time constants
+       * and can constant-fold them while (final or not) local variables are
+       * run time values.
+       */
+
+      static final Unsafe theUnsafe;
+
+      /**
+       * The offset to the first element in a byte array.
+       */
+      static final int BYTE_ARRAY_BASE_OFFSET;
+
+      static final int LONG_BYTES = 64 / 8;
+
+      static {
+        theUnsafe = (Unsafe) AccessController.doPrivileged(
+            new PrivilegedAction() {
+              @Override
+              public Object run() {
+                try {
+                  Field f = Unsafe.class.getDeclaredField("theUnsafe");
+                  f.setAccessible(true);
+                  return f.get(null);
+                } catch (NoSuchFieldException e) {
+                  // It doesn't matter what we throw;
+                  // it's swallowed in getBestComparator().
+                  throw new Error();
+                } catch (IllegalAccessException e) {
+                  throw new Error();
+                }
+              }
+            });
+
+        BYTE_ARRAY_BASE_OFFSET = theUnsafe.arrayBaseOffset(byte[].class);
+
+        // sanity check - this should never fail
+        if (theUnsafe.arrayIndexScale(byte[].class) != 1) {
+          throw new AssertionError();
+        }
+      }
+
+      @Override
+      public int compare(BytesRef leftRef, BytesRef rightRef) {
+        int minLength = Math.min(leftRef.length, rightRef.length);
+        int minWords = minLength / LONG_BYTES;
+
+        final byte[] left = leftRef.bytes;
+        final int leftOffset = leftRef.offset;
+        final byte[] right = rightRef.bytes;
+        final int rightOffset = rightRef.offset;
+
+
+        /*
+         * Compare 8 bytes at a time. Benchmarking shows comparing 8 bytes at a
+         * time is no slower than comparing 4 bytes at a time even on 32-bit.
+         * On the other hand, it is substantially faster on 64-bit.
+         */
+        int leftOffsetAdj = leftOffset + BYTE_ARRAY_BASE_OFFSET;
+        int rightOffsetAdj = rightOffset + BYTE_ARRAY_BASE_OFFSET;
+        for (int i = 0; i < minWords * LONG_BYTES; i += LONG_BYTES) {
+          long lw = theUnsafe.getLong(left, leftOffsetAdj + (long) i);
+          long rw = theUnsafe.getLong(right, rightOffsetAdj + (long) i);
+          long diff = lw ^ rw;
+
+          if (diff != 0) {
+            if (!littleEndian) {
+              // unsigned long comparison
+              lw = lw ^ Long.MIN_VALUE;
+              rw = rw ^ Long.MIN_VALUE;
+              return (lw < rw) ? -1 : ((lw > rw) ? 1 : 0);
+            }
+
+            // Use binary search
+            int n = 0;
+            int y;
+            int x = (int) diff;
+            if (x == 0) {
+              x = (int) (diff >>> 32);
+              n = 32;
+            }
+
+            y = x << 16;
+            if (y == 0) {
+              n += 16;
+            } else {
+              x = y;
+            }
+
+            y = x << 8;
+            if (y == 0) {
+              n += 8;
+            }
+            return (int) (((lw >>> n) & 0xFFL) - ((rw >>> n) & 0xFFL));
+          }
+        }
+
+        // The epilogue to cover the last (minLength % 8) elements.
+        int i = minWords * LONG_BYTES;
+        int leftI = i + leftOffset;
+        int rightI = i + rightOffset;
+        for (; i < minLength; i++) {
+          int aByte = left[leftI++] & 0xff;
+          int bByte = right[rightI++] & 0xff;
+
+          int diff = aByte - bByte;
+          if (diff != 0) {
+            return diff;
+          }
+        }
+        return leftRef.length - rightRef.length;
+      }
+    }
+
+    enum PureJavaComparator implements Comparator {
+      INSTANCE;
+
+      @Override
+      public int compare(BytesRef a, BytesRef b) {
+        final byte[] aBytes = a.bytes;
+        int aUpto = a.offset;
+        final byte[] bBytes = b.bytes;
+        int bUpto = b.offset;
+
+        final int aStop;
+        if (a.length < b.length) {
+          aStop = aUpto + a.length;
+        } else {
+          aStop = aUpto + b.length;
+        }
+
+        while (aUpto < aStop) {
+          int aByte = aBytes[aUpto++] & 0xff;
+          int bByte = bBytes[bUpto++] & 0xff;
+
+          int diff = aByte - bByte;
+          if (diff != 0) {
+            return diff;
+          }
+        }
+
+        // One is a prefix of the other, or, they are equal:
+        return a.length - b.length;
+      }
+    }
+
+    /**
+     * Returns the Unsafe-using Comparator, or falls back to the pure-Java
+     * implementation if unable to do so.
+     */
+    static Comparator getBestComparator() {
+      try {
+        Class theClass = Class.forName(UNSAFE_COMPARATOR_NAME);
+
+        // yes, UnsafeComparator does implement Comparator
+        @SuppressWarnings("unchecked")
+        Comparator comparator =
+            (Comparator) theClass.getEnumConstants()[0];
+        return comparator;
+      } catch (Throwable t) { // ensure we really catch *everything*
+        return lexicographicalComparatorJavaImpl();
+      }
+    }
+  }
+
 }