Index: lucene/CHANGES.txt =================================================================== --- lucene/CHANGES.txt (revision 1084632) +++ lucene/CHANGES.txt (working copy) @@ -368,6 +368,10 @@ * LUCENE-2936: PhraseQuery score explanations were not correctly identifying matches vs non-matches. (hossman) + +* LUCENE-2987: QueryParser was not parsing lowercase AND and OR token. The + AND and OR token definitions were update to accept lowercase AND and OR now. + (joecabrera) ======================= Lucene 3.x (not yet released) ======================= Index: lucene/src/java/org/apache/lucene/queryParser/QueryParser.java =================================================================== --- lucene/src/java/org/apache/lucene/queryParser/QueryParser.java (revision 1084632) +++ lucene/src/java/org/apache/lucene/queryParser/QueryParser.java (working copy) @@ -527,7 +527,7 @@ private int jj_gc = 0; /** Constructor with user supplied CharStream. */ - protected QueryParser(CharStream stream) { + public QueryParser(CharStream stream) { token_source = new QueryParserTokenManager(stream); token = new Token(); jj_ntk = -1; @@ -547,7 +547,7 @@ } /** Constructor with generated Token Manager. */ - protected QueryParser(QueryParserTokenManager tm) { + public QueryParser(QueryParserTokenManager tm) { token_source = tm; token = new Token(); jj_ntk = -1; @@ -655,7 +655,7 @@ for (int i = 0; i < jj_endpos; i++) { jj_expentry[i] = jj_lasttokens[i]; } - jj_entries_loop: for (java.util.Iterator it = jj_expentries.iterator(); it.hasNext();) { + jj_entries_loop: for (java.util.Iterator it = jj_expentries.iterator(); it.hasNext();) { int[] oldentry = (int[])(it.next()); if (oldentry.length == jj_expentry.length) { for (int i = 0; i < jj_expentry.length; i++) { Index: lucene/src/java/org/apache/lucene/queryParser/QueryParser.jj =================================================================== --- lucene/src/java/org/apache/lucene/queryParser/QueryParser.jj (revision 1084632) +++ lucene/src/java/org/apache/lucene/queryParser/QueryParser.jj (working copy) @@ -149,8 +149,8 @@ } TOKEN : { - -| + +| | | | Index: lucene/src/java/org/apache/lucene/queryParser/QueryParserTokenManager.java =================================================================== --- lucene/src/java/org/apache/lucene/queryParser/QueryParserTokenManager.java (revision 1084632) +++ lucene/src/java/org/apache/lucene/queryParser/QueryParserTokenManager.java (working copy) @@ -47,11 +47,11 @@ case 41: return jjStopAtPos(0, 15); case 42: - return jjStartNfaWithStates_2(0, 17, 43); + return jjStartNfaWithStates_2(0, 17, 48); case 43: - return jjStartNfaWithStates_2(0, 11, 15); + return jjStartNfaWithStates_2(0, 11, 20); case 45: - return jjStartNfaWithStates_2(0, 12, 15); + return jjStartNfaWithStates_2(0, 12, 20); case 58: return jjStopAtPos(0, 16); case 91: @@ -87,7 +87,7 @@ private int jjMoveNfa_2(int startState, int curPos) { int startsAt = 0; - jjnewStateCnt = 43; + jjnewStateCnt = 48; int i = 1; jjstateSet[0] = startState; int kind = 0x7fffffff; @@ -102,12 +102,20 @@ { switch(jjstateSet[--i]) { + case 48: + case 32: + if ((0xfbfffcf8ffffd9ffL & l) == 0L) + break; + if (kind > 23) + kind = 23; + jjCheckNAddTwoStates(32, 33); + break; case 0: if ((0xfbffd4f8ffffd9ffL & l) != 0L) { if (kind > 23) kind = 23; - jjCheckNAddTwoStates(27, 28); + jjCheckNAddTwoStates(32, 33); } else if ((0x100002600L & l) != 0L) { @@ -115,7 +123,7 @@ kind = 7; } else if ((0x280200000000L & l) != 0L) - jjstateSet[jjnewStateCnt++] = 15; + jjstateSet[jjnewStateCnt++] = 20; else if (curChar == 34) jjCheckNAddStates(0, 2); if ((0x7bffd0f8ffffd9ffL & l) != 0L) @@ -139,14 +147,6 @@ else if (curChar == 38) jjstateSet[jjnewStateCnt++] = 4; break; - case 43: - case 27: - if ((0xfbfffcf8ffffd9ffL & l) == 0L) - break; - if (kind > 23) - kind = 23; - jjCheckNAddTwoStates(27, 28); - break; case 4: if (curChar == 38 && kind > 8) kind = 8; @@ -155,104 +155,104 @@ if (curChar == 38) jjstateSet[jjnewStateCnt++] = 4; break; - case 13: + case 18: if (curChar == 33 && kind > 10) kind = 10; break; - case 14: + case 19: if ((0x280200000000L & l) != 0L) - jjstateSet[jjnewStateCnt++] = 15; + jjstateSet[jjnewStateCnt++] = 20; break; - case 15: + case 20: if ((0x100002600L & l) != 0L && kind > 13) kind = 13; break; - case 16: + case 21: if (curChar == 34) jjCheckNAddStates(0, 2); break; - case 17: + case 22: if ((0xfffffffbffffffffL & l) != 0L) jjCheckNAddStates(0, 2); break; - case 19: + case 24: jjCheckNAddStates(0, 2); break; - case 20: + case 25: if (curChar == 34 && kind > 19) kind = 19; break; - case 22: + case 27: if ((0x3ff000000000000L & l) == 0L) break; if (kind > 21) kind = 21; jjAddStates(11, 12); break; - case 23: + case 28: if (curChar == 46) - jjCheckNAdd(24); + jjCheckNAdd(29); break; - case 24: + case 29: if ((0x3ff000000000000L & l) == 0L) break; if (kind > 21) kind = 21; - jjCheckNAdd(24); + jjCheckNAdd(29); break; - case 25: + case 30: if (curChar == 42 && kind > 22) kind = 22; break; - case 26: + case 31: if ((0xfbffd4f8ffffd9ffL & l) == 0L) break; if (kind > 23) kind = 23; - jjCheckNAddTwoStates(27, 28); + jjCheckNAddTwoStates(32, 33); break; - case 29: + case 34: if (kind > 23) kind = 23; - jjCheckNAddTwoStates(27, 28); + jjCheckNAddTwoStates(32, 33); break; - case 30: - case 32: + case 35: + case 37: if (curChar == 47) jjCheckNAddStates(8, 10); break; - case 31: + case 36: if ((0xffff7fffffffffffL & l) != 0L) jjCheckNAddStates(8, 10); break; - case 34: + case 39: if (curChar == 47 && kind > 24) kind = 24; break; - case 35: + case 40: if ((0x7bffd0f8ffffd9ffL & l) == 0L) break; if (kind > 20) kind = 20; jjCheckNAddStates(3, 7); break; - case 36: + case 41: if ((0x7bfff8f8ffffd9ffL & l) == 0L) break; if (kind > 20) kind = 20; - jjCheckNAddTwoStates(36, 37); + jjCheckNAddTwoStates(41, 42); break; - case 38: + case 43: if (kind > 20) kind = 20; - jjCheckNAddTwoStates(36, 37); + jjCheckNAddTwoStates(41, 42); break; - case 39: + case 44: if ((0x7bfff8f8ffffd9ffL & l) != 0L) jjCheckNAddStates(13, 15); break; - case 41: + case 46: jjCheckNAddStates(13, 15); break; default : break; @@ -266,6 +266,16 @@ { switch(jjstateSet[--i]) { + case 48: + if ((0x97ffffff87ffffffL & l) != 0L) + { + if (kind > 23) + kind = 23; + jjCheckNAddTwoStates(32, 33); + } + else if (curChar == 92) + jjCheckNAddTwoStates(34, 34); + break; case 0: if ((0x97ffffff87ffffffL & l) != 0L) { @@ -279,33 +289,27 @@ { if (kind > 21) kind = 21; - jjstateSet[jjnewStateCnt++] = 22; + jjstateSet[jjnewStateCnt++] = 27; } if ((0x97ffffff87ffffffL & l) != 0L) { if (kind > 23) kind = 23; - jjCheckNAddTwoStates(27, 28); + jjCheckNAddTwoStates(32, 33); } if (curChar == 78) + jjstateSet[jjnewStateCnt++] = 16; + else if (curChar == 111) + jjstateSet[jjnewStateCnt++] = 13; + else if (curChar == 124) jjstateSet[jjnewStateCnt++] = 11; - else if (curChar == 124) - jjstateSet[jjnewStateCnt++] = 8; else if (curChar == 79) - jjstateSet[jjnewStateCnt++] = 6; + jjstateSet[jjnewStateCnt++] = 9; + else if (curChar == 97) + jjstateSet[jjnewStateCnt++] = 7; else if (curChar == 65) jjstateSet[jjnewStateCnt++] = 2; break; - case 43: - if ((0x97ffffff87ffffffL & l) != 0L) - { - if (kind > 23) - kind = 23; - jjCheckNAddTwoStates(27, 28); - } - else if (curChar == 92) - jjCheckNAddTwoStates(29, 29); - break; case 1: if (curChar == 68 && kind > 8) kind = 8; @@ -319,116 +323,136 @@ jjstateSet[jjnewStateCnt++] = 2; break; case 6: + if (curChar == 100 && kind > 8) + kind = 8; + break; + case 7: + if (curChar == 110) + jjstateSet[jjnewStateCnt++] = 6; + break; + case 8: + if (curChar == 97) + jjstateSet[jjnewStateCnt++] = 7; + break; + case 9: if (curChar == 82 && kind > 9) kind = 9; break; - case 7: + case 10: if (curChar == 79) - jjstateSet[jjnewStateCnt++] = 6; + jjstateSet[jjnewStateCnt++] = 9; break; - case 8: + case 11: if (curChar == 124 && kind > 9) kind = 9; break; - case 9: + case 12: if (curChar == 124) - jjstateSet[jjnewStateCnt++] = 8; + jjstateSet[jjnewStateCnt++] = 11; break; - case 10: + case 13: + if (curChar == 114 && kind > 9) + kind = 9; + break; + case 14: + if (curChar == 111) + jjstateSet[jjnewStateCnt++] = 13; + break; + case 15: if (curChar == 84 && kind > 10) kind = 10; break; - case 11: + case 16: if (curChar == 79) - jjstateSet[jjnewStateCnt++] = 10; + jjstateSet[jjnewStateCnt++] = 15; break; - case 12: + case 17: if (curChar == 78) - jjstateSet[jjnewStateCnt++] = 11; + jjstateSet[jjnewStateCnt++] = 16; break; - case 17: + case 22: if ((0xffffffffefffffffL & l) != 0L) jjCheckNAddStates(0, 2); break; - case 18: + case 23: if (curChar == 92) - jjstateSet[jjnewStateCnt++] = 19; + jjstateSet[jjnewStateCnt++] = 24; break; - case 19: + case 24: jjCheckNAddStates(0, 2); break; - case 21: + case 26: if (curChar != 126) break; if (kind > 21) kind = 21; - jjstateSet[jjnewStateCnt++] = 22; + jjstateSet[jjnewStateCnt++] = 27; break; - case 26: + case 31: if ((0x97ffffff87ffffffL & l) == 0L) break; if (kind > 23) kind = 23; - jjCheckNAddTwoStates(27, 28); + jjCheckNAddTwoStates(32, 33); break; - case 27: + case 32: if ((0x97ffffff87ffffffL & l) == 0L) break; if (kind > 23) kind = 23; - jjCheckNAddTwoStates(27, 28); + jjCheckNAddTwoStates(32, 33); break; - case 28: + case 33: if (curChar == 92) - jjCheckNAddTwoStates(29, 29); + jjCheckNAddTwoStates(34, 34); break; - case 29: + case 34: if (kind > 23) kind = 23; - jjCheckNAddTwoStates(27, 28); + jjCheckNAddTwoStates(32, 33); break; - case 31: + case 36: jjAddStates(8, 10); break; - case 33: + case 38: if (curChar == 92) - jjstateSet[jjnewStateCnt++] = 32; + jjstateSet[jjnewStateCnt++] = 37; break; - case 35: + case 40: if ((0x97ffffff87ffffffL & l) == 0L) break; if (kind > 20) kind = 20; jjCheckNAddStates(3, 7); break; - case 36: + case 41: if ((0x97ffffff87ffffffL & l) == 0L) break; if (kind > 20) kind = 20; - jjCheckNAddTwoStates(36, 37); + jjCheckNAddTwoStates(41, 42); break; - case 37: + case 42: if (curChar == 92) - jjCheckNAddTwoStates(38, 38); + jjCheckNAddTwoStates(43, 43); break; - case 38: + case 43: if (kind > 20) kind = 20; - jjCheckNAddTwoStates(36, 37); + jjCheckNAddTwoStates(41, 42); break; - case 39: + case 44: if ((0x97ffffff87ffffffL & l) != 0L) jjCheckNAddStates(13, 15); break; - case 40: + case 45: if (curChar == 92) - jjCheckNAddTwoStates(41, 41); + jjCheckNAddTwoStates(46, 46); break; - case 41: + case 46: jjCheckNAddStates(13, 15); break; - case 42: + case 47: if (curChar == 92) jjCheckNAddStates(16, 18); break; @@ -447,6 +471,14 @@ { switch(jjstateSet[--i]) { + case 48: + case 32: + if (!jjCanMove_2(hiByte, i1, i2, l1, l2)) + break; + if (kind > 23) + kind = 23; + jjCheckNAddTwoStates(32, 33); + break; case 0: if (jjCanMove_0(hiByte, i1, i2, l1, l2)) { @@ -457,7 +489,7 @@ { if (kind > 23) kind = 23; - jjCheckNAddTwoStates(27, 28); + jjCheckNAddTwoStates(32, 33); } if (jjCanMove_2(hiByte, i1, i2, l1, l2)) { @@ -466,67 +498,59 @@ jjCheckNAddStates(3, 7); } break; - case 43: - case 27: - if (!jjCanMove_2(hiByte, i1, i2, l1, l2)) - break; - if (kind > 23) - kind = 23; - jjCheckNAddTwoStates(27, 28); - break; - case 15: + case 20: if (jjCanMove_0(hiByte, i1, i2, l1, l2) && kind > 13) kind = 13; break; - case 17: - case 19: + case 22: + case 24: if (jjCanMove_1(hiByte, i1, i2, l1, l2)) jjCheckNAddStates(0, 2); break; - case 26: + case 31: if (!jjCanMove_2(hiByte, i1, i2, l1, l2)) break; if (kind > 23) kind = 23; - jjCheckNAddTwoStates(27, 28); + jjCheckNAddTwoStates(32, 33); break; - case 29: + case 34: if (!jjCanMove_1(hiByte, i1, i2, l1, l2)) break; if (kind > 23) kind = 23; - jjCheckNAddTwoStates(27, 28); + jjCheckNAddTwoStates(32, 33); break; - case 31: + case 36: if (jjCanMove_1(hiByte, i1, i2, l1, l2)) jjAddStates(8, 10); break; - case 35: + case 40: if (!jjCanMove_2(hiByte, i1, i2, l1, l2)) break; if (kind > 20) kind = 20; jjCheckNAddStates(3, 7); break; - case 36: + case 41: if (!jjCanMove_2(hiByte, i1, i2, l1, l2)) break; if (kind > 20) kind = 20; - jjCheckNAddTwoStates(36, 37); + jjCheckNAddTwoStates(41, 42); break; - case 38: + case 43: if (!jjCanMove_1(hiByte, i1, i2, l1, l2)) break; if (kind > 20) kind = 20; - jjCheckNAddTwoStates(36, 37); + jjCheckNAddTwoStates(41, 42); break; - case 39: + case 44: if (jjCanMove_2(hiByte, i1, i2, l1, l2)) jjCheckNAddStates(13, 15); break; - case 41: + case 46: if (jjCanMove_1(hiByte, i1, i2, l1, l2)) jjCheckNAddStates(13, 15); break; @@ -541,7 +565,7 @@ kind = 0x7fffffff; } ++curPos; - if ((i = jjnewStateCnt) == (startsAt = 43 - (jjnewStateCnt = startsAt))) + if ((i = jjnewStateCnt) == (startsAt = 48 - (jjnewStateCnt = startsAt))) return curPos; try { curChar = input_stream.readChar(); } catch(java.io.IOException e) { return curPos; } @@ -828,8 +852,8 @@ } } static final int[] jjnextStates = { - 17, 18, 20, 36, 39, 25, 40, 37, 31, 33, 34, 22, 23, 39, 25, 40, - 38, 41, 29, 0, 1, 2, 4, 5, + 22, 23, 25, 41, 44, 30, 45, 42, 36, 38, 39, 27, 28, 44, 30, 45, + 43, 46, 34, 0, 1, 2, 4, 5, }; private static final boolean jjCanMove_0(int hiByte, int i1, int i2, long l1, long l2) { @@ -893,8 +917,8 @@ 0x80L, }; protected CharStream input_stream; -private final int[] jjrounds = new int[43]; -private final int[] jjstateSet = new int[86]; +private final int[] jjrounds = new int[48]; +private final int[] jjstateSet = new int[96]; protected char curChar; /** Constructor. */ public QueryParserTokenManager(CharStream stream){ @@ -919,7 +943,7 @@ { int i; jjround = 0x80000001; - for (i = 43; i-- > 0;) + for (i = 48; i-- > 0;) jjrounds[i] = 0x80000000; }