Index: standard/StandardFilter.java =================================================================== --- standard/StandardFilter.java (revision 541610) +++ standard/StandardFilter.java (working copy) @@ -32,6 +32,7 @@ private static final String APOSTROPHE_TYPE = tokenImage[APOSTROPHE]; private static final String ACRONYM_TYPE = tokenImage[ACRONYM]; + private static final String PUNCTUATION_TYPE = tokenImage[PUNCTUATION]; /** Returns the next token in the stream, or null at EOS. *

Removes 's from the end of words. @@ -62,6 +63,8 @@ return new org.apache.lucene.analysis.Token (trimmed.toString(), t.startOffset(), t.endOffset(), type); + } else if (type == PUNCTUATION_TYPE) { + return next(); // ignore punctuation } else { return t; } Index: standard/StandardTokenizerTokenManager.java =================================================================== --- standard/StandardTokenizerTokenManager.java (revision 541610) +++ standard/StandardTokenizerTokenManager.java (working copy) @@ -101,7 +101,7 @@ { int[] nextStates; int startsAt = 0; - jjnewStateCnt = 75; + jjnewStateCnt = 76; int i = 1; jjstateSet[0] = startState; int j, kind = 0x7fffffff; @@ -123,276 +123,285 @@ kind = 1; jjCheckNAddStates(0, 11); } + else if ((0x8000500200000000L & l) != 0L) + { + if (kind > 15) + kind = 15; + } if ((0x3ff000000000000L & l) != 0L) jjCheckNAddStates(12, 17); if ((0x3ff000000000000L & l) != 0L) jjCheckNAddStates(18, 23); break; case 2: + if ((0x8000500200000000L & l) != 0L && kind > 15) + kind = 15; + break; + case 3: if ((0x3ff000000000000L & l) != 0L) jjCheckNAddStates(18, 23); break; - case 3: + case 4: if ((0x3ff000000000000L & l) != 0L) - jjCheckNAddTwoStates(3, 4); + jjCheckNAddTwoStates(4, 5); break; - case 4: case 5: + case 6: if ((0x3ff000000000000L & l) != 0L) - jjCheckNAddTwoStates(5, 6); + jjCheckNAddTwoStates(6, 7); break; - case 6: + case 7: if ((0xf00000000000L & l) != 0L) - jjCheckNAdd(7); + jjCheckNAdd(8); break; - case 7: + case 8: if ((0x3ff000000000000L & l) == 0L) break; if (kind > 7) kind = 7; - jjCheckNAdd(7); + jjCheckNAdd(8); break; - case 8: + case 9: if ((0x3ff000000000000L & l) != 0L) - jjCheckNAddTwoStates(8, 9); + jjCheckNAddTwoStates(9, 10); break; - case 9: case 10: + case 11: if ((0x3ff000000000000L & l) != 0L) - jjCheckNAddTwoStates(10, 11); + jjCheckNAddTwoStates(11, 12); break; - case 11: + case 12: if ((0xf00000000000L & l) != 0L) - jjCheckNAdd(12); + jjCheckNAdd(13); break; - case 12: + case 13: if ((0x3ff000000000000L & l) != 0L) - jjCheckNAddTwoStates(12, 13); + jjCheckNAddTwoStates(13, 14); break; - case 13: + case 14: if ((0xf00000000000L & l) != 0L) - jjCheckNAddTwoStates(14, 15); + jjCheckNAddTwoStates(15, 16); break; - case 14: + case 15: if ((0x3ff000000000000L & l) != 0L) - jjCheckNAddTwoStates(14, 15); + jjCheckNAddTwoStates(15, 16); break; - case 15: case 16: + case 17: if ((0x3ff000000000000L & l) == 0L) break; if (kind > 7) kind = 7; - jjCheckNAddTwoStates(11, 16); + jjCheckNAddTwoStates(12, 17); break; - case 17: + case 18: if ((0x3ff000000000000L & l) != 0L) - jjCheckNAddTwoStates(17, 18); + jjCheckNAddTwoStates(18, 19); break; - case 18: case 19: + case 20: if ((0x3ff000000000000L & l) != 0L) - jjCheckNAddTwoStates(19, 20); + jjCheckNAddTwoStates(20, 21); break; - case 20: + case 21: if ((0xf00000000000L & l) != 0L) - jjCheckNAdd(21); + jjCheckNAdd(22); break; - case 21: + case 22: if ((0x3ff000000000000L & l) != 0L) - jjCheckNAddTwoStates(21, 22); + jjCheckNAddTwoStates(22, 23); break; - case 22: + case 23: if ((0xf00000000000L & l) != 0L) - jjCheckNAddTwoStates(23, 24); + jjCheckNAddTwoStates(24, 25); break; - case 23: + case 24: if ((0x3ff000000000000L & l) != 0L) - jjCheckNAddTwoStates(23, 24); + jjCheckNAddTwoStates(24, 25); break; - case 24: case 25: + case 26: if ((0x3ff000000000000L & l) != 0L) - jjCheckNAddTwoStates(25, 26); + jjCheckNAddTwoStates(26, 27); break; - case 26: + case 27: if ((0xf00000000000L & l) != 0L) - jjCheckNAdd(27); + jjCheckNAdd(28); break; - case 27: + case 28: if ((0x3ff000000000000L & l) == 0L) break; if (kind > 7) kind = 7; - jjCheckNAddTwoStates(22, 27); + jjCheckNAddTwoStates(23, 28); break; - case 28: + case 29: if ((0x3ff000000000000L & l) != 0L) jjCheckNAddStates(12, 17); break; - case 29: + case 30: if ((0x3ff000000000000L & l) == 0L) break; if (kind > 1) kind = 1; jjCheckNAddStates(0, 11); break; - case 30: + case 31: if ((0x3ff000000000000L & l) == 0L) break; if (kind > 1) kind = 1; - jjCheckNAdd(30); + jjCheckNAdd(31); break; - case 31: + case 32: if ((0x3ff000000000000L & l) != 0L) jjCheckNAddStates(24, 26); break; - case 32: + case 33: if ((0x600000000000L & l) != 0L) - jjCheckNAdd(33); + jjCheckNAdd(34); break; - case 33: + case 34: if ((0x3ff000000000000L & l) != 0L) jjCheckNAddStates(27, 29); break; - case 35: + case 36: if ((0x3ff000000000000L & l) != 0L) - jjCheckNAddTwoStates(35, 36); + jjCheckNAddTwoStates(36, 37); break; - case 36: + case 37: if ((0x600000000000L & l) != 0L) - jjCheckNAdd(37); + jjCheckNAdd(38); break; - case 37: + case 38: if ((0x3ff000000000000L & l) == 0L) break; if (kind > 5) kind = 5; - jjCheckNAddTwoStates(36, 37); + jjCheckNAddTwoStates(37, 38); break; - case 38: + case 39: if ((0x3ff000000000000L & l) != 0L) - jjCheckNAddTwoStates(38, 39); + jjCheckNAddTwoStates(39, 40); break; - case 39: + case 40: if (curChar == 46) - jjCheckNAdd(40); + jjCheckNAdd(41); break; - case 40: + case 41: if ((0x3ff000000000000L & l) == 0L) break; if (kind > 6) kind = 6; - jjCheckNAddTwoStates(39, 40); + jjCheckNAddTwoStates(40, 41); break; - case 41: + case 42: if ((0x3ff000000000000L & l) != 0L) - jjCheckNAddTwoStates(41, 42); + jjCheckNAddTwoStates(42, 43); break; - case 42: + case 43: if ((0xf00000000000L & l) != 0L) - jjCheckNAddTwoStates(43, 44); + jjCheckNAddTwoStates(44, 45); break; - case 43: + case 44: if ((0x3ff000000000000L & l) != 0L) - jjCheckNAddTwoStates(43, 44); + jjCheckNAddTwoStates(44, 45); break; - case 44: case 45: + case 46: if ((0x3ff000000000000L & l) == 0L) break; if (kind > 7) kind = 7; - jjCheckNAdd(45); + jjCheckNAdd(46); break; - case 46: + case 47: if ((0x3ff000000000000L & l) != 0L) - jjCheckNAddTwoStates(46, 47); + jjCheckNAddTwoStates(47, 48); break; - case 47: + case 48: if ((0xf00000000000L & l) != 0L) - jjCheckNAddTwoStates(48, 49); + jjCheckNAddTwoStates(49, 50); break; - case 48: + case 49: if ((0x3ff000000000000L & l) != 0L) - jjCheckNAddTwoStates(48, 49); + jjCheckNAddTwoStates(49, 50); break; - case 49: case 50: + case 51: if ((0x3ff000000000000L & l) != 0L) - jjCheckNAddTwoStates(50, 51); + jjCheckNAddTwoStates(51, 52); break; - case 51: + case 52: if ((0xf00000000000L & l) != 0L) - jjCheckNAdd(52); + jjCheckNAdd(53); break; - case 52: + case 53: if ((0x3ff000000000000L & l) == 0L) break; if (kind > 7) kind = 7; - jjCheckNAddTwoStates(47, 52); + jjCheckNAddTwoStates(48, 53); break; - case 53: + case 54: if ((0x3ff000000000000L & l) != 0L) - jjCheckNAddTwoStates(53, 54); + jjCheckNAddTwoStates(54, 55); break; - case 54: + case 55: if ((0xf00000000000L & l) != 0L) - jjCheckNAddTwoStates(55, 56); + jjCheckNAddTwoStates(56, 57); break; - case 55: + case 56: if ((0x3ff000000000000L & l) != 0L) - jjCheckNAddTwoStates(55, 56); + jjCheckNAddTwoStates(56, 57); break; - case 56: case 57: + case 58: if ((0x3ff000000000000L & l) != 0L) - jjCheckNAddTwoStates(57, 58); + jjCheckNAddTwoStates(58, 59); break; - case 58: + case 59: if ((0xf00000000000L & l) != 0L) - jjCheckNAdd(59); + jjCheckNAdd(60); break; - case 59: + case 60: if ((0x3ff000000000000L & l) != 0L) - jjCheckNAddTwoStates(59, 60); + jjCheckNAddTwoStates(60, 61); break; - case 60: + case 61: if ((0xf00000000000L & l) != 0L) - jjCheckNAddTwoStates(61, 62); + jjCheckNAddTwoStates(62, 63); break; - case 61: + case 62: if ((0x3ff000000000000L & l) != 0L) - jjCheckNAddTwoStates(61, 62); + jjCheckNAddTwoStates(62, 63); break; - case 62: case 63: + case 64: if ((0x3ff000000000000L & l) == 0L) break; if (kind > 7) kind = 7; - jjCheckNAddTwoStates(58, 63); + jjCheckNAddTwoStates(59, 64); break; - case 66: + case 67: if (curChar == 39) - jjstateSet[jjnewStateCnt++] = 67; + jjstateSet[jjnewStateCnt++] = 68; break; - case 69: + case 70: if (curChar == 46) - jjCheckNAdd(70); + jjCheckNAdd(71); break; - case 71: + case 72: if (curChar != 46) break; if (kind > 3) kind = 3; - jjCheckNAdd(70); + jjCheckNAdd(71); break; - case 73: + case 74: if (curChar == 38) - jjstateSet[jjnewStateCnt++] = 74; + jjstateSet[jjnewStateCnt++] = 75; break; default : break; } @@ -417,273 +426,273 @@ if ((0x7fffffe07fffffeL & l) != 0L) jjCheckNAddStates(18, 23); break; - case 2: + case 3: if ((0x7fffffe07fffffeL & l) != 0L) jjCheckNAddStates(18, 23); break; - case 3: + case 4: if ((0x7fffffe07fffffeL & l) != 0L) - jjCheckNAddTwoStates(3, 4); + jjCheckNAddTwoStates(4, 5); break; - case 5: + case 6: if ((0x7fffffe07fffffeL & l) != 0L) jjAddStates(36, 37); break; - case 6: + case 7: if (curChar == 95) - jjCheckNAdd(7); + jjCheckNAdd(8); break; - case 7: + case 8: if ((0x7fffffe07fffffeL & l) == 0L) break; if (kind > 7) kind = 7; - jjCheckNAdd(7); + jjCheckNAdd(8); break; - case 8: + case 9: if ((0x7fffffe07fffffeL & l) != 0L) - jjCheckNAddTwoStates(8, 9); + jjCheckNAddTwoStates(9, 10); break; - case 10: + case 11: if ((0x7fffffe07fffffeL & l) != 0L) - jjCheckNAddTwoStates(10, 11); + jjCheckNAddTwoStates(11, 12); break; - case 11: + case 12: if (curChar == 95) - jjCheckNAdd(12); + jjCheckNAdd(13); break; - case 12: + case 13: if ((0x7fffffe07fffffeL & l) != 0L) - jjCheckNAddTwoStates(12, 13); + jjCheckNAddTwoStates(13, 14); break; - case 13: + case 14: if (curChar == 95) - jjCheckNAddTwoStates(14, 15); + jjCheckNAddTwoStates(15, 16); break; - case 14: + case 15: if ((0x7fffffe07fffffeL & l) != 0L) - jjCheckNAddTwoStates(14, 15); + jjCheckNAddTwoStates(15, 16); break; - case 16: + case 17: if ((0x7fffffe07fffffeL & l) == 0L) break; if (kind > 7) kind = 7; - jjCheckNAddTwoStates(11, 16); + jjCheckNAddTwoStates(12, 17); break; - case 17: + case 18: if ((0x7fffffe07fffffeL & l) != 0L) - jjCheckNAddTwoStates(17, 18); + jjCheckNAddTwoStates(18, 19); break; - case 19: + case 20: if ((0x7fffffe07fffffeL & l) != 0L) jjAddStates(38, 39); break; - case 20: + case 21: if (curChar == 95) - jjCheckNAdd(21); + jjCheckNAdd(22); break; - case 21: + case 22: if ((0x7fffffe07fffffeL & l) != 0L) - jjCheckNAddTwoStates(21, 22); + jjCheckNAddTwoStates(22, 23); break; - case 22: + case 23: if (curChar == 95) - jjCheckNAddTwoStates(23, 24); + jjCheckNAddTwoStates(24, 25); break; - case 23: + case 24: if ((0x7fffffe07fffffeL & l) != 0L) - jjCheckNAddTwoStates(23, 24); + jjCheckNAddTwoStates(24, 25); break; - case 25: + case 26: if ((0x7fffffe07fffffeL & l) != 0L) jjAddStates(40, 41); break; - case 26: + case 27: if (curChar == 95) - jjCheckNAdd(27); + jjCheckNAdd(28); break; - case 27: + case 28: if ((0x7fffffe07fffffeL & l) == 0L) break; if (kind > 7) kind = 7; - jjCheckNAddTwoStates(22, 27); + jjCheckNAddTwoStates(23, 28); break; - case 29: + case 30: if ((0x7fffffe07fffffeL & l) == 0L) break; if (kind > 1) kind = 1; jjCheckNAddStates(0, 11); break; - case 30: + case 31: if ((0x7fffffe07fffffeL & l) == 0L) break; if (kind > 1) kind = 1; - jjCheckNAdd(30); + jjCheckNAdd(31); break; - case 31: + case 32: if ((0x7fffffe07fffffeL & l) != 0L) jjCheckNAddStates(24, 26); break; - case 32: + case 33: if (curChar == 95) - jjCheckNAdd(33); + jjCheckNAdd(34); break; - case 33: + case 34: if ((0x7fffffe07fffffeL & l) != 0L) jjCheckNAddStates(27, 29); break; - case 34: + case 35: if (curChar == 64) - jjCheckNAdd(35); + jjCheckNAdd(36); break; - case 35: + case 36: if ((0x7fffffe07fffffeL & l) != 0L) - jjCheckNAddTwoStates(35, 36); + jjCheckNAddTwoStates(36, 37); break; - case 37: + case 38: if ((0x7fffffe07fffffeL & l) == 0L) break; if (kind > 5) kind = 5; - jjCheckNAddTwoStates(36, 37); + jjCheckNAddTwoStates(37, 38); break; - case 38: + case 39: if ((0x7fffffe07fffffeL & l) != 0L) - jjCheckNAddTwoStates(38, 39); + jjCheckNAddTwoStates(39, 40); break; - case 40: + case 41: if ((0x7fffffe07fffffeL & l) == 0L) break; if (kind > 6) kind = 6; - jjCheckNAddTwoStates(39, 40); + jjCheckNAddTwoStates(40, 41); break; - case 41: + case 42: if ((0x7fffffe07fffffeL & l) != 0L) - jjCheckNAddTwoStates(41, 42); + jjCheckNAddTwoStates(42, 43); break; - case 42: + case 43: if (curChar == 95) - jjCheckNAddTwoStates(43, 44); + jjCheckNAddTwoStates(44, 45); break; - case 43: + case 44: if ((0x7fffffe07fffffeL & l) != 0L) - jjCheckNAddTwoStates(43, 44); + jjCheckNAddTwoStates(44, 45); break; - case 45: + case 46: if ((0x7fffffe07fffffeL & l) == 0L) break; if (kind > 7) kind = 7; - jjstateSet[jjnewStateCnt++] = 45; + jjstateSet[jjnewStateCnt++] = 46; break; - case 46: + case 47: if ((0x7fffffe07fffffeL & l) != 0L) - jjCheckNAddTwoStates(46, 47); + jjCheckNAddTwoStates(47, 48); break; - case 47: + case 48: if (curChar == 95) - jjCheckNAddTwoStates(48, 49); + jjCheckNAddTwoStates(49, 50); break; - case 48: + case 49: if ((0x7fffffe07fffffeL & l) != 0L) - jjCheckNAddTwoStates(48, 49); + jjCheckNAddTwoStates(49, 50); break; - case 50: + case 51: if ((0x7fffffe07fffffeL & l) != 0L) jjAddStates(42, 43); break; - case 51: + case 52: if (curChar == 95) - jjCheckNAdd(52); + jjCheckNAdd(53); break; - case 52: + case 53: if ((0x7fffffe07fffffeL & l) == 0L) break; if (kind > 7) kind = 7; - jjCheckNAddTwoStates(47, 52); + jjCheckNAddTwoStates(48, 53); break; - case 53: + case 54: if ((0x7fffffe07fffffeL & l) != 0L) - jjCheckNAddTwoStates(53, 54); + jjCheckNAddTwoStates(54, 55); break; - case 54: + case 55: if (curChar == 95) - jjCheckNAddTwoStates(55, 56); + jjCheckNAddTwoStates(56, 57); break; - case 55: + case 56: if ((0x7fffffe07fffffeL & l) != 0L) - jjCheckNAddTwoStates(55, 56); + jjCheckNAddTwoStates(56, 57); break; - case 57: + case 58: if ((0x7fffffe07fffffeL & l) != 0L) - jjCheckNAddTwoStates(57, 58); + jjCheckNAddTwoStates(58, 59); break; - case 58: + case 59: if (curChar == 95) - jjCheckNAdd(59); + jjCheckNAdd(60); break; - case 59: + case 60: if ((0x7fffffe07fffffeL & l) != 0L) - jjCheckNAddTwoStates(59, 60); + jjCheckNAddTwoStates(60, 61); break; - case 60: + case 61: if (curChar == 95) - jjCheckNAddTwoStates(61, 62); + jjCheckNAddTwoStates(62, 63); break; - case 61: + case 62: if ((0x7fffffe07fffffeL & l) != 0L) - jjCheckNAddTwoStates(61, 62); + jjCheckNAddTwoStates(62, 63); break; - case 63: + case 64: if ((0x7fffffe07fffffeL & l) == 0L) break; if (kind > 7) kind = 7; - jjCheckNAddTwoStates(58, 63); + jjCheckNAddTwoStates(59, 64); break; - case 64: + case 65: if ((0x7fffffe07fffffeL & l) != 0L) jjCheckNAddStates(30, 35); break; - case 65: + case 66: if ((0x7fffffe07fffffeL & l) != 0L) - jjCheckNAddTwoStates(65, 66); + jjCheckNAddTwoStates(66, 67); break; - case 67: + case 68: if ((0x7fffffe07fffffeL & l) == 0L) break; if (kind > 2) kind = 2; - jjCheckNAddTwoStates(66, 67); + jjCheckNAddTwoStates(67, 68); break; - case 68: + case 69: if ((0x7fffffe07fffffeL & l) != 0L) - jjCheckNAddTwoStates(68, 69); + jjCheckNAddTwoStates(69, 70); break; - case 70: + case 71: if ((0x7fffffe07fffffeL & l) != 0L) jjAddStates(44, 45); break; - case 72: + case 73: if ((0x7fffffe07fffffeL & l) != 0L) - jjCheckNAddTwoStates(72, 73); + jjCheckNAddTwoStates(73, 74); break; - case 73: + case 74: if (curChar == 64) - jjCheckNAdd(74); + jjCheckNAdd(75); break; - case 74: + case 75: if ((0x7fffffe07fffffeL & l) == 0L) break; if (kind > 4) kind = 4; - jjCheckNAdd(74); + jjCheckNAdd(75); break; default : break; } @@ -728,262 +737,262 @@ if (jjCanMove_1(hiByte, i1, i2, l1, l2) && kind > 13) kind = 13; break; - case 2: + case 3: if (jjCanMove_2(hiByte, i1, i2, l1, l2)) jjCheckNAddStates(18, 23); break; - case 3: + case 4: if (jjCanMove_2(hiByte, i1, i2, l1, l2)) - jjCheckNAddTwoStates(3, 4); + jjCheckNAddTwoStates(4, 5); break; - case 4: + case 5: if (jjCanMove_3(hiByte, i1, i2, l1, l2)) - jjCheckNAddTwoStates(5, 6); + jjCheckNAddTwoStates(6, 7); break; - case 5: + case 6: if (jjCanMove_2(hiByte, i1, i2, l1, l2)) - jjCheckNAddTwoStates(5, 6); + jjCheckNAddTwoStates(6, 7); break; - case 7: + case 8: if (!jjCanMove_4(hiByte, i1, i2, l1, l2)) break; if (kind > 7) kind = 7; - jjstateSet[jjnewStateCnt++] = 7; + jjstateSet[jjnewStateCnt++] = 8; break; - case 8: + case 9: if (jjCanMove_2(hiByte, i1, i2, l1, l2)) - jjCheckNAddTwoStates(8, 9); + jjCheckNAddTwoStates(9, 10); break; - case 9: + case 10: if (jjCanMove_3(hiByte, i1, i2, l1, l2)) - jjCheckNAddTwoStates(10, 11); + jjCheckNAddTwoStates(11, 12); break; - case 10: + case 11: if (jjCanMove_2(hiByte, i1, i2, l1, l2)) - jjCheckNAddTwoStates(10, 11); + jjCheckNAddTwoStates(11, 12); break; - case 12: + case 13: if (jjCanMove_4(hiByte, i1, i2, l1, l2)) jjAddStates(46, 47); break; - case 14: + case 15: if (jjCanMove_2(hiByte, i1, i2, l1, l2)) jjAddStates(48, 49); break; - case 15: + case 16: if (!jjCanMove_3(hiByte, i1, i2, l1, l2)) break; if (kind > 7) kind = 7; - jjCheckNAddTwoStates(11, 16); + jjCheckNAddTwoStates(12, 17); break; - case 16: + case 17: if (!jjCanMove_2(hiByte, i1, i2, l1, l2)) break; if (kind > 7) kind = 7; - jjCheckNAddTwoStates(11, 16); + jjCheckNAddTwoStates(12, 17); break; - case 17: + case 18: if (jjCanMove_2(hiByte, i1, i2, l1, l2)) - jjCheckNAddTwoStates(17, 18); + jjCheckNAddTwoStates(18, 19); break; - case 18: + case 19: if (jjCanMove_3(hiByte, i1, i2, l1, l2)) - jjCheckNAddTwoStates(19, 20); + jjCheckNAddTwoStates(20, 21); break; - case 19: + case 20: if (jjCanMove_2(hiByte, i1, i2, l1, l2)) - jjCheckNAddTwoStates(19, 20); + jjCheckNAddTwoStates(20, 21); break; - case 21: + case 22: if (jjCanMove_4(hiByte, i1, i2, l1, l2)) - jjCheckNAddTwoStates(21, 22); + jjCheckNAddTwoStates(22, 23); break; - case 23: + case 24: if (jjCanMove_2(hiByte, i1, i2, l1, l2)) jjAddStates(50, 51); break; - case 24: + case 25: if (jjCanMove_3(hiByte, i1, i2, l1, l2)) - jjCheckNAddTwoStates(25, 26); + jjCheckNAddTwoStates(26, 27); break; - case 25: + case 26: if (jjCanMove_2(hiByte, i1, i2, l1, l2)) - jjCheckNAddTwoStates(25, 26); + jjCheckNAddTwoStates(26, 27); break; - case 27: + case 28: if (!jjCanMove_4(hiByte, i1, i2, l1, l2)) break; if (kind > 7) kind = 7; - jjCheckNAddTwoStates(22, 27); + jjCheckNAddTwoStates(23, 28); break; - case 28: + case 29: if (jjCanMove_3(hiByte, i1, i2, l1, l2)) jjCheckNAddStates(12, 17); break; - case 29: + case 30: if (!jjCanMove_4(hiByte, i1, i2, l1, l2)) break; if (kind > 1) kind = 1; jjCheckNAddStates(0, 11); break; - case 30: + case 31: if (!jjCanMove_4(hiByte, i1, i2, l1, l2)) break; if (kind > 1) kind = 1; - jjCheckNAdd(30); + jjCheckNAdd(31); break; - case 31: + case 32: if (jjCanMove_4(hiByte, i1, i2, l1, l2)) jjCheckNAddStates(24, 26); break; - case 33: + case 34: if (jjCanMove_4(hiByte, i1, i2, l1, l2)) jjCheckNAddStates(27, 29); break; - case 35: + case 36: if (jjCanMove_4(hiByte, i1, i2, l1, l2)) - jjCheckNAddTwoStates(35, 36); + jjCheckNAddTwoStates(36, 37); break; - case 37: + case 38: if (!jjCanMove_4(hiByte, i1, i2, l1, l2)) break; if (kind > 5) kind = 5; - jjCheckNAddTwoStates(36, 37); + jjCheckNAddTwoStates(37, 38); break; - case 38: + case 39: if (jjCanMove_4(hiByte, i1, i2, l1, l2)) - jjCheckNAddTwoStates(38, 39); + jjCheckNAddTwoStates(39, 40); break; - case 40: + case 41: if (!jjCanMove_4(hiByte, i1, i2, l1, l2)) break; if (kind > 6) kind = 6; - jjCheckNAddTwoStates(39, 40); + jjCheckNAddTwoStates(40, 41); break; - case 41: + case 42: if (jjCanMove_4(hiByte, i1, i2, l1, l2)) - jjCheckNAddTwoStates(41, 42); + jjCheckNAddTwoStates(42, 43); break; - case 43: + case 44: if (jjCanMove_2(hiByte, i1, i2, l1, l2)) jjAddStates(52, 53); break; - case 44: + case 45: if (!jjCanMove_3(hiByte, i1, i2, l1, l2)) break; if (kind > 7) kind = 7; - jjCheckNAdd(45); + jjCheckNAdd(46); break; - case 45: + case 46: if (!jjCanMove_2(hiByte, i1, i2, l1, l2)) break; if (kind > 7) kind = 7; - jjCheckNAdd(45); + jjCheckNAdd(46); break; - case 46: + case 47: if (jjCanMove_4(hiByte, i1, i2, l1, l2)) - jjCheckNAddTwoStates(46, 47); + jjCheckNAddTwoStates(47, 48); break; - case 48: + case 49: if (jjCanMove_2(hiByte, i1, i2, l1, l2)) jjAddStates(54, 55); break; - case 49: + case 50: if (jjCanMove_3(hiByte, i1, i2, l1, l2)) - jjCheckNAddTwoStates(50, 51); + jjCheckNAddTwoStates(51, 52); break; - case 50: + case 51: if (jjCanMove_2(hiByte, i1, i2, l1, l2)) - jjCheckNAddTwoStates(50, 51); + jjCheckNAddTwoStates(51, 52); break; - case 52: + case 53: if (!jjCanMove_4(hiByte, i1, i2, l1, l2)) break; if (kind > 7) kind = 7; - jjCheckNAddTwoStates(47, 52); + jjCheckNAddTwoStates(48, 53); break; - case 53: + case 54: if (jjCanMove_4(hiByte, i1, i2, l1, l2)) - jjCheckNAddTwoStates(53, 54); + jjCheckNAddTwoStates(54, 55); break; - case 55: + case 56: if (jjCanMove_2(hiByte, i1, i2, l1, l2)) jjAddStates(56, 57); break; - case 56: + case 57: if (jjCanMove_3(hiByte, i1, i2, l1, l2)) - jjCheckNAddTwoStates(57, 58); + jjCheckNAddTwoStates(58, 59); break; - case 57: + case 58: if (jjCanMove_2(hiByte, i1, i2, l1, l2)) - jjCheckNAddTwoStates(57, 58); + jjCheckNAddTwoStates(58, 59); break; - case 59: + case 60: if (jjCanMove_4(hiByte, i1, i2, l1, l2)) jjAddStates(58, 59); break; - case 61: + case 62: if (jjCanMove_2(hiByte, i1, i2, l1, l2)) jjAddStates(60, 61); break; - case 62: + case 63: if (!jjCanMove_3(hiByte, i1, i2, l1, l2)) break; if (kind > 7) kind = 7; - jjCheckNAddTwoStates(58, 63); + jjCheckNAddTwoStates(59, 64); break; - case 63: + case 64: if (!jjCanMove_2(hiByte, i1, i2, l1, l2)) break; if (kind > 7) kind = 7; - jjCheckNAddTwoStates(58, 63); + jjCheckNAddTwoStates(59, 64); break; - case 64: + case 65: if (jjCanMove_2(hiByte, i1, i2, l1, l2)) jjCheckNAddStates(30, 35); break; - case 65: + case 66: if (jjCanMove_2(hiByte, i1, i2, l1, l2)) - jjCheckNAddTwoStates(65, 66); + jjCheckNAddTwoStates(66, 67); break; - case 67: + case 68: if (!jjCanMove_2(hiByte, i1, i2, l1, l2)) break; if (kind > 2) kind = 2; - jjCheckNAddTwoStates(66, 67); + jjCheckNAddTwoStates(67, 68); break; - case 68: + case 69: if (jjCanMove_2(hiByte, i1, i2, l1, l2)) - jjCheckNAddTwoStates(68, 69); + jjCheckNAddTwoStates(69, 70); break; - case 70: + case 71: if (jjCanMove_2(hiByte, i1, i2, l1, l2)) jjAddStates(44, 45); break; - case 72: + case 73: if (jjCanMove_2(hiByte, i1, i2, l1, l2)) - jjCheckNAddTwoStates(72, 73); + jjCheckNAddTwoStates(73, 74); break; - case 74: + case 75: if (!jjCanMove_2(hiByte, i1, i2, l1, l2)) break; if (kind > 4) kind = 4; - jjstateSet[jjnewStateCnt++] = 74; + jjstateSet[jjnewStateCnt++] = 75; break; default : break; } @@ -996,17 +1005,17 @@ kind = 0x7fffffff; } ++curPos; - if ((i = jjnewStateCnt) == (startsAt = 75 - (jjnewStateCnt = startsAt))) + if ((i = jjnewStateCnt) == (startsAt = 76 - (jjnewStateCnt = startsAt))) return curPos; try { curChar = input_stream.readChar(); } catch(java.io.IOException e) { return curPos; } } } static final int[] jjnextStates = { - 30, 31, 32, 34, 38, 39, 41, 42, 46, 47, 53, 54, 5, 6, 10, 11, - 19, 20, 3, 4, 8, 9, 17, 18, 31, 32, 34, 32, 33, 34, 65, 66, - 68, 69, 72, 73, 5, 6, 19, 20, 25, 26, 50, 51, 70, 71, 12, 13, - 14, 15, 23, 24, 43, 44, 48, 49, 55, 56, 59, 60, 61, 62, + 31, 32, 33, 35, 39, 40, 42, 43, 47, 48, 54, 55, 6, 7, 11, 12, + 20, 21, 4, 5, 9, 10, 18, 19, 32, 33, 35, 33, 34, 35, 66, 67, + 69, 70, 73, 74, 6, 7, 20, 21, 26, 27, 51, 52, 71, 72, 13, 14, + 15, 16, 24, 25, 44, 45, 49, 50, 56, 57, 60, 61, 62, 63, }; private static final boolean jjCanMove_0(int hiByte, int i1, int i2, long l1, long l2) { @@ -1095,19 +1104,19 @@ } public static final String[] jjstrLiteralImages = { "", null, null, null, null, null, null, null, null, null, null, null, null, -null, null, null, }; +null, null, null, null, }; public static final String[] lexStateNames = { "DEFAULT", }; static final long[] jjtoToken = { - 0x30ffL, + 0xb0ffL, }; static final long[] jjtoSkip = { - 0x8000L, + 0x10000L, }; protected CharStream input_stream; -private final int[] jjrounds = new int[75]; -private final int[] jjstateSet = new int[150]; +private final int[] jjrounds = new int[76]; +private final int[] jjstateSet = new int[152]; protected char curChar; public StandardTokenizerTokenManager(CharStream stream) { @@ -1129,7 +1138,7 @@ { int i; jjround = 0x80000001; - for (i = 75; i-- > 0;) + for (i = 76; i-- > 0;) jjrounds[i] = 0x80000000; } public void ReInit(CharStream stream, int lexState) @@ -1189,9 +1198,9 @@ jjmatchedKind = 0x7fffffff; jjmatchedPos = 0; curPos = jjMoveStringLiteralDfa0_0(); - if (jjmatchedPos == 0 && jjmatchedKind > 15) + if (jjmatchedPos == 0 && jjmatchedKind > 16) { - jjmatchedKind = 15; + jjmatchedKind = 16; } if (jjmatchedKind != 0x7fffffff) { Index: standard/StandardTokenizerConstants.java =================================================================== --- standard/StandardTokenizerConstants.java (revision 541610) +++ standard/StandardTokenizerConstants.java (working copy) @@ -18,7 +18,8 @@ int CJ = 12; int KOREAN = 13; int DIGIT = 14; - int NOISE = 15; + int PUNCTUATION = 15; + int NOISE = 16; int DEFAULT = 0; @@ -38,6 +39,7 @@ "", "", "", + "", "", }; Index: standard/StandardTokenizer.java =================================================================== --- standard/StandardTokenizer.java (revision 541610) +++ standard/StandardTokenizer.java (working copy) @@ -58,6 +58,9 @@ case CJ: token = jj_consume_token(CJ); break; + case PUNCTUATION: + token = jj_consume_token(PUNCTUATION); + break; case 0: token = jj_consume_token(0); break; @@ -87,7 +90,7 @@ jj_la1_0(); } private static void jj_la1_0() { - jj_la1_0 = new int[] {0x10ff,}; + jj_la1_0 = new int[] {0x90ff,}; } public StandardTokenizer(CharStream stream) { @@ -166,8 +169,8 @@ public ParseException generateParseException() { jj_expentries.removeAllElements(); - boolean[] la1tokens = new boolean[16]; - for (int i = 0; i < 16; i++) { + boolean[] la1tokens = new boolean[17]; + for (int i = 0; i < 17; i++) { la1tokens[i] = false; } if (jj_kind >= 0) { @@ -183,7 +186,7 @@ } } } - for (int i = 0; i < 16; i++) { + for (int i = 0; i < 17; i++) { if (la1tokens[i]) { jj_expentry = new int[1]; jj_expentry[0] = i; Index: standard/StandardTokenizer.jj =================================================================== --- standard/StandardTokenizer.jj (revision 541610) +++ standard/StandardTokenizer.jj (working copy) @@ -158,6 +158,9 @@ "\u1040"-"\u1049" ] > + +| + } SKIP : { // skip unrecognized chars @@ -181,6 +184,7 @@ token = | token = | token = | + token = | token = ) {