args) {
+ super.init(args);
+ pattern = getPattern("pattern");
+ preserveOriginal = args.containsKey("preserve_original") ? Boolean.parseBoolean(args.get("preserve_original")) : true;
+ }
+ @Override
+ public PatternCaptureGroupTokenFilter create(TokenStream input) {
+ return new PatternCaptureGroupTokenFilter(input, preserveOriginal, pattern);
+ }
+}
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceFilter.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceFilter.java (revision 1444706)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceFilter.java (working copy)
@@ -38,7 +38,6 @@
* @see Pattern
*/
public final class PatternReplaceFilter extends TokenFilter {
- private final Pattern p;
private final String replacement;
private final boolean all;
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
@@ -60,7 +59,6 @@
String replacement,
boolean all) {
super(in);
- this.p=p;
this.replacement = (null == replacement) ? "" : replacement;
this.all=all;
this.m = p.matcher(termAtt);
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternCaptureGroupTokenFilter.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternCaptureGroupTokenFilter.java (revision 0)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternCaptureGroupTokenFilter.java (working copy)
@@ -0,0 +1,202 @@
+package org.apache.lucene.analysis.pattern;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.IOException;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.util.CharsRef;
+
+/**
+ * CaptureGroup uses Java regexes to emit multiple tokens - one for each capture
+ * group in one or more patterns.
+ *
+ *
+ * For example, a pattern like:
+ *
+ *
+ *
+ * "(https?://([a-zA-Z\-_0-9.]+))"
+ *
+ *
+ *
+ * when matched against the string "http://www.foo.com/index" would return the
+ * tokens "https://www.foo.com" and "www.foo.com".
+ *
+ *
+ *
+ * If none of the patterns match, or if preserveOriginal is true, the original
+ * token will be preserved.
+ *
+ *
+ * Each pattern is matched as often as it can be, so the pattern
+ * "(...)", when matched against "abcdefghi" would
+ * produce ["abc","def","ghi"]
+ *
+ *
+ * A camelCaseFilter could be written as:
+ *
+ *
+ *
+ * "([A-Z]{2,})",
+ * "(?<![A-Z])([A-Z][a-z]+)",
+ * "(?:^|\\b|(?<=[0-9_])|(?<=[A-Z]{2}))([a-z]+)",
+ * "([0-9]+)"
+ *
+ *
+ *
+ * plus if {@link #preserveOriginal} is true, it would also return
+ * "camelCaseFilter
+ *
+ */
+public final class PatternCaptureGroupTokenFilter extends TokenFilter {
+
+ private final CharTermAttribute charTermAttr = addAttribute(CharTermAttribute.class);
+ private final PositionIncrementAttribute posAttr = addAttribute(PositionIncrementAttribute.class);
+ private final OffsetAttribute offsetAttr = addAttribute(OffsetAttribute.class);
+ private final Matcher[] matchers;
+ private final CharsRef spare = new CharsRef();
+ private final int[] groupCounts;
+ private final boolean preserveOriginal;
+ private int[] currentGroup;
+ private int charOffsetStart;
+ private int currentMatcher;
+
+ /**
+ * @param input
+ * the input {@link TokenStream}
+ * @param preserveOriginal
+ * set to true to return the original token even if one of the
+ * patterns matches
+ * @param patterns
+ * an array of {@link Pattern} objects to match against each token
+ */
+ public PatternCaptureGroupTokenFilter(TokenStream input,
+ boolean preserveOriginal, Pattern... patterns) {
+ super(input);
+ this.preserveOriginal = preserveOriginal;
+ this.matchers = new Matcher[patterns.length];
+ this.groupCounts = new int[patterns.length];
+ this.currentGroup = new int[patterns.length];
+ for (int i = 0; i < patterns.length; i++) {
+ this.matchers[i] = patterns[i].matcher("");
+ this.groupCounts[i] = this.matchers[i].groupCount();
+ this.currentGroup[i] = -1;
+ }
+ }
+
+ private boolean nextCapture() {
+ int min_offset = Integer.MAX_VALUE;
+ currentMatcher = -1;
+ Matcher matcher;
+
+ for (int i = 0; i < matchers.length; i++) {
+ matcher = matchers[i];
+ if (currentGroup[i] == -1) {
+ currentGroup[i] = matcher.find() ? 1 : 0;
+ }
+ if (currentGroup[i] != 0) {
+ while (currentGroup[i] < groupCounts[i] + 1) {
+ final int start = matcher.start(currentGroup[i]);
+ final int end = matcher.end(currentGroup[i]);
+ if (start == end || preserveOriginal && start == 0
+ && spare.length == end) {
+ currentGroup[i]++;
+ continue;
+ }
+ if (start < min_offset) {
+ min_offset = start;
+ currentMatcher = i;
+ }
+ break;
+ }
+ if (currentGroup[i] == groupCounts[i] + 1) {
+ currentGroup[i] = -1;
+ i--;
+ }
+ }
+ }
+ return currentMatcher != -1;
+ }
+
+ @Override
+ public boolean incrementToken() throws IOException {
+
+ if (currentMatcher != -1 && nextCapture()) {
+
+ clearAttributes();
+
+ final int start = matchers[currentMatcher]
+ .start(currentGroup[currentMatcher]);
+ final int end = matchers[currentMatcher]
+ .end(currentGroup[currentMatcher]);
+
+ posAttr.setPositionIncrement(0);
+ charTermAttr.copyBuffer(spare.chars, start, end - start);
+ offsetAttr.setOffset(charOffsetStart + start, charOffsetStart + end);
+ currentGroup[currentMatcher]++;
+ return true;
+ }
+
+ if (!input.incrementToken()) {
+ return false;
+ }
+
+ char[] buffer = charTermAttr.buffer();
+ int length = charTermAttr.length();
+ spare.copyChars(buffer, 0, length);
+ charOffsetStart = offsetAttr.startOffset();
+
+ for (int i = 0; i < matchers.length; i++) {
+ matchers[i].reset(spare);
+ currentGroup[i] = -1;
+ }
+
+ if (preserveOriginal) {
+ currentMatcher = 0;
+ } else if (nextCapture()) {
+ final int start = matchers[currentMatcher]
+ .start(currentGroup[currentMatcher]);
+ final int end = matchers[currentMatcher]
+ .end(currentGroup[currentMatcher]);
+
+ // if we start at 0 we can simply set the length and save the copy
+ if (start == 0) {
+ charTermAttr.setLength(end);
+ } else {
+ charTermAttr.copyBuffer(spare.chars, start, end - start);
+ }
+ offsetAttr.setOffset(charOffsetStart + start, charOffsetStart + end);
+ currentGroup[currentMatcher]++;
+ }
+ return true;
+
+ }
+
+ @Override
+ public void reset() throws IOException {
+ super.reset();
+ currentMatcher = -1;
+ }
+
+}
Index: lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.util.TokenFilterFactory
===================================================================
--- lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.util.TokenFilterFactory (revision 1444706)
+++ lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.util.TokenFilterFactory (working copy)
@@ -69,6 +69,7 @@
org.apache.lucene.analysis.no.NorwegianLightStemFilterFactory
org.apache.lucene.analysis.no.NorwegianMinimalStemFilterFactory
org.apache.lucene.analysis.pattern.PatternReplaceFilterFactory
+org.apache.lucene.analysis.pattern.PatternCaptureGroupFilterFactory
org.apache.lucene.analysis.payloads.DelimitedPayloadTokenFilterFactory
org.apache.lucene.analysis.payloads.NumericPayloadTokenFilterFactory
org.apache.lucene.analysis.payloads.TokenOffsetPayloadTokenFilterFactory
Index: lucene/CHANGES.txt
===================================================================
--- lucene/CHANGES.txt (revision 1444706)
+++ lucene/CHANGES.txt (working copy)
@@ -155,6 +155,12 @@
to Lucene42DocValuesConsumer) if you want to make this tradeoff.
(Adrien Grand, Robert Muir)
+* LUCENE-4766: Added a Pattern TokenFilter which emits a token for every
+ capturing group. This filter accepts multiple patterns and emits tokens for
+ every capturing group that is matched in any pattern. Patterns are not
+ anchored to the beginning and end of the string, so each pattern can produce
+ multiple matches. (Clinton Gormley, Simon Willnauer)
+
API Changes
* LUCENE-4709: FacetResultNode no longer has a residue field. (Shai Erera)