Index: src/main/java/org/apache/jackrabbit/oak/index/IndexCommand.java =================================================================== --- src/main/java/org/apache/jackrabbit/oak/index/IndexCommand.java (revision 1869743) +++ src/main/java/org/apache/jackrabbit/oak/index/IndexCommand.java (working copy) @@ -31,6 +31,7 @@ import java.util.List; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.function.Predicate; import com.google.common.base.Joiner; import com.google.common.base.Stopwatch; @@ -43,6 +44,9 @@ import org.apache.jackrabbit.oak.api.CommitFailedException; import org.apache.jackrabbit.oak.index.indexer.document.DocumentStoreIndexer; import org.apache.jackrabbit.oak.plugins.index.importer.IndexDefinitionUpdater; +import org.apache.jackrabbit.oak.plugins.index.search.IndexNode; +import org.apache.jackrabbit.oak.plugins.index.search.SizeEstimator; +import org.apache.jackrabbit.oak.plugins.index.search.spi.query.FulltextIndex; import org.apache.jackrabbit.oak.run.cli.CommonOptions; import org.apache.jackrabbit.oak.run.cli.DocumentBuilderCustomizer; import org.apache.jackrabbit.oak.run.cli.NodeStoreFixture; @@ -50,6 +54,8 @@ import org.apache.jackrabbit.oak.run.cli.Options; import org.apache.jackrabbit.oak.run.commons.Command; import org.apache.jackrabbit.oak.run.commons.LoggingInitializer; +import org.apache.jackrabbit.oak.spi.query.Cursor; +import org.apache.jackrabbit.oak.spi.state.NodeState; import org.apache.jackrabbit.oak.spi.whiteboard.Registration; import org.apache.jackrabbit.util.ISO8601; import org.slf4j.Logger; @@ -60,6 +66,7 @@ import static java.util.Collections.emptyMap; public class IndexCommand implements Command { + private static final Logger log = LoggerFactory.getLogger(IndexCommand.class); private static final String LOG_SUFFIX = "indexing"; Index: src/main/java/org/apache/jackrabbit/oak/index/merge/IndexDefMerger.java =================================================================== --- src/main/java/org/apache/jackrabbit/oak/index/merge/IndexDefMerger.java (nonexistent) +++ src/main/java/org/apache/jackrabbit/oak/index/merge/IndexDefMerger.java (working copy) @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.merge; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map.Entry; +import java.util.Set; +import java.util.stream.Collectors; + +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; + +/** + * Utility that allows to merge index definitions. + */ +public class IndexDefMerger { + + private static HashSet IGNORE_LEVEL_0 = new HashSet<>(Arrays.asList( + "reindex", "refresh", "seed", "reindexCount")); + + /** + * Merge index definition changes. + * + * @param ancestor the common ancestor (the old product index, e.g. lucene) + * @param custom the latest customized version (e.g. lucene-custom-1) + * @param product the latest product index (e.g. lucene-2) + * @return the merged index definition (e.g. lucene-2-custom-1) + */ + public static JsonObject merge(JsonObject ancestor, JsonObject custom, JsonObject product) { + ArrayList conflicts = new ArrayList<>(); + JsonObject merged = merge(0, ancestor, custom, product, conflicts); + if (!conflicts.isEmpty()) { + throw new UnsupportedOperationException("Conflicts detected: " + conflicts); + } + return merged; + } + + private static boolean isSame(JsonElement a, JsonElement b) { + if (a == null || b == null) { + return a == b; + } + return a.equals(b); + } + + private static JsonElement get(JsonObject obj, String key) { + return obj == null ? null : obj.get(key); + } + + private static boolean isValue(JsonElement a) { + return a != null && !a.isJsonObject(); + } + + private static JsonObject merge(int level, JsonObject ancestor, JsonObject custom, JsonObject product, + ArrayList conflicts) { + JsonObject merged = new JsonObject(); + Set keys = new HashSet<>(); + keys.addAll(IndexJsonUtils.getKeys(ancestor)); + keys.addAll(IndexJsonUtils.getKeys(custom)); + keys.addAll(IndexJsonUtils.getKeys(product)); + for(String k : keys) { + if (level == 0 && IGNORE_LEVEL_0.contains(k)) { + // ignore some properties + continue; + } + if (k.startsWith(":")) { + // ignore hidden nodes or properties + continue; + } + JsonElement a = get(ancestor, k); + JsonElement c = get(custom, k); + JsonElement p = get(product, k); + JsonElement result; + if (isSame(a, p) || isSame(c, p)) { + result = c; + } else if (isSame(a, c)) { + result = p; + } else { + if (isValue(a) | isValue(c) | isValue(p)) { + conflicts.add("Could not merge value; key=" + k + "; ancestor=" + a + "; custom=" + c + "; product=" + p); + result = a; + } else { + result = merge(level + 1, (JsonObject) a, (JsonObject) c, (JsonObject) p, conflicts); + } + } + if (result != null) { + merged.add(k, result); + } + } + return merged; + } + + /** + * For indexes that were modified both by the customer and in the product, merge + * the changes, and create a new index. + * + * The new index (if any) is stored in the "newIndexes" object. + * + * @param newIndexes the new indexes + * @param allIndexes all index definitions (including the new ones) + */ + public static void merge(JsonObject newIndexes, JsonObject allIndexes) { + + // TODO when merging, we keep the product index, so two indexes are created. + // e.g. lucene, lucene-custom-1, lucene-2, lucene-2-custom-1 + // but if we don't have lucene-2, then we can't merge lucene-3. + + // TODO when merging, e.g. lucene-2-custom-1 is created. but + // it is only imported in the read-write repo, not in the read-only + // repository currently. so this new index won't be used; + // instead, lucene-2 will be used + + List newNames = IndexJsonUtils.getKeys(newIndexes).stream().map(s -> IndexName.parse(s)) + .collect(Collectors.toList()); + Collections.sort(newNames); + List allNames = IndexJsonUtils.getKeys(allIndexes).stream().map(s -> IndexName.parse(s)) + .collect(Collectors.toList()); + Collections.sort(allNames); + HashMap mergedMap = new HashMap<>(); + for (IndexName n : newNames) { + if (n.getCustomVersion() == 0) { + IndexName latest = n.getLatestCustomized(allNames); + IndexName ancestor = n.getLatestProduct(allNames); + if (latest != null && ancestor != null) { + if (n.compareTo(latest) <= 0 || n.compareTo(ancestor) <= 0) { + // ignore older versions of indexes + continue; + } + JsonObject latestCustomized = allIndexes.get(latest.getNodeName()).getAsJsonObject(); + JsonObject latestAncestor = allIndexes.get(ancestor.getNodeName()).getAsJsonObject(); + JsonObject newProduct = newIndexes.get(n.getNodeName()).getAsJsonObject(); + JsonObject merged = merge(latestAncestor, latestCustomized, newProduct); + mergedMap.put(n.nextCustomizedName(), merged); + } + } + } + for (Entry e : mergedMap.entrySet()) { + newIndexes.add(e.getKey(), e.getValue()); + } + } + +} \ No newline at end of file Index: src/main/java/org/apache/jackrabbit/oak/index/merge/IndexJsonUtils.java =================================================================== --- src/main/java/org/apache/jackrabbit/oak/index/merge/IndexJsonUtils.java (nonexistent) +++ src/main/java/org/apache/jackrabbit/oak/index/merge/IndexJsonUtils.java (working copy) @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.merge; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import com.google.gson.GsonBuilder; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; + +/** + * Json Utilities. + */ +public class IndexJsonUtils { + + /** + * Get the names of all keys in this object. + * + * @param json the json object + * @return the set of keys + */ + public static List getKeys(JsonObject json) { + if (json == null) { + return Collections.emptyList(); + } + return json.entrySet().stream().map(Map.Entry::getKey).collect(Collectors.toList()); + } + + /** + * Pretty-print a Json element. + * + * @param obj the json element + * @return the pretty-printed version + */ + public static String pretty(JsonElement obj) { + return new GsonBuilder().setPrettyPrinting().create().toJson(obj); + } + +} Index: src/main/java/org/apache/jackrabbit/oak/index/merge/IndexMerge.java =================================================================== --- src/main/java/org/apache/jackrabbit/oak/index/merge/IndexMerge.java (nonexistent) +++ src/main/java/org/apache/jackrabbit/oak/index/merge/IndexMerge.java (working copy) @@ -0,0 +1,278 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.merge; + +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringReader; +import java.io.StringWriter; +import java.lang.management.ManagementFactory; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +import org.apache.felix.inventory.Format; +import org.apache.jackrabbit.oak.Oak; +import org.apache.jackrabbit.oak.api.CommitFailedException; +import org.apache.jackrabbit.oak.api.Type; +import org.apache.jackrabbit.oak.commons.PathUtils; +import org.apache.jackrabbit.oak.plugins.index.IndexPathService; +import org.apache.jackrabbit.oak.plugins.index.IndexPathServiceImpl; +import org.apache.jackrabbit.oak.plugins.index.IndexUpdateProvider; +import org.apache.jackrabbit.oak.plugins.index.inventory.IndexDefinitionPrinter; +import org.apache.jackrabbit.oak.plugins.index.lucene.IndexTracker; +import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorProvider; +import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexProvider; +import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.DocumentQueue; +import org.apache.jackrabbit.oak.plugins.index.property.PropertyIndexEditorProvider; +import org.apache.jackrabbit.oak.run.cli.NodeStoreFixture; +import org.apache.jackrabbit.oak.run.cli.NodeStoreFixtureProvider; +import org.apache.jackrabbit.oak.run.cli.Options; +import org.apache.jackrabbit.oak.spi.commit.CommitInfo; +import org.apache.jackrabbit.oak.spi.commit.EditorHook; +import org.apache.jackrabbit.oak.spi.commit.EmptyHook; +import org.apache.jackrabbit.oak.spi.commit.Observer; +import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.apache.jackrabbit.oak.spi.state.NodeStore; +import org.apache.jackrabbit.oak.stats.StatisticsProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.util.concurrent.MoreExecutors; +import com.google.gson.Gson; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonPrimitive; + +import joptsimple.OptionParser; +import joptsimple.OptionSet; +import joptsimple.OptionSpec; + +/** + * Merge custom index definitions with out-of-the-box index definitions. + */ +public class IndexMerge { + + private final static Logger LOG = LoggerFactory.getLogger(IndexName.class); + + public static void main(String... args) throws Exception { + new IndexMerge().execute(args); + } + + private boolean quiet; + + /** + * Execute the command. + * + * @param args the command line arguments + */ + public void execute(String... args) throws Exception { + OptionParser parser = new OptionParser(); + OptionSpec quietOption = parser.accepts("quiet", "be less chatty"); + Options opts = new Options(); + OptionSet options = opts.parseAndConfigure(parser, args); + quiet = options.has(quietOption); + boolean isReadWrite = opts.getCommonOpts().isReadWrite(); + boolean success = true; + if (!isReadWrite) { + log("Repository connected in read-only mode. Use '--read-write' for write operations"); + } + try (NodeStoreFixture fixture = NodeStoreFixtureProvider.create(opts)) { + NodeStore nodeStore = fixture.getStore(); + + StatisticsProvider statisticsProvider = StatisticsProvider.NOOP; + Oak oak = new Oak(nodeStore).with(ManagementFactory.getPlatformMBeanServer()); + oak.getWhiteboard().register(StatisticsProvider.class, statisticsProvider, Collections.emptyMap()); + LuceneIndexProvider provider = createLuceneIndexProvider(); + oak.with((QueryIndexProvider) provider) + .with((Observer) provider) + .with(createLuceneIndexEditorProvider()); + + JsonObject indexes = getIndexDefinitions(nodeStore); + // the superseded indexes of the old repository + List supersededKeys = new ArrayList<>(getSupersededIndexDefs(indexes)); + Collections.sort(supersededKeys); + + // keep only new indexes that are not superseded + for (String superseded : supersededKeys) { + if (indexes.has(superseded)) { + log("Ignoring superseded index " + superseded); + indexes.remove(superseded); + } + } + Set indexKeys = new HashSet<>(IndexJsonUtils.getKeys(indexes)); + + IndexDefMerger.merge(indexes, indexes); + + Set newIndexKeys = new HashSet<>(IndexJsonUtils.getKeys(indexes)); + newIndexKeys.removeAll(indexKeys); + if (newIndexKeys.isEmpty()) { + log("No indexes to merge"); + } + for (String newIndexKey : newIndexKeys) { + log("New index: " + newIndexKey); + JsonObject merged = indexes.get(newIndexKey).getAsJsonObject(); + String def = IndexJsonUtils.pretty(merged); + log("Merged definition: " + def); + if (isReadWrite) { + storeIndex(nodeStore, newIndexKey, merged); + } + } + } + if (!success) { + System.exit(1); + } + } + + private void storeIndex(NodeStore ns, String newIndexName, JsonObject indexDef) { + NodeBuilder rootBuilder = ns.getRoot().builder(); + NodeBuilder b = rootBuilder; + for(String p : PathUtils.elements(newIndexName)) { + b = b.child(p); + } + build(" ", b, indexDef); + EditorHook hook = new EditorHook( + new IndexUpdateProvider(new PropertyIndexEditorProvider())); + try { + ns.merge(rootBuilder, hook, CommitInfo.EMPTY); + log("Added index " + newIndexName); + } catch (CommitFailedException e) { + LOG.error("Failed to add index " + newIndexName, e); + } + } + + private void build(String linePrefix, NodeBuilder builder, JsonObject json) { + for(Entry e : json.entrySet()) { + String k = e.getKey(); + JsonElement el = e.getValue(); + if (el.isJsonObject()) { + log(linePrefix + "child " + k); + build(linePrefix + " ", builder.child(k), (JsonObject) el); + } else if (el.isJsonArray()) { + Iterable it = StreamSupport.stream(el.getAsJsonArray().spliterator(), false) + .map(JsonElement::getAsString).collect((Collectors.toSet())); + log(linePrefix + "array " + k + " = " + it + " (String[])"); + builder.setProperty(k, it, Type.STRINGS); + } else if (el.isJsonPrimitive()) { + JsonPrimitive p = el.getAsJsonPrimitive(); + if (p.isBoolean()) { + log(linePrefix + "property " + k + " = " + p.getAsBoolean() + " (Boolean)"); + builder.setProperty(k, p.getAsBoolean()); + } else if (p.isNumber()) { + String num = p.getAsString(); + boolean isDouble = num.indexOf('.') >= 0; + if (isDouble) { + log(linePrefix + "property " + k + " = " + p.getAsDouble() + " (Double)"); + builder.setProperty(k, p.getAsDouble()); + } else { + log(linePrefix + "property " + k + " = " + p.getAsLong() + " (Long)"); + builder.setProperty(k, p.getAsLong()); + } + } else { + String value = el.getAsString(); + // TODO get the list of possible prefixes + if(value.startsWith("nam:")) { + String v = value.substring("nam:".length()); + log(linePrefix + "property " + k + " = " + v + " (Name)"); + builder.setProperty(k, v, Type.NAME); + } else if(value.startsWith("str:")) { + String v = value.substring("str:".length()); + log(linePrefix + "property " + k + " = " + v + " (String)"); + builder.setProperty(k, v); + } else { + log(linePrefix + "property " + k + " = " + value + " (String)"); + builder.setProperty(k, value); + } + } + } + } + } + + /** + * Get the names of the index definitions that are superseded in one of the + * indexes. + * + * @param indexDefs all index definitions + * @return the superseded indexes + */ + public static Set getSupersededIndexDefs(JsonObject indexDefs) { + return indexDefs.entrySet().stream() + .map(Map.Entry::getValue) + .map(e -> (JsonObject) e) + .filter(o -> o.has("supersedes")) + .map(o -> o.getAsJsonArray("supersedes")) + .flatMap(arr -> StreamSupport.stream(arr.spliterator(), false)) + .map(JsonElement::getAsString) + // ignore superseded properties + .filter(i -> !i.contains("/@")) + .collect(Collectors.toSet()); + } + + /** + * Get the the index definitions from a node store. It uses the index path + * service and index definition printer from Oak. + * + * @param nodeStore the source node store + * @return a JSON object with all index definitions + */ + private static JsonObject getIndexDefinitions(NodeStore nodeStore) throws IOException { + IndexPathService imageIndexPathService = new IndexPathServiceImpl(nodeStore); + IndexDefinitionPrinter indexDefinitionPrinter = new IndexDefinitionPrinter(nodeStore, imageIndexPathService); + StringWriter writer = new StringWriter(); + PrintWriter printWriter = new PrintWriter(writer); + indexDefinitionPrinter.print(printWriter, Format.JSON, false); + printWriter.flush(); + writer.flush(); + StringReader reader = new StringReader(writer.toString()); + return new Gson().fromJson(reader, JsonObject.class); + } + + private void log(String message) { + if(!quiet) { + System.out.println(message); + } + } + + private static LuceneIndexEditorProvider createLuceneIndexEditorProvider() { + LuceneIndexEditorProvider ep = new LuceneIndexEditorProvider(); + ScheduledExecutorService executorService = MoreExecutors.getExitingScheduledExecutorService( + (ScheduledThreadPoolExecutor) Executors.newScheduledThreadPool(5)); + StatisticsProvider statsProvider = StatisticsProvider.NOOP; + int queueSize = Integer.getInteger("queueSize", 1000); + IndexTracker tracker = new IndexTracker(); + DocumentQueue queue = new DocumentQueue(queueSize, tracker, executorService, statsProvider); + ep.setIndexingQueue(queue); + return ep; + } + + private static LuceneIndexProvider createLuceneIndexProvider() { + return new LuceneIndexProvider(); + } + +} Index: src/main/java/org/apache/jackrabbit/oak/index/merge/IndexName.java =================================================================== --- src/main/java/org/apache/jackrabbit/oak/index/merge/IndexName.java (nonexistent) +++ src/main/java/org/apache/jackrabbit/oak/index/merge/IndexName.java (working copy) @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.index.merge; + +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * An index name, which possibly contains two version numbers: the product + * version number, and the customer version number. + * + * The format of an index node name is: + * - The name of the index, + * - optionally a dash ('-') and the product version number, + * - optionally "-custom-" and the customer version number. + * + * If the node name doesn't contain version numbers / dashes, then version 0 is + * assumed (for both the product version number and customer version number). + */ +public class IndexName implements Comparable { + + private final static Logger LOG = LoggerFactory.getLogger(IndexName.class); + + private final String nodeName; + private final String baseName; + private final boolean isVersioned; + private final int productVersion; + private final int customerVersion; + private final boolean isLegal; + + /** + * Parse the node name. Both node names with version and without version are + * supported. + * + * @param nodeName the node name (starting from root; e.g. "/oak:index/lucene") + * @return the index name object + */ + public static IndexName parse(final String nodeName) { + String baseName = nodeName; + int index = baseName.lastIndexOf('-'); + if (index < 0) { + return new IndexName(nodeName, true); + } + String last = baseName.substring(index + 1); + baseName = baseName.substring(0, index); + try { + int v1 = Integer.parseInt(last); + if (!baseName.endsWith("-custom")) { + return new IndexName(nodeName, baseName, v1, 0); + } + baseName = baseName.substring(0, + baseName.length() - "-custom".length()); + index = baseName.lastIndexOf('-'); + if (index < 0) { + return new IndexName(nodeName, baseName, 0, v1); + } + last = baseName.substring(index + 1); + baseName = baseName.substring(0, index); + int v2 = Integer.parseInt(last); + return new IndexName(nodeName, baseName, v2, v1); + } catch (NumberFormatException e) { + LOG.warn("Index name format error: " + nodeName); + return new IndexName(nodeName, false); + } + } + + private IndexName(String nodeName, boolean isLegal) { + // not versioned + this.nodeName = nodeName; + this.baseName = nodeName; + this.isVersioned = false; + this.productVersion = 0; + this.customerVersion = 0; + this.isLegal = isLegal; + } + + private IndexName(String nodeName, String baseName, int productVersion, int customerVersion) { + // versioned + this.nodeName = nodeName; + this.baseName = baseName; + this.isVersioned = true; + this.productVersion = productVersion; + this.customerVersion = customerVersion; + this.isLegal = true; + } + + public String toString() { + return nodeName + + " base=" + baseName + + (isVersioned ? " versioned": "") + + " product=" + productVersion + + " custom=" + customerVersion + + (isLegal ? "" : " illegal"); + } + + @Override + public int compareTo(IndexName o) { + int comp = baseName.compareTo(o.baseName); + if (comp != 0) { + return comp; + } + comp = Integer.compare(productVersion, o.productVersion); + if (comp != 0) { + return comp; + } + return Integer.compare(customerVersion, o.customerVersion); + } + + public IndexName getLatestCustomized(List all) { + IndexName latest = null; + for (IndexName n : all) { + if (n.baseName.equals(baseName)) { + if (n.customerVersion > 0) { + if (latest == null || n.compareTo(latest) > 0) { + latest = n; + } + } + } + } + return latest; + } + + public IndexName getLatestProduct(List all) { + IndexName latest = null; + for (IndexName n : all) { + if (n.baseName.equals(baseName)) { + if (compareTo(n) > 0 && n.customerVersion == 0) { + if (latest == null || n.compareTo(latest) > 0) { + latest = n; + } + } + } + } + return latest; + } + + public String getNodeName() { + return nodeName; + } + + public int getCustomVersion() { + return customerVersion; + } + + public String nextCustomizedName() { + return baseName + "-" + productVersion + "-custom-" + (customerVersion + 1); + } + +} \ No newline at end of file Index: src/main/java/org/apache/jackrabbit/oak/run/AvailableModes.java =================================================================== --- src/main/java/org/apache/jackrabbit/oak/run/AvailableModes.java (revision 1869743) +++ src/main/java/org/apache/jackrabbit/oak/run/AvailableModes.java (working copy) @@ -45,6 +45,7 @@ .put("garbage", new GarbageCommand()) .put("help", new HelpCommand()) .put("history", new HistoryCommand()) + .put("index-merge", new IndexMergeCommand()) .put(IndexCommand.NAME, new IndexCommand()) .put(IOTraceCommand.NAME, new IOTraceCommand()) .put(JsonIndexCommand.INDEX, new JsonIndexCommand()) Index: src/main/java/org/apache/jackrabbit/oak/run/IndexMergeCommand.java =================================================================== --- src/main/java/org/apache/jackrabbit/oak/run/IndexMergeCommand.java (nonexistent) +++ src/main/java/org/apache/jackrabbit/oak/run/IndexMergeCommand.java (working copy) @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.run; + +import org.apache.jackrabbit.oak.index.merge.IndexMerge; +import org.apache.jackrabbit.oak.run.commons.Command; + +public class IndexMergeCommand implements Command { + + @Override + public void execute(String... args) throws Exception { + new IndexMerge().execute(args); + } + +} Index: src/test/java/org/apache/jackrabbit/oak/index/merge/IndexDefMergerTest.java =================================================================== --- src/test/java/org/apache/jackrabbit/oak/index/merge/IndexDefMergerTest.java (nonexistent) +++ src/test/java/org/apache/jackrabbit/oak/index/merge/IndexDefMergerTest.java (working copy) @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.merge; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.Map.Entry; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +import org.apache.jackrabbit.oak.api.CommitFailedException; +import org.apache.jackrabbit.oak.api.Type; +import org.apache.jackrabbit.oak.spi.commit.CommitHook; +import org.apache.jackrabbit.oak.spi.commit.CommitInfo; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.apache.jackrabbit.oak.spi.state.NodeStore; +import org.jetbrains.annotations.NotNull; +import org.junit.Test; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; + +/** + * Test merging index definitions. + */ +public class IndexDefMergerTest { + + @Test + public void merge() throws IOException, CommitFailedException { + JsonObject json = readFromResource("merge.txt"); + JsonArray array = json.get("tests").getAsJsonArray(); + for (int i = 0; i < array.size(); i++) { + JsonElement e = array.get(i); + merge(e.getAsJsonObject()); + } + } + + @Test + public void mergeIndexes() throws IOException, CommitFailedException { + JsonObject json = readFromResource("mergeIndexes.txt"); + JsonArray array = json.get("tests").getAsJsonArray(); + for (int i = 0; i < array.size(); i++) { + JsonElement e = array.get(i); + mergeIndexes(e.getAsJsonObject()); + } + } + + private void mergeIndexes(JsonObject e) { + JsonObject all = e.get("all").getAsJsonObject(); + JsonObject newDefs = e.get("new").getAsJsonObject(); + JsonObject expectedNew = e.get("expectedNew").getAsJsonObject(); + IndexDefMerger.merge(newDefs, all); + assertEquals("expected: " + IndexJsonUtils.pretty(expectedNew) + " got: " + + IndexJsonUtils.pretty(newDefs), newDefs, expectedNew); + } + + private void merge(JsonObject e) { + JsonObject ancestor = e.get("ancestor").getAsJsonObject(); + JsonObject custom = e.get("custom").getAsJsonObject(); + JsonObject product = e.get("product").getAsJsonObject(); + JsonElement expected = e.get("expected"); + try { + JsonObject got = IndexDefMerger.merge(ancestor, custom, product); + assertEquals("expected: " + IndexJsonUtils.pretty(expected) + " got: " + IndexJsonUtils.pretty(got), expected, got); + } catch (UnsupportedOperationException e2) { + assertEquals(expected.toString(), "\"" + e2.getMessage() + "\""); + } + } + + static JsonObject readFromResource(String resourceName) throws IOException { + try (InputStreamReader reader = new InputStreamReader( + IndexDefMergerTest.class.getResourceAsStream(resourceName))) { + return new Gson().fromJson(reader, JsonObject.class); + } + } + + static void commit(NodeStore ns, JsonObject json) throws CommitFailedException { + NodeState root = ns.getRoot(); + NodeBuilder rootBuilder = root.builder(); + build(rootBuilder, json); + ns.merge(rootBuilder, new CommitHook() { + @Override + public @NotNull NodeState processCommit(NodeState before, NodeState after, CommitInfo info) + throws CommitFailedException { + return after; + } + }, CommitInfo.EMPTY); + } + + private static void build(NodeBuilder builder, JsonObject json) { + for(Entry e : json.entrySet()) { + String k = e.getKey(); + JsonElement el = e.getValue(); + if (el.isJsonObject()) { + System.out.println("child " + k); + build(builder.child(k), (JsonObject) el); + } else if (el.isJsonArray()) { + Iterable it = StreamSupport.stream(el.getAsJsonArray().spliterator(), false) + .map(JsonElement::getAsString).collect((Collectors.toSet())); + System.out.println("array " + k + " = " + it); + builder.setProperty(k, it, Type.STRINGS); + } else if (el.isJsonPrimitive()) { + System.out.println("property " + k + " = " + el.getAsString()); + String value = el.getAsString(); + if(value.startsWith("nam:")) { + builder.setProperty(k, value.substring("nam:".length()), Type.NAME); + } else { + builder.setProperty(k, value); + } + } + } + System.out.println("done"); + } + +} Index: src/test/resources/org/apache/jackrabbit/oak/index/merge/merge.txt =================================================================== --- src/test/resources/org/apache/jackrabbit/oak/index/merge/merge.txt (nonexistent) +++ src/test/resources/org/apache/jackrabbit/oak/index/merge/merge.txt (working copy) @@ -0,0 +1,204 @@ +{"tests": +[ +{ + "ancestor": {"value": 1, a-old: 0}, + "custom": {"value": 2, "b-new": 3}, + "product": {"value": 1, "c-new": 4}, + "expected": {"value": 2, "b-new": 3, "c-new": 4} +}, +{ + "ancestor": {"o": {"a": 1}}, + "custom": {"o": {"a": 2, "c": 10}}, + "product": {"o": {"a": 1, "p": 20}}, + "expected": {"o": {"a": 2, "c": 10, "p": 20}} +}, +{ + "ancestor": {"o": {"a": 1}}, + "custom": {}, + "product": {"o": {"a": 2}}, + "expected":"Conflicts detected: [Could not merge value: ancestor=1; custom=null; product=2]" +}, +{ + "ancestor": {"a": 1}, + "custom": {"a": 2}, + "product": {"a": 3}, + "expected": "Conflicts detected: [Could not merge value: ancestor=1; custom=2; product=3]" +}, +{ + + + "ancestor": { + "jcr:primaryType": "nam:oak:QueryIndexDefinition", + "compatVersion": 2, + "includedPaths": ["/content/dam"], + "type": "lucene", + "async": ["async", "nrt"], + "evaluatePathRestrictions": true, + "aggregates": { + "jcr:primaryType": "nam:nt:unstructured", + "dam:Asset": { + "jcr:primaryType": "nam:nt:unstructured", + "include0": { + "jcr:primaryType": "nam:nt:unstructured", + "path": "str:jcr:content" + } + } + }, + "facets": { + "jcr:primaryType": "nam:nt:unstructured", + "topChildren": "100", + "secure": "statistical" + }, + "indexRules": { + "jcr:primaryType": "nam:nt:unstructured", + "dam:Asset": { + "jcr:primaryType": "nam:nt:unstructured", + "properties": { + "jcr:primaryType": "nam:nt:unstructured", + "cqTags": { + "jcr:primaryType": "nam:nt:unstructured", + "nodeScopeIndex": true, + "useInSuggest": true, + "propertyIndex": true, + "useInSpellcheck": true, + "name": "str:jcr:content/metadata/cq:tags" + } + } + } + } + } + + + , "custom": { + /* changed: facetc */ + "jcr:primaryType": "nam:oak:QueryIndexDefinition", + "compatVersion": 2, + "includedPaths": ["/content/dam"], + "type": "lucene", + "async": ["async", "nrt"], + "evaluatePathRestrictions": true, + "aggregates": { + "jcr:primaryType": "nam:nt:unstructured", + "dam:Asset": { + "jcr:primaryType": "nam:nt:unstructured", + "include0": { + "jcr:primaryType": "nam:nt:unstructured", + "path": "str:jcr:content" + } + } + }, + "facets": { + "jcr:primaryType": "nam:nt:unstructured", + "topChildren": "200", + "secure": "exact" + }, + "indexRules": { + "jcr:primaryType": "nam:nt:unstructured", + "dam:Asset": { + "jcr:primaryType": "nam:nt:unstructured", + "properties": { + "jcr:primaryType": "nam:nt:unstructured", + "cqTags": { + "jcr:primaryType": "nam:nt:unstructured", + "nodeScopeIndex": true, + "useInSuggest": true, + "propertyIndex": true, + "useInSpellcheck": true, + "name": "str:jcr:content/metadata/cq:tags" + } + } + } + } + } + + + , "product": + /* changed: cqTags/nodeScopeIndex = false */ + { + "jcr:primaryType": "nam:oak:QueryIndexDefinition", + "compatVersion": 2, + "includedPaths": ["/content/dam"], + "type": "lucene", + "async": ["async", "nrt"], + "evaluatePathRestrictions": true, + "aggregates": { + "jcr:primaryType": "nam:nt:unstructured", + "dam:Asset": { + "jcr:primaryType": "nam:nt:unstructured", + "include0": { + "jcr:primaryType": "nam:nt:unstructured", + "path": "str:jcr:content" + } + } + }, + "facets": { + "jcr:primaryType": "nam:nt:unstructured", + "topChildren": "100", + "secure": "statistical" + }, + "indexRules": { + "jcr:primaryType": "nam:nt:unstructured", + "dam:Asset": { + "jcr:primaryType": "nam:nt:unstructured", + "properties": { + "jcr:primaryType": "nam:nt:unstructured", + "cqTags": { + "jcr:primaryType": "nam:nt:unstructured", + "nodeScopeIndex": false, + "useInSuggest": true, + "propertyIndex": true, + "useInSpellcheck": true, + "name": "str:jcr:content/metadata/cq:tags" + } + } + } + } + } + + + , "expected": + { + "jcr:primaryType": "nam:oak:QueryIndexDefinition", + "compatVersion": 2, + "includedPaths": ["/content/dam"], + "type": "lucene", + "async": ["async", "nrt"], + "evaluatePathRestrictions": true, + "aggregates": { + "jcr:primaryType": "nam:nt:unstructured", + "dam:Asset": { + "jcr:primaryType": "nam:nt:unstructured", + "include0": { + "jcr:primaryType": "nam:nt:unstructured", + "path": "str:jcr:content" + } + } + }, + "facets": { + "jcr:primaryType": "nam:nt:unstructured", + "topChildren": "200", + "secure": "exact" + }, + "indexRules": { + "jcr:primaryType": "nam:nt:unstructured", + "dam:Asset": { + "jcr:primaryType": "nam:nt:unstructured", + "properties": { + "jcr:primaryType": "nam:nt:unstructured", + "cqTags": { + "jcr:primaryType": "nam:nt:unstructured", + "nodeScopeIndex": false, + "useInSuggest": true, + "propertyIndex": true, + "useInSpellcheck": true, + "name": "str:jcr:content/metadata/cq:tags" + } + } + } + } + } + + +} +] +} \ No newline at end of file Index: src/test/resources/org/apache/jackrabbit/oak/index/merge/mergeIndexes.txt =================================================================== --- src/test/resources/org/apache/jackrabbit/oak/index/merge/mergeIndexes.txt (nonexistent) +++ src/test/resources/org/apache/jackrabbit/oak/index/merge/mergeIndexes.txt (working copy) @@ -0,0 +1,17 @@ +{"tests": +[ + { + "all": { + "/oak:index/lucene": { "a": 1, "b": 10, "x": 1, "z": 2}, + "/oak:index/lucene-custom-1": { "a": 2, "b": 10, "c": 1, "x": 1} + }, + "new": { + "/oak:index/lucene-2": { "a": 1, "b": 11, "d": 100, "z": 2 } + }, + "expectedNew": { + "/oak:index/lucene-2": { "a": 1, "b": 11, "d": 100, "z": 2 }, + "/oak:index/lucene-2-custom-1": { "a": 2, "b": 11, "c": 1, "d": 100 } + } + } +] +} \ No newline at end of file