Index: oak-run/src/main/java/org/apache/jackrabbit/oak/index/merge/IndexDefMerger.java =================================================================== --- oak-run/src/main/java/org/apache/jackrabbit/oak/index/merge/IndexDefMerger.java (nonexistent) +++ oak-run/src/main/java/org/apache/jackrabbit/oak/index/merge/IndexDefMerger.java (working copy) @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.merge; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map.Entry; +import java.util.stream.Collectors; + +import org.apache.jackrabbit.oak.commons.json.JsonObject; +import org.apache.jackrabbit.oak.plugins.index.search.spi.query.IndexName; + +/** + * Utility that allows to merge index definitions. + */ +public class IndexDefMerger { + + private static HashSet IGNORE_LEVEL_0 = new HashSet<>(Arrays.asList( + "reindex", "refresh", "seed", "reindexCount")); + + /** + * Merge index definition changes. + * + * @param ancestor the common ancestor (the old product index, e.g. lucene) + * @param custom the latest customized version (e.g. lucene-custom-1) + * @param product the latest product index (e.g. lucene-2) + * @return the merged index definition (e.g. lucene-2-custom-1) + */ + public static JsonObject merge(JsonObject ancestor, JsonObject custom, JsonObject product) { + ArrayList conflicts = new ArrayList<>(); + JsonObject merged = merge(0, ancestor, custom, product, conflicts); + if (!conflicts.isEmpty()) { + throw new UnsupportedOperationException("Conflicts detected: " + conflicts); + } + return merged; + } + + private static boolean isSame(String a, String b) { + if (a == null || b == null) { + return a == b; + } + return a.equals(b); + } + + private static boolean isSame(JsonObject a, JsonObject b) { + if (a == null || b == null) { + return a == b; + } + return a.toString().equals(b.toString()); + } + + private static JsonObject merge(int level, JsonObject ancestor, JsonObject custom, JsonObject product, + ArrayList conflicts) { + JsonObject merged = new JsonObject(true); + LinkedHashMap properties = new LinkedHashMap<>(); + if (ancestor == null) { + ancestor = new JsonObject(); + } + for(String p : ancestor.getProperties().keySet()) { + properties.put(p, true); + } + if (custom == null) { + custom = new JsonObject(); + } + for(String p : custom.getProperties().keySet()) { + properties.put(p, true); + } + if (product == null) { + product = new JsonObject(); + } + for(String p : product.getProperties().keySet()) { + properties.put(p, true); + } + for(String k : properties.keySet()) { + if (level == 0 && IGNORE_LEVEL_0.contains(k)) { + // ignore some properties + continue; + } + if (k.startsWith(":")) { + // ignore hidden properties + continue; + } + String ap = ancestor.getProperties().get(k); + String cp = custom.getProperties().get(k); + String pp = product.getProperties().get(k); + String result; + if (isSame(ap, pp) || isSame(cp, pp)) { + result = cp; + } else if (isSame(ap, cp)) { + result = pp; + } else { + conflicts.add("Could not merge value; property=" + k + "; ancestor=" + ap + "; custom=" + cp + "; product=" + pp); + result = ap; + } + if (result != null) { + merged.getProperties().put(k, result); + } + } + LinkedHashMap children = new LinkedHashMap<>(); + for(String c : ancestor.getChildren().keySet()) { + children.put(c, true); + } + for(String c : custom.getChildren().keySet()) { + children.put(c, true); + } + for(String c : product.getChildren().keySet()) { + children.put(c, true); + } + for(String k : children.keySet()) { + if (k.startsWith(":")) { + // ignore hidden nodes + continue; + } + JsonObject a = ancestor.getChildren().get(k); + JsonObject c = custom.getChildren().get(k); + JsonObject p = product.getChildren().get(k); + JsonObject result; + if (isSame(a, p) || isSame(c, p)) { + result = c; + } else if (isSame(a, c)) { + result = p; + } else { + result = merge(level + 1, a, c, p, conflicts); + } + if (result != null) { + merged.getChildren().put(k, result); + } + } + return merged; + } + + /** + * For indexes that were modified both by the customer and in the product, merge + * the changes, and create a new index. + * + * The new index (if any) is stored in the "newIndexes" object. + * + * @param newIndexes the new indexes + * @param allIndexes all index definitions (including the new ones) + */ + public static void merge(JsonObject newIndexes, JsonObject allIndexes) { + + // TODO when merging, we keep the product index, so two indexes are created. + // e.g. lucene, lucene-custom-1, lucene-2, lucene-2-custom-1 + // but if we don't have lucene-2, then we can't merge lucene-3. + + // TODO when merging, e.g. lucene-2-custom-1 is created. but + // it is only imported in the read-write repo, not in the read-only + // repository currently. so this new index won't be used; + // instead, lucene-2 will be used + + List newNames = newIndexes.getChildren().keySet().stream().map(s -> IndexName.parse(s)) + .collect(Collectors.toList()); + Collections.sort(newNames); + List allNames = allIndexes.getChildren().keySet().stream().map(s -> IndexName.parse(s)) + .collect(Collectors.toList()); + Collections.sort(allNames); + HashMap mergedMap = new HashMap<>(); + for (IndexName n : newNames) { + if (n.getCustomerVersion() == 0) { + IndexName latest = n.getLatestCustomized(allNames); + IndexName ancestor = n.getLatestProduct(allNames); + if (latest != null && ancestor != null) { + if (n.compareTo(latest) <= 0 || n.compareTo(ancestor) <= 0) { + // ignore older versions of indexes + continue; + } + JsonObject latestCustomized = allIndexes.getChildren().get(latest.getNodeName()); + JsonObject latestAncestor = allIndexes.getChildren().get(ancestor.getNodeName()); + JsonObject newProduct = newIndexes.getChildren().get(n.getNodeName()); + JsonObject merged = merge(latestAncestor, latestCustomized, newProduct); + mergedMap.put(n.nextCustomizedName(), merged); + } + } + } + for (Entry e : mergedMap.entrySet()) { + newIndexes.getChildren().put(e.getKey(), e.getValue()); + } + } + +} \ No newline at end of file Index: oak-run/src/main/java/org/apache/jackrabbit/oak/index/merge/IndexMerge.java =================================================================== --- oak-run/src/main/java/org/apache/jackrabbit/oak/index/merge/IndexMerge.java (nonexistent) +++ oak-run/src/main/java/org/apache/jackrabbit/oak/index/merge/IndexMerge.java (working copy) @@ -0,0 +1,331 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.merge; + +import java.io.File; +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import javax.jcr.PropertyType; + +import org.apache.felix.inventory.Format; +import org.apache.jackrabbit.oak.api.CommitFailedException; +import org.apache.jackrabbit.oak.api.Type; +import org.apache.jackrabbit.oak.commons.PathUtils; +import org.apache.jackrabbit.oak.commons.json.JsonObject; +import org.apache.jackrabbit.oak.commons.json.JsopReader; +import org.apache.jackrabbit.oak.commons.json.JsopTokenizer; +import org.apache.jackrabbit.oak.json.TypeCodes; +import org.apache.jackrabbit.oak.plugins.index.CompositeIndexEditorProvider; +import org.apache.jackrabbit.oak.plugins.index.IndexPathService; +import org.apache.jackrabbit.oak.plugins.index.IndexPathServiceImpl; +import org.apache.jackrabbit.oak.plugins.index.IndexUpdateProvider; +import org.apache.jackrabbit.oak.plugins.index.counter.NodeCounterEditorProvider; +import org.apache.jackrabbit.oak.plugins.index.inventory.IndexDefinitionPrinter; +import org.apache.jackrabbit.oak.plugins.index.lucene.IndexCopier; +import org.apache.jackrabbit.oak.plugins.index.lucene.IndexTracker; +import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorProvider; +import org.apache.jackrabbit.oak.plugins.index.property.PropertyIndexEditorProvider; +import org.apache.jackrabbit.oak.plugins.index.reference.ReferenceEditorProvider; +import org.apache.jackrabbit.oak.plugins.memory.PropertyStates; +import org.apache.jackrabbit.oak.run.cli.NodeStoreFixture; +import org.apache.jackrabbit.oak.run.cli.NodeStoreFixtureProvider; +import org.apache.jackrabbit.oak.run.cli.Options; +import org.apache.jackrabbit.oak.spi.blob.BlobStore; +import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore; +import org.apache.jackrabbit.oak.spi.commit.CommitInfo; +import org.apache.jackrabbit.oak.spi.commit.EditorHook; +import org.apache.jackrabbit.oak.spi.mount.MountInfoProvider; +import org.apache.jackrabbit.oak.spi.mount.Mounts; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.apache.jackrabbit.oak.spi.state.NodeStore; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import joptsimple.OptionParser; +import joptsimple.OptionSet; +import joptsimple.OptionSpec; + +/** + * Merge custom index definitions with out-of-the-box index definitions. + */ +public class IndexMerge { + + private final static Logger LOG = LoggerFactory.getLogger(IndexMerge.class); + + public static final String OAK_CHILD_ORDER = ":childOrder"; + + private EditorHook hook; + private ExecutorService executorService; + + public static void main(String... args) throws Exception { + new IndexMerge().execute(args); + } + + private boolean quiet; + + /** + * Execute the command. + * + * @param args the command line arguments + */ + public void execute(String... args) throws Exception { + OptionParser parser = new OptionParser(); + OptionSpec quietOption = parser.accepts("quiet", "be less chatty"); + OptionSpec indexDirectory = parser.accepts("indexDir", "Index directory"). + withRequiredArg(); + Options opts = new Options(); + OptionSet options = opts.parseAndConfigure(parser, args); + quiet = options.has(quietOption); + boolean isReadWrite = opts.getCommonOpts().isReadWrite(); + boolean success = true; + String indexRootDir = indexDirectory.value(options); + if (indexRootDir == null) { + throw new IllegalArgumentException("Required argument indexDir missing"); + } + if (!isReadWrite) { + log("Repository connected in read-only mode. Use '--read-write' for write operations"); + } + try (NodeStoreFixture fixture = NodeStoreFixtureProvider.create(opts)) { + NodeStore nodeStore = fixture.getStore(); + BlobStore blobStore = fixture.getBlobStore(); + if (isReadWrite) { + if (blobStore == null) { + throw new IllegalArgumentException("No blob store specified"); + } + if (!(blobStore instanceof GarbageCollectableBlobStore)) { + throw new IllegalArgumentException("Not a garbage collectable blob store: " + blobStore); + } + } + initHook(indexRootDir, (GarbageCollectableBlobStore) blobStore); + + JsonObject indexes = getIndexDefinitions(nodeStore); + // the superseded indexes of the old repository + List supersededKeys = new ArrayList<>(getSupersededIndexDefs(indexes)); + Collections.sort(supersededKeys); + + // keep only new indexes that are not superseded + Map indexMap = indexes.getChildren(); + for (String superseded : supersededKeys) { + if (indexMap.containsKey(superseded)) { + log("Ignoring superseded index " + superseded); + indexMap.remove(superseded); + } + } + Set indexKeys = indexes.getChildren().keySet(); + + IndexDefMerger.merge(indexes, indexes); + + Set newIndexKeys = new HashSet<>(indexes.getChildren().keySet()); + newIndexKeys.removeAll(indexKeys); + if (newIndexKeys.isEmpty()) { + log("No indexes to merge"); + } + for (String newIndexKey : newIndexKeys) { + log("New index: " + newIndexKey); + JsonObject merged = indexMap.get(newIndexKey); + String def = merged.toString(); + log("Merged definition: " + def); + if (isReadWrite) { + storeIndex(nodeStore, newIndexKey, merged); + } + } + } + if (executorService != null) { + executorService.shutdown(); + } + if (!success) { + System.exit(1); + } + } + + private void storeIndex(NodeStore ns, String newIndexName, JsonObject indexDef) { + NodeBuilder rootBuilder = ns.getRoot().builder(); + NodeBuilder b = rootBuilder; + for(String p : PathUtils.elements(newIndexName)) { + b = b.child(p); + } + build(" ", b, indexDef); + try { + ns.merge(rootBuilder, hook, CommitInfo.EMPTY); + log("Added index " + newIndexName); + } catch (CommitFailedException e) { + LOG.error("Failed to add index " + newIndexName, e); + } + } + + private void build(String linePrefix, NodeBuilder builder, JsonObject json) { + for(Entry e : json.getProperties().entrySet()) { + String k = e.getKey(); + String value = e.getValue(); + JsopTokenizer tokenizer = new JsopTokenizer(value); + if (tokenizer.matches('[')) { + ArrayList list = new ArrayList<>(); + while (!tokenizer.matches(']')) { + String jsonString = tokenizer.getToken(); + list.add(jsonString); + tokenizer.matches(','); + } + log(linePrefix + "array " + k + " = " + list + " (String[])"); + builder.setProperty(k, list, Type.STRINGS); + } else if (tokenizer.matches(JsopReader.TRUE)) { + log(linePrefix + "property " + k + " = true (Boolean)"); + builder.setProperty(k, true); + } else if (tokenizer.matches(JsopReader.FALSE)) { + log(linePrefix + "property " + k + " = false (Boolean)"); + builder.setProperty(k, false); + } else if (tokenizer.matches(JsopReader.STRING)) { + String jsonString = tokenizer.getToken(); + int split = TypeCodes.split(jsonString); + if (split != -1) { + int type = TypeCodes.decodeType(split, jsonString); + String v = TypeCodes.decodeName(split, jsonString); + if (type == PropertyType.BINARY) { + throw new UnsupportedOperationException(); + } else { + builder.setProperty(PropertyStates.createProperty(k, v, type)); + } + } + } else if (tokenizer.matches(JsopReader.NUMBER)) { + String num = tokenizer.getToken(); + boolean isDouble = num.indexOf('.') >= 0; + if (isDouble) { + double d = Double.parseDouble(num); + log(linePrefix + "property " + k + " = " + d + " (Double)"); + builder.setProperty(k, d); + } else { + long x = Long.parseLong(num); + log(linePrefix + "property " + k + " = " + x + " (Long)"); + builder.setProperty(k, x); + } + } + } + ArrayList childOrder = new ArrayList<>(); + for(Entry e : json.getChildren().entrySet()) { + String k = e.getKey(); + JsonObject el = e.getValue(); + log(linePrefix + "child " + k); + build(linePrefix + " ", builder.child(k), (JsonObject) el); + childOrder.add(k); + } + if (!childOrder.isEmpty()) { + builder.setProperty(OAK_CHILD_ORDER, childOrder, Type.NAMES); + } + } + + /** + * Get the names of the index definitions that are superseded in one of the + * indexes. + * + * @param indexDefs all index definitions + * @return the superseded indexes + */ + public static Set getSupersededIndexDefs(JsonObject indexDefs) { + HashSet supersededIndexes = new HashSet<>(); + for(JsonObject d : indexDefs.getChildren().values()) { + String supersedes = d.getProperties().get("supersedes"); + if (supersedes != null) { + JsopTokenizer tokenizer = new JsopTokenizer(supersedes); + if (tokenizer.matches('[')) { + while (!tokenizer.matches(']')) { + if (tokenizer.matches(JsopReader.STRING)) { + String s = tokenizer.getToken(); + if (!s.contains("/@")) { + supersededIndexes.add(s); + } + } else { + throw new IllegalArgumentException("Unexpected token: " + tokenizer.getToken()); + } + tokenizer.matches(','); + } + } + } + } + return supersededIndexes; + } + + /** + * Get the the index definitions from a node store. It uses the index path + * service and index definition printer from Oak. + * + * @param nodeStore the source node store + * @return a JSON object with all index definitions + */ + private static JsonObject getIndexDefinitions(NodeStore nodeStore) throws IOException { + IndexPathService imageIndexPathService = new IndexPathServiceImpl(nodeStore); + IndexDefinitionPrinter indexDefinitionPrinter = new IndexDefinitionPrinter(nodeStore, imageIndexPathService); + StringWriter writer = new StringWriter(); + PrintWriter printWriter = new PrintWriter(writer); + indexDefinitionPrinter.print(printWriter, Format.JSON, false); + printWriter.flush(); + writer.flush(); + String json = writer.toString(); + return JsonObject.fromJson(json, true); + } + + private void log(String message) { + if(!quiet) { + System.out.println(message); + } + } + + private void initHook(String indexRootDir, GarbageCollectableBlobStore blobStore) throws IOException { + IndexTracker tracker = new IndexTracker(); + executorService = Executors.newFixedThreadPool(2); + IndexCopier indexCopier = new IndexCopier(executorService, new File(indexRootDir)); + MountInfoProvider mip = createMountInfoProvider(); + LuceneIndexEditorProvider luceneEditor = new LuceneIndexEditorProvider(indexCopier, tracker, null, null, mip); + luceneEditor.setBlobStore(blobStore); + CompositeIndexEditorProvider indexEditor = new CompositeIndexEditorProvider( + luceneEditor, + new PropertyIndexEditorProvider().with(mip), + new ReferenceEditorProvider().with(mip), + new NodeCounterEditorProvider().with(mip) + ); + IndexUpdateProvider updateProvider = new IndexUpdateProvider( + indexEditor, "async", false); + hook = new EditorHook(updateProvider); + } + + private static MountInfoProvider createMountInfoProvider() { + // TODO probably need the ability to configure mounts + return Mounts.newBuilder() + .mount("libs", true, Arrays.asList( + // pathsSupportingFragments + "/oak:index/*$" + ), Arrays.asList( + // mountedPaths + "/libs", + "/apps", + "/jcr:system/rep:permissionStore/oak:mount-libs-crx.default")) + .build(); + } + +} Index: oak-run/src/main/java/org/apache/jackrabbit/oak/run/AvailableModes.java =================================================================== --- oak-run/src/main/java/org/apache/jackrabbit/oak/run/AvailableModes.java (revision 1873893) +++ oak-run/src/main/java/org/apache/jackrabbit/oak/run/AvailableModes.java (working copy) @@ -45,6 +45,7 @@ .put("garbage", new GarbageCommand()) .put("help", new HelpCommand()) .put("history", new HistoryCommand()) + .put("index-merge", new IndexMergeCommand()) .put(IndexCommand.NAME, new IndexCommand()) .put(IOTraceCommand.NAME, new IOTraceCommand()) .put(JsonIndexCommand.INDEX, new JsonIndexCommand()) Index: oak-run/src/main/java/org/apache/jackrabbit/oak/run/IndexMergeCommand.java =================================================================== --- oak-run/src/main/java/org/apache/jackrabbit/oak/run/IndexMergeCommand.java (nonexistent) +++ oak-run/src/main/java/org/apache/jackrabbit/oak/run/IndexMergeCommand.java (working copy) @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.run; + +import org.apache.jackrabbit.oak.index.merge.IndexMerge; +import org.apache.jackrabbit.oak.run.commons.Command; + +public class IndexMergeCommand implements Command { + + @Override + public void execute(String... args) throws Exception { + new IndexMerge().execute(args); + } + +} Index: oak-run/src/test/java/org/apache/jackrabbit/oak/index/merge/IndexDefMergerTest.java =================================================================== --- oak-run/src/test/java/org/apache/jackrabbit/oak/index/merge/IndexDefMergerTest.java (nonexistent) +++ oak-run/src/test/java/org/apache/jackrabbit/oak/index/merge/IndexDefMergerTest.java (working copy) @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.merge; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.LineNumberReader; +import java.util.ArrayList; + +import org.apache.jackrabbit.oak.api.CommitFailedException; +import org.apache.jackrabbit.oak.commons.json.JsonObject; +import org.apache.jackrabbit.oak.commons.json.JsopTokenizer; +import org.junit.Test; + +/** + * Test merging index definitions. + */ +public class IndexDefMergerTest { + + @Test + public void merge() throws IOException, CommitFailedException { + String s = readFromResource("merge.txt"); + JsonObject json = JsonObject.fromJson(s, true); + for(JsonObject e : array(json.getProperties().get("tests"))) { + merge(e); + } + } + + @Test + public void mergeIndexes() throws IOException, CommitFailedException { + String s = readFromResource("mergeIndexes.txt"); + JsonObject json = JsonObject.fromJson(s, true); + for(JsonObject e : array(json.getProperties().get("tests"))) { + mergeIndexes(e); + } + } + + private void mergeIndexes(JsonObject e) { + JsonObject all = e.getChildren().get("all"); + JsonObject newDefs = e.getChildren().get("new"); + JsonObject expectedNew = e.getChildren().get("expectedNew"); + IndexDefMerger.merge(newDefs, all); + assertEquals( + expectedNew.toString(), + newDefs.toString()); + } + + private void merge(JsonObject e) { + JsonObject ancestor = e.getChildren().get("ancestor"); + JsonObject custom = e.getChildren().get("custom"); + JsonObject product = e.getChildren().get("product"); + try { + JsonObject got = IndexDefMerger.merge(ancestor, custom, product); + JsonObject expected = e.getChildren().get("expected"); + assertEquals(expected.toString(), got.toString()); + } catch (UnsupportedOperationException e2) { + String expected = e.getProperties().get("expected"); + assertEquals("" + expected, "\"" + e2.getMessage() + "\""); + } + } + + static String readFromResource(String resourceName) throws IOException { + try (InputStreamReader reader = new InputStreamReader( + IndexDefMergerTest.class.getResourceAsStream(resourceName))) { + StringBuilder buff = new StringBuilder(); + try (LineNumberReader l = new LineNumberReader(reader)) { + while (true) { + String s = l.readLine(); + if (s == null) { + break; + } + if (s.trim().startsWith("//")) { + // comment + continue; + } + buff.append(s); + buff.append('\n'); + } + } + return buff.toString(); + } + } + + private static ArrayList array(String json) { + ArrayList list = new ArrayList<>(); + JsopTokenizer tokenizer = new JsopTokenizer(json); + tokenizer.read('['); + while (!tokenizer.matches(']')) { + tokenizer.read('{'); + JsonObject j = JsonObject.create(tokenizer, true); + list.add(j); + tokenizer.matches(','); + } + return list; + } + +} Index: oak-run/src/test/resources/org/apache/jackrabbit/oak/index/merge/merge.txt =================================================================== --- oak-run/src/test/resources/org/apache/jackrabbit/oak/index/merge/merge.txt (nonexistent) +++ oak-run/src/test/resources/org/apache/jackrabbit/oak/index/merge/merge.txt (working copy) @@ -0,0 +1,204 @@ +{"tests": +[ +{ + "ancestor": {"value": 1, "a-old": 0}, + "custom": {"value": 2, "b-new": 3}, + "product": {"value": 1, "c-new": 4}, + "expected": {"value": 2, "b-new": 3, "c-new": 4} +}, +{ + "ancestor": {"o": {"a": 1}}, + "custom": {"o": {"a": 2, "c": 10}}, + "product": {"o": {"a": 1, "p": 20}}, + "expected": {"o": {"a": 2, "c": 10, "p": 20}} +}, +{ + "ancestor": {"o": {"a": 1}}, + "custom": {}, + "product": {"o": {"a": 2}}, + "expected":"Conflicts detected: [Could not merge value; property=a; ancestor=1; custom=null; product=2]" +}, +{ + "ancestor": {"a": 1}, + "custom": {"a": 2}, + "product": {"a": 3}, + "expected": "Conflicts detected: [Could not merge value; property=a; ancestor=1; custom=2; product=3]" +}, +{ + + + "ancestor": { + "jcr:primaryType": "nam:oak:QueryIndexDefinition", + "compatVersion": 2, + "includedPaths": ["/content/dam"], + "type": "lucene", + "async": ["async", "nrt"], + "evaluatePathRestrictions": true, + "aggregates": { + "jcr:primaryType": "nam:nt:unstructured", + "dam:Asset": { + "jcr:primaryType": "nam:nt:unstructured", + "include0": { + "jcr:primaryType": "nam:nt:unstructured", + "path": "str:jcr:content" + } + } + }, + "facets": { + "jcr:primaryType": "nam:nt:unstructured", + "topChildren": "100", + "secure": "statistical" + }, + "indexRules": { + "jcr:primaryType": "nam:nt:unstructured", + "dam:Asset": { + "jcr:primaryType": "nam:nt:unstructured", + "properties": { + "jcr:primaryType": "nam:nt:unstructured", + "cqTags": { + "jcr:primaryType": "nam:nt:unstructured", + "nodeScopeIndex": true, + "useInSuggest": true, + "propertyIndex": true, + "useInSpellcheck": true, + "name": "str:jcr:content/metadata/cq:tags" + } + } + } + } + } + + + , "custom": { +// changed: facetc + "jcr:primaryType": "nam:oak:QueryIndexDefinition", + "compatVersion": 2, + "includedPaths": ["/content/dam"], + "type": "lucene", + "async": ["async", "nrt"], + "evaluatePathRestrictions": true, + "aggregates": { + "jcr:primaryType": "nam:nt:unstructured", + "dam:Asset": { + "jcr:primaryType": "nam:nt:unstructured", + "include0": { + "jcr:primaryType": "nam:nt:unstructured", + "path": "str:jcr:content" + } + } + }, + "facets": { + "jcr:primaryType": "nam:nt:unstructured", + "topChildren": "200", + "secure": "exact" + }, + "indexRules": { + "jcr:primaryType": "nam:nt:unstructured", + "dam:Asset": { + "jcr:primaryType": "nam:nt:unstructured", + "properties": { + "jcr:primaryType": "nam:nt:unstructured", + "cqTags": { + "jcr:primaryType": "nam:nt:unstructured", + "nodeScopeIndex": true, + "useInSuggest": true, + "propertyIndex": true, + "useInSpellcheck": true, + "name": "str:jcr:content/metadata/cq:tags" + } + } + } + } + } + + + , "product": +// changed: cqTags/nodeScopeIndex = false + { + "jcr:primaryType": "nam:oak:QueryIndexDefinition", + "compatVersion": 2, + "includedPaths": ["/content/dam"], + "type": "lucene", + "async": ["async", "nrt"], + "evaluatePathRestrictions": true, + "aggregates": { + "jcr:primaryType": "nam:nt:unstructured", + "dam:Asset": { + "jcr:primaryType": "nam:nt:unstructured", + "include0": { + "jcr:primaryType": "nam:nt:unstructured", + "path": "str:jcr:content" + } + } + }, + "facets": { + "jcr:primaryType": "nam:nt:unstructured", + "topChildren": "100", + "secure": "statistical" + }, + "indexRules": { + "jcr:primaryType": "nam:nt:unstructured", + "dam:Asset": { + "jcr:primaryType": "nam:nt:unstructured", + "properties": { + "jcr:primaryType": "nam:nt:unstructured", + "cqTags": { + "jcr:primaryType": "nam:nt:unstructured", + "nodeScopeIndex": false, + "useInSuggest": true, + "propertyIndex": true, + "useInSpellcheck": true, + "name": "str:jcr:content/metadata/cq:tags" + } + } + } + } + } + + + , "expected": + { + "jcr:primaryType": "nam:oak:QueryIndexDefinition", + "compatVersion": 2, + "includedPaths": ["/content/dam"], + "type": "lucene", + "async": ["async", "nrt"], + "evaluatePathRestrictions": true, + "aggregates": { + "jcr:primaryType": "nam:nt:unstructured", + "dam:Asset": { + "jcr:primaryType": "nam:nt:unstructured", + "include0": { + "jcr:primaryType": "nam:nt:unstructured", + "path": "str:jcr:content" + } + } + }, + "facets": { + "jcr:primaryType": "nam:nt:unstructured", + "topChildren": "200", + "secure": "exact" + }, + "indexRules": { + "jcr:primaryType": "nam:nt:unstructured", + "dam:Asset": { + "jcr:primaryType": "nam:nt:unstructured", + "properties": { + "jcr:primaryType": "nam:nt:unstructured", + "cqTags": { + "jcr:primaryType": "nam:nt:unstructured", + "nodeScopeIndex": false, + "useInSuggest": true, + "propertyIndex": true, + "useInSpellcheck": true, + "name": "str:jcr:content/metadata/cq:tags" + } + } + } + } + } + + +} +] +} \ No newline at end of file Index: oak-run/src/test/resources/org/apache/jackrabbit/oak/index/merge/mergeIndexes.txt =================================================================== --- oak-run/src/test/resources/org/apache/jackrabbit/oak/index/merge/mergeIndexes.txt (nonexistent) +++ oak-run/src/test/resources/org/apache/jackrabbit/oak/index/merge/mergeIndexes.txt (working copy) @@ -0,0 +1,17 @@ +{"tests": +[ + { + "all": { + "/oak:index/lucene": { "a": 1, "b": 10, "x": 1, "z": 2}, + "/oak:index/lucene-custom-1": { "a": 2, "b": 10, "c": 1, "x": 1} + }, + "new": { + "/oak:index/lucene-2": { "a": 1, "b": 11, "d": 100, "z": 2 } + }, + "expectedNew": { + "/oak:index/lucene-2": { "a": 1, "b": 11, "d": 100, "z": 2 }, + "/oak:index/lucene-2-custom-1": { "a": 2, "b": 11, "c": 1, "d": 100 } + } + } +] +} \ No newline at end of file Index: oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/IndexName.java =================================================================== --- oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/IndexName.java (revision 1873893) +++ oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/IndexName.java (working copy) @@ -22,6 +22,7 @@ import java.util.Collection; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import org.apache.jackrabbit.oak.commons.PathUtils; import org.apache.jackrabbit.oak.spi.state.NodeState; @@ -143,6 +144,47 @@ } /** + * Get the latest index name object that matches this index base name, and + * is customized. + * + * @param all the list of all indexes + * @return the lastest customized index, or null if none + */ + public IndexName getLatestCustomized(List all) { + IndexName latest = null; + for (IndexName n : all) { + if (n.baseName.equals(baseName)) { + if (n.customerVersion > 0) { + if (latest == null || n.compareTo(latest) > 0) { + latest = n; + } + } + } + } + return latest; + } + + /** + * Get the latest product index that matches this index base name. + * + * @param all the list of all indexes + * @return the latest product index, or null if none + */ + public IndexName getLatestProduct(List all) { + IndexName latest = null; + for (IndexName n : all) { + if (n.baseName.equals(baseName)) { + if (compareTo(n) > 0 && n.customerVersion == 0) { + if (latest == null || n.compareTo(latest) > 0) { + latest = n; + } + } + } + } + return latest; + } + + /** * Filter out index that are replaced by another index with the same base * name but newer version. * @@ -181,6 +223,18 @@ return result; } + public String getNodeName() { + return nodeName; + } + + public int getCustomVersion() { + return customerVersion; + } + + public String nextCustomizedName() { + return baseName + "-" + productVersion + "-custom-" + (customerVersion + 1); + } + private static boolean isIndexActive(String indexPath, NodeState rootState) { NodeState indexNode = rootState; for(String e : PathUtils.elements(indexPath)) { @@ -194,14 +248,10 @@ return false; } - public String getNodeName() { - return nodeName; - } - public int getCustomerVersion() { return customerVersion; } - + public int getProductVersion() { return productVersion; } @@ -218,4 +268,4 @@ return isLegal; } -} \ No newline at end of file +}