From b3ef1c1fbc30ce767471bb02c2253fbc121f5222 Mon Sep 17 00:00:00 2001 From: Vikas Saurabh Date: Sat, 7 Jul 2018 19:42:08 +0530 Subject: [PATCH 1/3] OAK-7495: async,sync index not synchronous Add a lots of logs to assist in debugging --- .../plugins/index/lucene/IndexNodeManager.java | 3 +++ .../plugins/index/lucene/LucenePropertyIndex.java | 9 +++++---- .../index/lucene/hybrid/DocumentQueue.java | 15 ++++++++------- .../index/lucene/hybrid/LocalIndexObserver.java | 7 ++++++- .../index/lucene/hybrid/LuceneDocumentHolder.java | 6 ++++++ .../oak/plugins/index/lucene/hybrid/NRTIndex.java | 3 +++ 6 files changed, 31 insertions(+), 12 deletions(-) diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexNodeManager.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexNodeManager.java index 5e4ffaf768..4fa9b30a95 100644 --- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexNodeManager.java +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexNodeManager.java @@ -163,6 +163,7 @@ public class IndexNodeManager { } else { boolean success = false; try { + log.info("refreshOnReadIfRequired"); refreshPolicy.refreshOnReadIfRequired(refreshCallback); SearcherHolder local = searcherHolder; int tryCount = 0; @@ -209,6 +210,7 @@ public class IndexNodeManager { } private void refreshReadersOnWriteIfRequired() { + log.info("refreshOnWriteIfRequired"); refreshPolicy.refreshOnWriteIfRequired(refreshCallback); } @@ -221,6 +223,7 @@ public class IndexNodeManager { SearcherHolder old = searcherHolder; searcherHolder = createHolder(newNRTReaders); releaseHolder(old); + log.info("Refreshed reader for index [{}]", definition); PERF_LOGGER.end(start, 0, "Refreshed reader for index [{}]", definition); } } diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java index f863eeaeeb..4b3d3e61a1 100644 --- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java @@ -355,6 +355,7 @@ public class LucenePropertyIndex implements AdvancedQueryIndex, QueryIndex, Nati return queue.remove(); } releaseSearcher(); + LOG.info("EndOfData for {}", plan); return endOfData(); } @@ -388,7 +389,7 @@ public class LucenePropertyIndex implements AdvancedQueryIndex, QueryIndex, Nati } boolean shouldIncludeForHierarchy = shouldInclude(path, plan); - LOG.trace("Matched path {}; shouldIncludeForHierarchy: {}", path, shouldIncludeForHierarchy); + LOG.info("Matched path {}; shouldIncludeForHierarchy: {}", path, shouldIncludeForHierarchy); return shouldIncludeForHierarchy? new LuceneResultRow(path, doc.score, excerpts, facets, explanation) : null; } @@ -425,14 +426,14 @@ public class LucenePropertyIndex implements AdvancedQueryIndex, QueryIndex, Nati long start = PERF_LOGGER.start(); while (true) { if (lastDoc != null) { - LOG.debug("loading the next {} entries for query {}", nextBatchSize, query); + LOG.info("loading the next {} entries for query {}", nextBatchSize, query); if (sort == null) { docs = searcher.searchAfter(lastDoc, query, nextBatchSize); } else { docs = searcher.searchAfter(lastDoc, query, nextBatchSize, sort); } } else { - LOG.debug("loading the first {} entries for query {}", nextBatchSize, query); + LOG.info("loading the first {} entries for query {}", nextBatchSize, query); if (sort == null) { docs = searcher.search(query, nextBatchSize); } else { @@ -583,7 +584,7 @@ public class LucenePropertyIndex implements AdvancedQueryIndex, QueryIndex, Nati if (indexNodeId != indexNode.getIndexNodeId()){ //if already initialized then log about change if (indexNodeId > 0){ - LOG.debug("Change in index version detected. Query would be performed without offset"); + LOG.info("Change in index version detected. Query would be performed without offset"); } indexSearcher = indexNode.getSearcher(); diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DocumentQueue.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DocumentQueue.java index 936e83ec7b..714c35f32f 100644 --- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DocumentQueue.java +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DocumentQueue.java @@ -159,8 +159,8 @@ public class DocumentQueue implements Closeable, IndexingQueue { boolean added = docsQueue.offer(doc); if (added) { queueSizeStats.inc(); - if (log.isTraceEnabled()){ - log.trace("Adding {} without wait to queue at size {}", doc, docsQueue.size()); + if (true || log.isTraceEnabled()){ + log.info("Adding {} without wait to queue at size {}", doc, docsQueue.size()); } } return added; @@ -179,8 +179,8 @@ public class DocumentQueue implements Closeable, IndexingQueue { if (added) { queueSizeStats.inc(); - if (log.isTraceEnabled()){ - log.trace("Adding {} to queue at size {}", doc, docsQueue.size()); + if (true || log.isTraceEnabled()){ + log.info("Adding {} to queue at size {}", doc, docsQueue.size()); } } else { dropped.mark(); @@ -245,7 +245,7 @@ public class DocumentQueue implements Closeable, IndexingQueue { IndexNode indexNode = tracker.acquireIndexNode(indexPath); if (indexNode == null) { - log.debug("No IndexNode found for index [{}].", indexPath); + log.info("No IndexNode found for index [{}].", indexPath); return; } @@ -256,7 +256,7 @@ public class DocumentQueue implements Closeable, IndexingQueue { if (writer == null) { //IndexDefinition per IndexNode might have changed and local //indexing is disabled. Ignore - log.debug("No local IndexWriter found for index [{}]. Skipping index " + + log.info("No local IndexWriter found for index [{}]. Skipping index " + "entry for [{}]", indexPath, doc.docPath); return; } @@ -264,6 +264,7 @@ public class DocumentQueue implements Closeable, IndexingQueue { //Skip already processed doc entry continue; } else { + log.info("Processed {}", doc); doc.markProcessed(); } if (doc.delete) { @@ -273,7 +274,7 @@ public class DocumentQueue implements Closeable, IndexingQueue { } docAdded = true; String prefix = docsFromQueue ? "Queued" : "Direct"; - log.trace("[{}] Updated index with doc {}", prefix, doc); + log.info("[{}] Updated index with doc {}", prefix, doc); } if (docAdded) { indexNode.refreshReadersOnWriteIfRequired(); diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LocalIndexObserver.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LocalIndexObserver.java index d69b9e33b4..91f6ecafb0 100644 --- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LocalIndexObserver.java +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LocalIndexObserver.java @@ -30,6 +30,9 @@ import org.apache.jackrabbit.oak.stats.StatisticsProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Collection; +import java.util.Map; + public class LocalIndexObserver implements Observer{ private final Logger log = LoggerFactory.getLogger(getClass()); private final DocumentQueue docQueue; @@ -68,7 +71,9 @@ public class LocalIndexObserver implements Observer{ //After nrt docs add all sync indexed docs //Doing it *after* ensures thar nrt index might catch //up by the time sync one are finished - docQueue.addAllSynchronously(holder.getSyncIndexedDocs()); + Map> syncIndexedDocs = holder.getSyncIndexedDocs(); + log.info("Directly adding sync index docs: {}", syncIndexedDocs); + docQueue.addAllSynchronously(syncIndexedDocs); if (droppedCount > 0){ //TODO Ensure that log do not flood diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LuceneDocumentHolder.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LuceneDocumentHolder.java index 09eab8fd0f..9c514bb88d 100644 --- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LuceneDocumentHolder.java +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LuceneDocumentHolder.java @@ -49,6 +49,7 @@ public class LuceneDocumentHolder implements JournalProperty{ private boolean schedulingDone; public LuceneDocumentHolder(@Nonnull IndexingQueue documentQueue, int inMemoryDocsLimit) { + log.info("New holder"); this.documentQueue = checkNotNull(documentQueue); this.inMemoryDocsLimit = inMemoryDocsLimit; } @@ -60,7 +61,10 @@ public class LuceneDocumentHolder implements JournalProperty{ public Map> getSyncIndexedDocs(){ for (Map.Entry e : queuedSyncIndexedPath.entries()){ if (!e.getValue().isProcessed()){ + log.info("Returning: {}, {}", e.getKey(), e.getValue()); syncIndexedList.put(e.getKey(), e.getValue()); + } else { + log.info("Already processed: {}, {}", e.getKey(), e.getValue()); } } return syncIndexedList.asMap(); @@ -71,6 +75,7 @@ public class LuceneDocumentHolder implements JournalProperty{ //First try adding to queue in non blocking manner if (documentQueue.addIfNotFullWithoutWait(doc)){ if (sync){ + log.info("1. Holder adding: {}; sync: {}", doc, sync); queuedSyncIndexedPath.put(doc.indexPath, doc); } else { queuedNrtIndexedPath.put(doc.indexPath, doc.docPath); @@ -79,6 +84,7 @@ public class LuceneDocumentHolder implements JournalProperty{ } else { //Queue is full so keep it in memory if (sync) { + log.info("2. Holder adding: {}; sync: {}", doc, sync); syncIndexedList.put(doc.indexPath, doc); } else { if (queueSizeWithinLimits()) { diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/NRTIndex.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/NRTIndex.java index 7c94e15bb5..06d75f912e 100644 --- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/NRTIndex.java +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/NRTIndex.java @@ -147,7 +147,10 @@ public class NRTIndex implements Closeable { //reader not changed i.e. no change in index //reuse old readers if (latestReader == dirReader && readers != null){ + log.info("Reader not changed... using old reader"); return readers; + } else { + log.info("Updated reader"); } List newReaders = Lists.newArrayListWithCapacity(2); -- 2.17.1