Index: lucene/core/src/java/org/apache/lucene/index/BufferedDeletesStream.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/BufferedDeletesStream.java (revision 1525521) +++ lucene/core/src/java/org/apache/lucene/index/BufferedDeletesStream.java (working copy) @@ -421,6 +421,8 @@ // because on flush we apply all Term deletes to // each segment. So all Term deleting here is // against prior segments: + System.out.println("[" + Thread.currentThread().getName() + "] BDS.applyTermDeletes: reader=" + reader + " delTerm=" + term + " doc=" + docID + " docid=" + reader.document(docID).get("docid") + + " packID=" + reader.document(docID).get("packID")); // nocommit __REMOVE if (rld.delete(docID)) { delCount++; } Index: lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java (revision 1525521) +++ lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java (working copy) @@ -442,6 +442,7 @@ if (pendingDeletes.docIDs.size() > 0) { flushState.liveDocs = codec.liveDocsFormat().newLiveDocs(numDocsInRAM); for(int delDocID : pendingDeletes.docIDs) { + System.out.println("[" + Thread.currentThread().getName() + "] DWPT.flush delDocID=" + delDocID); // nocommit __REMOVE flushState.liveDocs.clear(delDocID); } flushState.delCountOnFlush = pendingDeletes.docIDs.size(); Index: lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriter.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriter.java (revision 1525521) +++ lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriter.java (working copy) @@ -23,6 +23,7 @@ import java.util.List; import java.util.Map; +import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.CollectionUtil; @@ -36,6 +37,7 @@ // flushed segment: if (state.segDeletes != null && state.segDeletes.terms.size() > 0) { Map segDeletes = state.segDeletes.terms; + System.out.println("[" + Thread.currentThread().getName() + "] FPTW.applyDelets seg=" + state.segmentInfo + " segDeletes=" + segDeletes); // nocommit __REMOVE List deleteTerms = new ArrayList(segDeletes.keySet()); Collections.sort(deleteTerms); String lastField = null; @@ -42,6 +44,7 @@ TermsEnum termsEnum = null; DocsEnum docsEnum = null; for(Term deleteTerm : deleteTerms) { + System.out.println("[" + Thread.currentThread().getName() + "] FPTW.applyDelets seg=" + state.segmentInfo + " delTerm=" + deleteTerm); // nocommit __REMOVE if (deleteTerm.field().equals(lastField) == false) { lastField = deleteTerm.field(); Terms terms = fields.terms(lastField); @@ -63,6 +66,22 @@ state.liveDocs = state.segmentInfo.getCodec().liveDocsFormat().newLiveDocs(state.segmentInfo.getDocCount()); } if (state.liveDocs.get(doc)) { + // nocommit __REMOVE + if (deleteTerm.field.equals("packID")) { + System.out.println("[" + Thread.currentThread().getName() + "] FPTW.applyDelets seg=" + state.segmentInfo + " delTerm=" + deleteTerm + " doc=" + doc); + Terms docTerms = fields.terms("docid"); + if (docTerms != null) { + TermsEnum docTermsEnum = docTerms.iterator(null); + BytesRef docIDTerm; + while ((docIDTerm = docTermsEnum.next()) != null) { + DocsEnum docDocs = docTermsEnum.docs(null, null, 0); + int docID; + while ((docID = docDocs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { + System.out.println("[" + Thread.currentThread().getName() + "] FPTW.applyDelets seg=" + state.segmentInfo + " delTerm=" + deleteTerm + " doc=" + docID + " docid=" + docIDTerm.utf8ToString()); + } + } + } + } state.delCountOnFlush++; state.liveDocs.clear(doc); } Index: lucene/core/src/java/org/apache/lucene/index/ReadersAndLiveDocs.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/ReadersAndLiveDocs.java (revision 1525521) +++ lucene/core/src/java/org/apache/lucene/index/ReadersAndLiveDocs.java (working copy) @@ -240,6 +240,12 @@ assert Thread.holdsLock(writer); assert docID >= 0 && docID < liveDocs.length() : "out of bounds: docid=" + docID + " liveDocsLength=" + liveDocs.length() + " seg=" + info.info.name + " docCount=" + info.info.getDocCount(); assert !liveDocsShared; + // nocommit __REMOVE + try { + System.out.println("[" + Thread.currentThread().getName() + "] RLD.delete: seg=" + info + " doc=" + docID + " docid=" + reader.document(docID).get("docid")); + } catch (IOException e) { + throw new RuntimeException(e); + } final boolean didDelete = liveDocs.get(docID); if (didDelete) { ((MutableBits) liveDocs).clear(docID); @@ -360,6 +366,13 @@ try { Codec codec = info.info.getCodec(); if (pendingDeleteCount > 0) { + // nocommit __REMOVE + System.out.println("[" + Thread.currentThread().getName() + "] RLD.writeLiveDocs: seg=" + info + " deletedDocs="); + for (int i = 0; i < liveDocs.length(); i++) { + if (!liveDocs.get(i)) { + System.out.println("doc=" + i + " docid=" + reader.document(i).get("docid")); + } + } codec.liveDocsFormat().writeLiveDocs((MutableBits)liveDocs, trackingDir, info, pendingDeleteCount, IOContext.DEFAULT); } Index: lucene/core/src/java/org/apache/lucene/index/SegmentReader.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/SegmentReader.java (revision 1525521) +++ lucene/core/src/java/org/apache/lucene/index/SegmentReader.java (working copy) @@ -89,6 +89,12 @@ if (si.hasDeletions()) { // NOTE: the bitvector is stored using the regular directory, not cfs liveDocs = codec.liveDocsFormat().readLiveDocs(directory(), si, IOContext.READONCE); + // nocommit __REMOVE + for (int i = 0; i < liveDocs.length(); i++) { + if (!liveDocs.get(i)) { + System.out.println("[" + Thread.currentThread().getName() + "] SR.init newReader(" + si + "): deleted doc=" + i + " docid=" + document(i).get("docid") + " packID=" + document(i).get("packID")); + } + } } else { assert si.getDelCount() == 0; liveDocs = null; @@ -154,6 +160,15 @@ // System.out.println("[" + Thread.currentThread().getName() + "] SR.init: sharing reader: " + sr + " for gens=" + sr.genDVProducers.keySet()); + // nocommit __REMOVE + if (liveDocs != null) { + for (int i = 0; i < liveDocs.length(); i++) { + if (!liveDocs.get(i)) { + System.out.println("[" + Thread.currentThread().getName() + "] SR.init shareReader(" + si + "): deleted doc=" + i + " docid=" + document(i).get("docid") + " packID=" + document(i).get("packID")); + } + } + } + // increment refCount of DocValuesProducers that are used by this reader boolean success = false; try { Index: lucene/test-framework/src/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java =================================================================== --- lucene/test-framework/src/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java (revision 1525521) +++ lucene/test-framework/src/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java (working copy) @@ -90,18 +90,42 @@ } protected void updateDocuments(Term id, List docs) throws Exception { + // nocommit __REMOVE + for (IndexDocument doc : docs) { + Document d = (Document) doc; + if (d.get("packID") != null) { + System.out.println("[" + Thread.currentThread().getName() + "] updateDocuments packID=" + d.get("packID") + " docid=" + d.get("docid")); + } + } writer.updateDocuments(id, docs); } protected void addDocuments(Term id, List docs) throws Exception { + // nocommit __REMOVE + for (IndexDocument doc : docs) { + Document d = (Document) doc; + if (d.get("packID") != null) { + System.out.println("[" + Thread.currentThread().getName() + "] addDocuments packID=" + d.get("packID") + " docid=" + d.get("docid")); + } + } writer.addDocuments(docs); } protected void addDocument(Term id, IndexDocument doc) throws Exception { + // nocommit __REMOVE + Document d = (Document) doc; + if (d.get("packID") != null) { + System.out.println("[" + Thread.currentThread().getName() + "] addDocument packID=" + d.get("packID") + " docid=" + d.get("docid")); + } writer.addDocument(doc); } protected void updateDocument(Term term, IndexDocument doc) throws Exception { + // nocommit __REMOVE + Document d = (Document) doc; + if (d.get("packID") != null) { + System.out.println("[" + Thread.currentThread().getName() + "] updateDocument packID=" + d.get("packID") + " docid=" + d.get("docid")); + } writer.updateDocument(term, doc); } @@ -208,11 +232,13 @@ if (VERBOSE) { System.out.println(Thread.currentThread().getName() + ": update pack packID=" + delSubDocs.packID + " count=" + docsList.size() + " docs=" + docIDs); } + System.out.println("[" + Thread.currentThread().getName() + "] delSubDocs!=null update pack packID=" + delSubDocs.packID + " count=" + docsList.size() + " docs=" + docIDs); // nocommit __REMOVE updateDocuments(packIDTerm, docsList); } else { if (VERBOSE) { System.out.println(Thread.currentThread().getName() + ": add pack packID=" + packID + " count=" + docsList.size() + " docs=" + docIDs); } + System.out.println("[" + Thread.currentThread().getName() + "] delSubDocs==null update pack packID=" + packID + " count=" + docsList.size() + " docs=" + docIDs); // nocommit __REMOVE addDocuments(packIDTerm, docsList); } doc.removeField("packID"); @@ -221,6 +247,7 @@ if (VERBOSE) { System.out.println(Thread.currentThread().getName() + ": buffer del id:" + packID); } + System.out.println("[" + Thread.currentThread().getName() + "] buffer del packID=" + packID); // nocommit __REMOVE toDeleteSubDocs.add(subDocs); } @@ -237,6 +264,7 @@ if (VERBOSE) { System.out.println(Thread.currentThread().getName() + ": buffer del id:" + doc.get("docid")); } + System.out.println("[" + Thread.currentThread().getName() + "] buffer del docid=" + doc.get("docid")); // nocommit __REMOVE toDeleteIDs.add(docid); } } @@ -249,6 +277,7 @@ System.out.println(Thread.currentThread().getName() + ": update doc id:" + doc.get("docid")); } final String docid = doc.get("docid"); + System.out.println("[" + Thread.currentThread().getName() + "] update-single docid=" + docid); // nocommit __REMOVE updateDocument(new Term("docid", docid), doc); addCount.getAndIncrement(); @@ -256,6 +285,7 @@ if (VERBOSE) { System.out.println(Thread.currentThread().getName() + ": buffer del id:" + doc.get("docid")); } + System.out.println("[" + Thread.currentThread().getName() + "] buffer-single del docid=" + docid); // nocommit __REMOVE toDeleteIDs.add(docid); } } @@ -268,6 +298,7 @@ if (VERBOSE) { System.out.println(Thread.currentThread().getName() + ": del term=id:" + id); } + System.out.println("[" + Thread.currentThread().getName() + "] deleteDocuments docid:" + id); // nocommit __REMOVE deleteDocuments(new Term("docid", id)); } final int count = delCount.addAndGet(toDeleteIDs.size()); @@ -275,11 +306,13 @@ System.out.println(Thread.currentThread().getName() + ": tot " + count + " deletes"); } delIDs.addAll(toDeleteIDs); + System.out.println("[" + Thread.currentThread().getName() + "] delIDs: " + delIDs); // nocommit __REMOVE toDeleteIDs.clear(); for(SubDocs subDocs : toDeleteSubDocs) { assert !subDocs.deleted; delPackIDs.add(subDocs.packID); + System.out.println("[" + Thread.currentThread().getName() + "] del docs subDocs=" + subDocs.subIDs + ", packID=" + subDocs.packID); // nocommit __REMOVE deleteDocuments(new Term("packID", subDocs.packID)); subDocs.deleted = true; if (VERBOSE) { @@ -599,6 +632,7 @@ if (!delIDs.contains(stringID)) { final TopDocs hits = s.search(new TermQuery(new Term("docid", stringID)), 1); if (hits.totalHits != 1) { + System.out.println("delIDs=" + delIDs); // nocommit __REMOVE System.out.println("doc id=" + stringID + " is not supposed to be deleted, but got hitCount=" + hits.totalHits); doFail = true; }