Lucene - Core
  1. Lucene - Core
  2. LUCENE-1399

NullPointerException issue with single IndexWriter instance shared among multiple Thread

    Details

    • Type: Bug Bug
    • Status: Closed
    • Priority: Major Major
    • Resolution: Duplicate
    • Affects Version/s: 2.3.2
    • Fix Version/s: 2.4
    • Component/s: core/index
    • Labels:
      None
    • Environment:

      Windows XP, JDK 1.6

    • Lucene Fields:
      New

      Description

      NullPointerException is thrown while indexing within multiple threads but using one single IndexWriter instance. According to "Lucene in Action" this should do: "...a single instance of either class can be shared among multiple threads, and all calls to its index-modifying methods will be properly synchronized so that index modifications are executed one after the other."

      Following my exception trace:

      java.lang.NullPointerException
      at org.apache.lucene.index.IndexFileDeleter.decRef(IndexFileDeleter.java:475)
      at org.apache.lucene.index.IndexWriter.commitTransaction(IndexWriter.java:1986)
      at org.apache.lucene.index.IndexWriter.addIndexes(IndexWriter.java:2219)
      at com.shi.marketingspy.test.SHILuceneWritingFromThreadsTest$testthread.run(SHILuceneWritingFromThreadsTest.java:48)
      at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:885)
      at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
      at java.lang.Thread.run(Thread.java:619)

      Following my test program:

      package com.shi.marketingspy.test;

      import org.apache.lucene.analysis.Analyzer;
      import org.apache.lucene.document.Document;
      import org.apache.lucene.document.Field;
      import org.apache.lucene.document.Field.Store;
      import org.apache.lucene.index.IndexWriter;

      public class SHILuceneWritingFromThreadsTest
      {
      public static void log(String s, com.shi.jipmisc misc)

      { misc.__Log(s); }

      public static class testthread implements Runnable
      {
      public org.apache.lucene.index.IndexWriter indexwriter;
      public int iThread;
      public boolean bWithOptimize;
      public com.shi.jipmisc misc;

      public testthread(org.apache.lucene.index.IndexWriter indexwriter, int iThread, boolean bWithOptimize, com.shi.jipmisc misc)

      { this.indexwriter = indexwriter; this.iThread = iThread; this.bWithOptimize = bWithOptimize; this.misc = misc; }

      public void run()
      {
      try
      {
      Analyzer anal = new com.shi.lucene.index.SHIAnalyser(null,null);
      org.apache.lucene.store.Directory tmpidxdirectory = new org.apache.lucene.store.RAMDirectory();
      org.apache.lucene.index.IndexWriter tempindexwriter = new IndexWriter(tmpidxdirectory, anal, true);

      Document doc = new Document();
      for(int ifld=0; ifld < 9; ifld++)

      { doc.add(new Field("field_"+ifld, "iThread: "+iThread+" field: "+ifld+" "+System.currentTimeMillis()+" "+misc.getDateString()+" "+misc.getDateString2()+" "+com.shi.jipmisc.getTimeStamp()+" "+misc.getClass()+" "+misc.getDebug(), Store.COMPRESS, Field.Index.UN_TOKENIZED)); }

      tempindexwriter.addDocument(doc);
      tempindexwriter.optimize();
      org.apache.lucene.store.Directory[] idxs = new org.apache.lucene.store.Directory[1];
      idxs[0] = tmpidxdirectory;
      //todo: del-call...
      this.indexwriter.addIndexes(idxs);
      if (this.bWithOptimize)
      this.indexwriter.optimize();
      }
      catch(Throwable e)

      { log("testthread:run\tERR:2(bWithOpti:"+bWithOptimize+")(iThread:"+iThread+"):"+e.getMessage()+"\n\t"+com.shi.jipmisc.stack2string(e), misc); }

      }
      }

      public SHILuceneWritingFromThreadsTest()
      {

      }

      public static void main(String[] args)

      { testluceneconcurrency(false); testluceneconcurrency(true ); }

      public static java.util.concurrent.ExecutorService threadpool_optimized = java.util.concurrent.Executors.newFixedThreadPool(9999999);
      public static java.util.concurrent.ExecutorService threadpool_not_optimized = java.util.concurrent.Executors.newFixedThreadPool(9999999);

      public static void testluceneconcurrency(boolean bWithOptimize)
      {
      com.shi.jipmisc.createDir("temp", "c:");
      com.shi.jipmisc.createDir("testluceneconcurrency", "c:/temp");
      com.shi.jipmisc.createDir(""+bWithOptimize, "c:/temp/testluceneconcurrency");
      com.shi.jipmisc misc = new com.shi.jipmisc();
      misc.setMyPath("C:/temp/testluceneconcurrency");
      misc.setDebug(true);
      misc.emptyDir("c:/temp/testluceneconcurrency/"+bWithOptimize);
      Analyzer anal = new com.shi.lucene.index.SHIAnalyser(null,null);
      org.apache.lucene.index.IndexWriter indexwriter = null;
      try

      { indexwriter = new IndexWriter("c:/temp/testluceneconcurrency/"+bWithOptimize, anal, true); }

      catch(Throwable e)

      { log("SHILuceneWritingFromThreadsTest:testluceneconcurrency\tERR:0("+bWithOptimize+"):"+e.getMessage(), misc); return; }

      log("SHILuceneWritingFromThreadsTest:testluceneconcurrency\tINF:step0("bWithOptimize")(BEGIN)-------------------------------------", misc);

      for(int idoc=0; idoc < 9999; idoc++)
      {
      Document doc = new Document();
      for(int ifld=0; ifld < 9; ifld++)

      { doc.add(new Field("field_"+ifld, "doc: "+idoc+" field: "+ifld+" "+System.currentTimeMillis()+" "+misc.getDateString()+" "+misc.getDateString2()+" "+com.shi.jipmisc.getTimeStamp()+" "+misc.getClass()+" "+misc.getDebug(), Store.COMPRESS, Field.Index.UN_TOKENIZED)); }

      try

      { indexwriter.addDocument(doc); }

      catch(Throwable e)

      { log("SHILuceneWritingFromThreadsTest:testluceneconcurrency\tERR:1("+bWithOptimize+"):"+e.getMessage(), misc); return; }

      }
      log("SHILuceneWritingFromThreadsTest:testluceneconcurrency\tINF:step1("bWithOptimize")", misc);
      try

      { indexwriter.optimize(); }

      catch(Throwable e)

      { log("SHILuceneWritingFromThreadsTest:testluceneconcurrency\tERR:2("+bWithOptimize+"):"+e.getMessage(), misc); return; }

      log("SHILuceneWritingFromThreadsTest:testluceneconcurrency\tINF:step2("bWithOptimize")", misc);
      java.util.concurrent.ExecutorService threadpool = bWithOptimize ? threadpool_optimized : threadpool_not_optimized;
      for(int it=0; it < 999; it++)

      { threadpool.execute(new testthread(indexwriter, it, bWithOptimize, misc)); }

      log("SHILuceneWritingFromThreadsTest:testluceneconcurrency\tINF:step3("bWithOptimize")", misc);
      threadpool.shutdown(); //=execute threads and wait til all threads finish
      com.shi.jipmisc.wait(999, misc);
      log("SHILuceneWritingFromThreadsTest:testluceneconcurrency\tINF:step4("bWithOptimize")", misc);

      try

      { //indexwriter.optimize(); //indexwriter.close(); }

      catch(Throwable e)

      { log("SHILuceneWritingFromThreadsTest:testluceneconcurrency\tERR:9("+bWithOptimize+"):"+e.getMessage(), misc); return; }

      log("SHILuceneWritingFromThreadsTest:testluceneconcurrency\tINF:step5("bWithOptimize")(END)-----------------------------", misc);
      }
      }

        Activity

        Hide
        Michael McCandless added a comment -

        Thanks for reporting this!

        It should already be fixed on 2.4 (this is a dup of LUCENE-1335). Could you run your test program on 2.4 to confirm?

        Show
        Michael McCandless added a comment - Thanks for reporting this! It should already be fixed on 2.4 (this is a dup of LUCENE-1335 ). Could you run your test program on 2.4 to confirm?
        Hide
        Rudi Quark added a comment -

        This issue seems to be fixed by 2.4 version.

        Show
        Rudi Quark added a comment - This issue seems to be fixed by 2.4 version.
        Hide
        Michael McCandless added a comment -

        Super thanks Rudi!

        Show
        Michael McCandless added a comment - Super thanks Rudi!
        Hide
        Michael McCandless added a comment -

        Dup of LUCENE-1335.

        Show
        Michael McCandless added a comment - Dup of LUCENE-1335 .

          People

          • Assignee:
            Unassigned
            Reporter:
            Rudi Quark
          • Votes:
            0 Vote for this issue
            Watchers:
            0 Start watching this issue

            Dates

            • Created:
              Updated:
              Resolved:

              Development