--- Before Optimization --------------------------------------------------------------------------------------------------------------------------------------------------------------------

2009-10-24 17:00:32,335 INFO org.apache.lucene.index.CheckIndex - Segments file=segments_5aa numSegments=14 version=FORMAT_DIAGNOSTICS [Lucene 2.9]
2009-10-24 17:00:32,337 INFO org.apache.lucene.index.CheckIndex -   1 of 14: name=_diu docCount=623534
2009-10-24 17:00:32,337 INFO org.apache.lucene.index.CheckIndex -     compound=true
2009-10-24 17:00:32,337 INFO org.apache.lucene.index.CheckIndex -     hasProx=true
2009-10-24 17:00:32,337 INFO org.apache.lucene.index.CheckIndex -     numFiles=2
2009-10-24 17:00:32,338 INFO org.apache.lucene.index.CheckIndex -     size (MB)=5,549.038
2009-10-24 17:00:32,339 INFO org.apache.lucene.index.CheckIndex -     diagnostics = {java.version=1.5.0_21, lucene.version=2.9.0 817268P - 2009-09-21 10:25:09, os=SunOS, os.arch=amd64, optimize=true, mergeDocStores=true, java.vendor=Sun Microsystems Inc., os.version=5.10, source=merge, mergeFactor=10}
2009-10-24 17:00:32,339 INFO org.apache.lucene.index.CheckIndex -     has deletions [delFileName=_diu_28.del]
2009-10-24 17:00:37,018 INFO org.apache.lucene.index.CheckIndex -     test: open reader.........OK [12331 deleted docs]
2009-10-24 17:00:37,058 INFO org.apache.lucene.index.CheckIndex -     test: fields..............OK [15111 fields]
2009-10-24 17:00:58,731 INFO org.apache.lucene.index.CheckIndex -     test: field norms.........OK [15111 fields]
2009-10-24 17:03:51,063 INFO org.apache.lucene.index.CheckIndex -     test: terms, freq, prox...OK [54938535 terms; 564885379 terms/docs pairs; 652753888 tokens]
2009-10-24 17:07:39,417 INFO org.apache.lucene.index.CheckIndex -     test: stored fields.......OK [32159982 total field count; avg 52.618 fields per doc]
2009-10-24 17:08:07,324 INFO org.apache.lucene.index.CheckIndex -     test: term vectors........OK [577624 total vector count; avg 0.945 term/freq vector fields per doc]
2009-10-24 17:08:07,325 INFO org.apache.lucene.index.CheckIndex -   2 of 14: name=_dkk docCount=9528
2009-10-24 17:08:07,326 INFO org.apache.lucene.index.CheckIndex -     compound=true
2009-10-24 17:08:07,326 INFO org.apache.lucene.index.CheckIndex -     hasProx=true
2009-10-24 17:08:07,326 INFO org.apache.lucene.index.CheckIndex -     numFiles=2
2009-10-24 17:08:07,344 INFO org.apache.lucene.index.CheckIndex -     size (MB)=98.222
2009-10-24 17:08:07,344 INFO org.apache.lucene.index.CheckIndex -     diagnostics = {java.version=1.5.0_21, lucene.version=2.9.0 817268P - 2009-09-21 10:25:09, os=SunOS, os.arch=amd64, optimize=false, mergeDocStores=true, java.vendor=Sun Microsystems Inc., os.version=5.10, source=merge, mergeFactor=10}
2009-10-24 17:08:07,345 INFO org.apache.lucene.index.CheckIndex -     has deletions [delFileName=_dkk_a.del]
2009-10-24 17:08:07,410 INFO org.apache.lucene.index.CheckIndex -     test: open reader.........OK [86 deleted docs]
2009-10-24 17:08:07,410 INFO org.apache.lucene.index.CheckIndex -     test: fields..............OK [632 fields]
2009-10-24 17:08:07,488 INFO org.apache.lucene.index.CheckIndex -     test: field norms.........OK [632 fields]
2009-10-24 17:08:09,330 INFO org.apache.lucene.index.CheckIndex -     test: terms, freq, prox...OK [606245 terms; 10922412 terms/docs pairs; 13375599 tokens]
2009-10-24 17:08:13,182 INFO org.apache.lucene.index.CheckIndex -     test: stored fields.......OK [614639 total field count; avg 65.096 fields per doc]
2009-10-24 17:08:13,533 INFO org.apache.lucene.index.CheckIndex -     test: term vectors........OK [9084 total vector count; avg 0.962 term/freq vector fields per doc]
2009-10-24 17:08:13,534 INFO org.apache.lucene.index.CheckIndex -   3 of 14: name=_dlq docCount=1351
2009-10-24 17:08:13,534 INFO org.apache.lucene.index.CheckIndex -     compound=true
2009-10-24 17:08:13,534 INFO org.apache.lucene.index.CheckIndex -     hasProx=true
2009-10-24 17:08:13,534 INFO org.apache.lucene.index.CheckIndex -     numFiles=2
2009-10-24 17:08:13,534 INFO org.apache.lucene.index.CheckIndex -     size (MB)=22.289
2009-10-24 17:08:13,535 INFO org.apache.lucene.index.CheckIndex -     diagnostics = {java.version=1.5.0_21, lucene.version=2.9.0 817268P - 2009-09-21 10:25:09, os=SunOS, os.arch=amd64, optimize=false, mergeDocStores=true, java.vendor=Sun Microsystems Inc., os.version=5.10, source=merge, mergeFactor=10}
2009-10-24 17:08:13,535 INFO org.apache.lucene.index.CheckIndex -     has deletions [delFileName=_dlq_4.del]
2009-10-24 17:08:13,588 INFO org.apache.lucene.index.CheckIndex -     test: open reader.........OK [94 deleted docs]
2009-10-24 17:08:13,588 INFO org.apache.lucene.index.CheckIndex -     test: fields..............OK [471 fields]
2009-10-24 17:08:13,612 INFO org.apache.lucene.index.CheckIndex -     test: field norms.........OK [471 fields]
2009-10-24 17:08:14,253 INFO org.apache.lucene.index.CheckIndex -     test: terms, freq, prox...OK [196441 terms; 2213399 terms/docs pairs; 2842075 tokens]
2009-10-24 17:08:15,180 INFO org.apache.lucene.index.CheckIndex -     test: stored fields.......OK [131798 total field count; avg 104.851 fields per doc]
2009-10-24 17:08:15,263 INFO org.apache.lucene.index.CheckIndex -     test: term vectors........OK [1257 total vector count; avg 1 term/freq vector fields per doc]
2009-10-24 17:08:15,264 INFO org.apache.lucene.index.CheckIndex -   4 of 14: name=_dlo docCount=5
2009-10-24 17:08:15,264 INFO org.apache.lucene.index.CheckIndex -     compound=true
2009-10-24 17:08:15,264 INFO org.apache.lucene.index.CheckIndex -     hasProx=true
2009-10-24 17:08:15,264 INFO org.apache.lucene.index.CheckIndex -     numFiles=2
2009-10-24 17:08:15,265 INFO org.apache.lucene.index.CheckIndex -     size (MB)=0.059
2009-10-24 17:08:15,265 INFO org.apache.lucene.index.CheckIndex -     diagnostics = {java.version=1.5.0_21, lucene.version=2.9.0 817268P - 2009-09-21 10:25:09, os=SunOS, os.arch=amd64, java.vendor=Sun Microsystems Inc., os.version=5.10, source=flush}
2009-10-24 17:08:15,265 INFO org.apache.lucene.index.CheckIndex -     has deletions [delFileName=_dlo_1.del]
2009-10-24 17:08:15,356 INFO org.apache.lucene.index.CheckIndex -     test: open reader.........OK [5 deleted docs]
2009-10-24 17:08:15,356 INFO org.apache.lucene.index.CheckIndex -     test: fields..............OK [136 fields]
2009-10-24 17:08:15,357 INFO org.apache.lucene.index.CheckIndex -     test: field norms.........OK [136 fields]
2009-10-24 17:08:15,372 INFO org.apache.lucene.index.CheckIndex -     test: terms, freq, prox...OK [1698 terms; 4236 terms/docs pairs; 0 tokens]
2009-10-24 17:08:15,373 INFO org.apache.lucene.index.CheckIndex -     test: stored fields.......OK [0 total field count; avg ? fields per doc]
2009-10-24 17:08:15,373 INFO org.apache.lucene.index.CheckIndex -     test: term vectors........OK [0 total vector count; avg ? term/freq vector fields per doc]
2009-10-24 17:08:15,374 INFO org.apache.lucene.index.CheckIndex -   5 of 14: name=_dlp docCount=1000
2009-10-24 17:08:15,374 INFO org.apache.lucene.index.CheckIndex -     compound=true
2009-10-24 17:08:15,374 INFO org.apache.lucene.index.CheckIndex -     hasProx=true
2009-10-24 17:08:15,374 INFO org.apache.lucene.index.CheckIndex -     numFiles=3
2009-10-24 17:08:15,374 INFO org.apache.lucene.index.CheckIndex -     size (MB)=3.37
2009-10-24 17:08:15,374 INFO org.apache.lucene.index.CheckIndex -     diagnostics = {java.version=1.5.0_21, lucene.version=2.9.0 817268P - 2009-09-21 10:25:09, os=SunOS, os.arch=amd64, java.vendor=Sun Microsystems Inc., os.version=5.10, source=flush}
2009-10-24 17:08:15,375 INFO org.apache.lucene.index.CheckIndex -     docStoreOffset=0
2009-10-24 17:08:15,375 INFO org.apache.lucene.index.CheckIndex -     docStoreSegment=_dlp
2009-10-24 17:08:15,375 INFO org.apache.lucene.index.CheckIndex -     docStoreIsCompoundFile=true
2009-10-24 17:08:15,375 INFO org.apache.lucene.index.CheckIndex -     has deletions [delFileName=_dlp_1.del]
2009-10-24 17:08:15,440 INFO org.apache.lucene.index.CheckIndex -     test: open reader.........OK [1 deleted docs]
2009-10-24 17:08:15,440 INFO org.apache.lucene.index.CheckIndex -     test: fields..............OK [167 fields]
2009-10-24 17:08:15,441 INFO org.apache.lucene.index.CheckIndex -     test: field norms.........OK [167 fields]
2009-10-24 17:08:15,738 INFO org.apache.lucene.index.CheckIndex -     test: terms, freq, prox...OK [102586 terms; 987048 terms/docs pairs; 1113284 tokens]
2009-10-24 17:08:16,092 INFO org.apache.lucene.index.CheckIndex -     test: stored fields.......OK [65222 total field count; avg 65.287 fields per doc]
2009-10-24 17:08:16,143 INFO org.apache.lucene.index.CheckIndex -     test: term vectors........OK [999 total vector count; avg 1 term/freq vector fields per doc]
2009-10-24 17:08:16,143 INFO org.apache.lucene.index.CheckIndex -   6 of 14: name=_dlr docCount=441
2009-10-24 17:08:16,143 INFO org.apache.lucene.index.CheckIndex -     compound=true
2009-10-24 17:08:16,143 INFO org.apache.lucene.index.CheckIndex -     hasProx=true
2009-10-24 17:08:16,144 INFO org.apache.lucene.index.CheckIndex -     numFiles=1
2009-10-24 17:08:16,144 INFO org.apache.lucene.index.CheckIndex -     size (MB)=4.972
2009-10-24 17:08:16,144 INFO org.apache.lucene.index.CheckIndex -     diagnostics = {java.version=1.5.0_21, lucene.version=2.9.0 817268P - 2009-09-21 10:25:09, os=SunOS, os.arch=amd64, java.vendor=Sun Microsystems Inc., os.version=5.10, source=flush}
2009-10-24 17:08:16,144 INFO org.apache.lucene.index.CheckIndex -     no deletions
2009-10-24 17:08:16,167 INFO org.apache.lucene.index.CheckIndex -     test: open reader.........OK
2009-10-24 17:08:16,167 INFO org.apache.lucene.index.CheckIndex -     test: fields..............OK [167 fields]
2009-10-24 17:08:16,168 INFO org.apache.lucene.index.CheckIndex -     test: field norms.........OK [167 fields]
2009-10-24 17:08:16,304 INFO org.apache.lucene.index.CheckIndex -     test: terms, freq, prox...OK [31791 terms; 596030 terms/docs pairs; 702936 tokens]
2009-10-24 17:08:16,523 INFO org.apache.lucene.index.CheckIndex -     test: stored fields.......OK [32619 total field count; avg 73.966 fields per doc]
2009-10-24 17:08:16,540 INFO org.apache.lucene.index.CheckIndex -     test: term vectors........OK [441 total vector count; avg 1 term/freq vector fields per doc]
2009-10-24 17:08:16,541 INFO org.apache.lucene.index.CheckIndex -   7 of 14: name=_dm2 docCount=189
2009-10-24 17:08:16,541 INFO org.apache.lucene.index.CheckIndex -     compound=true
2009-10-24 17:08:16,541 INFO org.apache.lucene.index.CheckIndex -     hasProx=true
2009-10-24 17:08:16,541 INFO org.apache.lucene.index.CheckIndex -     numFiles=2
2009-10-24 17:08:16,541 INFO org.apache.lucene.index.CheckIndex -     size (MB)=3.205
2009-10-24 17:08:16,541 INFO org.apache.lucene.index.CheckIndex -     diagnostics = {java.version=1.5.0_21, lucene.version=2.9.0 817268P - 2009-09-21 10:25:09, os=SunOS, os.arch=amd64, optimize=false, mergeDocStores=true, java.vendor=Sun Microsystems Inc., os.version=5.10, source=merge, mergeFactor=10}
2009-10-24 17:08:16,542 INFO org.apache.lucene.index.CheckIndex -     has deletions [delFileName=_dm2_2.del]
2009-10-24 17:08:16,599 INFO org.apache.lucene.index.CheckIndex -     test: open reader.........OK [71 deleted docs]
2009-10-24 17:08:16,599 INFO org.apache.lucene.index.CheckIndex -     test: fields..............OK [271 fields]
2009-10-24 17:08:16,613 INFO org.apache.lucene.index.CheckIndex -     test: field norms.........OK [271 fields]
2009-10-24 17:08:16,839 INFO org.apache.lucene.index.CheckIndex -     test: terms, freq, prox...OK [68684 terms; 281857 terms/docs pairs; 185552 tokens]
2009-10-24 17:08:16,907 INFO org.apache.lucene.index.CheckIndex -     test: stored fields.......OK [9069 total field count; avg 76.856 fields per doc]
2009-10-24 17:08:16,917 INFO org.apache.lucene.index.CheckIndex -     test: term vectors........OK [118 total vector count; avg 1 term/freq vector fields per doc]
2009-10-24 17:08:16,918 INFO org.apache.lucene.index.CheckIndex -   8 of 14: name=_dm3 docCount=6
2009-10-24 17:08:16,918 INFO org.apache.lucene.index.CheckIndex -     compound=true
2009-10-24 17:08:16,918 INFO org.apache.lucene.index.CheckIndex -     hasProx=true
2009-10-24 17:08:16,918 INFO org.apache.lucene.index.CheckIndex -     numFiles=1
2009-10-24 17:08:16,918 INFO org.apache.lucene.index.CheckIndex -     size (MB)=0.194
2009-10-24 17:08:16,919 INFO org.apache.lucene.index.CheckIndex -     diagnostics = {java.version=1.5.0_21, lucene.version=2.9.0 817268P - 2009-09-21 10:25:09, os=SunOS, os.arch=amd64, java.vendor=Sun Microsystems Inc., os.version=5.10, source=flush}
2009-10-24 17:08:16,919 INFO org.apache.lucene.index.CheckIndex -     no deletions
2009-10-24 17:08:16,938 INFO org.apache.lucene.index.CheckIndex -     test: open reader.........OK
2009-10-24 17:08:16,939 INFO org.apache.lucene.index.CheckIndex -     test: fields..............OK [150 fields]
2009-10-24 17:08:16,940 INFO org.apache.lucene.index.CheckIndex -     test: field norms.........OK [150 fields]
2009-10-24 17:08:16,959 INFO org.apache.lucene.index.CheckIndex -     test: terms, freq, prox...OK [5274 terms; 13407 terms/docs pairs; 20984 tokens]
2009-10-24 17:08:16,979 INFO org.apache.lucene.index.CheckIndex -     test: stored fields.......OK [902 total field count; avg 150.333 fields per doc]
2009-10-24 17:08:16,980 INFO org.apache.lucene.index.CheckIndex -     test: term vectors........OK [6 total vector count; avg 1 term/freq vector fields per doc]
2009-10-24 17:08:16,981 INFO org.apache.lucene.index.CheckIndex -   9 of 14: name=_dm4 docCount=68
2009-10-24 17:08:16,981 INFO org.apache.lucene.index.CheckIndex -     compound=true
2009-10-24 17:08:16,981 INFO org.apache.lucene.index.CheckIndex -     hasProx=true
2009-10-24 17:08:16,981 INFO org.apache.lucene.index.CheckIndex -     numFiles=1
2009-10-24 17:08:16,982 INFO org.apache.lucene.index.CheckIndex -     size (MB)=1.535
2009-10-24 17:08:16,982 INFO org.apache.lucene.index.CheckIndex -     diagnostics = {java.version=1.5.0_21, lucene.version=2.9.0 817268P - 2009-09-21 10:25:09, os=SunOS, os.arch=amd64, java.vendor=Sun Microsystems Inc., os.version=5.10, source=flush}
2009-10-24 17:08:16,982 INFO org.apache.lucene.index.CheckIndex -     no deletions
2009-10-24 17:08:17,012 INFO org.apache.lucene.index.CheckIndex -     test: open reader.........OK
2009-10-24 17:08:17,012 INFO org.apache.lucene.index.CheckIndex -     test: fields..............OK [151 fields]
2009-10-24 17:08:17,020 INFO org.apache.lucene.index.CheckIndex -     test: field norms.........OK [151 fields]
2009-10-24 17:08:17,114 INFO org.apache.lucene.index.CheckIndex -     test: terms, freq, prox...OK [38735 terms; 146075 terms/docs pairs; 198370 tokens]
2009-10-24 17:08:17,197 INFO org.apache.lucene.index.CheckIndex -     test: stored fields.......OK [5823 total field count; avg 85.632 fields per doc]
2009-10-24 17:08:17,207 INFO org.apache.lucene.index.CheckIndex -     test: term vectors........OK [66 total vector count; avg 0.971 term/freq vector fields per doc]
2009-10-24 17:08:17,207 INFO org.apache.lucene.index.CheckIndex -   10 of 14: name=_dm5 docCount=21
2009-10-24 17:08:17,208 INFO org.apache.lucene.index.CheckIndex -     compound=true
2009-10-24 17:08:17,208 INFO org.apache.lucene.index.CheckIndex -     hasProx=true
2009-10-24 17:08:17,208 INFO org.apache.lucene.index.CheckIndex -     numFiles=2
2009-10-24 17:08:17,208 INFO org.apache.lucene.index.CheckIndex -     size (MB)=0.285
2009-10-24 17:08:17,208 INFO org.apache.lucene.index.CheckIndex -     diagnostics = {java.version=1.5.0_21, lucene.version=2.9.0 817268P - 2009-09-21 10:25:09, os=SunOS, os.arch=amd64, java.vendor=Sun Microsystems Inc., os.version=5.10, source=flush}
2009-10-24 17:08:17,209 INFO org.apache.lucene.index.CheckIndex -     has deletions [delFileName=_dm5_1.del]
2009-10-24 17:08:17,233 INFO org.apache.lucene.index.CheckIndex -     test: open reader.........OK [15 deleted docs]
2009-10-24 17:08:17,233 INFO org.apache.lucene.index.CheckIndex -     test: fields..............OK [177 fields]
2009-10-24 17:08:17,234 INFO org.apache.lucene.index.CheckIndex -     test: field norms.........OK [177 fields]
2009-10-24 17:08:17,258 INFO org.apache.lucene.index.CheckIndex -     test: terms, freq, prox...OK [7748 terms; 23094 terms/docs pairs; 8788 tokens]
2009-10-24 17:08:17,289 INFO org.apache.lucene.index.CheckIndex -     test: stored fields.......OK [391 total field count; avg 65.167 fields per doc]
2009-10-24 17:08:17,289 INFO org.apache.lucene.index.CheckIndex -     test: term vectors........OK [6 total vector count; avg 1 term/freq vector fields per doc]
2009-10-24 17:08:17,290 INFO org.apache.lucene.index.CheckIndex -   11 of 14: name=_dm6 docCount=15
2009-10-24 17:08:17,290 INFO org.apache.lucene.index.CheckIndex -     compound=true
2009-10-24 17:08:17,290 INFO org.apache.lucene.index.CheckIndex -     hasProx=true
2009-10-24 17:08:17,290 INFO org.apache.lucene.index.CheckIndex -     numFiles=1
2009-10-24 17:08:17,290 INFO org.apache.lucene.index.CheckIndex -     size (MB)=0.182
2009-10-24 17:08:17,291 INFO org.apache.lucene.index.CheckIndex -     diagnostics = {java.version=1.5.0_21, lucene.version=2.9.0 817268P - 2009-09-21 10:25:09, os=SunOS, os.arch=amd64, java.vendor=Sun Microsystems Inc., os.version=5.10, source=flush}
2009-10-24 17:08:17,291 INFO org.apache.lucene.index.CheckIndex -     no deletions
2009-10-24 17:08:17,316 INFO org.apache.lucene.index.CheckIndex -     test: open reader.........OK
2009-10-24 17:08:17,316 INFO org.apache.lucene.index.CheckIndex -     test: fields..............OK [179 fields]
2009-10-24 17:08:17,316 INFO org.apache.lucene.index.CheckIndex -     test: field norms.........OK [179 fields]
2009-10-24 17:08:17,329 INFO org.apache.lucene.index.CheckIndex -     test: terms, freq, prox...OK [4698 terms; 14864 terms/docs pairs; 17583 tokens]
2009-10-24 17:08:17,337 INFO org.apache.lucene.index.CheckIndex -     test: stored fields.......OK [1028 total field count; avg 68.533 fields per doc]
2009-10-24 17:08:17,338 INFO org.apache.lucene.index.CheckIndex -     test: term vectors........OK [15 total vector count; avg 1 term/freq vector fields per doc]
2009-10-24 17:08:17,338 INFO org.apache.lucene.index.CheckIndex -   12 of 14: name=_dm7 docCount=1
2009-10-24 17:08:17,338 INFO org.apache.lucene.index.CheckIndex -     compound=true
2009-10-24 17:08:17,338 INFO org.apache.lucene.index.CheckIndex -     hasProx=true
2009-10-24 17:08:17,339 INFO org.apache.lucene.index.CheckIndex -     numFiles=1
2009-10-24 17:08:17,339 INFO org.apache.lucene.index.CheckIndex -     size (MB)=0.009
2009-10-24 17:08:17,339 INFO org.apache.lucene.index.CheckIndex -     diagnostics = {java.version=1.5.0_21, lucene.version=2.9.0 817268P - 2009-09-21 10:25:09, os=SunOS, os.arch=amd64, java.vendor=Sun Microsystems Inc., os.version=5.10, source=flush}
2009-10-24 17:08:17,339 INFO org.apache.lucene.index.CheckIndex -     no deletions
2009-10-24 17:08:17,346 INFO org.apache.lucene.index.CheckIndex -     test: open reader.........OK
2009-10-24 17:08:17,346 INFO org.apache.lucene.index.CheckIndex -     test: fields..............OK [34 fields]
2009-10-24 17:08:17,346 INFO org.apache.lucene.index.CheckIndex -     test: field norms.........OK [34 fields]
2009-10-24 17:08:17,347 INFO org.apache.lucene.index.CheckIndex -     test: terms, freq, prox...OK [322 terms; 322 terms/docs pairs; 358 tokens]
2009-10-24 17:08:17,347 INFO org.apache.lucene.index.CheckIndex -     test: stored fields.......OK [15 total field count; avg 15 fields per doc]
2009-10-24 17:08:17,348 INFO org.apache.lucene.index.CheckIndex -     test: term vectors........OK [1 total vector count; avg 1 term/freq vector fields per doc]
2009-10-24 17:08:17,348 INFO org.apache.lucene.index.CheckIndex -   13 of 14: name=_dm8 docCount=7
2009-10-24 17:08:17,348 INFO org.apache.lucene.index.CheckIndex -     compound=true
2009-10-24 17:08:17,348 INFO org.apache.lucene.index.CheckIndex -     hasProx=true
2009-10-24 17:08:17,348 INFO org.apache.lucene.index.CheckIndex -     numFiles=2
2009-10-24 17:08:17,348 INFO org.apache.lucene.index.CheckIndex -     size (MB)=0.165
2009-10-24 17:08:17,349 INFO org.apache.lucene.index.CheckIndex -     diagnostics = {java.version=1.5.0_21, lucene.version=2.9.0 817268P - 2009-09-21 10:25:09, os=SunOS, os.arch=amd64, java.vendor=Sun Microsystems Inc., os.version=5.10, source=flush}
2009-10-24 17:08:17,349 INFO org.apache.lucene.index.CheckIndex -     has deletions [delFileName=_dm8_1.del]
2009-10-24 17:08:17,373 INFO org.apache.lucene.index.CheckIndex -     test: open reader.........OK [7 deleted docs]
2009-10-24 17:08:17,373 INFO org.apache.lucene.index.CheckIndex -     test: fields..............OK [150 fields]
2009-10-24 17:08:17,373 INFO org.apache.lucene.index.CheckIndex -     test: field norms.........OK [150 fields]
2009-10-24 17:08:17,381 INFO org.apache.lucene.index.CheckIndex -     test: terms, freq, prox...OK [4401 terms; 14774 terms/docs pairs; 0 tokens]
2009-10-24 17:08:17,382 INFO org.apache.lucene.index.CheckIndex -     test: stored fields.......OK [0 total field count; avg ? fields per doc]
2009-10-24 17:08:17,382 INFO org.apache.lucene.index.CheckIndex -     test: term vectors........OK [0 total vector count; avg ? term/freq vector fields per doc]
2009-10-24 17:08:17,383 INFO org.apache.lucene.index.CheckIndex -   14 of 14: name=_dm9 docCount=60
2009-10-24 17:08:17,383 INFO org.apache.lucene.index.CheckIndex -     compound=true
2009-10-24 17:08:17,383 INFO org.apache.lucene.index.CheckIndex -     hasProx=true
2009-10-24 17:08:17,383 INFO org.apache.lucene.index.CheckIndex -     numFiles=1
2009-10-24 17:08:17,383 INFO org.apache.lucene.index.CheckIndex -     size (MB)=0.895
2009-10-24 17:08:17,383 INFO org.apache.lucene.index.CheckIndex -     diagnostics = {java.version=1.5.0_21, lucene.version=2.9.0 817268P - 2009-09-21 10:25:09, os=SunOS, os.arch=amd64, java.vendor=Sun Microsystems Inc., os.version=5.10, source=flush}
2009-10-24 17:08:17,383 INFO org.apache.lucene.index.CheckIndex -     no deletions
2009-10-24 17:08:17,414 INFO org.apache.lucene.index.CheckIndex -     test: open reader.........OK
2009-10-24 17:08:17,414 INFO org.apache.lucene.index.CheckIndex -     test: fields..............OK [178 fields]
2009-10-24 17:08:17,415 INFO org.apache.lucene.index.CheckIndex -     test: field norms.........OK [178 fields]
2009-10-24 17:08:17,443 INFO org.apache.lucene.index.CheckIndex -     test: terms, freq, prox...OK [10517 terms; 104446 terms/docs pairs; 125888 tokens]
2009-10-24 17:08:17,497 INFO org.apache.lucene.index.CheckIndex -     test: stored fields.......OK [5613 total field count; avg 93.55 fields per doc]
2009-10-24 17:08:17,501 INFO org.apache.lucene.index.CheckIndex -     test: term vectors........OK [60 total vector count; avg 1 term/freq vector fields per doc]
2009-10-24 17:08:17,501 INFO org.apache.lucene.index.CheckIndex - No problems were detected with this index.




--- Lucene 2.9: ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------

middlew@pansrv1:~/pvindex$ time ./optimize.sh 
real    8m7.949s
user    2m53.612s
sys     2m28.776s

2009-10-24 16:50:37,244 INFO org.apache.lucene.index.CheckIndex - Segments file=segments_5ab numSegments=1 version=FORMAT_DIAGNOSTICS [Lucene 2.9]
2009-10-24 16:50:37,248 INFO org.apache.lucene.index.CheckIndex -   1 of 1: name=_dmb docCount=623616
2009-10-24 16:50:37,248 INFO org.apache.lucene.index.CheckIndex -     compound=true
2009-10-24 16:50:37,249 INFO org.apache.lucene.index.CheckIndex -     hasProx=true
2009-10-24 16:50:37,249 INFO org.apache.lucene.index.CheckIndex -     numFiles=1
2009-10-24 16:50:37,256 INFO org.apache.lucene.index.CheckIndex -     size (MB)=5,550.325
2009-10-24 16:50:37,256 INFO org.apache.lucene.index.CheckIndex -     diagnostics = {java.version=1.5.0_21, lucene.version=2.9.0 817268P - 2009-09-21 10:25:09, os=SunOS, os.arch=amd64, optimize=true, mergeDocStores=true, java.vendor=Sun Microsystems Inc., os.version=5.10, source=merge, mergeFactor=5}
2009-10-24 16:50:37,256 INFO org.apache.lucene.index.CheckIndex -     no deletions
2009-10-24 16:50:40,856 INFO org.apache.lucene.index.CheckIndex -     test: open reader.........OK
2009-10-24 16:50:40,884 INFO org.apache.lucene.index.CheckIndex -     test: fields..............OK [15135 fields]
2009-10-24 16:50:52,530 INFO org.apache.lucene.index.CheckIndex -     test: field norms.........OK [15135 fields]
2009-10-24 16:52:58,802 INFO org.apache.lucene.index.CheckIndex -     test: terms, freq, prox...OK [54949269 terms; 564946256 terms/docs pairs; 671345305 tokens]
2009-10-24 16:56:44,957 INFO org.apache.lucene.index.CheckIndex -     test: stored fields.......OK [33027101 total field count; avg 52.961 fields per doc]
2009-10-24 16:57:11,331 INFO org.apache.lucene.index.CheckIndex -     test: term vectors........OK [589677 total vector count; avg 0.946 term/freq vector fields per doc]
2009-10-24 16:57:11,332 INFO org.apache.lucene.index.CheckIndex - No problems were detected with this index.

Update 38 documents and then optimize again:

middlew@pansrv1:~/pvindex$ time ./optimize.sh
real    7m59.934s
user    3m12.261s
sys     2m35.106s

2009-10-24 17:54:53,682 INFO org.apache.lucene.index.CheckIndex - Segments file=segments_5ad numSegments=1 version=FORMAT_DIAGNOSTICS [Lucene 2.9]
2009-10-24 17:54:53,691 INFO org.apache.lucene.index.CheckIndex -   1 of 1: name=_dmd docCount=623616
2009-10-24 17:54:53,691 INFO org.apache.lucene.index.CheckIndex -     compound=true
2009-10-24 17:54:53,691 INFO org.apache.lucene.index.CheckIndex -     hasProx=true
2009-10-24 17:54:53,692 INFO org.apache.lucene.index.CheckIndex -     numFiles=1
2009-10-24 17:54:53,697 INFO org.apache.lucene.index.CheckIndex -     size (MB)=5,550.328
2009-10-24 17:54:53,697 INFO org.apache.lucene.index.CheckIndex -     diagnostics = {java.version=1.5.0_21, lucene.version=2.9.0 817268P - 2009-09-21 10:25:09, os=SunOS, os.arch=amd64, optimize=true, mergeDocStores=true, java.vendor=Sun Microsystems Inc., os.version=5.10, source=merge, mergeFactor=2}
2009-10-24 17:54:53,697 INFO org.apache.lucene.index.CheckIndex -     no deletions
2009-10-24 17:54:57,285 INFO org.apache.lucene.index.CheckIndex -     test: open reader.........OK
2009-10-24 17:54:57,316 INFO org.apache.lucene.index.CheckIndex -     test: fields..............OK [15135 fields]
2009-10-24 17:55:09,821 INFO org.apache.lucene.index.CheckIndex -     test: field norms.........OK [15135 fields]
2009-10-24 17:57:23,990 INFO org.apache.lucene.index.CheckIndex -     test: terms, freq, prox...OK [54949266 terms; 564946271 terms/docs pairs; 671345294 tokens]
2009-10-24 18:01:21,896 INFO org.apache.lucene.index.CheckIndex -     test: stored fields.......OK [33027101 total field count; avg 52.961 fields per doc]
2009-10-24 18:01:49,528 INFO org.apache.lucene.index.CheckIndex -     test: term vectors........OK [589677 total vector count; avg 0.946 term/freq vector fields per doc]
2009-10-24 18:01:49,529 INFO org.apache.lucene.index.CheckIndex - No problems were detected with this index.




--- Lucene 3.0 + patch buschmi -------------------------------------------------------------------------------------------------------------------------------------------------------------

middlew@pansrv1:~/pvindex.2$ time ./optimize.sh 
real    17m52.670s
user    8m54.373s
sys     4m55.338s

2009-10-24 17:28:52,872 INFO org.apache.lucene.index.CheckIndex - Segments file=segments_5ab numSegments=1 version=FORMAT_DIAGNOSTICS [Lucene 2.9]
2009-10-24 17:28:52,873 INFO org.apache.lucene.index.CheckIndex -   1 of 1: name=_dmb docCount=623616
2009-10-24 17:28:52,873 INFO org.apache.lucene.index.CheckIndex -     compound=true
2009-10-24 17:28:52,873 INFO org.apache.lucene.index.CheckIndex -     hasProx=true
2009-10-24 17:28:52,873 INFO org.apache.lucene.index.CheckIndex -     numFiles=1
2009-10-24 17:28:52,878 INFO org.apache.lucene.index.CheckIndex -     size (MB)=11,481.688
2009-10-24 17:28:52,878 INFO org.apache.lucene.index.CheckIndex -     diagnostics = {java.version=1.5.0_21, lucene.version=3.0-dev ${svnversion} - 2009-10-24 10:25:18, os=SunOS, os.arch=amd64, optimize=true, mergeDocStores=true, java.vendor=Sun Microsystems Inc., os.version=5.10, source=merge, mergeFactor=5}
2009-10-24 17:28:52,878 INFO org.apache.lucene.index.CheckIndex -     no deletions
2009-10-24 17:28:56,356 INFO org.apache.lucene.index.CheckIndex -     test: open reader.........OK
2009-10-24 17:28:56,380 INFO org.apache.lucene.index.CheckIndex -     test: fields..............OK [15135 fields]
2009-10-24 17:29:07,772 INFO org.apache.lucene.index.CheckIndex -     test: field norms.........OK [15135 fields]
2009-10-24 17:31:50,412 INFO org.apache.lucene.index.CheckIndex -     test: terms, freq, prox...OK [54949269 terms; 564946256 terms/docs pairs; 671345305 tokens]
2009-10-24 17:33:52,029 INFO org.apache.lucene.index.CheckIndex -     test: stored fields.......OK [33027101 total field count; avg 52.961 fields per doc]
2009-10-24 17:34:17,286 INFO org.apache.lucene.index.CheckIndex -     test: term vectors........OK [589677 total vector count; avg 0.946 term/freq vector fields per doc]
2009-10-24 17:34:17,287 INFO org.apache.lucene.index.CheckIndex - No problems were detected with this index.

Update 38 documents and then optimize again:

middlew@pansrv1:~/pvindex.2$ time ./optimize.sh
real    11m55.246s
user    3m18.471s
sys     4m38.736s

2009-10-24 18:34:47,818 INFO org.apache.lucene.index.CheckIndex - Segments file=segments_5af numSegments=1 version=FORMAT_DIAGNOSTICS [Lucene 2.9]
2009-10-24 18:34:47,819 INFO org.apache.lucene.index.CheckIndex -   1 of 1: name=_dmf docCount=623616
2009-10-24 18:34:47,819 INFO org.apache.lucene.index.CheckIndex -     compound=true
2009-10-24 18:34:47,819 INFO org.apache.lucene.index.CheckIndex -     hasProx=true
2009-10-24 18:34:47,820 INFO org.apache.lucene.index.CheckIndex -     numFiles=1
2009-10-24 18:34:47,821 INFO org.apache.lucene.index.CheckIndex -     size (MB)=11,481.693
2009-10-24 18:34:47,821 INFO org.apache.lucene.index.CheckIndex -     diagnostics = {java.version=1.5.0_21, lucene.version=3.0-dev ${svnversion} - 2009-10-24 10:25:18, os=SunOS, os.arch=amd64, optimize=true, mergeDocStores=true, java.vendor=Sun Microsystems Inc., os.version=5.10, source=merge, mergeFactor=2}
2009-10-24 18:34:47,821 INFO org.apache.lucene.index.CheckIndex -     no deletions
2009-10-24 18:34:51,486 INFO org.apache.lucene.index.CheckIndex -     test: open reader.........OK
2009-10-24 18:34:51,514 INFO org.apache.lucene.index.CheckIndex -     test: fields..............OK [15135 fields]
2009-10-24 18:35:02,688 INFO org.apache.lucene.index.CheckIndex -     test: field norms.........OK [15135 fields]
2009-10-24 18:37:10,748 INFO org.apache.lucene.index.CheckIndex -     test: terms, freq, prox...OK [54949265 terms; 564946271 terms/docs pairs; 671345294 tokens]
2009-10-24 18:38:55,191 INFO org.apache.lucene.index.CheckIndex -     test: stored fields.......OK [33027101 total field count; avg 52.961 fields per doc]
2009-10-24 18:39:24,843 INFO org.apache.lucene.index.CheckIndex -     test: term vectors........OK [589677 total vector count; avg 0.946 term/freq vector fields per doc]
2009-10-24 18:39:24,844 INFO org.apache.lucene.index.CheckIndex - No problems were detected with this index.