Index: lucene/common-build.xml =================================================================== --- lucene/common-build.xml (revision 934796) +++ lucene/common-build.xml (working copy) @@ -441,12 +441,6 @@ - - - Index: lucene/src/test/org/apache/lucene/collation/CollationTestBase.java =================================================================== --- lucene/src/test/org/apache/lucene/collation/CollationTestBase.java (revision 934796) +++ lucene/src/test/org/apache/lucene/collation/CollationTestBase.java (working copy) @@ -42,7 +42,7 @@ import java.io.IOException; -public class CollationTestBase extends LuceneTestCase { +public abstract class CollationTestBase extends LuceneTestCase { protected String firstRangeBeginningOriginal = "\u062F"; protected String firstRangeEndOriginal = "\u0698"; Index: lucene/src/test/org/apache/lucene/util/cache/BaseTestLRU.java =================================================================== --- lucene/src/test/org/apache/lucene/util/cache/BaseTestLRU.java (revision 934796) +++ lucene/src/test/org/apache/lucene/util/cache/BaseTestLRU.java (working copy) @@ -19,7 +19,7 @@ import org.apache.lucene.util.LuceneTestCase; -public class BaseTestLRU extends LuceneTestCase { +public abstract class BaseTestLRU extends LuceneTestCase { protected void testCache(Cache cache, int n) throws Exception { Object dummy = new Object(); Index: lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/BenchmarkTestCase.java =================================================================== --- lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/BenchmarkTestCase.java (revision 934796) +++ lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/BenchmarkTestCase.java (working copy) @@ -18,21 +18,83 @@ */ import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.StringReader; +import org.apache.lucene.benchmark.byTask.Benchmark; import org.apache.lucene.util.LuceneTestCase; /** Base class for all Benchmark unit tests. */ -public class BenchmarkTestCase extends LuceneTestCase { +public abstract class BenchmarkTestCase extends LuceneTestCase { + + public File getWorkDir() { + return TEMP_DIR; + } + + /** Copy a resource into the workdir */ + public void copyToWorkDir(String resourceName) throws IOException { + InputStream resource = getClass().getResourceAsStream(resourceName); + OutputStream dest = new FileOutputStream(new File(getWorkDir(), resourceName)); + byte[] buffer = new byte[8192]; + int len; + + while ((len = resource.read(buffer)) > 0) { + dest.write(buffer, 0, len); + } - private static final File workDir; - - static { - workDir = new File(System.getProperty("benchmark.work.dir", "test/benchmark")).getAbsoluteFile(); - workDir.mkdirs(); + resource.close(); + dest.close(); } - public File getWorkDir() { - return workDir; + /** Return a path, suitable for a .alg config file, for a resource in the workdir */ + public String getWorkDirResourcePath(String resourceName) { + return new File(getWorkDir(), resourceName).getAbsolutePath().replace("\\", "/"); } + /** Return a path, suitable for a .alg config file, for the workdir */ + public String getWorkDirPath() { + return getWorkDir().getAbsolutePath().replace("\\", "/"); + } + + // create the benchmark and execute it. + public Benchmark execBenchmark(String[] algLines) throws Exception { + String algText = algLinesToText(algLines); + logTstLogic(algText); + Benchmark benchmark = new Benchmark(new StringReader(algText)); + benchmark.execute(); + return benchmark; + } + + // properties in effect in all tests here + final String propLines [] = { + "work.dir=" + getWorkDirPath(), + "directory=RAMDirectory", + "print.props=false", + }; + + static final String NEW_LINE = System.getProperty("line.separator"); + + // catenate alg lines to make the alg text + private String algLinesToText(String[] algLines) { + String indent = " "; + StringBuffer sb = new StringBuffer(); + for (int i = 0; i < propLines.length; i++) { + sb.append(indent).append(propLines[i]).append(NEW_LINE); + } + for (int i = 0; i < algLines.length; i++) { + sb.append(indent).append(algLines[i]).append(NEW_LINE); + } + return sb.toString(); + } + + private static void logTstLogic (String txt) { + if (!VERBOSE) + return; + System.out.println("Test logic of:"); + System.out.println(txt); + } + } Index: lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/quality/TestQualityRun.java =================================================================== --- lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/quality/TestQualityRun.java (revision 934796) +++ lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/quality/TestQualityRun.java (working copy) @@ -19,7 +19,6 @@ import java.io.BufferedReader; import java.io.File; -import java.io.FileReader; import java.io.InputStream; import java.io.InputStreamReader; import java.io.PrintWriter; @@ -36,7 +35,6 @@ import org.apache.lucene.benchmark.quality.utils.SubmissionReport; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.store.FSDirectory; -import org.apache.lucene.util.LuceneTestCase; /** * Test that quality run does its job. @@ -47,6 +45,12 @@ */ public class TestQualityRun extends BenchmarkTestCase { + @Override + protected void setUp() throws Exception { + super.setUp(); + copyToWorkDir("reuters.578.lines.txt.bz2"); + } + public void testTrecQuality() throws Exception { // first create the partial reuters index createReutersIndex(); @@ -173,7 +177,7 @@ String algLines[] = { "# ----- properties ", "content.source=org.apache.lucene.benchmark.byTask.feeds.LineDocSource", - "docs.file=" + getReuters578LinesFile(), + "docs.file=" + getWorkDirResourcePath("reuters.578.lines.txt.bz2"), "content.source.log.step=2500", "doc.term.vector=false", "content.source.forever=false", @@ -188,11 +192,6 @@ }; // 2. execute the algorithm (required in every "logic" test) - TestPerfTasksLogic.execBenchmark(algLines); + execBenchmark(algLines); } - - private static String getReuters578LinesFile() { - return System.getProperty("lucene.common.dir").replace('\\','/') + - "/contrib/benchmark/src/test/org/apache/lucene/benchmark/quality/reuters.578.lines.txt.bz2"; - } } Index: lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/reuters.first20.lines.txt =================================================================== --- lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/reuters.first20.lines.txt (revision 934796) +++ lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/reuters.first20.lines.txt (working copy) @@ -1,20 +0,0 @@ -BAHIA COCOA REVIEW 19870226200101 Showers continued throughout the week in the Bahia cocoa zone, alleviating the drought since early January and improving prospects for the coming temporao, although normal humidity levels have not been restored, Comissaria Smith said in its weekly review. The dry period means the temporao will be late this year. Arrivals for the week ended February 22 were 155,221 bags of 60 kilos making a cumulative total for the season of 5.93 mln against 5.81 at the same stage last year. Again it seems that cocoa delivered earlier on consignment was included in the arrivals figures. Comissaria Smith said there is still some doubt as to how much old crop cocoa is still available as harvesting has practically come to an end. With total Bahia crop estimates around 6.4 mln bags and sales standing at almost 6.2 mln there are a few hundred thousand bags still in the hands of farmers, middlemen, exporters and processors. There are doubts as to how much of this cocoa would be fit for export as shippers are now experiencing dificulties in obtaining +Bahia superior+ certificates. In view of the lower quality over recent weeks farmers have sold a good part of their cocoa held on consignment. Comissaria Smith said spot bean prices rose to 340 to 350 cruzados per arroba of 15 kilos. Bean shippers were reluctant to offer nearby shipment and only limited sales were booked for March shipment at 1,750 to 1,780 dlrs per tonne to ports to be named. New crop sales were also light and all to open ports with June/July going at 1,850 and 1,880 dlrs and at 35 and 45 dlrs under New York july, Aug/Sept at 1,870, 1,875 and 1,880 dlrs per tonne FOB. Routine sales of butter were made. March/April sold at 4,340, 4,345 and 4,350 dlrs. April/May butter went at 2.27 times New York May, June/July at 4,400 and 4,415 dlrs, Aug/Sept at 4,351 to 4,450 dlrs and at 2.27 and 2.28 times New York Sept and Oct/Dec at 4,480 dlrs and 2.27 times New York Dec, Comissaria Smith said. Destinations were the U.S., Covertible currency areas, Uruguay and open ports. Cake sales were registered at 785 to 995 dlrs for March/April, 785 dlrs for May, 753 dlrs for Aug and 0.39 times New York Dec for Oct/Dec. Buyers were the U.S., Argentina, Uruguay and convertible currency areas. Liquor sales were limited with March/April selling at 2,325 and 2,380 dlrs, June/July at 2,375 dlrs and at 1.25 times New York July, Aug/Sept at 2,400 dlrs and at 1.25 times New York Sept and Oct/Dec at 1.25 times New York Dec, Comissaria Smith said. Total Bahia sales are currently estimated at 6.13 mln bags against the 1986/87 crop and 1.06 mln bags against the 1987/88 crop. Final figures for the period to February 28 are expected to be published by the Brazilian Cocoa Trade Commission after carnival which ends midday on February 27. Reuter  -STANDARD OIL TO FORM FINANCIAL UNIT 19870226200220 Standard Oil Co and BP North America Inc said they plan to form a venture to manage the money market borrowing and investment activities of both companies. BP North America is a subsidiary of British Petroleum Co Plc , which also owns a 55 pct interest in Standard Oil. The venture will be called BP/Standard Financial Trading and will be operated by Standard Oil under the oversight of a joint management committee. Reuter  -COBANCO INC YEAR NET 19870226201859 Shr 34 cts vs 1.19 dlrs Net 807,000 vs 2,858,000 Assets 510.2 mln vs 479.7 mln Deposits 472.3 mln vs 440.3 mln Loans 299.2 mln vs 327.2 mln Note: 4th qtr not available. Year includes 1985 extraordinary gain from tax carry forward of 132,000 dlrs, or five cts per shr. Reuter  -WORLD MARKET PRICE FOR UPLAND COTTON - USDA 19870226213846 The U.S. Agriculture Department announced the prevailing world market price, adjusted to U.S. quality and location, for Strict Low Middling, 1-1/16 inch upland cotton at 52.69 cts per lb, to be in effect through midnight March 5. The adjusted world price is at average U.S. producing locations (near Lubbock, Texas) and will be further adjusted for other qualities and locations. The price will be used in determining First Handler Cotton Certificate payment rates. Based on data for the week ended February 26, the adjusted world price for upland cotton is determined as follows, in cts per lb -- Northern European Price 66.32 Adjustments -- Average U.S. spot mkt location 10.42 SLM 1-1/16 inch cotton 1.80 Average U.S. location 0.53 Sum of adjustments 12.75 Adjusted world price 53.57 Reuter  -SUGAR QUOTA IMPORTS DETAILED -- USDA 19870226213854 The U.S. Agriculture Department said cumulative sugar imports from individual countries during the 1987 quota year, which began January 1, 1987 and ends December 31, 1987 were as follows, with quota allocations for the quota year in short tons, raw value -- CUMULATIVE QUOTA 1987 IMPORTS ALLOCATIONS ARGENTINA nil 39,130 AUSTRALIA nil 75,530 BARBADOS nil 7,500 BELIZE nil 10,010 BOLIVIA nil 7,500 BRAZIL nil 131,950 CANADA nil 18,876 QUOTA 1987 IMPORTS ALLOCATIONS COLOMBIA 103 21,840 CONGO nil 7,599 COSTA RICA nil 17,583 IVORY COAST nil 7,500 DOM REP 5,848 160,160 ECUADOR nil 10,010 EL SALVADOR nil 26,019.8 FIJI nil 25,190 GABON nil 7,500 QUOTA 1987 IMPORTS ALLOCATIONS GUATEMALA nil 43,680 GUYANA nil 10,920 HAITI nil 7,500 HONDURAS nil 15,917.2 INDIA nil 7,500 JAMAICA nil 10,010 MADAGASCAR nil 7,500 MALAWI nil 9,,100 QUOTA 1987 IMPORTS ALLOCATIONS MAURITIUS nil 10,920 MEXICO 37 7,500 MOZAMBIQUE nil 11,830 PANAMA nil 26,390 PAPUA NEW GUINEA nil 7,500 PARAGUAY nil 7,500 PERU nil 37,310 PHILIPPINES nil 143,780 ST.CHRISTOPHER- NEVIS nil 7,500 QUOTA 1987 IMPORTS ALLOCATIONS SWAZILAND nil 14,560 TAIWAN nil 10,920 THAILAND nil 12,740 TRINIDAD-TOBAGO nil 7,500 URUGUAY nil 7,500 ZIMBABWE nil 10,920 Reuter  -GRAIN SHIPS LOADING AT PORTLAND 19870226213903 There were seven grain ships loading and six ships were waiting to load at Portland, according to the Portland Merchants Exchange. Reuter  -IRAN ANNOUNCES END OF MAJOR OFFENSIVE IN GULF WAR 19870226214000 Iran announced tonight that its major offensive against Iraq in the Gulf war had ended after dealing savage blows against the Baghdad government. The Iranian news agency IRNA, in a report received in London, said the operation code-named Karbala-5 launched into Iraq on January 9 was now over. It quoted a joint statewment by the Iranian Army and Revolutionary Guards Corps as saying that their forces had "dealt one of the severest blows on the Iraqi war machine in the history of the Iraq-imposed war." The statement by the Iranian High Command appeared to herald the close of an assault on the port city of Basra in southern Iraq. "The operation was launched at a time when the Baghdad government was spreading extensive propaganda on the resistance power of its army...," said the statement quoted by IRNA. It claimed massive victories in the seven-week offensive and called on supporters of Baghdad to "come to their senses" and discontinue support for what it called the tottering regime in Iraq. Iran said its forces had "liberated" 155 square kilometers of enemy-occupied territory during the 1987 offensive and taken over islands, townships, rivers and part of a road leading into Basra. The Iranian forces "are in full control of these areas," the statement said. It said 81 Iraqi brigades and battalions were totally destroyed, along with 700 tanks and 1,500 other vehicles. The victory list also included 80 warplanes downed, 250 anti- aircraft guns and 400 pieces of military hardware destroyed and the seizure of 220 tanks and armoured personnel carriers. Reuter  -MERIDIAN BANCORP INC SETS REGULAR PAYOUT 19870226214034 Qtly div 25 cts vs 25 cts prior Pay April one Record March 15 Reuter  -U.S. BANK DISCOUNT BORROWINGS 310 MLN DLRS 19870226214134 U.S. bank discount window borrowings less extended credits averaged 310 mln dlrs in the week to Wednesday February 25, the Federal Reserve said. The Fed said that overall borrowings in the week fell 131 mln dlrs to 614 mln dlrs, with extended credits up 10 mln dlrs at 304 mln dlrs. The week was the second half of a two-week statement period. Net borrowings in the prior week averaged 451 mln dlrs. Commenting on the two-week statement period ended February 25, the Fed said that banks had average net free reserves of 644 mln dlrs a day, down from 1.34 billion two weeks earlier. A Federal Reserve spokesman told a press briefing that there were no large single day net misses in the Fed's reserve projections in the week to Wednesday. He said that natural float had been "acting a bit strangely" for this time of year, noting that there had been poor weather during the latest week. The spokesman said that natural float ranged from under 500 mln dlrs on Friday, for which he could give no reason, to nearly one billion dlrs on both Thursday and Wednesday. The Fed spokeman could give no reason for Thursday's high float, but he said that about 750 mln dlrs of Wednesday's float figure was due to holdover and transportation float at two widely separated Fed districts. For the week as a whole, he said that float related as of adjustments were "small," adding that they fell to a negative 750 mln dlrs on Tuesday due to a number of corrections for unrelated cash letter errors in six districts around the country. The spokesman said that on both Tuesday and Wednesday, two different clearing banks had system problems and the securities and Federal funds wires had to be held open until about 2000 or 2100 EST on both days. However, he said that both problems were cleared up during both afternoons and there was no evidence of any reserve impact. During the week ended Wednesday, 45 pct of net discount window borrowings were made by the smallest banks, with 30 pct by the 14 large money center banks and 25 pct by large regional institutions. On Wednesday, 55 pct of the borrowing was accounted for by the money center banks, with 30 pct by the large regionals and 15 pct by the smallest banks. The Fed spokesman said the banking system had excess reserves on Thursday, Monday and Tuesday and a deficit on Friday and Wedndsday. That produced a small daily average deficit for the week as a whole. For the two-week period, he said there were relatively high excess reserves on a daily avearge, almost all of which were at the smallest banks. Reuter  -AMERICAN EXPRESS SEEN IN POSSIBLE SPINNOFF 19870226214313 American Express Co remained silent on market rumors it would spinoff all or part of its Shearson Lehman Brothers Inc, but some analysts said the company may be considering such a move because it is unhappy with the market value of its stock. American Express stock got a lift from the rumor, as the market calculated a partially public Shearson may command a good market value, thereby boosting the total value of American Express. The rumor also was accompanied by talk the financial services firm would split its stock and boost its dividend. American Express closed on the New York Stock Exchange at 72-5/8, up 4-1/8 on heavy volume. American Express would not comment on the rumors or its stock activity. Analysts said comments by the company at an analysts' meeting Tuesday helped fuel the rumors as did an announcement yesterday of management changes. At the meeting, company officials said American Express stock is undervalued and does not fully reflect the performance of Shearson, according to analysts. Yesterday, Shearson said it was elevating its chief operating officer, Jeffery Lane, to the added position of president, which had been vacant. It also created four new positions for chairmen of its operating divisions. Analysts speculated a partial spinoff would make most sense, contrary to one variation on market rumors of a total spinoff. Some analysts, however, disagreed that any spinoff of Shearson would be good since it is a strong profit center for American Express, contributing about 20 pct of earnings last year. "I think it is highly unlikely that American Express is going to sell shearson," said Perrin Long of Lipper Analytical. He questioned what would be a better investment than "a very profitable securities firm." Several analysts said American Express is not in need of cash, which might be the only reason to sell a part of a strong asset. But others believe the company could very well of considered the option of spinning out part of Shearson, and one rumor suggests selling about 20 pct of it in the market. Larry Eckenfelder of Prudential-Bache Securities said he believes American Express could have considered a partial spinoff in the past. "Shearson being as profitable as it is would have fetched a big premium in the market place. Shearson's book value is in the 1.4 mln dlr range. Shearson in the market place would probably be worth three to 3.5 bilion dlrs in terms of market capitalization," said Eckenfelder. Some analysts said American Express could use capital since it plans to expand globally. "They have enormous internal growth plans that takes capital. You want your stock to reflect realistic valuations to enhance your ability to make all kinds of endeavors down the road," said E.F. Hutton Group analyst Michael Lewis. "They've outlined the fact that they're investing heavily in the future, which goes heavily into the international arena," said Lewis. "...That does not preclude acquisitions and divestitures along the way," he said. Lewis said if American Express reduced its exposure to the brokerage business by selling part of shearson, its stock might better reflect other assets, such as the travel related services business. "It could find its true water mark with a lesser exposure to brokerage. The value of the other components could command a higher multiple because they constitute a higher percentage of the total operating earnings of the company," he said. Lewis said Shearson contributed 316 mln in after-tax operating earnings, up from about 200 mln dlrs in 1985. Reuter  -OHIO MATTRESS MAY HAVE LOWER 1ST QTR NET 19870226201915 Ohio Mattress Co said its first quarter, ending February 28, profits may be below the 2.4 mln dlrs, or 15 cts a share, earned in the first quarter of fiscal 1986. The company said any decline would be due to expenses related to the acquisitions in the middle of the current quarter of seven licensees of Sealy Inc, as well as 82 pct of the outstanding capital stock of Sealy. Because of these acquisitions, it said, first quarter sales will be substantially higher than last year's 67.1 mln dlrs. Noting that it typically reports first quarter results in late march, said the report is likely to be issued in early April this year. It said the delay is due to administrative considerations, including conducting appraisals, in connection with the acquisitions. Reuter  -U.S. M-1 MONEY SUPPLY ROSE 2.1 BILLION DLRS 19870226214435 U.S. M-1 money supply rose 2.1 billion dlrs to a seasonally adjusted 736.7 billion dlrs in the February 16 week, the Federal Reserve said. The previous week's M-1 level was revised to 734.6 billion dlrs from 734.2 billion dlrs, while the four-week moving average of M-1 rose to 735.0 billion dlrs from 733.5 billion. Economists polled by Reuters said that M-1 should be anywhere from down four billion dlrs to up 2.3 billion dlrs. The average forecast called for a 300 mln dlr M-1 rise. Reuter  -GENERAL BINDING IN MARKETING AGREEMENT 19870226214508 General Binding Corp said it reached a marketing agreement with Varitronic Systems Inc, a manufacturer and marketer of electronic lettering systems. Under terms of the agreement, General Binding will carry Varitronics' Merlin Express Presentation Lettering System, a portable, battery-operated lettering system which produces type on adhesive-backed tape. Reuter  -LIBERTY ALL-STAR SETS INITIAL PAYOUT 19870226214544 Liberty All-Star Equity Fund said it declared an initial dividend of five cts per share, payable April two to shareholders of record March 20. It said the dividend includes a quarterly dividend of three cts a share and a special payout of two cts a share, which covers the period from November three, 1986, when the fund began operations, to December 31, 1986. The fund said its quarterly dividend rate may fluctuate in the future. Reuter  -COCA COLA UNIT AND WORLD FILM IN VENTURE 19870226214745 Coca-Cola Co's Entertainment Business Sector Inc unit said it formed a joint venture with an affiliate of World Film Services to acquire, produce and distribute television programming around the world. World Film Services was formed by chairman John Heyman in 1963 to produce films. Reuter  -FORD MOTOR CREDIT TO REDEEM DEBENTURES 19870226214753 Ford Motor Co said its Ford Motor Credit Co on April One will redeem 4.0 mln dlrs of its 8.70 pct debentures due April 1, 1999. It said the debentures are redeemable at a price of 100 pct of the principal. Because April 1, 1987 is an interest payment date on the debentures, no accrued interest will be payable on the redemption date as part of the redemption proceeds. Debentures will be selected for redemption on a pro rata basis, Ford said. Reuter  -STERLING SOFTWARE NOTE HOLDERS OK BUY 19870226214802 Sterling Software Inc said it received consent of a majority of the holders of its eight pct convertible sernior subordinated debentures required to purchase shares of its common. The company said it may now buy its stock at its discretion depending on market conditions. Reuter  - MAKES INITIAL STOCK OFFER 19870226214818 Schult Homes Corp announced an initial public offering of 833,334 units at five dlrs per unit, said Janney Montgomery Scott Inc and Woolcott and Co, managing underwriters of the offering. They said each unit consists of one common share and one warrant to buy one-half share of common. The warrant will entitle holders to buy one-half common share at 5.50 dlrs per full share from March one, 1988, to September one, 1989, and thereafter at 6.50 dlrs per full share until March 1991, they said. Reuter  -FLUOR UNIT GETS CONSTRUCTION CONTRACT 19870226214826 Fluor Corp said its Fluor Daniel unit received a contract from Union Carbide Corp covering design, procurement and construction of a 108 megawatt combined cycle cogeneration facility in Seadrift, Texas. The value of the contract was not disclosed. Reuter  -SUFFIELD FINANCIAL CORP SELLS STOCK 19870226214835 Suffield Financial Corp said Jon Googel and Benjamin Sisti of Colonial Realty, West Hartford, Conn., purchased 175,900 shares of its stock for 3,416,624. The company said the purchase equals 5.2 pct of its outstanding shares. Reuter  Index: lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java =================================================================== --- lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java (revision 934796) +++ lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java (working copy) @@ -29,6 +29,7 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.WhitespaceAnalyzer; import org.apache.lucene.analysis.tokenattributes.TermAttribute; +import org.apache.lucene.benchmark.BenchmarkTestCase; import org.apache.lucene.benchmark.byTask.feeds.DocMaker; import org.apache.lucene.benchmark.byTask.feeds.ReutersQueryMaker; import org.apache.lucene.benchmark.byTask.tasks.CountingSearchTestTask; @@ -50,26 +51,16 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.search.FieldCache.StringIndex; import org.apache.lucene.search.FieldCache; -import org.apache.lucene.util.LuceneTestCase; /** * Test very simply that perf tasks - simple algorithms - are doing what they should. */ -public class TestPerfTasksLogic extends LuceneTestCase { +public class TestPerfTasksLogic extends BenchmarkTestCase { - static final String NEW_LINE = System.getProperty("line.separator"); - - // properties in effect in all tests here - static final String propLines [] = { - "directory=RAMDirectory", - "print.props=false", - }; - - /** - * @param name test name - */ - public TestPerfTasksLogic(String name) { - super(name); + @Override + protected void setUp() throws Exception { + super.setUp(); + copyToWorkDir("reuters.first20.lines.txt"); } /** @@ -531,35 +522,7 @@ ir.close(); } - // create the benchmark and execute it. - public static Benchmark execBenchmark(String[] algLines) throws Exception { - String algText = algLinesToText(algLines); - logTstLogic(algText); - Benchmark benchmark = new Benchmark(new StringReader(algText)); - benchmark.execute(); - return benchmark; - } - - // catenate alg lines to make the alg text - private static String algLinesToText(String[] algLines) { - String indent = " "; - StringBuffer sb = new StringBuffer(); - for (int i = 0; i < propLines.length; i++) { - sb.append(indent).append(propLines[i]).append(NEW_LINE); - } - for (int i = 0; i < algLines.length; i++) { - sb.append(indent).append(algLines[i]).append(NEW_LINE); - } - return sb.toString(); - } - private static void logTstLogic (String txt) { - if (!VERBOSE) - return; - System.out.println("Test logic of:"); - System.out.println(txt); - } - /** * Test that exhaust in loop works as expected (LUCENE-1115). */ @@ -851,7 +814,7 @@ assertEquals("Missing some tasks to check!",3,nChecked); } - private static String[] disableCountingLines (boolean disable) { + private String[] disableCountingLines (boolean disable) { String dis = disable ? "-" : ""; return new String[] { "# ----- properties ", @@ -901,7 +864,7 @@ assertEquals(new Locale("no", "NO", "NY"), benchmark.getRunData().getLocale()); } - private static String[] getLocaleConfig(String localeParam) { + private String[] getLocaleConfig(String localeParam) { String algLines[] = { "# ----- properties ", "content.source=org.apache.lucene.benchmark.byTask.feeds.LineDocSource", @@ -966,7 +929,7 @@ ts2.close(); } - private static String[] getCollatorConfig(String localeParam, + private String[] getCollatorConfig(String localeParam, String collationParam) { String algLines[] = { "# ----- properties ", @@ -1048,7 +1011,7 @@ stream.close(); } - private static String[] getShingleConfig(String params) { + private String[] getShingleConfig(String params) { String algLines[] = { "content.source=org.apache.lucene.benchmark.byTask.feeds.LineDocSource", "docs.file=" + getReuters20LinesFile(), @@ -1061,8 +1024,7 @@ return algLines; } - private static String getReuters20LinesFile() { - return System.getProperty("lucene.common.dir").replace('\\','/') + - "/contrib/benchmark/src/test/org/apache/lucene/benchmark/reuters.first20.lines.txt"; + private String getReuters20LinesFile() { + return getWorkDirResourcePath("reuters.first20.lines.txt"); } } Index: lucene/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java =================================================================== --- lucene/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java (revision 934796) +++ lucene/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java (working copy) @@ -18,548 +18,192 @@ */ import java.io.BufferedReader; -import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FilenameFilter; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; -import java.nio.ByteBuffer; -import java.nio.charset.Charset; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.LinkedHashSet; -import java.util.List; +import java.util.HashSet; +import java.util.Random; +import java.util.Set; -import org.apache.lucene.analysis.BaseTokenStreamTestCase; import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.BaseTokenStreamTestCase; +import org.apache.lucene.analysis.KeywordAnalyzer; import org.apache.lucene.analysis.SimpleAnalyzer; import org.apache.lucene.analysis.StopAnalyzer; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.document.Fieldable; import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.queryParser.ParseException; import org.apache.lucene.queryParser.QueryParser; -import org.apache.lucene.search.Collector; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.Scorer; -import org.apache.lucene.store.Directory; +import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.RAMDirectory; -import org.apache.lucene.index.TermDocs; /** -Verifies that Lucene MemoryIndex and RAMDirectory have the same behaviour, -returning the same results for any given query. -Runs a set of queries against a set of files and compares results for identity. -Can also be used as a simple benchmark. -

-Example usage: -

-cd lucene-svn
-java -server -cp ~/unix/java/share/misc/junit/junit.jar:build/classes:build/lucene-core-2.1-dev.jar:build/contrib/memory/classes/test:build/contrib/memory/classes/java org.apache.lucene.index.memory.MemoryIndexTest 1 1 memram @contrib/memory/src/test/org/apache/lucene/index/memory/testqueries.txt *.txt *.html *.xml xdocs/*.xml src/test/org/apache/lucene/queryParser/*.java contrib/memory/src/java/org/apache/lucene/index/memory/*.java
-
-where testqueries.txt is a file with one query per line, such as: -
-#
-# queries extracted from TestQueryParser.java
-#
-Apache
-Apach~ AND Copy*
-
-a AND b
-(a AND b)
-c OR (a AND b)
-a AND NOT b
-a AND -b
-a AND !b
-a && b
-a && ! b
-
-a OR b
-a || b
-a OR !b
-a OR ! b
-a OR -b
-
-+term -term term
-foo:term AND field:anotherTerm
-term AND "phrase phrase"
-"hello there"
-
-germ term^2.0
-(term)^2.0
-(germ term)^2.0
-term^2.0
-term^2
-"germ term"^2.0
-"term germ"^2
-
-(foo OR bar) AND (baz OR boo)
-((a OR b) AND NOT c) OR d
-+(apple "steve jobs") -(foo bar baz)
-+title:(dog OR cat) -author:"bob dole"
-
-
-a&b
-a&&b
-.NET
-
-"term germ"~2
-"term germ"~2 flork
-"term"~2
-"~2 germ"
-"term germ"~2^2
-
-3
-term 1.0 1 2
-term term1 term2
-
-term*
-term*^2
-term~
-term~0.7
-term~^2
-term^2~
-term*germ
-term*germ^3
-
-
-term*
-Term*
-TERM*
-term*
-Term*
-TERM*
-
-// Then 'full' wildcard queries:
-te?m
-Te?m
-TE?M
-Te?m*gerM
-te?m
-Te?m
-TE?M
-Te?m*gerM
-
-term term term
-term +stop term
-term -stop term
-drop AND stop AND roll
-term phrase term
-term AND NOT phrase term
-stop
-
-
-[ a TO c]
-[ a TO c ]
-{ a TO c}
-{ a TO c }
-{ a TO c }^2.0
-[ a TO c] OR bar
-[ a TO c] AND bar
-( bar blar { a TO c}) 
-gack ( bar blar { a TO c}) 
-
-
-+weltbank +worlbank
-+weltbank\n+worlbank
-weltbank \n+worlbank
-weltbank \n +worlbank
-+weltbank\r+worlbank
-weltbank \r+worlbank
-weltbank \r +worlbank
-+weltbank\r\n+worlbank
-weltbank \r\n+worlbank
-weltbank \r\n +worlbank
-weltbank \r \n +worlbank
-+weltbank\t+worlbank
-weltbank \t+worlbank
-weltbank \t +worlbank
-
-
-term term term
-term +term term
-term term +term
-term +term +term
--term term term
-
-
-on^1.0
-"hello"^2.0
-hello^2.0
-"on"^1.0
-the^3
-
- -*/ + * Verifies that Lucene MemoryIndex and RAMDirectory have the same behaviour, + * returning the same results for queries on some randomish indexes. + */ public class MemoryIndexTest extends BaseTokenStreamTestCase { + private Set queries = new HashSet(); + private Random random; - private Analyzer analyzer; - - private static final String FIELD_NAME = "content"; + public static final int ITERATIONS = 100; - /** Runs the tests and/or benchmark */ - public static void main(String[] args) throws Throwable { - new MemoryIndexTest().run(args); - } - - /* all files will be open relative to this */ - public String fileDir; @Override - protected void setUp() throws Exception { + public void setUp() throws Exception { super.setUp(); - fileDir = System.getProperty("lucene.common.dir", null); + queries.addAll(readQueries("testqueries.txt")); + queries.addAll(readQueries("testqueries2.txt")); + random = newRandom(); } -// public void tearDown() {} - - public void testMany() throws Throwable { - String[] files = listFiles(new String[] { - "*.txt", "*.html", "*.xml", "xdocs/*.xml", - "src/java/test/org/apache/lucene/queryParser/*.java", - "contrib/memory/src/java/org/apache/lucene/index/memory/*.java", - }); - if (VERBOSE) System.out.println("files = " + java.util.Arrays.asList(files)); - String[] xargs = new String[] { - "1", "1", "memram", - "@contrib/memory/src/test/org/apache/lucene/index/memory/testqueries.txt", - }; - String[] args = new String[xargs.length + files.length]; - System.arraycopy(xargs, 0, args, 0, xargs.length); - System.arraycopy(files, 0, args, xargs.length, files.length); - run(args); - } - - private void run(String[] args) throws Throwable { - int k = -1; - - int iters = 1; - if (args.length > ++k) iters = Math.max(1, Integer.parseInt(args[k])); - - int runs = 1; - if (args.length > ++k) runs = Math.max(1, Integer.parseInt(args[k])); - - String cmd = "memram"; - if (args.length > ++k) cmd = args[k]; - boolean useMemIndex = cmd.indexOf("mem") >= 0; - boolean useRAMIndex = cmd.indexOf("ram") >= 0; - - String[] queries = { "term", "term*", "term~", "Apache", "Apach~ AND Copy*" }; - if (args.length > ++k) { - String arg = args[k]; - if (arg.startsWith("@")) - queries = readLines(new File(fileDir, arg.substring(1))); - else - queries = new String[] { arg }; - } - - File[] files = new File[] {new File("CHANGES.txt"), new File("LICENSE.txt") }; - if (args.length > ++k) { - files = new File[args.length - k]; - for (int i=k; i < args.length; i++) { - files[i-k] = new File(args[i]); - } - } - -// boolean toLowerCase = false; -// Set stopWords = null; - - Analyzer[] analyzers = new Analyzer[] { - new SimpleAnalyzer(TEST_VERSION_CURRENT), - new StopAnalyzer(TEST_VERSION_CURRENT), - new StandardAnalyzer(TEST_VERSION_CURRENT), -// new WhitespaceAnalyzer(TEST_VERSION_CURRENT), -// new PatternAnalyzer(PatternAnalyzer.NON_WORD_PATTERN, false, null), -// new PatternAnalyzer(PatternAnalyzer.NON_WORD_PATTERN, true, stopWords), -// new SnowballAnalyzer("English", StopAnalyzer.ENGLISH_STOP_WORDS), - }; - - boolean first = true; - - for (int iter=0; iter < iters; iter++) { - if (VERBOSE) System.out.println("\n########### iteration=" + iter); - long start = System.currentTimeMillis(); - long bytes = 0; - - for (int anal=0; anal < analyzers.length; anal++) { - this.analyzer = analyzers[anal]; - - for (int i=0; i < files.length; i++) { - File file = files[i]; - if (!file.exists() || file.isDirectory()) continue; // ignore - bytes += file.length(); - String text = toString(new FileInputStream(file), null); - Document doc = createDocument(text); - if (VERBOSE) System.out.println("\n*********** FILE=" + file); - - boolean measureIndexing = false; // toggle this to measure query performance - MemoryIndex memind = null; - IndexSearcher memsearcher = null; - if (useMemIndex && !measureIndexing) { - memind = createMemoryIndex(doc); - memsearcher = memind.createSearcher(); - } - - if (first) { - IndexSearcher s = memind.createSearcher(); - TermDocs td = s.getIndexReader().termDocs(null); - assertTrue(td.next()); - assertEquals(0, td.doc()); - assertEquals(1, td.freq()); - td.close(); - s.close(); - first = false; - } - - RAMDirectory ramind = null; - IndexSearcher ramsearcher = null; - if (useRAMIndex && !measureIndexing) { - ramind = createRAMIndex(doc); - ramsearcher = new IndexSearcher(ramind); - } - - for (int q=0; q < queries.length; q++) { - try { - Query query = parseQuery(queries[q]); - for (int run=0; run < runs; run++) { - float score1 = 0.0f; float score2 = 0.0f; - if (useMemIndex && measureIndexing) { - memind = createMemoryIndex(doc); - memsearcher = memind.createSearcher(); - } - if (useMemIndex) score1 = query(memsearcher, query); - if (useRAMIndex && measureIndexing) { - ramind = createRAMIndex(doc); - ramsearcher = new IndexSearcher(ramind); - } - if (useRAMIndex) score2 = query(ramsearcher, query); - if (useMemIndex && useRAMIndex) { - if (VERBOSE) System.out.println("diff="+ (score1-score2) + ", query=" + queries[q] + ", s1=" + score1 + ", s2=" + score2); - if (score1 != score2 || score1 < 0.0f || score2 < 0.0f || score1 > 1.0f || score2 > 1.0f) { - throw new IllegalStateException("BUG DETECTED:" + (i*(q+1)) + " at query=" + queries[q] + ", file=" + file + ", anal=" + analyzer); - } - } - } - - } catch (Throwable t) { - if (t instanceof OutOfMemoryError) t.printStackTrace(); - if (VERBOSE) System.out.println("Fatal error at query=" + queries[q] + ", file=" + file + ", anal=" + analyzer); - throw t; - } - } - } - } - long end = System.currentTimeMillis(); - if (VERBOSE) { - System.out.println("\nsecs = " + ((end-start)/1000.0f)); - System.out.println("queries/sec= " + - (1.0f * runs * queries.length * analyzers.length * files.length - / ((end-start)/1000.0f))); - float mb = (1.0f * bytes * queries.length * runs) / (1024.0f * 1024.0f); - System.out.println("MB/sec = " + (mb / ((end-start)/1000.0f))); - } - } - - if (!VERBOSE) return; - - if (useMemIndex && useRAMIndex) - System.out.println("No bug found. done."); - else - System.out.println("Done benchmarking (without checking correctness)."); - } - - // returns file line by line, ignoring empty lines and comments - private String[] readLines(File file) throws Exception { - BufferedReader reader = new BufferedReader(new InputStreamReader( - new FileInputStream(file))); - List lines = new ArrayList(); - String line; + /** + * read a set of queries from a resource file + */ + private Set readQueries(String resource) throws IOException { + Set queries = new HashSet(); + InputStream stream = getClass().getResourceAsStream(resource); + BufferedReader reader = new BufferedReader(new InputStreamReader(stream, "UTF-8")); + String line = null; while ((line = reader.readLine()) != null) { - String t = line.trim(); - if (t.length() > 0 && t.charAt(0) != '#' && (!t.startsWith("//"))) { - lines.add(line); + line = line.trim(); + if (line.length() > 0 && !line.startsWith("#") && !line.startsWith("//")) { + queries.add(line); } } - reader.close(); - - String[] result = new String[lines.size()]; - lines.toArray(result); - return result; + return queries; } - private Document createDocument(String content) { - Document doc = new Document(); - doc.add(new Field(FIELD_NAME, content, Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS)); - return doc; - } - private MemoryIndex createMemoryIndex(Document doc) { - MemoryIndex index = new MemoryIndex(); - Iterator iter = doc.getFields().iterator(); - while (iter.hasNext()) { - Fieldable field = iter.next(); - index.addField(field.name(), field.stringValue(), analyzer); - } - return index; + /** + * runs random tests, up to ITERATIONS times. + */ + public void testRandomQueries() throws Exception { + for (int i = 0; i < ITERATIONS; i++) + assertAgainstRAMDirectory(); } - - private RAMDirectory createRAMIndex(Document doc) { - RAMDirectory dir = new RAMDirectory(); - IndexWriter writer = null; - try { - writer = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, analyzer)); - writer.addDocument(doc); - writer.optimize(); - return dir; - } catch (IOException e) { // should never happen (RAMDirectory) - throw new RuntimeException(e); - } finally { - try { - if (writer != null) writer.close(); - } catch (IOException e) { // should never happen (RAMDirectory) - throw new RuntimeException(e); - } + + /** + * Build a randomish document for both RAMDirectory and MemoryIndex, + * and run all the queries against it. + */ + public void assertAgainstRAMDirectory() throws Exception { + StringBuilder fooField = new StringBuilder(); + StringBuilder termField = new StringBuilder(); + + // add up to 250 terms to field "foo" + for (int i = 0; i < random.nextInt(250); i++) { + fooField.append(" "); + fooField.append(randomTerm()); } - } - final float[] scores = new float[1]; // inits to 0.0f (no match) - - private float query(IndexSearcher searcher, Query query) { -// System.out.println("MB=" + (getMemorySize(index) / (1024.0f * 1024.0f))); - try { - searcher.search(query, new Collector() { - private Scorer scorer; - - @Override - public void collect(int doc) throws IOException { - scores[0] = scorer.score(); - } - - @Override - public void setScorer(Scorer scorer) throws IOException { - this.scorer = scorer; - } - - @Override - public boolean acceptsDocsOutOfOrder() { - return true; - } - - @Override - public void setNextReader(IndexReader reader, int docBase) { } - }); - float score = scores[0]; -// Hits hits = searcher.search(query); -// float score = hits.length() > 0 ? hits.score(0) : 0.0f; - return score; - } catch (IOException e) { // should never happen (RAMDirectory) - throw new RuntimeException(e); + // add up to 250 terms to field "term" + for (int i = 0; i < random.nextInt(250); i++) { + termField.append(" "); + termField.append(randomTerm()); } + + RAMDirectory ramdir = new RAMDirectory(); + Analyzer analyzer = randomAnalyzer(); + IndexWriter writer = new IndexWriter(ramdir, analyzer, + IndexWriter.MaxFieldLength.UNLIMITED); + Document doc = new Document(); + Field field1 = new Field("foo", fooField.toString(), Field.Store.NO, Field.Index.ANALYZED); + Field field2 = new Field("term", termField.toString(), Field.Store.NO, Field.Index.ANALYZED); + doc.add(field1); + doc.add(field2); + writer.addDocument(doc); + writer.close(); + + MemoryIndex memory = new MemoryIndex(); + memory.addField("foo", fooField.toString(), analyzer); + memory.addField("term", termField.toString(), analyzer); + assertAllQueries(memory, ramdir, analyzer); } - // for debugging purposes - int getMemorySize(Object index) { - if (index instanceof Directory) { - try { - Directory dir = (Directory) index; - int size = 0; - String[] fileNames = dir.listAll(); - for (int i=0; i < fileNames.length; i++) { - size += dir.fileLength(fileNames[i]); - } - return size; - } - catch (IOException e) { // can never happen (RAMDirectory) - throw new RuntimeException(e); - } + /** + * Run all queries against both the RAMDirectory and MemoryIndex, ensuring they are the same. + */ + public void assertAllQueries(MemoryIndex memory, RAMDirectory ramdir, Analyzer analyzer) throws Exception { + IndexSearcher ram = new IndexSearcher(ramdir); + IndexSearcher mem = memory.createSearcher(); + QueryParser qp = new QueryParser(TEST_VERSION_CURRENT, "foo", analyzer); + for (String query : queries) { + TopDocs ramDocs = ram.search(qp.parse(query), 1); + TopDocs memDocs = mem.search(qp.parse(query), 1); + assertEquals(ramDocs.totalHits, memDocs.totalHits); } - else { - return ((MemoryIndex) index).getMemorySize(); - } } - private Query parseQuery(String expression) throws ParseException { - QueryParser parser = new QueryParser(TEST_VERSION_CURRENT, FIELD_NAME, analyzer); -// parser.setPhraseSlop(0); - return parser.parse(expression); - } - - /** returns all files matching the given file name patterns (quick n'dirty) */ - static String[] listFiles(String[] fileNames) { - LinkedHashSet allFiles = new LinkedHashSet(); - for (int i=0; i < fileNames.length; i++) { - int k; - if ((k = fileNames[i].indexOf("*")) < 0) { - allFiles.add(fileNames[i]); - } else { - String prefix = fileNames[i].substring(0, k); - if (prefix.length() == 0) prefix = "."; - final String suffix = fileNames[i].substring(k+1); - File[] files = new File(prefix).listFiles(new FilenameFilter() { - public boolean accept(File dir, String name) { - return name.endsWith(suffix); - } - }); - if (files != null) { - for (int j=0; j < files.length; j++) { - allFiles.add(files[j].getPath()); - } - } - } + /** + * Return a random analyzer (Simple, Stop, Standard) to analyze the terms. + */ + private Analyzer randomAnalyzer() { + switch(random.nextInt(3)) { + case 0: return new SimpleAnalyzer(TEST_VERSION_CURRENT); + case 1: return new StopAnalyzer(TEST_VERSION_CURRENT); + default: return new StandardAnalyzer(TEST_VERSION_CURRENT); } - - String[] result = new String[allFiles.size()]; - allFiles.toArray(result); - return result; } - // trick to detect default platform charset - private static final Charset DEFAULT_PLATFORM_CHARSET = - Charset.forName(new InputStreamReader(new ByteArrayInputStream(new byte[0])).getEncoding()); + /** + * Some terms to be indexed, in addition to random words. + * These terms are commonly used in the queries. + */ + private static final String[] TEST_TERMS = {"term", "Term", "tErm", "TERM", + "telm", "stop", "drop", "roll", "phrase", "a", "c", "bar", "blar", + "gack", "weltbank", "worlbank", "hello", "on", "the", "apache", "Apache", + "copyright", "Copyright"}; - // the following utility methods below are copied from Apache style Nux library - see http://dsd.lbl.gov/nux - private static String toString(InputStream input, Charset charset) throws IOException { - if (charset == null) charset = DEFAULT_PLATFORM_CHARSET; - byte[] data = toByteArray(input); - return charset.decode(ByteBuffer.wrap(data)).toString(); + + /** + * half of the time, returns a random term from TEST_TERMS. + * the other half of the time, returns a random unicode string. + */ + private String randomTerm() { + if (random.nextBoolean()) { + // return a random TEST_TERM + return TEST_TERMS[random.nextInt(TEST_TERMS.length)]; + } else { + // return a random unicode term + return randomString(); + } } - private static byte[] toByteArray(InputStream input) throws IOException { - try { - // safe and fast even if input.available() behaves weird or buggy - int len = Math.max(256, input.available()); - byte[] buffer = new byte[len]; - byte[] output = new byte[len]; - - len = 0; - int n; - while ((n = input.read(buffer)) >= 0) { - if (len + n > output.length) { // grow capacity - byte tmp[] = new byte[Math.max(output.length << 1, len + n)]; - System.arraycopy(output, 0, tmp, 0, len); - System.arraycopy(buffer, 0, tmp, len, n); - buffer = output; // use larger buffer for future larger bulk reads - output = tmp; - } else { - System.arraycopy(buffer, 0, output, len, n); - } - len += n; + /** + * Return a random unicode term, like TestStressIndexing. + */ + private String randomString() { + final int end = random.nextInt(20); + if (buffer.length < 1 + end) { + char[] newBuffer = new char[(int) ((1 + end) * 1.25)]; + System.arraycopy(buffer, 0, newBuffer, 0, buffer.length); + buffer = newBuffer; + } + for (int i = 0; i < end - 1; i++) { + int t = random.nextInt(6); + if (0 == t && i < end - 1) { + // Make a surrogate pair + // High surrogate + buffer[i++] = (char) nextInt(0xd800, 0xdc00); + // Low surrogate + buffer[i] = (char) nextInt(0xdc00, 0xe000); + } else if (t <= 1) buffer[i] = (char) random.nextInt(0x80); + else if (2 == t) buffer[i] = (char) nextInt(0x80, 0x800); + else if (3 == t) buffer[i] = (char) nextInt(0x800, 0xd800); + else if (4 == t) buffer[i] = (char) nextInt(0xe000, 0xffff); + else if (5 == t) { + // Illegal unpaired surrogate + if (random.nextBoolean()) buffer[i] = (char) nextInt(0xd800, 0xdc00); + else buffer[i] = (char) nextInt(0xdc00, 0xe000); } - - if (len == output.length) return output; - buffer = null; // help gc - buffer = new byte[len]; - System.arraycopy(output, 0, buffer, 0, len); - return buffer; - } finally { - input.close(); } + return new String(buffer, 0, end); } + private char buffer[] = new char[20]; + // start is inclusive and end is exclusive + private int nextInt(int start, int end) { + return start + random.nextInt(end - start); + } }