Index: src/test/java/org/apache/http/nio/URList.java =================================================================== --- src/test/java/org/apache/http/nio/URList.java (版本 0) +++ src/test/java/org/apache/http/nio/URList.java (版本 0) @@ -0,0 +1,236 @@ +/* + * $HeadURL: http://svn.apache.org/repos/asf/httpcomponents/httpcore/tags/4.0-beta1/module-nio/src/test/java/org/apache/http/nio/TestAll.java $ + * $Revision: 613298 $ + * $Date: 2008-01-19 06:09:22 +0800 (六, 2008-01-19) $ + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ + +package org.apache.http.nio; + +public class URList { + public final static String[] LIST = new String[] {"http://www.apache.org", + "http://www.google.com", + "http://www.yahoo.com", + "http://www.facebook.com", + "http://www.myspace.com", + "http://v.blog.sohu.com", + "http://sohucallcenter.blog.sohu.com", + "http://cslr1840.blog.sohu.com", + "http://zhushuai2008.blog.sohu.com", + "http://loloB.blog.sohu.com", + "http://caotxe06.blog.sohu.com", + "http://dandanan.blog.sohu.com", + "http://konggreen.blog.sohu.com", + "http://yeiyu2000.blog.sohu.com", + "http://anglic319.blog.sohu.com", + "http://haocheng81687.blog.sohu.com", + "http://yanyan-zhu.blog.sohu.com", + "http://vogueman1069.blog.sohu.com", + "http://loverss0703.blog.hexun.com", + "http://fox0099.blog.sohu.com", + "http://lovelxx.blog.sohu.com", + "http://elaine19791214.blog.sohu.com", + "http://tigers.blog.sohu.com", + "http://huan82.blog.sohu.com", + "http://bqzm6194319.blog.sohu.com", + "http://eejuba.blog.sohu.com", + "http://zhaofuyu7143875.blog.sohu.com", + "http://sudachao.blog.sohu.com", + "http://wqqqqq.blog.sohu.com", + "http://libman.blog.sohu.com", + "http://snownini.blog.sohu.com", + "http://wendyzy100000.blog.sohu.com", + "http://hhy707.blog.sohu.com", + "http://naaifeng.blog.sohu.com", + "http://xiaoyu2008160.blog.sohu.com", + "http://akiraishidasama.blog.sohu.com", + "http://snls.blog.sohu.com", + "http://glory-2.blog.sohu.com", + "http://geduosniper.blog.sohu.com", + "http://emilyscui.blog.sohu.com", + "http://ylwdbk.blog.sohu.com", + "http://xyl8405.blog.sohu.com", + "http://yuxuan2008.blog.sohu.com", + "http://yaoguoyin3511.blog.sohu.com", + "http://wyq550730.blog.sohu.com", + "http://wonder-kid.blog.sohu.com", + "http://yoowoojun.blog.sohu.com", + "http://daozhongyang.blog.sohu.com", + "http://lixueqhd.blog.sohu.com", + "http://zhuziranxu.blog.sohu.com", + "http://ailyer.blog.sohu.com", + "http://jiaxi2005.blog.sohu.com", + "http://hbxingzi.blog.sohu.com", + "http://maoyuan.blog.sohu.com", + "http://jljxhgq.blog.sohu.com", + "http://lwwym.blog.sohu.com", + "http://jxhshys.blog.sohu.com", + "http://ymnygq.blog.sohu.com", + "http://ldj781228.blog.sohu.com", + "http://qiujia2279.blog.sohu.com", + "http://wxlayy.blog.sohu.com", + "http://yuyunzhufeng.blog.sohu.com", + "http://yuerfeii.blog.sohu.com", + "http://free-fr.blog.sohu.com", + "http://my911.blog.sohu.com", + "http://lixia29161823.blog.sohu.com", + "http://51music.blog.sohu.com", + "http://jinxuefen.blog.sohu.com", + "http://fuchong.blog.sohu.com", + "http://xiaoyu413.blog.sohu.com", + "http://fengf0621.blog.sohu.com", + "http://free222.blog.sohu.com", + "http://askhng.blog.sohu.com", + "http://chilangjiaoshou.blog.sohu.com", + "http://jasonxiao250.blog.sohu.com", + "http://ciflefdsf.blog.sohu.com", + "http://foerfedf.blog.sohu.com", + "http://xinhaonanren1978.blog.sohu.com", + "http://zouxun2003.blog.sohu.com", + "http://rokeyhu.blog.sohu.com", + "http://luxunzailai.blog.sohu.com", + "http://loverss.blog.hexun.com", + "http://liuyifei9999.blog.sohu.com", + "http://qyqmsj.blog.sohu.com", + "http://rizhisucai.blog.sohu.com", + "http://zhangyu01.blog.sohu.com", + "http://niuniu0068.blog.sohu.com", + "http://mihu-hutu.blog.sohu.com", + "http://zhanghan5921.blog.sohu.com", + "http://pangbiandeeryu.blog.sohu.com", + "http://gurenqing.blog.sohu.com", + "http://zhujuanjuan0817.blog.sohu.com", + "http://huanyou0326.blog.sohu.com", + "http://kekaiming.blog.sohu.com", + "http://jiacg.blog.sohu.com", + "http://200651368.blog.sohu.com", + "http://kun1020.blog.sohu.com", + "http://bien.blog.sohu.com", + "http://rose617.blog.sohu.com", + "http://renzhe0703.blog.sohu.com", + "http://jplr.blog.sohu.com", + "http://bujin.blog.sohu.com", + "http://32dexiyang.blog.sohu.com", + "http://chaixiaona119.blog.sohu.com", + "http://cwjdl.blog.sohu.com", + "http://vail076.blog.sohu.com", + "http://caci.blog.sohu.com", + "http://nanxinzuo.blog.sohu.com", + "http://huayf.blog.sohu.com", + "http://bairi.blog.sohu.com", + "http://scopettg.blog.sohu.com", + "http://yyrlcn110.blog.sohu.com", + "http://chinesesurname.blog.sohu.com", + "http://hyyhb.blog.sohu.com", + "http://see-the-sun.blog.sohu.com", + "http://mrxiaonan.blog.sohu.com", + "http://mlqk.blog.sohu.com", + "http://xiaoqian-lz.blog.sohu.com", + "http://shjunzheng.blog.sohu.com", + "http://215shop.blog.sohu.com", + "http://baihualin2006.blog.sohu.com", + "http://wangsanshang.blog.sohu.com", + "http://zheyangderizi.blog.sohu.com", + "http://hongioupo.blog.sohu.com", + "http://ruoshui364.blog.sohu.com", + "http://wg2006.blog.sohu.com", + "http://handsomehaole.blog.sohu.com", + "http://yimoyangguang.blog.sohu.com", + "http://xiefeng555.blog.sohu.com", + "http://qvbipyd.blog.sohu.com", + "http://ygxgjc2268.blog.sohu.com", + "http://maomit.blog.sohu.com", + "http://jingcaiyanyi.blog.sohu.com", + "http://hn53.blog.sohu.com", + "http://lnth.blog.sohu.com", + "http://yyrlcn.blog.sohu.com", + "http://xas0017.blog.sohu.com", + "http://jgtian2001.blog.sohu.com", + "http://huangcao.blog.sohu.com", + "http://275699.blog.sohu.com", + "http://hbjjxytdk616.blog.sohu.com", + "http://qianewen.blog.sohu.com", + "http://wangxianni.blog.sohu.com", + "http://dhqiang.blog.sohu.com", + "http://ppyyt.blog.sohu.com", + "http://yayazhilv.blog.sohu.com", + "http://shaqiaoer.blog.sohu.com", + "http://bbs52014.blog.sohu.com", + "http://neverlandjoyce.blog.sohu.com", + "http://dongfang929.blog.sohu.com", + "http://littleredboy.blog.sohu.com", + "http://welovemove.blog.sohu.com", + "http://welovemovie.blog.sohu.com", + "http://19860117.blog.sohu.com", + "http://happynancy.blog.sohu.com", + "http://nmqihuai.blog.sohu.com", + "http://allsyo2000.blog.sohu.com", + "http://larry98000.blog.sohu.com", + "http://redcoverage.blog.sohu.com", + "http://happyjimmy777.blog.sohu.com", + "http://anuo8520.blog.sohu.com", + "http://jiangbiehe123.blog.sohu.com", + "http://jingme.blog.sohu.com", + "http://huijie06.blog.sohu.com", + "http://houlaiwawa.blog.sohu.com", + "http://yongmeiboke.blog.sohu.com", + "http://huangshahe.blog.sohu.com", + "http://sunshuoly126.blog.sohu.com", + "http://zengjq.blog.sohu.com", + "http://shituscv.blog.sohu.com", + "http://zhangyinqiu.blog.sohu.com", + "http://xiangxueerbindou.blog.sohu.com", + "http://yumin-gao.blog.sohu.com", + "http://alcazar.blog.sohu.com", + "http://xraiyl.blog.sohu.com", + "http://DAOZHONGYANG.blog.sohu.com", + "http://019571005.blog.sohu.com", + "http://wyg7116.blog.sohu.com", + "http://wayneilsky.blog.sohu.com", + "http://bzwq007.blog.sohu.com", + "http://qingbaicai.blog.sohu.com", + "http://helen520530.blog.sohu.com", + "http://nandehutu16.blog.sohu.com", + "http://ysljl.blog.sohu.com", + "http://qlhh123456.blog.sohu.com", + "http://3572486.blog.sohu.com", + "http://elvating.blog.sohu.com", + "http://huipengsun.blog.sohu.com", + "http://YMNYGQ.blog.sohu.com", + "http://860531.blog.sohu.com", + "http://bdandbd.blog.sohu.com", + "http://tanshanqiang.blog.sohu.com", + "http://llaoqi202.blog.sohu.com", + "http://yoyolqq.blog.sohu.com", + "http://linjunxian.blog.sohu.com", + "http://takehujie.blog.sohu.com", + "http://maggie-mark.blog.sohu.com", + "http://fzy3.blog.sohu.com", + "http://liyangaisha.blog.sohu.com", + "http://mc99.blog.sohu.com", + "http://clibaobei.blog.sohu.com"}; + +} Index: src/test/java/org/apache/http/nio/TestAll.java =================================================================== --- src/test/java/org/apache/http/nio/TestAll.java (版本 662663) +++ src/test/java/org/apache/http/nio/TestAll.java (工作副本) @@ -44,9 +44,10 @@ public static Test suite() { TestSuite suite = new TestSuite(); - suite.addTest(TestAllUtil.suite()); - suite.addTest(TestAllImpl.suite()); - suite.addTest(TestAllProtocol.suite()); +// suite.addTest(TestAllUtil.suite()); +// suite.addTest(TestAllImpl.suite()); +// suite.addTest(TestAllProtocol.suite()); + suite.addTest(new TestSuite(OOMTest.class)); return suite; } Index: src/test/java/org/apache/http/nio/OOMTest.java =================================================================== --- src/test/java/org/apache/http/nio/OOMTest.java (版本 0) +++ src/test/java/org/apache/http/nio/OOMTest.java (版本 0) @@ -0,0 +1,318 @@ +/* + * $HeadURL: http://svn.apache.org/repos/asf/httpcomponents/httpcore/tags/4.0-beta1/module-nio/src/test/java/org/apache/http/nio/TestAll.java $ + * $Revision: 613298 $ + * $Date: 2008-01-19 06:09:22 +0800 (六, 2008-01-19) $ + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ + +package org.apache.http.nio; + +import java.io.*; +import java.util.concurrent.Executors; +import java.net.*; + +import org.apache.http.*; +import org.apache.http.impl.DefaultConnectionReuseStrategy; +import org.apache.http.params.BasicHttpParams; +import org.apache.http.impl.nio.DefaultClientIOEventDispatch; +import org.apache.http.impl.nio.reactor.DefaultConnectingIOReactor; +import org.apache.http.message.BasicHttpRequest; +import org.apache.http.nio.NHttpConnection; +import org.apache.http.nio.protocol.*; +import org.apache.http.nio.reactor.*; +import org.apache.http.params.*; +import org.apache.http.protocol.*; +import org.apache.http.util.EntityUtils; + +import junit.framework.*; +import java.io.InterruptedIOException; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.regex.Pattern; +import java.util.regex.Matcher; +import java.util.List; +import java.util.ArrayList; + +public class OOMTest extends TestCase { + HttpParams params = new BasicHttpParams(); + static ConnectingIOReactor ioReactor; + + public void testThrottling() throws Exception { + + params + .setIntParameter(CoreConnectionPNames.SO_TIMEOUT, 5000) + .setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, 10000) + .setIntParameter(CoreConnectionPNames.SOCKET_BUFFER_SIZE, 8 * 1024) + .setBooleanParameter(CoreConnectionPNames.STALE_CONNECTION_CHECK, false) + .setBooleanParameter(CoreConnectionPNames.TCP_NODELAY, true) + .setParameter(CoreProtocolPNames.USER_AGENT, "HttpComponents/1.1"); + + ioReactor = new DefaultConnectingIOReactor(2, params); + + BasicHttpProcessor httpproc = new BasicHttpProcessor(); + httpproc.addInterceptor(new RequestContent()); + httpproc.addInterceptor(new RequestTargetHost()); + httpproc.addInterceptor(new RequestConnControl()); + httpproc.addInterceptor(new RequestUserAgent()); + httpproc.addInterceptor(new RequestExpectContinue()); + + // We are going to use this object to synchronize between the + // I/O event and main threads + RequestCount requestCount = new RequestCount(3); + + ThrottlingHttpClientHandler handler = new ThrottlingHttpClientHandler( + httpproc, + new MyHttpRequestExecutionHandler(requestCount), + new DefaultConnectionReuseStrategy(), + Executors.newCachedThreadPool(), + params); + + handler.setEventListener(new EventLogger()); + + final IOEventDispatch ioEventDispatch = new DefaultClientIOEventDispatch(handler, params); + + Thread t = new Thread(new Runnable() { + public void run() { + try { + ioReactor.execute(ioEventDispatch); + } catch (InterruptedIOException ex) { + System.err.println("Interrupted"); + } catch (IOException e) { + System.err.println("I/O error: " + e.getMessage()); + } + System.out.println("Shutdown"); + } + + }); + t.start(); + + SessionRequest[] reqs = new SessionRequest[requestCount.getValue()]; + + for (String link : URList.LIST) { + URL url = new URL(link); + ioReactor.connect( + new InetSocketAddress(url.getHost(), 80), + null, + new HttpHost(url.getHost()), + null); + } + + // Block until all connections signal + // completion of the request execution + synchronized (requestCount) { + while (requestCount.getValue() > 0) { + requestCount.wait(); + } + } + + System.out.println("Shutting down I/O reactor"); + + ioReactor.shutdown(); + + System.out.println("Done"); + } + + + static class MyHttpRequestExecutionHandler implements HttpRequestExecutionHandler { + + private final static String REQUEST_SENT = "request-sent"; + private final static String RESPONSE_RECEIVED = "response-received"; + + private final RequestCount requestCount; + + public MyHttpRequestExecutionHandler(final RequestCount requestCount) { + super(); + this.requestCount = requestCount; + } + + public void initalizeContext(final HttpContext context, final Object attachment) { + HttpHost targetHost = (HttpHost) attachment; + context.setAttribute(ExecutionContext.HTTP_TARGET_HOST, targetHost); + } + + public void finalizeContext(final HttpContext context) { + Object flag = context.getAttribute(RESPONSE_RECEIVED); + if (flag == null) { + // Signal completion of the request execution + // Don't decrement, keep it running + // synchronized (this.requestCount) { + // this.requestCount.decrement(); + // this.requestCount.notifyAll(); + // } + } + } + + public HttpRequest submitRequest(final HttpContext context) { + HttpHost targetHost = (HttpHost) context.getAttribute( + ExecutionContext.HTTP_TARGET_HOST); + Object flag = context.getAttribute(REQUEST_SENT); + if (flag == null) { + // Stick some object into the context + context.setAttribute(REQUEST_SENT, Boolean.TRUE); + + System.out.println("--------------"); + System.out.println("Sending request to " + targetHost); + System.out.println("--------------"); + + return new BasicHttpRequest("GET", "/"); + } else { + // No new request to submit + return null; + } + } + + public void handleResponse(final HttpResponse response, final HttpContext context) { + HttpHost targetHost = (HttpHost) context.getAttribute(ExecutionContext.HTTP_TARGET_HOST); + HttpEntity entity = response.getEntity(); + try { + String content = EntityUtils.toString(entity); + + System.out.println("--------------"); + System.out.println(response.getStatusLine()); + System.out.println("--------------"); + System.out.println("Document length: " + content.length()); + System.out.println("--------------"); + + System.out.println(); + + for (String link : URLFetch.fetch(content, new URL(targetHost.toURI()))) { + URL url = new URL(link); + ioReactor.connect( + new InetSocketAddress(url.getHost(), 80), + null, + new HttpHost(url.getHost()), + null); + } + } catch (IOException ex) { + System.err.println("I/O error: " + ex.getMessage()); + } + + context.setAttribute(RESPONSE_RECEIVED, Boolean.TRUE); + + // Signal completion of the request execution + // Don't decrement, keep it running + // synchronized (this.requestCount) { + // this.requestCount.decrement(); + // this.requestCount.notifyAll(); + // } + } + + } + + static class EventLogger implements EventListener { + + public void connectionOpen(final NHttpConnection conn) { + System.out.println("Connection open: " + conn); + } + + public void connectionTimeout(final NHttpConnection conn) { + System.out.println("Connection timed out: " + conn); + } + + public void connectionClosed(final NHttpConnection conn) { + System.out.println("Connection closed: " + conn); + } + + public void fatalIOException(final IOException ex, final NHttpConnection conn) { + System.err.println("I/O error: " + ex.getMessage()); + } + + public void fatalProtocolException(final HttpException ex, final NHttpConnection conn) { + System.err.println("HTTP error: " + ex.getMessage()); + } + + } + + static class RequestCount { + + private int value; + + public RequestCount(int initialValue) { + this.value = initialValue; + } + + public int getValue() { + return this.value; + } + + public void decrement() { + this.value--; + } + } + + static class URLFetch { + public static List fetch(String pageContents, URL pageUrl) { + Pattern p = Pattern.compile("]", Pattern.CASE_INSENSITIVE); + Matcher m = p.matcher(pageContents); + + ArrayList linkList = new ArrayList(); + while (m.find()) { + String link = m.group(1).trim(); + // Skip empty links. + if (link.length() < 1) { + continue; + } + // Skip links that are just page anchors. + if (link.charAt(0) == '#') { + continue; + } + // Skip mailto links. + if (link.indexOf("mailto:") != -1) { + continue; + } + // Skip JavaScript links. + if (link.toLowerCase().indexOf("javascript") != -1) { + continue; + } + // Prefix absolute and relative URLs if necessary. + if (link.indexOf("://") == -1) { + // Handle absolute URLs. + if (link.charAt(0) == '/') { + link = "http://" + pageUrl.getHost() + link; + // Handle relative URLs. + } else { + String file = pageUrl.getFile(); + if (file.indexOf('/') == -1) { + link = "http://" + pageUrl.getHost() + "/" + link; + } else { + String path = + file.substring(0, file.lastIndexOf('/') + 1); + link = "http://" + pageUrl.getHost() + path + link; + } + } + } + // Remove anchors from link. + int index = link.indexOf('#'); + if (index != -1) { + link = link.substring(0, index); + } + linkList.add(link); + } + return linkList; + } + } +} +