Httpclient 4.3 connection pool stuck

100 0
static {
 try {
 SSLContextBuilder builder = new SSLContextBuilder();
 builder.loadTrustMaterial(null, new TrustSelfSignedStrategy());
 SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(builder.build());
 CookieSpecProvider easySpecProvider = new CookieSpecProvider() {
 public CookieSpec create(HttpContext context) {
 return new BrowserCompatSpec() {
 @Override
 public void validate(Cookie cookie, CookieOrigin origin) throws MalformedCookieException {
//Oh, I am easy
 }
 };
 }
 };
 Registry<CookieSpecProvider> r = RegistryBuilder.<CookieSpecProvider> create()
. register(CookieSpecs.BEST_MATCH, new BestMatchSpecFactory())
. register(CookieSpecs.BROWSER_COMPATIBILITY, new BrowserCompatSpecFactory())
. register("easy", easySpecProvider).build();
//5秒超时
 RequestConfig requestConfig = RequestConfig.custom().setConnectionRequestTimeout(5000)
. setSocketTimeout(10000).setConnectTimeout(10000).setCookieSpec("easy").setRedirectsEnabled(false)
. build();
 PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
 cm.setMaxTotal(100);//连接池最大并发连接数
 cm.setDefaultMaxPerRoute(10);//单路由最大并发数
 client = HttpClients.custom().setConnectionManager(cm).setDefaultCookieSpecRegistry(r)
. setSSLSocketFactory(sslsf).setDefaultRequestConfig(requestConfig).build();
 } catch (Exception e) {
 logger.error("http client 初始化失败!", e);
 }
}
public static String execute(HttpRequest httpRequest) {
 CloseableHttpResponse response = null;
 HttpGet httpGet = null;
 HttpEntity httpEntity = null;
 try {
 httpGet = new HttpGet(httpRequest.getUrl());
 httpGet.setHeader("Connection","close");//短链接
 if (httpRequest.isUseGzip()) {
 httpGet.addHeader("Accept-Encoding","gzip,deflate,sdch");
 }
 if (!StringUtils.isEmpty(httpRequest.getContentType())) {
 httpRequest.setContentType(httpRequest.getContentType());
 }
 httpGet.addHeader("User-Agent",
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63");
 response = client.execute(httpGet);
 httpEntity = response.getEntity();
//不管状态如何, response 都需要读取
 byte[] bytes = null;
 try {
 bytes = EntityUtils.toByteArray(httpEntity);
 } catch (Exception e) {
 return null;
 }
 if (response.getStatusLine().getStatusCode()!= 200) {
 logger.warn("请求异常! StatusCode:" + response.getStatusLine().getStatusCode() +", url:"
 + httpRequest.getUrl());
 return null;
 }
//确认网页编码
 @SuppressWarnings("deprecation")
 String charset = EntityUtils.getContentCharSet(httpEntity);
 if (StringUtils.isEmpty(charset)) {
 Matcher match = charsetPatterm.matcher(new String(bytes));
 if (match.find()) {
 charset = match.group(1);
 }
 }
 if (!StringUtils.isEmpty(charset)) {
 String strUtf8 = new String(new String(bytes, charset).getBytes(), GlobalConfig.ENCODING);
 return StringEscapeUtils.unescapeHtml4(strUtf8);
 }
 } catch (Exception e) {
 logger.error("抓取页面异常! url [" + httpRequest.getUrl() +"]", e);
 } finally {
 try {
 if (httpEntity!= null) {
 EntityUtils.consume(httpEntity);
 }
 if (response!= null) {
 response.close();
 }
 if (httpGet!= null) {
 httpGet.abort();
 }
 } catch (Exception e) {
//ignore
 }
 }
 return null;
}

It's the code used by httpclient 4.3, and the program runs for a while using jstack to view thread status discovery.

"pool-1-thread-10" prio=10 tid=0x00007f7168003000 nid=0x3e4d waiting on condition [0x00007f717c398000]
 java.lang.Thread.State: WAITING (parking)
 at sun.misc.Unsafe.park(Native Method)
 - parking to wait for <0x00000000e69d7350> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
 at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
 at org.apache.http.pool.PoolEntryFuture.await(PoolEntryFuture.java:133)
 at org.apache.http.pool.AbstractConnPool.getPoolEntryBlocking(AbstractConnPool.java:282)
 at org.apache.http.pool.AbstractConnPool.access$000(AbstractConnPool.java:64)
 at org.apache.http.pool.AbstractConnPool$2.getPoolEntry(AbstractConnPool.java:177)
 at org.apache.http.pool.AbstractConnPool$2.getPoolEntry(AbstractConnPool.java:170)
 at org.apache.http.pool.PoolEntryFuture.get(PoolEntryFuture.java:102)
 at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.leaseConnection(PoolingHttpClientConnectionManager.jav
a:244)
 at org.apache.http.impl.conn.PoolingHttpClientConnectionManager$1.get(PoolingHttpClientConnectionManager.java:231)
 at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:173)
 at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:195)
 at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:86)
 at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:108)
 at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
 at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
 at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)

Some of these threads have been stuck in here, which is why.

3 Answers

121 5

I've got this problem with httpclient4. 5, and use the loop to adjust an address, and find that it'll soon be stuck. The trace is full of connections, and there are no more links available, so the thread is waiting for a link. The default timeout is never because no link timeout is set. Thread will be blocked.
1, the solution is simple to set time and then capture the exception, when, turn off the connection pool and create a new linked pool.

try{
//http invoke
}catch(Exception ex){
 httpClientConnectionManager.shutdown();
 httpClientConnectionManager = new.. .;
}

2, set up the connection recovery policy ( with a thread to recycle )

2.5. 连接回收策略
经典阻塞I/O模型的一个主要缺点就是只有当组侧I/O时,socket才能对I/O事件做出反应。当连接被管理器收回后,这个连接仍然存活,但是却无法监控socket的状态,也无法对I/O事件做出反馈。如果连接被服务器端关闭了,客户端监测不到连接的状态变化(也就无法根据连接状态的变化,关闭本地的socket)。
HttpClient为了缓解这一问题造成的影响,会在使用某个连接前,监测这个连接是否已经过时,如果服务器端关闭了连接,那么连接就会失效。这种过时检查并不是100%有效,并且会给每个请求增加10到30毫秒额外开销。唯一一个可行的,且does not involve a one thread per socket model for idle connections的解决办法,是建立一个监控线程,来专门回收由于长时间不活动而被判定为失效的连接。这个监控线程可以周期性的调用ClientConnectionManager类的closeExpiredConnections()方法来关闭过期的连接,回收连接池中被关闭的连接。它也可以选择性的调用ClientConnectionManager类的closeIdleConnections()方法来关闭一段时间内不活动的连接。
 public static class IdleConnectionMonitorThread extends Thread {
 private final HttpClientConnectionManager connMgr;
 private volatile boolean shutdown;
 public IdleConnectionMonitorThread(HttpClientConnectionManager connMgr) {
 super();
 this.connMgr = connMgr;
 }
 @Override
 public void run() {
 try {
 while (!shutdown) {
 synchronized (this) {
 wait(5000);
//关闭失效的连接
 connMgr.closeExpiredConnections();
//可选的, 关闭30秒内不活动的连接
 connMgr.closeIdleConnections(30, TimeUnit.SECONDS);
 }
 }
 } catch (InterruptedException ex) {
//terminate
 }
 }
 public void shutdown() {
 shutdown = true;
 synchronized (this) {
 notifyAll();
 }
 }
 }
...