为什么使用HTTP连接池?
随着系统架构风格逐渐向前后端分离架构,微服务架构转变,RestFul风格API的开发与设计,同时SpringMVC也很好的支持了REST风格接口。各个系统之间服务的调用大多采用HTTP+JSON或HTTPS+JSON方式。
HTTP1.1默认是持久连接,HTTP1.0也可以通过在请求头中设置Connection:keep-alive使得连接成为长连接。既然HTTP协议支持长连接,那么HTTP连接同样可以使用连接池技术来管理和维护连接建立和销毁。 但是由于每次HTTP连接请求实际上都是在传输层建立的TCP连接,利用的socket进行通信,HTTP连接的保持和关闭实际上都同TCP连接的建立和关闭有关,所有每次HTTP请求都有经过TCP连接的三次握手(建立连接)和四次挥手(释放连接)的过程。所以采用HTTP连接池有以下优势:降低了频繁建立HTTP连接的时间开销,减少了TCP连接建立和释放时socket通信服务器端资源的浪费;
支持更高的并发量;
常用HttpClient连接池API
PoolingHttpClientConnectionManager连接池管理实现类
PoolingHttpClientConnectionManager是一个HttpClient连接池实现类,实现了HttpClientConnectionManager和ConnPoolControl接口。构造方法:
PoolingHttpClientConnectionManager():无参构造方法,从源码中可以看到该方法调用了getDefaultRegistry()来注册http协议和https协议。常用方法:
public void setMaxTotal(int max):该方法定义在ConnPoolControl接口中,表示设置最大连接数为max。 public void setDefaultMaxPerRoute(int max):该方法也是定义在ConnPoolControl接口中,表示将每个路由的默认最大连接数设置为max public void setMaxPerRoute(HttpRoute route,int max):设置某个指定路由的最大连接数,这个配置会覆盖setDefaultMaxPerRoute的某个路由的值。常用方法
static RequestConfig.Builder custom():静态方法,用于构建Builder 对象,然后设置相应的参数; int getConnectionRequestTimeout():获取从连接池获取连接的最长时间,单位是毫秒; int getConnectTimeout():获取创建连接的最长时间,单位是毫秒; int getSocketTimeout():获取数据传输的最长时间,单位是毫秒;RequestConfig有一个静态内部类Builder,用于构建RequestConfig对象并设置请求参数,该类有以下常用方法:
public RequestConfig.Builder setConnectionRequestTimeout(int connectionRequestTimeout):设置从连接池获取连接的最长时间,单位是毫秒; public RequestConfig.Builder setConnectTimeout(int connectTimeout):设置创建连接的最长时间,单位是毫秒; public RequestConfig.Builder setSocketTimeout(int socketTimeout):设置数据传输的最长时间,单位是毫秒;HttpRequestRetryHandler请求重试接口
boolean retryRequest(IOException exception, int executionCount, org.apache.http.protocol.HttpContext context):实现该接口的,必须实现该方法,决定了如果一个方法执行时发生了IO异常,是否应该重试,重试executionCount次。单线程-使用连接池管理HTTP请求
主要步骤:
1.创建HTTP的连接池管理对象cm,如下所示
PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
2.设置最大连接数和每个路由的默认最大连接数等参数,如下所示
//将最大连接数增加到200
cm.setMaxTotal(200); //将每个路由的默认最大连接数增加到20 cm.setDefaultMaxPerRoute(20);
3.模拟发送HttpGet请求或者HttpPost请求,注意这里创建HttpClients对象的时候需要从连接池中获取,即要设置连接池对象,当执行发生IO异常需要处理时,要实现HttpRequestRetryHandler接口;需要注意的是这里处理完请求响应后,不能关闭HttpClient对象,如果关闭连接池也会销毁。HttpClients对象创建对象所示:
CloseableHttpClient httpClient = HttpClients.custom()
.setConnectionManager(connectionManager) .setRetryHandler(retryHandler(5)).build();
4.可以开启线程来监听连接池中空闲连接,并清理无效连接,线程监听频率可以自行设置。
Java实现源码:package com.liangpj.develop.httpclient;
import org.apache.http.HttpEntityEnclosingRequest; import org.apache.http.HttpRequest; import org.apache.http.NoHttpResponseException; import org.apache.http.client.HttpRequestRetryHandler; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.protocol.HttpClientContext; import org.apache.http.conn.ConnectTimeoutException; import org.apache.http.conn.HttpClientConnectionManager; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.protocol.HttpContext; import org.apache.http.util.EntityUtils; import javax.net.ssl.SSLException; import javax.net.ssl.SSLHandshakeException; import java.io.IOException; import java.io.InterruptedIOException; import java.net.UnknownHostException; /** * 单线程-使用连接池管理HTTP请求 * @author: liangpengju * @version: 1.0 */ public class HttpConnectManager { public static void main(String[] args) throws Exception { //创建HTTP的连接池管理对象 PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager(); //将最大连接数增加到200 connectionManager.setMaxTotal(200); //将每个路由的默认最大连接数增加到20 connectionManager.setDefaultMaxPerRoute(20); //将http://www.baidu.com:80的最大连接增加到50 //HttpHost httpHost = new HttpHost("http://www.baidu.com",80); //connectionManager.setMaxPerRoute(new HttpRoute(httpHost),50); //发起3次GET请求 String url ="https://www.baidu.com/s?word=java"; long start = System.currentTimeMillis(); for (int i=0;i<100;i++){ doGet(connectionManager,url); } long end = System.currentTimeMillis(); System.out.println("consume -> " + (end - start)); //清理无效连接 new IdleConnectionEvictor(connectionManager).start(); } /** * 请求重试处理 * @param tryTimes 重试次数 * @return */ public static HttpRequestRetryHandler retryHandler(final int tryTimes){ HttpRequestRetryHandler httpRequestRetryHandler = new HttpRequestRetryHandler() { @Override public boolean retryRequest(IOException exception, int executionCount, HttpContext context) { // 如果已经重试了n次,就放弃 if (executionCount >= tryTimes) { return false; } // 如果服务器丢掉了连接,那么就重试 if (exception instanceof NoHttpResponseException) { return true; } // 不要重试SSL握手异常 if (exception instanceof SSLHandshakeException) { return false; } // 超时 if (exception instanceof InterruptedIOException) { return false; } // 目标服务器不可达 if (exception instanceof UnknownHostException) { return true; } // 连接被拒绝 if (exception instanceof ConnectTimeoutException) { return false; } // SSL握手异常 if (exception instanceof SSLException) { return false; } HttpClientContext clientContext = HttpClientContext .adapt(context); HttpRequest request = clientContext.getRequest(); // 如果请求是幂等的,就再次尝试 if (!(request instanceof HttpEntityEnclosingRequest)) { return true; } return false; } }; return httpRequestRetryHandler; } /** * doGet * @param url 请求地址 * @param connectionManager * @throws Exception */ public static void doGet(HttpClientConnectionManager connectionManager,String url) throws Exception { //从连接池中获取client对象,多例 CloseableHttpClient httpClient = HttpClients.custom() .setConnectionManager(connectionManager) .setRetryHandler(retryHandler(5)).build(); // 创建http GET请求 HttpGet httpGet = new HttpGet(url); // 构建请求配置信息 RequestConfig config = RequestConfig.custom().setConnectTimeout(1000) // 创建连接的最长时间 .setConnectionRequestTimeout(500) // 从连接池中获取到连接的最长时间 .setSocketTimeout(10 * 1000) // 数据传输的最长时间10s .setStaleConnectionCheckEnabled(true) // 提交请求前测试连接是否可用 .build(); // 设置请求配置信息 httpGet.setConfig(config); CloseableHttpResponse response = null; try { // 执行请求 response = httpClient.execute(httpGet); // 判断返回状态是否为200 if (response.getStatusLine().getStatusCode() == 200) { String content = EntityUtils.toString(response.getEntity(), "UTF-8"); System.out.println("内容长度:" + content.length()); } } finally { if (response != null) { response.close(); } // 此处不能关闭httpClient,如果关闭httpClient,连接池也会销毁 // httpClient.close(); } } /** * 监听连接池中空闲连接,清理无效连接 */ public static class IdleConnectionEvictor extends Thread { private final HttpClientConnectionManager connectionManager; private volatile boolean shutdown; public IdleConnectionEvictor(HttpClientConnectionManager connectionManager) { this.connectionManager = connectionManager; } @Override public void run() { try { while (!shutdown) { synchronized (this) { //3s检查一次 wait(3000); // 关闭失效的连接 connectionManager.closeExpiredConnections(); } } } catch (InterruptedException ex) { // 结束 ex.printStackTrace(); } } public void shutdown() { shutdown = true; synchronized (this) { notifyAll(); } } } }多线程-HttpClient连接池管理HTTP请求实例
主要步骤:
1.创建HTTP的连接池管理对象cm,如下所示
PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
2.设置最大连接数和每个路由的默认最大连接数等参数,如下所示
//将最大连接数增加到200
cm.setMaxTotal(200); //将每个路由的默认最大连接数增加到20 cm.setDefaultMaxPerRoute(20);
3.创建HttpClients对象的并设置连接池对象,HttpClients对象创建对象所示:
CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(connectionManager).build();
4.继承Thread类实现一个执行Get请求线程类GetThread,重载run()方法,执行HttpGet请求;
5.定义要请求的URI地址为数组形式,为每一个URI创建一个GetThread线程,并启动所有线程; Java实现源码:package com.liangpj.develop.httpclient;
import org.apache.http.HttpEntity; import org.apache.http.client.ClientProtocolException; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.protocol.HttpClientContext; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.protocol.HttpContext; import java.io.IOException; /** * 多线程-HttpClient连接池管理HTTP请求实例 */ public class MultiThreadHttpConnManager { public static void main(String[] args) { //连接池对象 PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager(); //将最大连接数增加到200 connectionManager.setMaxTotal(200); //将每个路由的默认最大连接数增加到20 connectionManager.setDefaultMaxPerRoute(20); //HttpClient对象 CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(connectionManager).build(); //URIs to DoGet String[] urisToGet = { "https://www.baidu.com/s?word=java", "https://www.baidu.com/s?word=java", "https://www.baidu.com/s?word=java", "https://www.baidu.com/s?word=java" }; //为每一个URI创建一个线程 GetThread[] threads = new GetThread[urisToGet.length]; for (int i=0;i<threads.length;i++){ HttpGet httpGet = new HttpGet(urisToGet[i]); threads[i] = new GetThread(httpClient,httpGet); } //启动线程 for (int j=0;j<threads.length;j++){ threads[j].start(); } //join 线程 for(int k=0;k<threads.length;k++){ try { threads[k].join(); } catch (InterruptedException e) { e.printStackTrace(); } } } /** * 执行Get请求线程 */ public static class GetThread extends Thread{ private final CloseableHttpClient httpClient; private final HttpContext context; private final HttpGet httpget; public GetThread(CloseableHttpClient httpClient, HttpGet httpget) { this.httpClient = httpClient; this.context = HttpClientContext.create(); this.httpget = httpget; } @Override public void run() { try { CloseableHttpResponse response = httpClient.execute(httpget,context); try { HttpEntity entity = response.getEntity(); }finally { response.close(); } }catch (ClientProtocolException ex){ //处理客户端协议异常 }catch (IOException ex){ //处理客户端IO异常 } } } }