httpclient模拟需要登陆之后才能访问第三方网站(并且需要一些人工参与的操作)

httpclient模拟用户登陆并访问目标页面....作个个人记录！

最近接了个功能，调用第三方的API获取数据，入库到我们自己平台！但是提供方网站上的东西比较简陋：

简单说明下：

1.获取数据的接口需要唯一标识access_token；

2.access_token的获取需要通过code去获取；

3.code的获取需要引导用户(第三方平台的登陆用户)打开一个授权页面进行授权之后，重定向地址上 而假如没有登陆还需要你先进行登陆；

难点就在这个需要引导用户打开授权页面进行授权

1). 这里面我们直接获取数据，用户就是我们自己在该第三方平台的账号；

2).  没有登陆直接去访问会跳转到登陆页面；

3). 登陆了之后，会有个授权页面，需要手动去点击授权按钮才真正跳转；

如下列图：







然后查看上述页面的源代码，可找到发送请求的URL以及参数：

登陆的地址和参数：



点击授权连接按钮所作的操作：



上代码(HttpClients模拟 + Jsoup解析html)

-----此例是比较简单的没有验证码的登录页面

package com.fulihui.daweixinke.test.conn;

import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.NameValuePair;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.cookie.Cookie;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.junit.Test;
import org.near.toolkit.security.codec.MD5Coder;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import static java.lang.System.out;

/**
* <pre>
* 模拟需要登陆之后才能访问第三方网站
* 并且需要一些人工参与的操作
* </pre>
*
* @author wang_wx
* @Date 2017-08-24
*/
public class EasyLoginNextPageA {

private String userName = "我是用户名";

private String password = "我是密码";

@Test
public void loginNext() throws IOException {

BasicCookieStore cookieStore = new BasicCookieStore();
// 全局用这一个httpClient对象模拟真实的一个浏览器中操作
CloseableHttpClient httpClient = HttpClients.custom().setDefaultCookieStore(cookieStore).build();
try {
// 模拟用户登录
HttpPost httpLogin = new HttpPost("xxxxxxxxxxxxxx/index.php/auth/auth/login");// 指向一个没有验证码的登录页面
List<NameValuePair> nvps = new ArrayList<>();
nvps.add(new BasicNameValuePair("username", userName));// 用户名对应的key
nvps.add(new BasicNameValuePair("password", MD5Coder.md5Encode(password)));// 密码对应的key
httpLogin.setEntity(new UrlEncodedFormEntity(nvps));
CloseableHttpResponse respLogin = httpClient.execute(httpLogin);
try {
HttpEntity entity = respLogin.getEntity();
out.println("respLogin------------>>" + respLogin.toString());
out.println("Login form get: " + respLogin.getStatusLine());
EntityUtils.consume(entity);
out.println("Initial set of cookies:");
List<Cookie> cookies = cookieStore.getCookies();
if (cookies.isEmpty()) {
out.println("None");
} else {
for (int i = 0; i < cookies.size(); i++) {
out.println("Cookie-" + i + "==" + cookies.get(i).toString());
}
}
} finally {
respLogin.close();
}

// 利用会话保持，继续访问目标地址
HttpGet httpGetAuth = new HttpGet("xxxxxxxxxxxxxxx/index.php/Auth/Auth/auth?app_id=xxxxxx&redirect_uri=回调地址&response_type=code");
CloseableHttpResponse respAuth = httpClient.execute(httpGetAuth);
String entityAuthStr = "";
try {
out.println(respAuth.getStatusLine());
HttpEntity entityAuth = respAuth.getEntity();
entityAuthStr = EntityUtils.toString(entityAuth);
out.println("get Auth cookies:");
List<Cookie> cookies = cookieStore.getCookies();
if (cookies.isEmpty()) {
out.println("None");
} else {
for (int i = 0; i < cookies.size(); i++) {
out.println("- " + cookies.get(i).toString());
}
}
out.println("访问目标地址的结果--------------------->>" + entityAuthStr);//把结果打印出来看一下
EntityUtils.consume(entityAuth);
} finally {
respAuth.close();
}
// 解析登陆之后访问目标地址的html页面 获取目标form表单元素
Document doc = Jsoup.parseBodyFragment(entityAuthStr);
String title = doc.title();
out.println("title-------->>" + title);
Element element = doc.body();
Elements form = element.select("[name='authForm']");// 授权的form表单
out.println(form);
String actionUrl = form.attr("action");
String app_id = element.select("[name='app_id']").val();
String redirect_uri = element.select("[name='redirect_uri']").val();
String state = element.select("[name='state']").val();
String key = element.select("[name='key']").val();
String format = element.select("[name='format']").val();

// 利用会话保持，继续模拟点击授权按钮 提交form表单
HttpPost httpPostGetCode = new HttpPost(actionUrl);
List<NameValuePair> nvps2 = new ArrayList<>();
nvps2.add(new BasicNameValuePair("app_id", app_id));
nvps2.add(new BasicNameValuePair("redirect_uri", redirect_uri));
nvps2.add(new BasicNameValuePair("state", state));
nvps2.add(new BasicNameValuePair("key", key));
nvps2.add(new BasicNameValuePair("format", format));
httpPostGetCode.setEntity(new UrlEncodedFormEntity(nvps2));
CloseableHttpResponse respGetCode = httpClient.execute(httpPostGetCode);
try {
HttpEntity entityGetCode = respGetCode.getEntity();
out.println("respAuth------------>>" + respGetCode.toString());
// 最终目的 获取Location中的url中的某一值
Header location = respGetCode.getFirstHeader("Location");
out.println("location------------>>" + location.getValue());
EntityUtils.consume(entityGetCode);
} finally {
respGetCode.close();
}
} finally {
httpClient.close();
}
}

@Test
public void test111() {
String password = "我是密码";
String encode = MD5Coder.md5Encode(password);
out.println("encode的值是：---" + encode + "，当前方法=EasyLoginNextPageA.test111()");
}
}

打印日志：