国际dhl与gatl运单号透过httpClient抓取数据
国际dhl与gatl运单号通过httpClient抓取数据
package com.choice.ehr.utils;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import net.sf.json.JSONArray;
import net.sf.json.JSONObject;
import org.apache.commons.httpclient.DefaultHttpMethodRetryHandler;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.params.HttpMethodParams;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.codehaus.jackson.map.ObjectMapper;
import com.alibaba.fastjson.JSON;
import com.choice.ehr.jobs.exchange.Checkpoints;
import com.choice.ehr.jobs.exchange.Courier;
import com.choice.ehr.jobs.exchange.Orders;
import com.choice.weixin.bean.ShptInfoWeiXinBean;
import com.sun.org.apache.xalan.internal.xsltc.compiler.sym;
import com.sunrise.vivo.utils.DateUtil;
public class DHLExchange {
private static final Log log = LogFactory.getLog(DHLExchange.class);
//dhl 请求返回json
public static String QueryDHL(String dhlID) throws Exception {
String gjson="";
//String strURL="http://api.open.baidu.com/pae/channel/data/asyncqury?cb=jQuery11020536401330732589_1417141855576&appid=4001&com=dhl&nu=96974175220&_=1417141855582";
//String strURL="http://www.cn.dhl.com/shipmentTracking?AWB=9697417520&countryCode=cn&languageCode=zh&_=1416213138381";
String strURL="http://www.cn.dhl.com/shipmentTracking?AWB="+dhlID+"&countryCode=cn&languageCode=en&_=1417403812558";
// http://www.cn.dhl.com/shipmentTracking?AWB=9697417520%2C9697426421%2C9697426340%2C9697426314%2C9697426270%2C9697426233%2C9697426222%2C9697426115%2C9697426071%2C9697426244&countryCode=cn&languageCode=zh&_=1417582317698
//String strURL="http://www.cn.dhl.com/shipmentTracking?AWB=9697417523&countryCode=cn&languageCode=zh&_=1416213138381";
URL url = new URL(strURL);
HttpURLConnection httpConn = (HttpURLConnection) url.openConnection();
InputStreamReader input = new InputStreamReader(httpConn
.getInputStream(), "utf-8");
BufferedReader bufReader = new BufferedReader(input);
String line = "";
StringBuilder contentBuf = new StringBuilder();
while ((line = bufReader.readLine()) != null) {
contentBuf.append(line);
}
System.out.println("captureJavascript()的结果：\n" + contentBuf.toString());
gjson=contentBuf.toString();
return gjson;
}
//gatl 运单号 国外网站
public static List<Checkpoints> QueryGatlen(String fRefNo) {
List returnList = new ArrayList();
List<Checkpoints> poinsList= new ArrayList<Checkpoints>();
// 构造HttpClient的实例
chttpClient = new HttpClient();
// 处理代理服务器
//httpClient.getHostConfiguration().setProxy("10.200.1.19", 886);
// 创建GET方法的实例
//GetMethod getMethod = new GetMethod(
// "http://www.boc.cn/sourcedb/whpj/index.html");
GetMethod getMethod= new GetMethod("http://www.gati.com/gatitrck.jsp?dktNo="+fRefNo+"");
getMethod.getResponseCharSet();
// 使用系统提供的默认的恢复策略
getMethod.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
new DefaultHttpMethodRetryHandler());
try {
// 执行getMethod
int statusCode = httpClient.executeMethod(getMethod);
if (statusCode != HttpStatus.SC_OK) {
log.error("微信gati请求网站出错：" + getMethod.getStatusLine());
System.err.println("Method failed: "
+ getMethod.getStatusLine());
}
// 读取内容
byte[] responseBody = getMethod.getResponseBody();
// 处理内容
String returnStr = new String(responseBody, "UTF-8");
String regex;
List<String> list = new ArrayList<String>();
//<table border=\"0\" cellpadding=\"1\" cellspacing=\"1\" width=\"98%\" align=\"center\" class=\"form_table\">
//regex="<table class=\"result-checkpoints show result-has-pieces\" summary=\"DHL Express shipments checkpoints\">.*?</table>";
// regex = "<table cellpadding=\"0\" align=\"left\" cellspacing=\"0\" width=\"100%\">.*?</table>";
//regex="<table border=\"0\" cellpadding=\"1\" cellspacing=\"1\" width=\"98%\" align=\"center\" class=\"form_table\">.*?</table>";
regex="<tr bgcolor='#05a6c3' class=\"textbold\">.*?</table>";
final Pattern pa = Pattern.compile(regex, Pattern.DOTALL);
final Matcher ma = pa.matcher(returnStr);
while (ma.find()) {
list.add(ma.group());
}
for ( String s:list) {
String regex1;
String ye=s.replaceAll(" colspan='2'", "");
List<String> list1 = new ArrayList<String>();
regex1 = "<td>.*?</td>";
final Pattern pa1 = Pattern.compile(regex1, Pattern.DOTALL);
final Matcher ma1 = pa1.matcher(ye);
while (ma1.find()) {
list1.add(ma1.group());
}
int i = 0;
String date="";
String description="";
for ( String s1:list1) {
i ++;
String str = s1.substring(4, s1.lastIndexOf("<"));
Checkpoints points=new Checkpoints();
if ( i >3) {
if(i%4==0){
date=DateUtil.getEnsYYMM(str);
}
if(i%4==1){
date+=" "+str;
points.setDate(date);
}
if(i%4==2){
description=str;
}
if(i%4==3){
points.setDate(date+":00");
points.setDescription(str);
points.setLocation(description);
poinsList.add(points);
}
}
}
}
} catch (HttpException e) {
// 发生致命的异常，可能是协议不对或者返回的内容有问题
System.out.println("Please check your provided http address!");
e.printStackTrace();
} catch (IOException e) {
// 发生网络异常
log.error("微信gati网站网络出错："+e, e);
e.printStackTrace();
} finally {
// 释放连接
getMethod.releaseConnection();
return poinsList;
}
}
//国内gatl运单号抓取页面的快递信息
public static List<Checkpoints> QueryGatlzg(String fRefNo) {
List returnList = new ArrayList();
List<Checkpoints> poinsList= new ArrayList<Checkpoints>();
// 构造HttpClient的实例
HttpClient httpClient = new HttpClient();
GetMethod getMethod= new GetMethod("http://218.244.150.40/cgi-bin/GInfo.dll?EmmisTrack&cno="+fRefNo+"");
getMethod.getResponseCharSet();
// 使用系统提供的默认的恢复策略
getMethod.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
new DefaultHttpMethodRetryHandler());
try {
// 执行getMethod
int statusCode = httpClient.executeMethod(getMethod);
if (statusCode != HttpStatus.SC_OK) {
log.error("微信gati请求网站出错：" + getMethod.getStatusLine());
System.err.println("Method failed: "
+ getMethod.getStatusLine());
}
// 读取内容
byte[] responseBody = getMethod.getResponseBody();
// 处理内容
String returnStr = new String(responseBody, "gb2312");
String regex;
List<String> list = new ArrayList<String>();
//<table border=\"0\" cellpadding=\"1\" cellspacing=\"1\" width=\"98%\" align=\"center\" class=\"form_table\">
regex="<table width='98%' border='1' align='center' cellpadding='1' cellspacing=0 class='trackContentTable' id='oTHtable'>.*?</table>";
final Pattern pa = Pattern.compile(regex, Pattern.DOTALL);
final Matcher ma = pa.matcher(returnStr);
while (ma.find()) {
list.add(ma.group());
}
for ( String s:list) {
String regex1;
String ye=s.replaceAll(" align='left' bgcolor='#F5F9FA' class='trackListOdd'", "");
ye=ye.replaceAll(" align='center' bgcolor='#FFFFFF' class='trackListEven'", "");
ye=ye.replaceAll(" align='center' bgcolor='#F5F9FA' class='trackListOdd'", "");
ye=ye.replaceAll(" align='left' bgcolor='#FFFFFF' class='trackListEven'", "");
ye=ye.replaceAll(" align='center' bgcolor='#FFFFFF'", "");
ye=ye.replaceAll(" align='center' bgcolor='#F5F9FA'", "");
List<String> list1 = new ArrayList<String>();
regex1 = "<td>.*?</td>";
final Pattern pa1 = Pattern.compile(regex1, Pattern.DOTALL);
final Matcher ma1 = pa1.matcher(ye);
while (ma1.find()) {
list1.add(ma1.group());
}
int i = 3;
String date="";
String description="";
for ( String s1:list1) {
String str = s1.substring(4, s1.lastIndexOf("<"));
Checkpoints points=new Checkpoints();
if(i%3==0){
date=DateUtil.getcgYYMM1(str);
}
if(i%3==1){
description=str.trim();
}
if(i%3==2){
points.setDate(date);
System.out.println();
points.setDescription(str.substring(0, str.trim().length()-1));
points.setLocation(description.substring(0, description.length()-1));
poinsList.add(points);
}
i ++;
}
}
} catch (HttpException e) {
// 发生致命的异常，可能是协议不对或者返回的内容有问题
System.out.println("Please check your provided http address!");
e.printStackTrace();
} catch (IOException e) {
// 发生网络异常
log.error("微信gati网站网络出错："+e, e);
e.printStackTrace();
} finally {
// 释放连接
getMethod.releaseConnection();
// 将ArrayLista中的元素进行倒序
Collections.reverse(poinsList);
return poinsList;
}
}
public static void main(String[] args) throws Exception {
List<Checkpoints> list= DHLExchange.QueryGatlzg("641449633");
for (Checkpoints opint : list) {
System.out.println(opint.getDate()+"="+opint.getDescription()+"=="+opint.getLocation());
}
}
}