请求中设置代理IP_JAVA_程序员俱乐部

您所在的位置：程序员俱乐部 > 编程开发 > JAVA > 请求中设置代理IP

请求中设置代理IP

2018/4/18 18:27:55 st4024589553 程序员俱乐部我要评论(0)

摘要：(不能直接JVM设置代理System.getProperties().setProperty("http.proxyHost",ip);System.getProperties().setProperty("http.proxyPort","80");设置JVM代理后无法再次获取GetDlIp.dlip();)-----------------------------------------设置请求的代理IPpackagecom.bageer.meituan
标签：代理

(不能直接JVM设置代理
System.getProperties().setProperty("http.proxyHost", ip);
System.getProperties().setProperty("http.proxyPort", "80");
设置JVM代理后无法再次获取GetDlIp.dlip();   )

-----------------------------------------设置请求的代理IP

package com.bageer.meituan;
//9f27a0700a6b26192380a791807639bf
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Random;

import org.apache.http.Consts;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.NameValuePair;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import com.bageer.dao.DataInfoDao;
import com.bageer.dao.DataInfoDaoImpl;
import com.bageer.model.Shop_Info_Model;
public class GetMtLhfj {
public static DataInfoDao dao=new DataInfoDaoImpl();
public static void main(String[] args) {
int page=1;
while(true){
List list=dao.getData("select SHOP_URL ,Rownum rn from o2o_shop_info t where city='邢台市' and pt_name='美团外卖' and shop_url like '%access_source%' and is_lhfj is null", "1");
if(list.size()==0){
System.exit(0);
}
for (int i = 0; i < list.size(); i++) {
Map map=(Map)list.get(i);
String shop_url=map.get("SHOP_URL").toString();

Shop_Info_Model shopIfo=search(shop_url.split("\\?")[0].replace("restaurant", "qualification"));

if(shopIfo!=null){
if(shopIfo.getDeath()==null){
shopIfo.setShop_url(shop_url);
dao.update(shopIfo);
}else{
//死链接，为什么要单独更新死链接
//单独更新死链接可以只更新死链接的状态并不修改数据库中的数据
}
}
System.out.println(page+"==========================");
/*try {
Random rnd=new Random();
Thread.sleep((rnd.nextInt(1000)+1000));
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}*/
}
page++;
}
}

public static Shop_Info_Model search(String url){
String ip=GetDlIp.dlip();
        CloseableHttpClient httpClient = HttpClients.createDefault();
        CloseableHttpResponse response = null;
        InputStream is = null;
        //封装请求参数
        List<NameValuePair> params = new ArrayList<NameValuePair>();
        params.add(new BasicNameValuePair("access_source", "1004"));
        String str = "";
        try {
            //转换为键值对
            str = EntityUtils.toString(new UrlEncodedFormEntity(params, Consts.UTF_8));
            System.out.println(str);
          //创建Get请求
            HttpGet httpGet = new HttpGet(url+"?"+str);
            httpGet.setHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
            httpGet.setHeader("Accept-Encoding", "gzip, deflate, sdch");
            httpGet.setHeader("Accept-Language", "zh-CN,zh;q=0.8");
            httpGet.setHeader("Cache-Control", "max-age=0");
            httpGet.setHeader("Connection", "keep-alive");
            httpGet.setHeader("Cookie", "w_uuid=jIr1mG-pCMj69WwvZxpD7LvaHf2k9xnJPzGB9aeIPDWEG-mSYOur_6UIOYG-DZfP; _lxsdk_cuid=162cc2b4462c8-0977acd7dae47d-6b1b1279-100200-162cc2b4463c8; _lxsdk=162cc2b4462c8-0977acd7dae47d-6b1b1279-100200-162cc2b4463c8; _lx_utm=utm_campaign%3Dbaidu%26utm_source%3D1522; _ga=GA1.2.1772268467.1523843811; _gid=GA1.2.1093233036.1524031032; w_cid=110108; w_cpy_cn=\"%E6%B5%B7%E6%B7%80%E5%8C%BA\"; w_cpy=haidianqu; waddrname=\"%E7%B4%AB%E9%87%91%E5%A4%A7%E5%8E%A6\"; w_geoid=wx4equc9mmtp; w_ah=\"39.972626846283674,116.31454780697823,%E7%B4%AB%E9%87%91%E5%A4%A7%E5%8E%A6\"; JSESSIONID=jn7psbnnfzlg11c9ckvs7gfzd; _ga=GA1.3.1772268467.1523843811; _gid=GA1.3.1093233036.1524031032; _gat=1; w_utmz=\"utm_campaign=baidu&utm_source=1522&utm_medium=(none)&utm_content=(none)&utm_term=(none)\"; w_visitid=90067f92-aa84-4233-8ded-4943e427773c; __mta=251558201.1523843813548.1524031660063.1524031728661.43; _lxsdk_s=162d7540a1a-567-6e4-74b%7C%7C22");
            httpGet.setHeader("Host", "waimai.meituan.com");
            httpGet.setHeader("Upgrade-Insecure-Requests", "1");
            httpGet.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36");
            //执行Get请求，
            System.out.println(ip);
            HttpHost proxy = new HttpHost(ip.split(":")[0],Integer.parseInt( ip.split(":")[1].split(",")[0]));
            RequestConfig requestConfig = RequestConfig.custom()
                .setProxy(proxy)
                .setConnectTimeout(10000)
                .setSocketTimeout(10000)
                .setConnectionRequestTimeout(3000)
                .build();
            httpGet.setConfig(requestConfig);
            response = httpClient.execute(httpGet);
            //得到响应体
            HttpEntity entity = response.getEntity();
            if(entity != null){
                is = entity.getContent();
                //转换为字节输入流
                BufferedReader br = new BufferedReader(new InputStreamReader(is, Consts.UTF_8));
                String body = null;
                StringBuilder sb=new StringBuilder();
                while((body=br.readLine()) != null){
                sb.append(body);
                }
             Document doc= Jsoup.parse(sb.toString());
             Shop_Info_Model shopinfo=new Shop_Info_Model();
             if(doc.text().indexOf("返回首页")>0){
            shopinfo.setDeath("1");
            return shopinfo;
             }
         String csdj = null;// 场所等级
    String gldj = null;// 管理等级
    String jcrq = null;// 检查日期
    String dwmc = null;// 单位名称
    String xkzh = null;// 许可证号
    String fddbr = null;// 法定代表人
    String jydz = null;// 经营地址
    String ztyt = null;// 主体业态
    String jyfw = null;// 经营范围
    String yxq = null;// 有效期
     int indexOf = sb.indexOf("营业执照");
     String yyzh = null;
     String xkz = null;
     if (indexOf > -1) {
     yyzh = "1";
     } else {
     yyzh = "0";
     }

     int indexOf2 = sb.indexOf("餐饮服务许可证");
     if (indexOf2 > -1) {
     xkz = "1";
     } else {
     indexOf2 = sb.indexOf("特许证件");
     if (indexOf2 > -1) {
     xkz = "1";
     } else {
     xkz = "0";
     }
     }
     shopinfo.setIs_yyzz(yyzh);
     shopinfo.setIs_xkz(xkz);
     String text = doc.getElementsByAttributeValue("class", "new-zizhi").text();
     if (text.length() > 2) {
     shopinfo.setIs_lhfj("1");
     if (text.indexOf("场所等级：") > -1) {
     csdj = text.split("场所等级：")[1].trim().split(" ")[0].trim();
     System.out.println(csdj);
     shopinfo.setCsdj(csdj);
     }
     if (text.indexOf("管理等级：") > -1) {
     gldj = text.split("管理等级：")[1].trim().split(" ")[0].trim();
     System.out.println(gldj);
     shopinfo.setGldj(gldj);;
     }
     if (text.indexOf("检查日期：") > -1) {
     jcrq = text.split("检查日期：")[1].trim().split(" ")[0].trim();
     System.out.println(jcrq);
     shopinfo.setJcrq(jcrq);
     }
     if (text.indexOf("单位名称：") > -1) {
     dwmc = text.split("单位名称：")[1].trim().split(" ")[0].trim();
     System.out.println(dwmc);
     shopinfo.setDwmc(dwmc);
     }
     if (text.indexOf("许可证号：") > -1) {
     xkzh = text.split("许可证号：")[1].trim().split(" ")[0].trim();
     System.out.println(xkzh);
     shopinfo.setXkzh(xkzh);
     }
     if (text.indexOf("法定代表人") > -1) {

     fddbr = text.split("法定代表人")[1].trim().split(" ")[0].trim();
     System.out.println(fddbr);
     shopinfo.setFddbr(fddbr);
     }
     if (text.indexOf("经营地址：") > -1) {

     jydz = text.split("经营地址：")[1].trim().split(" ")[0].trim();
     System.out.println(jydz);
     shopinfo.setJydz(jydz);
     }
     if (text.indexOf("主体业态：") > -1) {

     ztyt = text.split("主体业态：")[1].trim().split(" ")[0].trim();
     System.out.println(ztyt);
     shopinfo.setZtyt(ztyt);
     }
     if (text.indexOf("经营项目：") > -1) {

     jyfw = text.split("经营项目：")[1].trim().split(" ")[0].trim();
     System.out.println(jyfw);
     shopinfo.setJyfw(jyfw);
     }
     if (text.indexOf("有效期：") > -1) {

     yxq = text.split("有效期：")[1].trim().split(" ")[0].trim();
     System.out.println(yxq);
     shopinfo.setYxq(yxq);
     }
     } else {
     System.out.println("没有量化分级"+"================"+sb.length());
     System.out.println("没有量化分级"+"================"+sb.indexOf("营业执照"));
     System.out.println("没有量化分级"+"================"+sb.indexOf("许可证"));
     shopinfo.setIs_lhfj("0");
     if(sb.length()<500){
     return null;
     }
     }
     return shopinfo;
            }
        } catch (Exception e) {
            e.printStackTrace();
        } finally{
            //关闭输入流，释放资源
            if(is != null){
                try {
                    is.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
            //消耗实体内容
            if(response != null){
                try {
                    response.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
            //关闭相应丢弃http连接
            if(httpClient != null){
                try {
                    httpClient.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }

        }
        return null;
}

}

-----------------------------------获取代理IP

package com.bageer.meituan;

import java.io.BufferedInputStream;
import java.io.InputStream;
import java.net.HttpURLConnection;

public class GetDlIp {
public static String dlip(){
// 每隔几秒提取一次IP
// 请填写无忧代理IP订单号，填写之后才可以提取到IP哦
String order = "xxxx";
try {
java.net.URL url = new java.net.URL("http://api.ip.data5u.com/dynamic/get.html?order=" + order + "&ttl&random=true");
//java.net.URL url = new java.net.URL("http://103.30.41.108/Tools/proxyIP.ashx?Type=YYYYY888888885545555555556&qty=5");

HttpURLConnection connection = (HttpURLConnection)url.openConnection();
    connection.setConnectTimeout(3000);
    connection = (HttpURLConnection)url.openConnection();
        InputStream raw = connection.getInputStream();
        InputStream in = new BufferedInputStream(raw);
        byte[] data = new byte[in.available()];
        int bytesRead = 0;
        int offset = 0;
        while(offset < data.length) {
            bytesRead = in.read(data, offset, data.length - offset);
            if(bytesRead == -1) {
                break;
            }
            offset += bytesRead;
        }
        in.close();
        raw.close();
String[] res = new String(data, "UTF-8").split("\n");
System.out.println(">>>>>>>>>>>>>>当前返回IP量 " + res.length);
return res[0];
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return null;
}
}

上一篇： Idea修改文件编码格式下一篇：没有下一篇了!