java jsoup 网络爬虫 学习例子(六)京东和当当商品比价_JAVA_编程开发_程序员俱乐部

中国优秀的程序员网站程序员频道CXYCLUB技术地图
热搜:
更多>>
 
您所在的位置: 程序员俱乐部 > 编程开发 > JAVA > java jsoup 网络爬虫 学习例子(六)京东和当当商品比价

java jsoup 网络爬虫 学习例子(六)京东和当当商品比价

 2016/5/12 5:33:54  InJavaWeTrust  程序员俱乐部  我要评论(0)
  • 摘要:javajsoup网络爬虫学习例子(六)京东和当当商品比价packagecom.iteye.injavawetrust.jdvsdd;importjava.util.List;/****@authorInJavaWeTrust**/publicinterfaceProductList{/***爬取商品列表*@return*/publicList<ProductInfo>getProductList();}packagecom.iteye.injavawetrust.jdvsdd
  • 标签:学习 例子 Java 网络 京东 当当 网络爬虫 JS

?

java jsoup 网络爬虫 学习例子(六)京东和当当商品比价

?

?

?

?

class="java" name="code">package com.iteye.injavawetrust.jdvsdd;

import java.util.List;

/**
 * 
 * @author InJavaWeTrust
 *
 */
public interface ProductList {
	
	/**
	 * 爬取商品列表
	 * @return
	 */
	public List<ProductInfo> getProductList();

}

package com.iteye.injavawetrust.jdvsdd;

import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

/**
 * 
 * @author InJavaWeTrust
 *
 */
public class JDProductList implements ProductList{
	
	private String jdUrl;
	
	private String productName;
	
	private static PriceCheckUtil pcu = PriceCheckUtil.getInstance();
	
	public JDProductList(String jdUrl, String productName){
		this.jdUrl = jdUrl;
		this.productName = productName;
	}

	@Override
	public List<ProductInfo> getProductList() {
		List<ProductInfo> jdProductList = new ArrayList<ProductInfo>();
		ProductInfo productInfo = null;
		String url = "";
		for(int i = 0; i < 10; i++){
			try {
				System.out.println("JD Product 第[" + (i + 1) + "]页");
				if(i == 0) {
					url = jdUrl;
				}else{
					url = Constants.JDURL + pcu.getGbk(productName) + Constants.JDENC + Constants.JDPAGE + (i + 1);
				}
				System.out.println(url);
				Document document = Jsoup.connect(url).timeout(5000).get();
				Elements uls = document.select("ul[class=gl-warp clearfix]");
				Iterator<Element> ulIter = uls.iterator();
				while(ulIter.hasNext()) {
					Element ul = ulIter.next();
					Elements lis = ul.select("li[data-sku]");
					Iterator<Element> liIter = lis.iterator();
					while(liIter.hasNext()) {
						Element li = liIter.next();
						Element div = li.select("div[class=gl-i-wrap]").first();
						Elements title = div.select("div[class=p-name p-name-type-2]>a");
						String productName = title.attr("title"); //得到商品名称
						Elements price = div.select(".p-price>strong");
						String productPrice =price.attr("data-price"); //得到商品价格
						productInfo = new ProductInfo();
						productInfo.setProductName(productName);
						productInfo.setProductPrice(productPrice);
						jdProductList.add(productInfo);
					}
				}
			} catch(Exception e) {
				System.out.println("Get JD product has error [" + url + "]");
				System.out.println(e.getMessage());
			}
		}
		return jdProductList;
	}
	
	public static void main(String[] args) {
		try {
			String productName = "书包";
			String jdUrl = Constants.JDURL + pcu.getGbk(productName)  + Constants.JDENC;
			List<ProductInfo> list = new JDProductList(jdUrl, productName).getProductList();
			System.out.println(list.size());
			for(ProductInfo pi : list){
				System.out.println(pi.getProductName() + "  " + pi.getProductPrice());
			}
		} catch (Exception e) {
			e.printStackTrace();
		}
	}

}


package com.iteye.injavawetrust.jdvsdd;

import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

/**
 * 
 * @author InJavaWeTrust
 *
 */
public class DDProductList implements ProductList{
	
	private String ddUrl;
	
	private String productName;
	
	public DDProductList(String ddUrl, String productName) {
		this.ddUrl = ddUrl;
		this.productName = productName;
	}

	@Override
	public List<ProductInfo> getProductList() {
		List<ProductInfo> ddProductList = new ArrayList<ProductInfo>();
		ProductInfo productInfo = null;
		String url = "";
		for(int i = 0; i < 10; i++){
			try{
				System.out.println("DD Product 第[" + (i + 1) + "]页");
				if(i == 0){
					url = ddUrl;
				}else{
					url = Constants.DDURL + productName + Constants.ACT + Constants.DDPAGE + (i + 1);
				}
				System.out.println(url);
				Document document = Jsoup.connect(url).timeout(5000).get();
				Elements uls = document.select("ul[class=bigimg cloth_shoplist]");
				Iterator<Element> ulIter = uls.iterator();
				while(ulIter.hasNext()){
					Element ul = ulIter.next();
					Elements lis = ul.select("li");
					Iterator<Element> liIter = lis.iterator();
					while(liIter.hasNext()){
						Element li = liIter.next();
						Elements price = li.select("p[class=price]>span");
						String productPrice = price.html().replaceAll("&yen;", "");
						Elements title = li.select("p[class=name]>a");
						String productName = title.attr("title");
						productInfo = new ProductInfo();
						productInfo.setProductName(productName);
						productInfo.setProductPrice(productPrice);
						ddProductList.add(productInfo);
					}
					
				}
			} catch(Exception e){
				System.out.println("Get DD product has error [" + url + "]");
				System.out.println(e.getMessage());
			}
		}
		return ddProductList;
	}
	
	public static void main(String[] args) {
		String productName = "学生铅笔";
		String ddUrl = Constants.DDURL + productName;
		List<ProductInfo> list = new DDProductList(ddUrl, productName).getProductList();
		System.out.println(list.size());
		for(ProductInfo pi : list){
			System.out.println(pi.getProductName() + "  " + pi.getProductPrice());
		}
	}

}


package com.iteye.injavawetrust.jdvsdd;

import java.io.Serializable;
import java.util.Date;

/**
 * 
 * @author InJavaWeTrust
 *
 */
public class ProductInfo implements Serializable{

	private static final long serialVersionUID = 8179244535272774089L;
	
	/**
	 * 商品ID
	 */
	private String productid;
	/**
	 * 商品名称
	 */
	private String productName;
	/**
	 * 商品价格
	 */
	private String productPrice;
	/**
	 * 月销售笔数
	 */
	private String tradeNum;
	/**
	 * 商品URL
	 */
	private String productUrl;
	/**
	 * 商品网店名称
	 */
	private String shopName;
	/**
	 * 电商名称
	 */
	private String ecName;
	/**
	 * 爬取入库日期
	 */
	private Date date;
	
	public String getProductid() {
		return productid;
	}
	public void setProductid(String productid) {
		this.productid = productid;
	}
	public String getProductName() {
		return productName;
	}
	public void setProductName(String productName) {
		this.productName = productName;
	}
	public String getProductPrice() {
		return productPrice;
	}
	public void setProductPrice(String productPrice) {
		this.productPrice = productPrice;
	}
	public String getTradeNum() {
		return tradeNum;
	}
	public void setTradeNum(String tradeNum) {
		this.tradeNum = tradeNum;
	}
	public String getProductUrl() {
		return productUrl;
	}
	public void setProductUrl(String productUrl) {
		this.productUrl = productUrl;
	}
	public String getShopName() {
		return shopName;
	}
	public void setShopName(String shopName) {
		this.shopName = shopName;
	}
	public String getEcName() {
		return ecName;
	}
	public void setEcName(String ecName) {
		this.ecName = ecName;
	}
	public Date getDate() {
		return date;
	}
	public void setDate(Date date) {
		this.date = date;
	}
	
}


package com.iteye.injavawetrust.jdvsdd;

/**
 * 
 * @author InJavaWeTrust
 *
 */
public class Constants {
	
	/**
	 * JDURL
	 */
	public static String JDURL = "http://search.jd.com/Search?keyword=";
	/**
	 * JD汉字编码格式
	 */
	public static String JDENC = "&enc=utf-8";
	/**
	 * JD分页
	 */
	public static String JDPAGE ="&page=";
	/**
	 * 当当URL
	 */
	public static String DDURL = "http://search.dangdang.com/?key=";
	/**
	 * 当当ACT
	 */
	public static String ACT = "&act=input";
	/**
	 *  当当分页
	 */
	public static String DDPAGE = "&page_index=";

}


package com.iteye.injavawetrust.jdvsdd;

import java.io.UnsupportedEncodingException;
import java.text.SimpleDateFormat;
import java.util.List;
import java.util.TimeZone;

/**
 * 
 * @author InJavaWeTrust
 *
 */
public class PriceCheckUtil {
	
	private PriceCheckUtil() {
		
	}
	
	private static final PriceCheckUtil instance = new PriceCheckUtil();
	
	public static PriceCheckUtil getInstance() {
		return instance;
	}
	
	/**
	 * 商品汉字转码
	 * @param productName 商品名称
	 * @return
	 */
	public String getGbk(String productName){
		String retGbk = "";
		try {
			retGbk = new String(productName.getBytes("UTF-8"), "GBK");
		} catch (UnsupportedEncodingException e) {
			e.printStackTrace();
		}
		return retGbk;
	}
	
	/**
     * 从列表list中找到与productName相似度最高的ProductInfo
     *
     * @param productName
     * @param list
     * @return 相似度最高的productName
     */
	public ProductInfo getSimilarity(String productName, List<ProductInfo> list) {
		ProductInfo productInfo = null;
		/**
		 * 找到list中所有的productName与字符串productName的相似度,保存在lens数组中
		 */
		double lens[] = new double[list.size()];
		for (int i = 0; i < list.size() - 1; i++) {
			lens[i] = sim(productName, list.get(i).getProductName());
		}
		/**
		 * 遍历出最大的相似度maxLen
		 */
		double maxLen = 0.0;
		for (int i = 0; i < lens.length; i++) {
			if (maxLen < lens[i]) {
				maxLen = lens[i];
			}
		}
		/**
		 * 遍历出最大的相似度的索引maxLenIndex
		 */
		int maxLenIndex = 0;
		for (int i = 0; i < lens.length; i++) {
			if (maxLen == lens[i]) {
				maxLenIndex = i;
			}
		}
		productInfo = list.get(maxLenIndex);
		return productInfo;
	}
	
	/**
     * 求三个数中最小的一个
     * @param one
     * @param two
     * @param three
     * @return
     */
    public int min(int one, int two, int three) {
        int min = one;
        if(two < min) {
            min = two;
        }
        if(three < min) {
            min = three;
        }
        return min;
    }

    /**
     * 计算矢量距离
     * Levenshtein Distance(LD)
     * @param str1
     * @param str2
     * @return
     */
    public int ld(String str1, String str2) {
        int d[][];    //矩阵
        int n = str1.length();
        int m = str2.length();
        int i;    //遍历str1的
        int j;    //遍历str2的
        char ch1;    //str1的
        char ch2;    //str2的
        int temp;    //记录相同字符,在某个矩阵位置值的增量,不是0就是1
        if(n == 0) {
            return m;
        }
        if(m == 0) {
            return n;
        }
        d = new int[n+1][m+1];
        for(i=0; i<=n; i++) {    //初始化第一列
            d[i][0] = i;
        }
        for(j=0; j<=m; j++) {    //初始化第一行
            d[0][j] = j;
        }
        for(i=1; i<=n; i++) {    //遍历str1
            ch1 = str1.charAt(i-1);
            //去匹配str2
            for(j=1; j<=m; j++) {
                ch2 = str2.charAt(j-1);
                if(ch1 == ch2) {
                    temp = 0;
                } else {
                    temp = 1;
                }
                //左边+1,上边+1, 左上角+temp取最小
                d[i][j] = min(d[i-1][j]+1, d[i][j-1]+1, d[i-1][j-1]+temp);
            }
        }
        return d[n][m];
    }

    /**
     * 计算相似度
     * @param str1
     * @param str2
     * @return
     */
	public double sim(String str1, String str2) {
		int ld = ld(str1, str2);
		return 1 - (double) ld / Math.max(str1.length(), str2.length());
	}
	
	/** 
     * 毫秒转换成hhmmss 
     * @param ms 毫秒 
     * @return hh:mm:ss 
     */  
    public String msToss(long ms) {
        SimpleDateFormat formatter = new SimpleDateFormat("HH:mm:ss");  
        formatter.setTimeZone(TimeZone.getTimeZone("GMT+00:00"));  
        String ss = formatter.format(ms);  
        return ss;  
    }  

}


package com.iteye.injavawetrust.jdvsdd;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Scanner;

/**
 * 
 * @author InJavaWeTrust
 *
 */
public class PriceCheckMain {
	
	private static PriceCheckUtil pcu = PriceCheckUtil.getInstance();
	
	public List<Map<String, ProductInfo>> getProductList(String productName) {
		
		String jdUrl = Constants.JDURL + pcu.getGbk(productName) + Constants.JDENC;
		
		String ddUrl = Constants.DDURL + productName;
		
		return getProductFromUrls(jdUrl, ddUrl, productName);
	}
	
	public List<Map<String, ProductInfo>> getProductFromUrls(String jdUrl, String ddUrl, String productName) {
		List<Map<String, ProductInfo>> retListMap = new ArrayList<Map<String,ProductInfo>>();
		List<ProductInfo> jdProductList = new JDProductList(jdUrl, productName).getProductList();
		List<ProductInfo> ddProductList = new DDProductList(ddUrl, productName).getProductList();
		for(int i = 0; i < jdProductList.size(); i++){
			String jdProductName = jdProductList.get(i).getProductName();
			Map<String, ProductInfo> map = new HashMap<String, ProductInfo>();
			map.put("JD", jdProductList.get(i));
			ProductInfo ddProduct = pcu.getSimilarity(jdProductName, ddProductList);
			map.put("DD", ddProduct);
			retListMap.add(map);
		}
		
		return retListMap;
	}
	
	
	public static void main(String[] args) {
		System.out.println("输入商品名称:");
		Scanner scanner = new Scanner(System.in);
		String productName = scanner.next();
		scanner.close();
		System.out.println("京东和当当[" + productName + "]商品比价开始。。。。。。");
		try{
			long starTime = System.currentTimeMillis();
			List<Map<String, ProductInfo>> list = new PriceCheckMain().getProductList(productName);
			for(Map<String, ProductInfo> map : list) {
				
				String jdName = map.get("JD").getProductName();
				String jdPrice = map.get("JD").getProductPrice();
				String ddName = map.get("DD").getProductName();
				String ddPrice = map.get("DD").getProductPrice();
				
				System.out.println("[" + jdName + "]  [" + ddName + "]");
				System.out.println("[" + jdPrice + "]  [" + ddPrice + "]");
				System.out.println("-----------------------------------------------------------");
			}
			long endTime = System.currentTimeMillis();
			System.out.println("用时 [" + pcu.msToss(endTime - starTime) + "]");
		}catch(Exception e){
			System.out.println("error");
			System.out.println(e.getMessage());
		}
	}

}

?

运行结果:

?

?输入商品名称:
铅笔
京东和当当[铅笔]商品比价开始。。。。。。
JD Product 第[1]页
http://search.jd.com/Search?keyword=閾呯瑪&enc=utf-8
JD Product 第[2]页
http://search.jd.com/Search?keyword=閾呯瑪&enc=utf-8&page=2

?

..............................

?

DD Product 第[1]页
http://search.dangdang.com/?key=铅笔
DD Product 第[2]页
http://search.dangdang.com/?key=铅笔&act=input&page_index=2
DD Product 第[3]页

?

...................................

?

[得力(deli) S908 木世界系列六角笔杆原木HB铅笔/素描绘图学生铅笔 50支/桶]? [ 【开学必备文具】正品 得力文具S907/s908原木HB/2B铅笔绘图素描儿童学生铅笔50支装 ]
[18.00]? [16.80]
-----------------------------------------------------------
[得力(deli) 7084 安全考试专用填涂答题卡2B木质铅笔/学生铅笔 12支/盒]? [ 得力文具(deli) 0641 削笔器 削笔刀 卷笔刀 削笔机学习用品手摇转笔刀笔刨文具 ]
[10.00]? [13.50]
-----------------------------------------------------------
[得力(deli)7083 安全石墨铅芯素描 绘图HB铅笔/学生铅笔 12支/盒]? [ 六一儿童节礼物!三菱 绘图HB/2B铅笔12支装 ]
[9.00]? [35.00]
-----------------------------------------------------------
[辉柏嘉(Faber-castell)114468 水溶性彩色铅笔 水溶彩铅 48色套装(赠毛笔+笔刨)]? [ 德国Faber-castell辉柏嘉三角杆学生铅笔 儿童铅笔 HB 2H 2B 12支 ]
[109.00]? [12.00]

?

................................................

?

[中华6725桶装彩色铅笔36色24色18色12色原木三角杆彩色铅笔 美术绘画涂鸦涂色彩铅 12色]? [ 晨光彩色铅笔36色 24色绘图涂鸦 桶装绘画彩铅彩笔 儿童绘画 ]
[8.80]? [12.00]
-----------------------------------------------------------
用时 [00:00:10]

?

?

?

?

?

发表评论
用户名: 匿名