《Java网络爬虫简单实现》.docVIP

首先介绍每个类的功能：DownloadPage.java的功能是下载此超链接的页面源代码.FunctionUtils.java 的功能是提供不同的静态方法，包括：页面链接正则表达式匹配,获取URL链接的元素,判断是否创建文件,获取页面的Url并将其转换为规范的Url,截取网页网页源文件的目标内容。HrefOfPage.java 的功能是获取页面源代码的超链接。UrlDataHanding.java 的功能是整合各个给类，实现url到获取数据到数据处理类。UrlQueue.java 的未访问Url队列。VisitedUrlQueue.java 已访问过的URL队列。下面介绍一下每个类的源代码：DownloadPage.java 此类要用到HttpClient组件。? package com.sreach.spider; import java.io.IOException; import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; import org.apache.http.client.ClientProtocolException; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.DefaultHttpClient; import org.apache.http.util.EntityUtils; public class DownloadPage { ???? /** ??????* 根据URL抓取网页内容 ??????*? ??????* @param url ??????* @return ??????*/ ???? public static String getContentFormUrl(String url) ???? { ???????? /* 实例化一个HttpClient客户端 */ ???????? HttpClient client = new DefaultHttpClient(); ???????? HttpGet getHttp = new HttpGet(url); ???????? String content = null; ???????? HttpResponse response; ???????? try ???????? { ???????????? /*获得信息载体*/ ???????????? response = client.execute(getHttp); ???????????? HttpEntity entity = response.getEntity(); ???????????? VisitedUrlQueue.addElem(url); ???????????? if (entity != null) ???????????? { ???????????????? /* 转化为文本信息 */ ???????????????? content = EntityUtils.toString(entity); ???????????????? /* 判断是否符合下载网页源代码到本地的条件 */ ???????????????? if (FunctionUtils.isCreateFile(url) ???????????????????????? FunctionUtils.isHasGoalContent(content) != -1) ???????????????? { ???????????????????? FunctionUtils.createFile(FunctionUtils ???????????????????????????? .getGoalContent(content), url); ???????????????? } ???????????? } ???????? } catch (ClientProtocolException e) ???????? { ???????????? e.printStackTrace(); ???????? } catch (IOException e) ???????? { ???????????? e.printStackTrace(); ???????? } finally ???????? { ???????????? client.getConnectionManager().sh