spider简单的爬虫程序!!!经典.docVIP

spider简单的爬虫程序 2008-10-10 16:29 spider简单的爬虫程序 1、基础准备htmlparser首页：/projects/htmlparser/下载：/project/showfiles.php?group_id=24399文件：htmlparser1_6zipdependencygroupIdorg.htmlparser/groupIdartifactIdhtmlparser/artifactIdversion1.6/version/dependency cpdetector首页：/下载：/project/showfiles.php?group_id=114421文件：cpdetector_eclipse_project_1.0.7.zip dependencygroupIdcpdetector/groupIdartifactIdcpdetector/artifactIdversion1.0.5/version/dependency spindle首页：/projects/spindle/?（但是已经无法访问） 2 修改spindle代码得到的spider简单的将URL打印出来了，解析的内容等等都没有处理 解析HTML的基类HtmlParserUtil.javapackage mons.utils.html; import java.io.BufferedReader;import java.io.FileNotFoundException;import java.io.IOException;import java.io.InputStream;import java.io.InputStreamReader;import java.io.UnsupportedEncodingException;import .MalformedURLException;import .SocketException;import .SocketTimeoutException;import .URL;import .UnknownHostException;import java.nio.charset.Charset; import org.htmlparser.Parser;import org.htmlparser.util.NodeList;import org.htmlparser.util.ParserException;import org.htmlparser.visitors.HtmlPage; import?cpdetector.io.ASCIIDetector;import?cpdetector.io.CodepageDetectorProxy;import?cpdetector.io.JChardetFacade;import?cpdetector.io.ParsingDetector;import?cpdetector.io.UnicodeDetector; public class HtmlParserUtil { /* StringBuffer的缓冲区大小 */public static int TRANSFER_SIZE = 4096; /* 当前平台的行分隔符 */public static String lineSep = System.getProperty(line.separator); /* 自动探测页面编码，避免中文乱码的出现 */public static String autoDetectCharset(URL url) { ?? CodepageDetectorProxy detector = CodepageDetectorProxy.getInstance();?? /**?? * ParsingDetector可用于检查HTML、XML等文件或字符流的编码 构造方法中的参数用于指示是否显示探测过程的详细信息?? * 为false则不显示?? */?? detector.add(new ParsingDetector(false));?? detector.add(JChardetFacade.getInstance());?? detector.add(ASCIIDetector.getInstance());?? detector.add(UnicodeDetector.getInstance()); ?? Charset charset = null;?? try {??? charset = detector.detectCode