nutch源码分析1------inject.docVIP

nutch源码分析1------inject 分类：?nutch2011-08-04 15:34?170人阅读?评论(0)?收藏?举报 最近在学习nutch，了解了nutch的一个大概的工作原理之后，就把源码下载下来，了解一些比较细的细节，今天先来看inject过程。 我用的使1.3版本的nutch 当在命令行中输入 bin/nutch crawl urls -dir crawl -depth 3 -topN 5 时，首先会进入Crawl.java的main方法，代码如下： public static void main(String args[]) throws Exception {??? Configuration conf = NutchConfiguration.create();??? int res = ToolRunner.run(conf, new Crawl(), args);??? System.exit(res);? } 很容易看到当调用 ToolRunner.run方法时会调用Crawl的run方法： ?public int run(String[] args) throws Exception {??? if (args.length 1) {????? System.out.println????? (Usage: Crawl urlDir -solr solrURL [-dir d] [-threads n] [-depth i] [-topN N]);????? return -1;??? }??? Path rootUrlDir = null;??? Path dir = new Path(crawl- + getDate());??? int threads = getConf().getInt(fetcher.threads.fetch, 10);??? int depth = 5;??? long topN = Long.MAX_VALUE;??? String solrUrl = null;?? ???? for (int i = 0; i args.length; i++) {????? if (-dir.equals(args[i])) {??????? dir = new Path(args[i+1]);??????? i++;????? } else if (-threads.equals(args[i])) {??????? threads = Integer.parseInt(args[i+1]);??????? i++;????? } else if (-depth.equals(args[i])) {??????? depth = Integer.parseInt(args[i+1]);??????? i++;????? } else if (-topN.equals(args[i])) {????????? topN = Integer.parseInt(args[i+1]);????????? i++;????? } else if (-solr.equals(args[i])) {??????? solrUrl = StringUtils.lowerCase(args[i + 1]);??????? i++;????? } else if (args[i] != null) {??????? rootUrlDir = new Path(args[i]);????? }??? }?? ???? JobConf job = new NutchJob(getConf());??? if (solrUrl == null) {????? LOG.warn(solrUrl is not set, indexing will be skipped...);??? }??? FileSystem fs = FileSystem.get(job);??? if (LOG.isInfoEnabled()) {????? LOG.info(crawl started in: + dir);????? LOG.info(rootUrlDir = + rootUrlDir);????? LOG.info(threads = + threads);????? LOG.info(depth = + depth);???? ?????? LOG.info(solrUrl= + solrUrl);????? if (topN != Long.MAX_VALUE)???????