Hadoop倒排索引例子.docVIP

package cn.yws; import java.io.IOException; import java.util.StringTokenizer; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.Reducer; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.input.FileSplit; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.util.GenericOptionsParser; //倒排索引 请在hadoop index_in目录下放置file1,file2测试文件 public class MyInvertedIndex { public static class Map extends MapperObject, Text, Text, Text { private Text keyinfo=new Text(); private Text valueinfo=new Text(); private FileSplit split; //映射 @Override protected void map(Object key, Text value, Context context) throws IOException, InterruptedException { //super.map(key, value, context); //获取文件分词 split=(FileSplit) context.getInputSplit(); StringTokenizer tokenizer=new StringTokenizer(value.toString()); while(tokenizer.hasMoreTokens()) { int splitindex=split.getPath().toString().indexOf(file); keyinfo.set(tokenizer.nextToken()+:+split.getPath().toString().substring(splitindex)); valueinfo.set(1); //file3:1; context.write(keyinfo, valueinfo); } } } public static class Combine extends ReducerText, Text, Text, Text { private Text infoText=new Text(); @Override protected void reduce(Text key, IterableText values, Context context) throws IOException, InterruptedException { //super.reduce(key, values, context); int sum=0; for(Text value:values) { sum+=Integer.parseInt(value.toString()); } int splitindex=key.toString().indexOf(:); //file2:1;file3:2;file1:1 infoText.set(key.toString().substring(splitindex+1)+:+sum); key.set(key.toString().substring(0,splitindex)); context.write(key, infoText)