南京大学计算机科学与技术系主讲人：黄宜华，杨晓亮 2011年春季学期 MapReduce海量数据并行处理教学内容.pptVIP

 //we need to customize a new Writable format to represent a //term’s Info public class SingleRecordWritable implements WritableComparableSingleRecordWritable { private LongWritable DID=new LongWritable(); //DID private FloatWritable rank=new FloatWritable(); //Rank private Text positions=new Text(); //Positions ….. } Build Inverted Index --Pseudo Code(1)  public class Mapper extends MapperLongWritable, Text, Text, SingleRecordWritable{ public HashMapTerm,Rank_Positions TokenMap; //TMP Info public void map(LongWritable offset,Text val,Context context) { String[] str=val.toString().split(\007, 5); //Split post to get title and content HandleToken(str[3],true); HandleToken(str[4],false); //store line’s terms in TMP Info this.EmitMapValue(key,context); //After scaned the whole text,emit key and value } //Handle input line:segement into words,handle the words one by one. public HandleToken (Iine,IsTitle){ while(line.hasnext()){ if(TokenMap.contians(line.GetTerm()) TokenMap.Update(line.GetTerm());//if TMP don’t contains the word put in TMP else TokenMap.Put(line.GetTerm()); //Update the term’s info in TMP.} } public void EmitMapValue(LongWritable key,Context context) { for(Map.EntryString,Rank_Positions entry:TokenMap.entrySet()) context.write(word, singleRecord); //emit the key/value pair one by one } Build Inverted Index --Pseudo Code(2)  public class LineIndexerReducer extends ReducerText,SingleRecordWritable,Text,Text { //reduce’s input key is a term,input value is a List of Infos that //contains the same term,each info represent a Doc. public void reduce(Text key, IterableSingleRecordWritable values,Context context){ StringBuilder Info; for(SingleRecordWritable val:values) Info.append(ResetFinalRank(val,values.len)+;); out.set(info.toString()); context.write(key,out); } //The Num of doucment that a t