hadoop Yarn作业提交流程代码解析.docxVIP

1.上篇 根据wordcount进行分析： import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.Reducer; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import java.io.IOException; /** * @author: LUGH1 * @date: 2019-4-8 * @description: */ public class WordCount { public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException { Configuration conf = new Configuration(); conf.set(fs.defaultFS,hdfs://30:9000); Job job = Job.getInstance(conf); job.setJarByClass(WordCount.class); job.setMapperClass(WdMapper.class); job.setReducerClass(WdReducer.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(IntWritable.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); FileInputFormat.setInputPaths(job, new Path(/test/word.txt)); FileOutputFormat.setOutputPath(job, new Path(/test/output)); boolean result = job.waitForCompletion(true); System.exit(result?0:1); System.out.println(good job); } } class WdMapper extends MapperObject, Text, Text, IntWritable { @Override protected void map(Object key, Text value, Context context) throws IOException, InterruptedException { String line = value.toString(); String[] split = line.split( ); for(String word : split){ context.write(new Text(word), new IntWritable(1)); } } } class WdReducer extends ReducerText, IntWritable, Text, IntWritable { @Override protected void reduce(Text key, IterableIntWritable values, Context context) throws IOException, InterruptedException { int count