Hadoop自定义序列化类.pptx

Hadoop自定义序列化类1. 理解Hadoop序列化2. 了解序列化特点3. 掌握自定义序列化类1. Hadoop序列化2. 序列化特点3. 自定义序列化类Hadoop序列化1、序列化（Serialization）是指把结构化对象转化为字节流。2、反序列化（Deserialization）是序列化的逆过程。即把字节流转回结构化对象。3、Java序列化（java.io.Serializable）Hadoop序列化作用：1、序列化在分布式环境的两大作用：进程间通信，永久存储。2、Hadoop节点间通信。Hadoop的序列化格式：Writable序列化格式特点1、紧凑：高效使用存储空间。2、快速：读写数据的额外开销小3、可扩展：可透明地读取老格式的数据4、互操作：支持多语言的交互自定义序列化类public class TrafficApp { public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException { Job job = Job.getInstance(new Configuration(),TrafficApp.class.getSimpleName()); job.setJarByClass(TrafficApp.class); FileInputFormat.setInputPaths(job, args[0]); job.setMapperClass(MyMapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(TrafficWritable.class); job.setReducerClass(MyReduce.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(TrafficWritable.class); FileOutputFormat.setOutputPath(job, new Path(args[1])); job.waitForCompletion(true); }自定义序列化类public static class MyMapper extends MapperLongWritable, Text, Text, TrafficWritable{ Text k2 = new Text(); TrafficWritable v2 = new TrafficWritable(); @Override protected void map(LongWritable key, Text value, MapperLongWritable, Text, Text, TrafficWritable.Context context) throws IOException, InterruptedException {String line = value.toString();String[] splited = line.split(\t);k2.set(splited[1]);v2.set(splited[6], splited[7], splited[8], splited[9]);context.write(k2, v2); } }自定义序列化类public static class MyReduce extends ReducerText, TrafficWritable, Text, TrafficWritable{ TrafficWritable v3 = new TrafficWritable(); @Override protected void reduce(Text k2, IterableTrafficWritable v2s, ReducerText, TrafficWritable, Text, TrafficWritable.Context context) throws IOException, InterruptedException { long t1 = 0L;自定义序列化类long t2 = 0L;long t3 = 0L;long t4 = 0L;for(TrafficWritable v2 : v2s){t1 += v2.t1;t2 += v2.t2;t3 += v2.t3;t4 += v2.t4; }v3.set(t1, t2, t3, t4);context.wr