Spark Python API函数学习.docx

Spark Python API函数学习pyspark version1print(pyspark version:?+?str(sc.version))23?4pyspark version:1.2.2map01# map02# sc = spark context, parallelize creates an RDD from the passed object03x?=?sc.parallelize([1,2,3])04y?=?x.map(lambda?x: (x,x**2))05?06# collect copies RDD elements to a list on the driver07print(x.collect())08print(y.collect())09?10[1,?2,?3]11[(1,?1), (2,?4), (3,?9)]flatMap1# flatMap2x?=?sc.parallelize([1,2,3])3y?=?x.flatMap(lambda?x: (x,?100*x, x**2))4print(x.collect())5print(y.collect())6?7[1,?2,?3]8[1,?100,?1,?2,?200,?4,?3,?300,?9]mapPartitions01# mapPartitions02x?=?sc.parallelize([1,2,3],?2)03def?f(iterator):?yield?sum(iterator)04y?=?x.mapPartitions(f)05# glom() flattens elements on the same partition06print(x.glom().collect())?07print(y.glom().collect())08?09[[1], [2,?3]]10[[1], [5]]mapPartitionsWithIndex01# mapPartitionsWithIndex02x?=?sc.parallelize([1,2,3],?2)03def?f(partitionIndex, iterator):?yield?(partitionIndex,sum(iterator))04y?=?x.mapPartitionsWithIndex(f)05?06# glom() flattens elements on the same partition07print(x.glom().collect())?08print(y.glom().collect())09?10[[1], [2,?3]]11[[(0,?1)], [(1,?5)]]getNumPartitions1# getNumPartitions2x?=?sc.parallelize([1,2,3],?2)3y?=?x.getNumPartitions()4print(x.glom().collect())5print(y)6?7[[1], [2,?3]]82filter1# filter2x?=?sc.parallelize([1,2,3])3y?=?x.filter(lambda?x: x%2?==?1)??# filters out even elements4print(x.collect())5print(y.collect())6?7[1,?2,?3]8[1,?3]distinct1# distinct2x?=?sc.parallelize([A,A,B])3y?=?x.distinct()4print(x.collect())5print(y.collect())6?7[A,?A,?B]8[A,?B]sample01# sample02x?=?sc.parallelize(range(7))03# call sample 5 times04ylist?=?[x.sample(withReplacement=False, fraction=0.5)?for?i?inrange(5)]05print(x = ?+?str(x.collect()))06for?cnt,y?in?zip(range(len(ylist)), ylist):07????print(sample:?+?str(cnt)?+? y = ?+??str(y.collect()))08?09x?=?[0,?1,?2,?3,?4,?5,?6]10sample:0?y?=?[0,?2,?5,?6]11sample:1?y?=?[2,?6]12sample:2?y?=?[0,?4,?5,?6]13sample:3?y?=?[0,?2,?6]14sample:4?y?=?[0,?3