import org.apache.spark.{SparkConf, SparkContext}
/**
* RDD再分区
*/
object RDDRepartition {
def main(args: Array[String]): Unit = {
val conf = new SparkConf()
conf.setAppName("rdddemo")
conf.setMaster("local[3]")
val sc = new SparkContext(conf)
val rdd1 = sc.makeRDD((1 to 10) , 5)
println("rdd1'pars : " + rdd1.partitions.length)
val rdd2 = rdd1.map(e=>{
val tid = Thread.currentThread().getId
val tname = Thread.currentThread().getName
printf("%d/%s : %d\r\n" , tid , tname , e)
e
})
println("rdd2'pars : " + rdd2.partitions.length)
rdd2.collect()
while(true){
Thread.sleep(1000)
}
}
}
//执行结果
46/Executor task launch worker for task 0 : 1
46/Executor task launch worker for task 0 : 2
46/Executor task launch worker for task 3 : 7
46/Executor task launch worker for task 3 : 8
47/Executor task launch worker for task 1 : 3
47/Executor task launch worker for task 1 : 4
47/Executor task launch worker for task 4 : 9
47/Executor task launch worker for task 4 : 10
48/Executor task launch worker for task 2 : 5
48/Executor task launch worker for task 2 : 6
版权声明：本文内容由互联网用户自发贡献，该文观点与技术仅代表作者本人。本站仅提供信息存储空间服务，不拥有所有权，不承担相关法律责任。如发现本站有涉嫌侵权/违法违规的内容， 请发送邮件至 [email protected] 举报，一经查实，本站将立刻删除。
