spark基本框架(本地读取)

scala样板(度本地文件执行文件)

import org.apache.spark.{SparkConf, SparkContext}
object wordCount {
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setMaster("local").setAppName("My App")
    val sc = new SparkContext(conf)
    # 文件输入
    val input = sc.textFile("D:\\Java_code\\bigDataLogTest_new\\pom.xml")
	# 数据处理过程
	.....
	# 结束sparkContext
    sc.stop()
  }
}

java样板(读本地文件执行任务)

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.FlatMapFunction;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.api.java.function.VoidFunction;
import scala.Tuple2;
import java.util.*;

public class wordCount {
    public static void main(String[] args) {
        SparkConf conf = new SparkConf().setMaster("local").setAppName("wordCount");
        JavaSparkContext jsc = new JavaSparkContext(conf);
        ## 文件读取
        JavaRDD<String> inputRDD = jsc.textFile("D:\\Java_code\\bigDataLogTest_new\\pom.xml");
        ## 数据处理过程
        .......
        ## javaSparkContext 结束
        jsc.stop();
    }
}