spark on k8s

spark on k8s

quick start

• 打包spark镜像

./bin/docker-image-tool.sh -t 3.2.1.quickstart build

• 安装minikube
• 创建运行环境

# 命名空间创建
kubectl create namespace spark
 
# 创建service账户以及绑定角色
kubectl create serviceaccount spark-serviceaccount --namespace spark
kubectl create clusterrolebinding spark-rolebinding --clusterrole=edit --serviceaccount=spark:spark-serviceaccount --namespace=spark

• 获得master地址

kubectl cluster-info

• 提交任务

# 任务提交

./bin/spark-submit \
    --master k8s://https://127.0.0.1:8443 \
    --deploy-mode cluster \
    --name spark-pi \
    --class org.apache.spark.examples.SparkPi \
    --conf spark.executor.instances=10 \
    --conf spark.kubernetes.namespace=spark \
    --conf spark.kubernetes.authenticate.driver.serviceAccountName=spark-serviceaccount \
    --conf spark.kubernetes.container.image.pullPolicy=Always \
    --conf spark.kubernetes.container.image=spark:3.2.1.quickstart \
    local:///${SPARK_HOME}/examples/jars/spark-examples_2.11-2.4.7.jar