基于开源工具构建 EMR 数据分析平台（四）使用 Kyuubi 进行 Spark SQL 任务提交

# 指定s3路径
s3_path=s3://xxxx/tmp/
 
# 例如以下载1.9.2版本为例
wget https://dlcdn.apache.org/kyuubi/kyuubi-1.9.2/apache-kyuubi-1.9.2-bin.tgz
 
# 使用aws cli将tar包上传到指定的s3路径
aws s3 cp apache-kyuubi-1.9.2-bin.tgz  $s3_path

#!/bin/bash
# 保存脚本为kyuubi_install.sh，将脚本上传到S3
kyuubi_s3_path=$1
# Wait for EMR provisioning to become  successful.
while [[ $(sed '/localInstance {/{:1;  /}/!{N; b1}; /nodeProvision/p}; d'  /emr/instance-controller/lib/info/job-flow-state.txt | sed '/nodeProvisionCheckinRecord  {/{:1; /}/!{N; b1}; /status/p}; d' | awk '/SUCCESSFUL/' | xargs) !=  "status: SUCCESSFUL" ]];
do
   sleep 3
done
# Now the EMR cluster is ready. Do your  work here.
 
IS_MASTER=$(cat  /mnt/var/lib/info/instance.json | jq -r ".isMaster" | grep "true"  || true);
 
if [ -z $IS_MASTER ]; then
     exit 0
fi;
 
# download
cd /mnt/
mkdir kyuubi && cd kyuubi
aws s3 cp $kyuubi_s3_path .
tar -xvzf apache-kyuubi-1.9.2-bin.tgz
ln -s apache-kyuubi-1.9.2-bin kyuubi
 
# config
cd /mnt/kyuubi/kyuubi/conf
mv kyuubi-env.sh.template kyuubi-env.sh
sed -i -e '$a\export  SPARK_HOME=/usr/lib/spark'   kyuubi-env.sh
sed -i -e '$a\export  HADOOP_CONF_DIR=/etc/hadoop/conf'   kyuubi-env.sh
# env中kyuubi中的jvm内存可以调大
mv  /mnt/kyuubi/kyuubi/conf/kyuubi-defaults.conf.template /mnt/kyuubi/kyuubi/conf/kyuubi-defaults.conf
export zk=hostname
 
sed -i -e '$a\kyuubi.ha.addresses  '$zk':2181'  kyuubi-defaults.conf
sed -i -e  '$a\kyuubi.session.engine.idle.timeout PT10M'   kyuubi-defaults.conf
#   USER, CONNECTION, GROUP, SERVER
sed -i -e '$a\kyuubi.engine.share.level CONNECTION'  kyuubi-defaults.conf
 
# kyuubi-defaults.conf 添加
sed -i -e  '$a\spark.dynamicAllocation.initialExecutors=1'  kyuubi-defaults.conf
sed -i -e  '$a\spark.dynamicAllocation.minExecutors=1'   kyuubi-defaults.conf
sed -i -e '$a\spark.sql.legacy.allowNonEmptyLocationInCTAS=true'  kyuubi-defaults.conf
sed -i -e  '$a\spark.serializer=org.apache.spark.serializer.KryoSerializer'  kyuubi-defaults.conf
sed -i -e  '$a\spark.sql.hive.convertMetastoreParquet=false'  kyuubi-defaults.conf
sed -i -e  '$a\spark.sql.storeAssignmentPolicy=LEGACY'   kyuubi-defaults.conf
sed -i -e  '$a\spark.sql.legacy.parquet.datetimeRebaseModeInRead=CORRECTED'  kyuubi-defaults.conf
sed -i -e  '$a\spark.sql.legacy.parquet.datetimeRebaseModeInWrite=CORRECTED'  kyuubi-defaults.conf
sed -i -e  '$a\spark.sql.session.timeZone=GMT+8'   kyuubi-defaults.conf
sed -i -e  '$a\spark.sql.filesourceTableRelationCacheSize=0'  kyuubi-defaults.conf
sed -i -e  '$a\spark.sql.sources.default=parquet'   kyuubi-defaults.conf
sed -i -e '$a\spark.sql.parquet.datetimeRebaseModeInRead=CORRECTED'  kyuubi-defaults.conf
sed -i -e  '$a\spark.hadoop.hive.exec.dynamic.partition.mode=nonstrict'  kyuubi-defaults.conf
sed -i -e  '$a\spark.hadoop.hive.exec.dynamic.partition=true'  kyuubi-defaults.conf
sed -i -e '$a\spark.sql.sources.partitionOverwriteMode=DYNAMIC'  kyuubi-defaults.conf
sed -i -e  '$a\spark.driver.maxResultSize=10G'   kyuubi-defaults.conf
sed -i -e  '$a\spark.sql.parquet.int96RebaseModeInWrite=CORRECTED'  kyuubi-defaults.conf
sed -i -e  '$a\spark.sql.parquet.int96RebaseModeInRead=CORRECTED'  kyuubi-defaults.conf
sed -i -e '$a\spark.master=yarn'  kyuubi-defaults.conf
sed -i -e  '$a\spark.submit.deployMode=cluster'   kyuubi-defaults.conf
sed -i -e  '$a\kyuubi.session.engine.startup.waitCompletion=false'  kyuubi-defaults.conf
# spark engine初始化的超时时间，默认180000ms，比如集群没有资源，spark submit提交等待资源，如果超时这个超时时间，application会被kill, client链接超时失败，可以调大
sed -i -e  '$a\kyuubi.session.engine.initialize.timeout=3600000'  kyuubi-defaults.conf
 
 
# 这是当前executor的内存设定，可以根据需求调整
sed -i -e  '$a\spark.executor.memory=4096M'   kyuubi-defaults.conf
sed -i -e  '$a\spark.emr.default.executor.memory=4096M'   kyuubi-defaults.conf
sed -i -e  '$a\spark.executor.cores=2'   kyuubi-defaults.conf
sed -i -e  '$a\spark.emr.default.executor.cores=2'   kyuubi-defaults.conf
 
# kyuubi-env KYUUBI_JAVA_OPTS -Xmx10g, 下面命令去掉kyuubi默认配置JVM 10GB配置的注释
sed -i '/^#.*export  KYUUBI_JAVA_OPTS/s/^#//g' kyuubi-env.sh
 
# spark driver memory，也可以在kyuubi-defaults.conf设置，可以针对每个不通用户设置，改值是可以被kyuubi-defaults.conf中设置的driver内存覆盖
sudo sed -i -e  '$a\spark.driver.memory  1024M'  /etc/spark/conf/spark-defaults.conf
 
# start
sudo chown -R  hadoop:hadoop /mnt/kyuubi
sudo -u hadoop  /mnt/kyuubi/kyuubi/bin/kyuubi start
exit 0 

#!/bin/bash
# 保存脚本为kyuubi_bootstrap.sh  将脚本上传到S3
kyuubi_script_s3_path=$1
kyuubi_s3_path=$2
aws s3 cp $kyuubi_script_s3_path .
chmod +x kyuubi_install.sh
nohup ./kyuubi_install.sh  $kyuubi_s3_path &>/dev/null &