当前位置：首页 > news >正文

hive自动安装脚本

news 2025/7/17 4:58:09

使用该脚本注意事项

安装hive之前确定机子有网络。或者yum 更改为本地源，因为会使用epel仓库下载一个pv的软件
使用该脚本前提是自行安装好mysql数据库
准备好tomcat软件包，该脚本使用tomcat9.x版本测试过能正常执行安装成功，其他版本没有测试过不知道有没这个问题
hive安装脚本中自动更换成tez引擎了，需要准备好tez安装软件，测试时使用的是tez-0.9.2版本
安装之前请自行更改脚本内容前面定义的software_dir(所有安装软件包存放目录)、install_dir(安装软件目录)、mysql_user、mysql_password 这个几个变量根据自已情况进行更改
使用该脚本之前需要把hadoop先安装好并且把hadoop集群启动起来，hadoop集群未启动切勿执行该脚本

#!/bin/bashsoftware_dir=/root/hadoop/
install_dir=/opt/
hadoop_path=$(which hadoop|sed 's|/bin.*||')
tomcat_path=${install_dir}tomcat
mysql_user=root
mysql_password=1234yum repolist |grep epel
if [ $? -ne 0 ] 
thenwget -O /etc/yum.repos.d/epel.repo https://mirrors.aliyun.com/repo/epel-7.repo &>/dev/null
firpm -qa |grep pv
if [ $? -ne 0 ]
thenyum install -y pv &>/dev/nullrpm -qa |grep pv[ $? -ne 0 ] && echo " Please resolve the network problem !!!" && exit 1
fiecho ============ Start Hive Configuration =============
hive_path=${install_dir}hive
hive_conf_path=$hive_path/conf
tez_path=${install_dir}tezrm -rf $tez_path
pv ${software_dir}*tez* |tar zxf - -C $install_dir
mv ${install_dir}*tez* $tez_path
chown -R root:root $tez_pathrm -rf $hive_path
pv ${software_dir}*hive* |tar zxf - -C $install_dir
mv ${install_dir}*hive* ${install_dir}hiveif [ -f "$hive_conf_path/hive-log4j2.properties.template" ]
then mv $hive_conf_path/hive-log4j2.properties.template $hive_conf_path/hive-log4j2.properties
fiif [ -f "$hive_conf_path/hive-exec-log4j2.properties.template" ] 
thenmv $hive_conf_path/hive-exec-log4j2.properties.template $hive_conf_path/hive-exec-log4j2.properties
fi
cp ${software_dir}mysql-connector-java-5.1.44-bin.jar $hive_path/lib
sed -i "/property.hive.log.dir/c\property.hive.log.dir=$hive_path/logs" $hive_conf_path/hive-log4j2.properties
sed -i "/property.hive.log.dir/c\property.hive.log.dir=$hive_path/logs" $hive_conf_path/hive-exec-log4j2.propertiescat > $hive_conf_path/hive-site.xml <<EOF
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?><configuration><!-- 配置hive元数据在hdfs的存储路径 --><property><name>hive.metastore.warehouse.dir</name><value>/hive/database</value></property><!-- 对于小数据量，自动使用本地模式执行 MR job 加快执行过程 ，默认是false --><property><name>hive.exec.mode.local.auto</name><value>true</value></property><!-- 取消列名前面的表名 --><property><name>hive.resultset.use.unique.column.names</name><value>false</value></property><!-- 更换计算引擎 默认MR -->  <property><name>hive.execution.engine</name><value>tez</value></property><!-- 关闭元数据校验 --><property><name>hive.metastore.schema.verification</name><value>false</value></property><property><name>javax.jdo.option.ConnectionURL</name><value>jdbc:mysql://${HOSTNAME}:3306/hive?createDatabaseIfNotExist=true&amp;useUnicode=true&amp;characterEncodeing=UTF-8&amp;useSSL=false</value></property><property><name>javax.jdo.option.ConnectionDriverName</name><value>com.mysql.jdbc.Driver</value></property><property><name>javax.jdo.option.ConnectionUserName</name><value>$mysql_user</value></property><property><name>javax.jdo.option.ConnectionPassword</name><value>$mysql_password</value></property><!-- 下面三个设置解决 tez-ui中hive queries不显示数据的问题--><property><name>hive.exec.pre.hooks</name><value>org.apache.hadoop.hive.ql.hooks.ATSHook</value></property><property><name>hive.exec.post.hooks</name><value>org.apache.hadoop.hive.ql.hooks.ATSHook</value></property><property><name>hive.exec.failure.hooks</name><value>org.apache.hadoop.hive.ql.hooks.ATSHook</value></property></configuration>
EOF
rm -rf ${hive_path}/lib/log4j-slf4j-impl-*.jar
docker exec -it mysql mysql -u $mysql_user -p$mysql_password -e "drop database if exists hive;" &>/dev/null
schematool -dbType mysql -initSchemahdfs dfs -rm -r /tez
hdfs dfs -mkdir /tez
tez_name=$(ls $tez_path/share)
hdfs dfs -put $tez_path/share/$tez_name /tez
rm -rf ${tez_path}/lib/slf4j-log4j12-*.jarcat > $hive_conf_path/tez-site.xml <<EOF
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration><property><name>tez.lib.uris</name><value>\${fs.defaultFS}/tez/$tez_name</value></property><property><name>tez.use.cluster.hadoop-libs</name><value>true</value></property><property><name>tez.history.logging.service.class</name><value>org.apache.tez.dag.history.logging.ats.ATSHistoryLoggingService</value></property><property><name>tez.tez-ui.history-url.base</name><value>http://localhost:8080/tez-ui/</value></property>  
</configuration>
EOFmv $hive_conf_path/hive-env.sh.template $hive_conf_path/hive-env.shcat >> $hive_conf_path/hive-env.sh <<EOF
export TEZ_HOME=$tez_path
export HADOOP_CLASSPATH=\$HADOOP_CLASSPATH:\$TEZ_HOME/*:\$TEZ_HOME/lib/*
EOFsed -i '/^<\/configuration>$/d' $hadoop_path/etc/hadoop/yarn-site.xml
sed -i '/<!-- 开启timelineserver -->/,$d' $hadoop_path/etc/hadoop/yarn-site.xml
cat >> $hadoop_path/etc/hadoop/yarn-site.xml <<EOF<!-- 开启timelineserver --><property><name>yarn.timeline-service.enabled</name><value>true</value></property><!-- 配置timeline安装在指定节点上 --><property><name>yarn.timeline-service.hostname</name><value>$HOSTNAME</value></property><!-- 开启跨域 --><property><name>yarn.timeline-service.http-cross-origin.enabled</name><value>true</value></property><!-- 发送RM的系统指标到timeline服务上 --><property><name>yarn.resourcemanager.system-metrics-publisher.enabled</name><value>true</value></property><!-- 开启通用历史服务 -->  <property><name>yarn.timeline-service.generic-application-history.enabled</name><value>true</value></property><!-- 设置RPC请求的处理程序线程数  默认 10 --><property><name>yarn.timeline-service.handler-thread-count</name><value>24</value></property></configuration>
EOFxsync $hadoop_path/etc/hadoop/yarn-site.xml
jps |egrep "NodeManager|ResourceManager"
if [ $? -eq 0 ]
thenstop-yarn.shsleep 2start-yarn.sh
elsestart-all.sh
fijps |grep "ApplicationHistoryServer"
if [ $? -eq 0 ]
then yarn-daemon.sh stop timelineserver
fi
yarn-daemon.sh start timelineserverif [ -x $tomcat_path/bin/shutdown.sh ]
then$tomcat_path/bin/shutdown.sh &>/dev/nullrm -rf $tomcat_path
fipv ${software_dir}*tomcat* |tar zxf - -C $install_dir
mv ${install_dir}*tomcat* $tomcat_pathmkdir -p $tomcat_path/webapps/tez-ui/
unzip -d $tomcat_path/webapps/tez-ui/ $tez_path/*.war &>/dev/null
sed -i ' s|//timeline: "http://localhost:8188"|timeline: "http://master:8188"|' $tomcat_path/webapps/tez-ui/config/configs.env
sed -i 's|//rm: "http://localhost:8088"|rm: "http://node1:8088"|' $tomcat_path/webapps/tez-ui/config/configs.env
sed -i 's|//timeZone: "UTC"|timeZone: "Asia/Shanghai"|' $tomcat_path/webapps/tez-ui/config/configs.envcat > /usr/lib/systemd/system/tomcat.service <<EOF
[Unit]
Description=tomcat
After=syslog.target network.target remote-fs.target nss-lookup.target[Service]
Type=forkingEnvironment="JAVA_HOME=/opt/jdk1.8.0_201"PIDFile=$tomcat_path/tomcat.pid
ExecStart=$tomcat_path/bin/startup.sh
ExecStop=$tomcat_path/bin/shutdown.sh
ExecReload=/bin/kill -s HUP $MAINPID
PrivateTmp=true[Install]
WantedBy=multi-user.targetEOFecho "CATALINA_PID=\$CATALINA_BASE/tomcat.pid"  >  $tomcat_path/bin/setenv.shsystemctl daemon-reload
systemctl enable --now tomcat