2020-03-17

【数仓项目】常用shell脚本编写

zk.sh:用于zookeeper的启动，状态，停止

#! /bin/bash

case $1 in
"start"){
	for i in node02 node03 node04
	do
		echo "========================="$i"========================="
		ssh $i "source /etc/profile;/opt/module/zookeeper-3.4.10/bin/zkServer.sh start"
	done
};;
"stop"){
	for i in node02 node03 node04
	do
		echo "========================="$i"========================="
		ssh $i "source /etc/profile;/opt/module/zookeeper-3.4.10/bin/zkServer.sh stop"
	done
};;
"status"){
	for i in node02 node03 node04
	do
		echo "========================="$i"========================="
		ssh $i "source /etc/profile;/opt/module/zookeeper-3.4.10/bin/zkServer.sh status"
	done
};;
esac

lg.sh：用于调用java代码根据时间戳生成数据

#! /bin/bash

for i in node02 node03 
do
	echo "========================="$i"========================="
	ssh $i "source /etc/profile;java -classpath /opt/module/log-collector-1.0-SNAPSHOT-jar-with-dependencies.jar com.sumiya.appclient.AppMain $1 $2 >/opt/module/test.log &"
done

dt.sh：用于统一集群时间（方便数据日志生成）

#!/bin/bash
log_date=$1
for i in node02 node03 node04
do
	ssh -t $i "source /etc/profile;sudo date -s $log_date"
done


## f1.sh：用于2，3节点flume的启动停止
#! /bin/bash
case $1 in
"start"){
        for i in node02 node03
        do
                echo " --------启动 $i 采集flume-------"
                ssh $i "source /etc/profile;nohup /opt/module/flume/bin/flume-ng agent --conf-file /opt/module/flume/conf/file-flume-kafka.conf --name a1 -Dflume.root.logger=INFO,LOGFILE >/dev/null 2>&1 &"
        done
};;	
"stop"){
        for i in node02 node03
        do
                echo " --------停止 $i 采集flume-------"
                ssh $i "source /etc/profile;ps -ef | grep file-flume-kafka | grep -v grep |awk '{print \$2}' | xargs kill"
        done

};;
esac

说明1：nohup，该命令可以在你退出帐户/关闭终端之后继续运行相应的进程。nohup就是不挂起的意思，不挂断地运行命令。
说明2：/dev/null代表linux的空设备文件，所有往这个文件里面写入的内容都会丢失，俗称“黑洞”。
标准输入0：从键盘获得输入 /proc/self/fd/0
标准输出1：输出到屏幕（即控制台） /proc/self/fd/1
错误输出2：输出到屏幕（即控制台） /proc/self/fd/2

kf.sh：用于kafka的启动停止

#! /bin/bash
case $1 in
"start"){
        for i in node02 node03 node04
        do
                echo " --------启动 $i Kafka-------"
                # 用于KafkaManager监控
                ssh $i "source /etc/profile;export JMX_PORT=9988 && /opt/module/kafka/bin/kafka-server-start.sh -daemon /opt/module/kafka/config/server.properties "
        done
};;
"stop"){
        for i in node02 node03 node04
        do
                echo " --------停止 $i Kafka-------"
                ssh $i "source /etc/profile;/opt/module/kafka/bin/kafka-server-stop.sh stop"
        done
};;
esac

说明：启动Kafka时要先开启JMX端口，是用于后续KafkaManager监控。

km.sh ：用于启动停止kafkamanager

#! /bin/bash

case $1 in
"start"){
        echo " -------- 启动 KafkaManager -------"
        source /etc/profile;nohup /opt/module/kafka-manager-1.3.3.22/bin/kafka-manager   -Dhttp.port=7456 >start.log 2>&1 &
};;
"stop"){
        echo " -------- 停止 KafkaManager -------"
        ssh node02 "source /etc/profile;ps -ef | grep ProdServerStart | grep -v grep |awk '{print $2}' | xargs kill -9" 


};;
esac

f2.sh：用于启动3节点的flume

#! /bin/bash

case $1 in
"start"){
        for i in node04
        do
                echo " --------启动 $i 消费flume-------"
                ssh $i "nohup /opt/module/flume/bin/flume-ng agent --conf-file /opt/module/flume/conf/kafka-flume-hdfs.conf --name a1 -Dflume.root.logger=INFO,LOGFILE >/opt/module/flume/log.txt   2>&1 &"
        done
};;
"stop"){
        for i in node04
        do
                echo " --------停止 $i 消费flume-------"
                ssh $i "ps -ef | grep kafka-flume-hdfs | grep -v grep |awk '{print \$2}' | xargs kill"
        done

};;
esac

cluster.sh：一键启动停止整个集群

#! /bin/bash

case $1 in
"start"){
	echo " -------- 启动 集群 -------"

	echo " -------- 启动 hadoop集群 -------"
	/opt/module/hadoop-2.7.2/sbin/start-dfs.sh 
	ssh node03 "/opt/module/hadoop-2.7.2/sbin/start-yarn.sh"

	#启动 Zookeeper集群
	zk.sh start

sleep 4s;

	#启动 Flume采集集群
	f1.sh start

	#启动 Kafka采集集群
	kf.sh start

sleep 6s;

	#启动 Flume消费集群
	f2.sh start

	#启动 KafkaManager
	km.sh start
};;
"stop"){
    echo " -------- 停止 集群 -------"

	#停止 KafkaManager
	km.sh stop

    #停止 Flume消费集群
	f2.sh stop

	#停止 Kafka采集集群
	kf.sh stop

    sleep 6s;

	#停止 Flume采集集群
	f1.sh stop

	#停止 Zookeeper集群
	zk.sh stop

	echo " -------- 停止 hadoop集群 -------"
	ssh node03 "/opt/module/hadoop-2.7.2/sbin/stop-yarn.sh"
	/opt/module/hadoop-2.7.2/sbin/stop-dfs.sh 
};;
esac