hadoop memo

master,slave1,slave2master,slave1,slave2//opt represent wechat hadoop bigdata dev//2019 :bigdata competitionhadoop 50070hbase 16010storm 8080
#hostnamehostnamectl set-hostname master && bash hostname  master && bash hostname  slave1/slave2 && bash vim /etc/hostname   master/slave1/slave2vim /etc/hosts  ip master   ip slave1  ipslave2
#yumcd /etc/yum.repos.d && rm -rf *wget http://172.16.47.240/bigdata/repofile/bigdata.repoyum clean all
#firewallsystemctl stop firewalldsystemctl status firewalld
#timezonetzselect  5-9-1-1echo "TZ='Asia/Shanghai'; export TZ" >> /etc/profile && source /etc/profile
#ntpyum install -y ntpvim /etc/ntp.conf//#server 0~3.centos.pool.ntp.org iburstserver 127.127.1.0fudge 127.127.1.0 stratum 10/bin/systemctl restart ntpd.servicentpdate master (slave1,slave2)
#crontabservice crond status/sbin/service crond startcrontab -e*/30 8-17 * * * /usr/sbin/ntpdate mastercrontab –l
#ssh passwordssh-keygen -t dsa -P '' -f _{/.ssh/id_dsacat /root/.ssh/id_dsa.pub >> /root/.ssh/authorized_keysscp ~/.ssh/authorized_keys root@slave1:}/.ssh/scp _{/.ssh/authorized_keys root@slave2:}/.ssh/
ssh-copy-id masterssh-copy-id slave1ssh-copy-id slave2
#install jdkmkdir -p /usr/javatar -zxvf jdk-8u171-linux-x64.tar.gz -C /usr/java/
vim /etc/profileexport JAVA_HOME=/usr/java/jdk1.8.0_171export CLASSPATH=$JAVA_HOME/lib/export PATH=$PATH:$JAVA_HOME/bin
source /etc/profile && java -version
scp -r /usr/java root@slave1:/usr/scp -r /usr/java root@slave2:/usr/
#install hadoopmkdir -p /usr/hadoop && cd /usr/hadooptar -zxvf /usr/hadoop/hadoop-2.7.3.tar.gz -C /usr/hadoop/rm -rf /usr/hadoop/hadoop-2.7.3.tar.gzvim /etc/profileexport HADOOP_HOME=/usr/hadoop/hadoop-2.7.3export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbinhadoop  //test
hadoop-env.sh/mapred-env.sh/yarn-en.shexport JAVA_HOME=/usr/java/jdk1.8.0_171
##vim core-site.xmlfs.default.name {hdfs://master:9000}hadoop.tmp.dir {/usr/hadoop/hadoop-2.7.3/hdfs/tmp}io.file.buffer.size {131072}fs.checkpoint.period {60}fs.checkpoint.size {67108864}
##hdfs-site.xmldfs.replication {2}dfs.namenode.name.dir {file:/usr/hadoop/hadoop-2.7.3/hdfs/name}dfs.datanode.data.dir {file:/usr/hadoop/hadoop-2.7.3/hdfs/data}
##vim yarn-env.shyarn.resourcemanager.address {master:18040}yarn.resourcemanager.scheduler.address {master:18030}yarn.resourcemanager.webapp.address {master:18088}yarn.resourcemanager.resource-tracker.address {18025}yarn.resourcemanager.admin.address {master:18141}yarn.nodemanager.aux-services {mapreduce_shuffle}yarn.nodemanager.auxservices.mapreduce.shuffle.class {org.apache.hadoop.mapred.ShuffleHandler}
#vim mapred-site.xmlmapreduce.framework.name {yarn}
#slaves fileecho master > master && echo slave1 > slaves && echo slave2 >> slaves
#hadoop formathadoop namenode -format (master)   //has been successfully#start hadoopstart-all.shmaster :NameNode,SecondaryNameNode,ResourceManagerslave1~2:DataNode,NodeManager
start-dfs.shstart-yarn.shhadoop-daemon.sh start namenodehadoop-daemon.sh start datanodehadoop-daemon.sh start secondarynamenodehadoop-daemon.sh start resourcemanagerhadoop-daemon.sh start nodemanager
test hdfs& mapreducehadoop fs -mkdir /inputhadoop fs -put $HADOOP_HOME/README.txt /input
http://master:50070hadoop jar $HADOOP_HOME/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.8.4.jar
#install hiveyum -y install mysql-community-server slave2:mysqlserver slave1:hiveserver master:hiveclientsystemctl daemon-reloadsystemctl start mysqldcat /var/log/mysqld.log grep "temporary password"mysql -uroot -pset global validate_password_policy=0;set global validate_password_length=4;alter user 'root'@'localhost' identified by '123456';mysql -uroot -p123456create user 'root'@'%' identified by '123456';grant all privileges on . to 'root'@'%' with grant option;flush privileges;
mkdir -p /usr/hive tar -zxvf /usr/hive/apache-hive-2.1.1-bin.tar.gz -C /usr/hive/
vim /etc/profile   //for hiveexport HIVE_HOME=/usr/hive/apache-hive-2.1.1-binexport PATH=$PATH:$HIVE_HOME/binsource /etc/profile
vim hive-env.shcd $HIVE_HOME/conf && vim hive-env.shexport HADOOP_HOME=/usr/hadoop/hadoop-2.7.3export HIVE_CONF_DIR=/usr/hive/apache-hive-2.1.1-bin/confexport HIVE_AUX_JARS_PATH=/usr/hive/apache-hive-2.1.1-bin/lib
cp $HIVE_HOME/lib/jline-2.12.jar $HADOOP_HOME/share/hadoop/yarn/lib/
##slave1 hive-servercd $HIVE_HOME/lib && wget or cp mysql-connector-java-5.1.47-bin.jar
hive-site.xml (hive-server)hive.metastore.warehouse.dir {/user/hive_remote/warehouse}javax.jdo.option.ConnectionDriverName {com.mysql.jdbc.Driver}javax.jdo.option.ConnectionURL {jdbc:mysql://slave2:3306/hive?createDatabaseIfNotExist=true&useSSL=false}javax.jdo.option.ConnectionUserName {root}javax.jdo.option.ConnectionPassword {123456}
hive-site.xml (hive client)hive.metastore.warehouse.dir {/user/hive_remote/warehouse}hive.metastore.local {false}hive.metastore.uris {thrift://slave1:9083}
//opt  hadoop fs -mkdir /tmp  hadoop fs -mkdir -p /user/hive/warehousehadoop fs -chmod g+w /tmp& /user/hive/warehouse//opt modify log locationvim hive-log4j2.propertiesproperty.hive.log.dir=/home/hadoop
//opt hive-site.xmlhive.cli.print.current.db {true}hive.cli.print.header {true}
beeline connect shoud config core-site.xmlhadoop.proxyuser.hadoop.hosts ..groups {}bin/beeline->!connect jdbc:hive2://centos01:10000or bin/beeline -u jdbc:hive2://centos01:10000 -n hadoop
desc table/desc database/desc database extended testdb;

hive-serverschematool -dbType mysql -initSchemahive --service metastore
hive-clienthive \ncreate database hongyaa;

#install zookeepermkdir -p /usr/zookeeper && cd /usr/zookeepertar -zxvf /usr/zookeeper/zookeeper-3.4.10.tar.gz -C /usr/zookeeper/vi /etc/profile  //for zookeeperexport ZOOKEEPER_HOME=/usr/zookeeper/zookeeper-3.4.10PATH=$PATH:$ZOOKEEPER_HOME/binsource /etc/profile
cd /usr/zookeeper/zookeeper-3.4.10/conf/ && mv zoo_sample.cfg zoo.cfg && vimzoo.cfgtickTime=2000initLimit=10syncLimit=5dataDir=/usr/zookeeper/zookeeper-3.4.10/zkdataclientPort=2181dataLogDir=/usr/zookeeper/zookeeper-3.4.10/zkdatalogserver.1=master:2888:3888server.2=slave1:2888:3888server.3=slave2:2888:3888

cd /usr/zookeeper/zookeeper-3.4.10 && mkdir zkdata zkdatalogcd /usr/zookeeper/zookeeper-3.4.10/zkdata && echo "1" >> myid
zkServer.sh start(master,slave1,slave2)netstat -antp|grep 2181

#install hbase//opttar -zxf hbase-1.2.6.1-bin.tar.gz -C /opt/modules/export JAVA_HOME=/usr/java/jdk1.8.0_171export HBASE_MANAGES_ZK=true
vim hbase-site.xmlhbase.rootdir {hdfs://centos01:9000}hbase.cluster.distributed {true}hbase.zookeeper.quorum {centos01,centos02,centos03}hbase.zookeeper.property.dataDir {/opt/modules/hbase-1.2.6/zkData}

#install kafkatar -zxvf kafka_2.11-2.0.0.tgz -C /opt/modulesvim config/server.propertiesbroker.id=1num.partitions=2default.replication.factor=1lisheners=PLAINTEXT://centos01:9092log.dirs=/opt/modules/kafka_2.11-2.0.0/kafka-logszookeeper.connect=centos01:2181,centos02:2181,centos03:2181scp to centos02,centos03  and modify id
bin/kafka-server-start.sh -daemon config/server.properties/kafka-server-start.sh -daemon config/server.properties
create topicbin/kafka-topics.sh --create --zookeeper centos01:2181,centos02:2181,centos03:2181 --replication-factor 2 --partitions 2 --topic topictest
bin/kafka-topics.sh --list --zookeeper centos01:2181
bin/kafka-topics.sh --describe --zookeeper centos01:2181//生产者bin/kafka-console-producer.sh --broker-list centos01:9092,centos02:9092,centos03:9092 --topic topictest//消费者bin/kafka-console-consumer.sh --bootstrap-server centos01:9092,centos02:9092,centos03:9092 --topic topictest
java -Xms256m -Xmx512m -classpath .:./lib/* com.cmcc.hsd.loader.ScannerV2p9Loader -t 6 -d 202008250000

#install stormtar zxvf apache-storm-1.1.0.tar.gz -C /opt/modulesmv apache-storm-1.1.0/ storm-1.1.0vim /etc/profileexport STORM_HOME=/opt/modules/storm-1.1.0export PATH=$PATH:$STORM_HOME/bin
storm-env.shexport JAVA_HOME=/opt/softwares/jdk1.8.0_201export STORM_CONF_DIR="/opt/modules/storm-1.1.0/conf"vim storm.yaml storm.zookeeper.servers:     - "centos01"     - "centos02"     - "centos03" supervisor.slots.ports:     - 6700     - 6701     - 6702     - 6703 storm.zookeeper.port: 2181 storm.local.dir: "/opt/modules/storm-1.1.0/data"  scp -r to centos01,centos02,centos03 storm nimbus > /dev/null 2>&1 &   (master)storm ui > /dev/null 2>&1 & storm supervisor > /dev/null 2>&1 &  (slave)
jpsnimbus/Supervisor

#install sparktar zxvf spark-2.4.0-bin-hadoop2.7.tgz -C /opt/modulesvim conf/slavescentos02centos03vim spark-env.shexport JAVA_HOME=/opt/softwares/jdk1.8.0_201export SPARK_MASTER_IP=centos01export SPARK_MASTER_PORT=7077

export HADOOP_HOME=/opt/modules/hadoop-2.8.4export HADOOP_CONF_DIR=$HADOOP_HOME/etc/haoop
scp -r to other node
spark ha  delete SPARK_MASTER_IP  add export SPARK_DAEMON_JAVA_OPT="-Dspark.deploy.recoveryMode=ZOOKEEPER-Dspark.deploy.zookeeper.url=centos01:2181,centos02:2181,centos03:2181-Dspark.deploy.zookeepr.dir=/spark"

bin/spark-submit --class org.apache.spark.examples.SparkPi --master yarn --deploy-mode cluster \/opt/modules/spark-2.4.0-bin-hadoop2.7/examples/jars/spark-examples_2.11-2.4.0.jar
bin/spark-shell --master spark://centos02:7077
bin/spark-submit --class org.apache.spark.examples.SparkPi --master yarn --deploy-mode cluster --drive-memory 512m --executor-memory 1g --executor-cores 2 \/opt/modules/spark-2.4.0-bin-hadoop2.7/examples/jars/spark-examples_2.11-2.4.0.jar
yarn-site.xml ofen kill by yarnpmem-check-enablevmem-check-enable

#analysis data  *****create database hive;use hive;create table bike (duration int,startdate timestamp,enddate timestamp,type string) row format delimitedfields terminated by ',';
LOAD DATA LOCAL INPATH './load.csv'  INTO TABLE loan;insert overwrite local directory '/home/hadoop/college001/'row format delimited fields terminated by 't'select count from bike;##about hiveDateType ,UDAF UDTFselect from_unixtime cast(concat(startdate,':00') as timestamp),'yyyyMMdd000000')  from bike limit 10;select date_format('2019-10-07', 'yyyyMMdd000000')select from_unixtime(unix_timestamp('2019-10-07', 'yyyy-MM-dd'), 'yyyyMMdd000000')select unix_timestamp('2019-10-07 13:24:20','yyyy-MM-dd HH:mm:ss')

JSON.stringify(option)
iconv -f gbk -t utf8 a.txt –o b.txt  
mapreduce output rw-rr   2 hadoop supergroup          0 2021-10-26 07:06 /output2/_SUCCESS-rw-rr-   2 hadoop supergroup       1306 2021-10-26 07:06 /output2/part-r-00000

#add node   slave3yum install -y ntp && ntpdate mastervim /etc/hostsslave3scp _{/.ssh/authorized_keys root@slave3:}/.ssh/yum install -y ntp && ntpdate mastervim  /etc/profile && source /etc/profileexport JAVA...export HADOOP...vim $HADOOP_HOME/etc/hadoop/slaves  add slave3
$HADOOP_HOME/sbin/hadoop-daemon.sh start datanode$HADOOP_HOME/sbin/yarn-daemon.sh start nodemanagerhdfs dfsadmin -refreshNodessbin/start-balancer.shhdfs dfsadmin -report
echo "slave3" > excludes
hdfs-site.xml  (master)
dfs.hosts.exclude {/usr/hadoop/hadoop-2.7.3/etc/hadoop/excludes}
hdfs dfsadmin -refreshNodeshdfs dfsadmin -reporthadoop-daemon.sh stop datanodeyarn-daemon.sh stop nodemanagerhdfs dfsadmin -reportsbin/start-balancer.sh