hadoop fully distributed

Article directory

Fully distributed

1.1 installation configuration

1)To configure mapred-site(nn01 Operation above)

    [root@nn01 ~]cd /usr/local/hadoop/etc/hadoop/
    [root@nn01 hadoop] mv mapred-site.xml.template mapred-site.xml
    [root@nn01 hadoop vim mapred-site.xml
    <configuration>
    <property>
            <name>mapreduce.framework.name</name>
            <value>yarn</value>
        </property>
    </configuration>
shujshuj
2)To configure yarguanln-site(nn01 Operation above)
peiz
    [root@nn01 hadoop] vim yarn-site.xml
    <configuration>5)web Visit hadoop

    http://192.168.1.60:50070 / / / -- namenode web page (nn01)
    http://192.168.1.60:50090 / / / -- second namenode web page (nn01)
    http://192.168.1.61:50075 / / / -- datanode web page (node1,node2,node3)
    http://192.168.1.60:8088 / / / -- resource manager web page (nn01)
    http://192.168.1.61:8042 / / / -- nodemanager web page (node1,node2,node3) do rsync -aSH --delete /usr/local/hadoop/
\ 192.168.1.$i:/usr/local/hadoop/  -e 'ssh' & done        //Synchronous configuration do rsync -aSH --delete /usr/local/hadoop/
\ 192.168.1.$i:/usr/local/hadoop/  -e 'ssh' & done        //Synchronous configuration do rsync -aSH --delete /usr/local/hadoop/
\ 192.168.1.$i:/usr/local/hadoop/  -e 'ssh' & done        //Synchronous configuration
    <!-- Site specific YARN configuration properties -->
    <property>
            <name>yarn.resourcemanager.hostname</name>
            <value>nn01</value>
        </property>
        <property>
            <name>yarn.nodemanager.aux-services</name>
            <value>mapreduce_shuffle</value>
        </property>
    </configuration>
 do rsync -aSH --delete /usr/local/hadoop/
\ 192.168.1.$i:/usr/local/hadoop/  -e 'ssh' & done        //Synchronous configuration
3)Synchronization configuration( nn01 Operation above)

    [root@nn01 hadoop] for i in {62..64}; do rsync -aSH --delete /usr/local/hadoop/ 192.168.1.$i:/usr/local/hadoop/  -e 'ssh' & done
    [1] 712
    [2] 713
    [3] 714

4)Verify configuration( nn01 Operation above)

    [root@nn01 hadoop] cd /usr/local/hadoop
    [root@nn01 hadoop] ./sbin/start-dfs.sh
    Starting namenodes on [nn01]
    nn01: namenode running as process 23408. Stop it first.
    node1: datanode running as process 22409. Stop it first.
    node2: datanode running as process 22367. Stop it first.
    node3: datanode running as process 22356. Stop it first.
    Starting secondary namenodes [nn01]
    nn01: secondarynamenode running as process 23591. Stop it first.
    [root@nn01 hadoop] ./sbin/start-yarn.sh
    starting yarn daemons
    starting resourcemanager, logging to /usr/local/hadoop/logs/yarn-root-resourcemanager-nn01.out
    node2: starting nodemanager, logging to /usr/local/hadoop/logs/yarn-root-nodemanager-node2.out
    node3: starting nodemanager, logging to /usr/local/hadoop/logs/yarn-root-nodemanager-node3.out
    node1: starting nodemanager, logging to /usr/local/hadoop/logs/yarn-root-nodemanager-node1.out
    [root@nn01 hadoop] jps    //nn01 view ResourceManager
    23408 NameNode
    1043 ResourceManager
    1302 Jps
    23591 SecondaryNameNode
    [root@nn01 hadoop] ssh node1 jps        //Node1view NodeManager
    25777 Jps
    22409 DataNode
    25673 NodeManager
    [root@nn01 hadoop] ssh node2 jps        //Node1view NodeManager
    25729 Jps
    25625 NodeManager
    22367 DataNode
    [root@nn01 hadoop ssh node3 jps        //Node1view NodeManager
    22356 DataNode
    25620 NodeManager
    25724 Jps
5)web Visit hadoop

    http://192.168.1.60:50070 / / / -- namenode web page (nn01)
    http://192.168.1.60:50090 / / / -- second namenode web page (nn01)
    http://192.168.1.61:50075 / / / -- datanode web page (node1,node2,node3)
    http://192.168.1.60:8088 / / / -- resource manager web page (nn01)
    http://192.168.1.61:8042 / / / -- nodemanager web page (node1,node2,node3)


1.2 solve problems

cd /usr/local/hadoop
./sbin/stop-all.sh
rm -rf /var/hadoop/*
./bin/hdfs namenode -format
./sbin/start-all.sh
./bin/hdfs dfsadmin -report
./bin/yarn node -list


1.3 hadoop word frequency statistics

  ./bin/hadoop fs -mkdir /input
  ./bin/hadoop fs -put *.txt /input
  ./bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-2.7.7.jar \ 
  wordcount /input /output Analysis data
 ./bin/hadoop fs -ls /output
  ./bin/hadoop fs -cat /output/part-r-00000  View data

2HDFS node management

2.1 add nodes

Start a new system, set SSH anti password login
Modify / etc/hosts in all nodes to add host information of new nodes
Install Java runtime environment (java-1.8.0-openjdk-devel java-1.8.0-openjdk)
Modify the slave file of NameNode to add the node
Copy / usr/local/hadoop of NameNode to the local machine
DataNode started on this node
./sbin/hadoop-daemon.sh start datanode

Set synchronization bandwidth and synchronize data
./sbin/hadoop-daemon.sh start datanode
./sbin/start-balancer.sh
 . / bin/hdfs dfsadmin -report view cluster status

2.2 deleting nodes

[root@node4 hadoop] ./sbin/hadoop-daemon.sh stop datanode    //Stop datanode
stopping datanode
[root@node4 hadoop]./sbin/yarn-daemon.sh start nodemanager             
//yarn adds nodemanager
[root@node4 hadoop] ./sbin/yarn-daemon.sh stop  nodemanager  //Stop nodemanager
stopping nodemanager
[root@node4 hadoop] ./bin/yarn node -list        
//yarn looks at the node status. There are still node4 nodes. It will take a while before they disappear
Total Nodes:4
         Node-Id         Node-State    Node-Http-Address    Number-of-Running-Containers
     node3:34628            RUNNING           node3:8042                               0
     node2:36300            RUNNING           node2:8042                               0
     node4:42459            RUNNING           node4:8042                               0
     node1:39196            RUNNING           node1:8042                               0


3NFS gateway

3.1nfs gateway

purpose

Users can browse the HDFS file system through the local NFSv3 client compatible with the operating system
Users can download documents from HDFS file system to local file system
Users can directly stream data through mount points, supporting file attachment, but not random writing
nfs gateway supports NFSv3 and allows hdfs to be mounted as part of the client file system

Characteristic

The HDFS super user can be a user with the same identity as the NameNode process itself, and the super user can perform any operation, because the permission check will not consider the super user to fail

Matters needing attention
In non secure mode, the user running the NIC process is the agent user
In secure mode, the user in Kerberos Ketab is a proxy user

3.2 configure users

  Adding users to the namenode is the same as the group nfs gateway
    [root@nn01 hadoop] groupadd -g 800 nfsuser
    [root@nn01 hadoop] useradd -u 800 -g 800 -r -d /var/hadoop nfsuser

3.3 configure core-site.xml

vim core-site.xml
    <property>
        <name>hadoop.proxyuser.nfsuser.groups</name>
        <value>*</value>
    </property>
    <property>
        <name>hadoop.proxyuser.nfsuser.hosts</name>
        <value>*</value>
    </property>
Synchronize configuration to node1´╝înode2´╝înode3 

Start cluster  namenode upper
    [root@nn01 hadoop] /usr/local/hadoop/sbin/start-dfs.sh
 View state
    [root@nn01 hadoop] /usr/local/hadoop/bin/hdfs  dfsadmin -report

node)

3.4 NFSGW configuration

1)install java-1.8.0-openjdk-devel and rsync

    [root@nfsgw ~]# yum -y install java-1.8.0-openjdk-devel
    [root@nn01 hadoop]# rsync -avSH --delete \ 
    /usr/local/hadoop/ 192.168.1.65:/usr/local/hadoop/  -e 'ssh'

2)Create data root /var/hadoop(stay NFSGW Operation on the host)

    [root@nfsgw ~]# mkdir /var/hadoop

3)Create a dump directory and give it to the user nfs Empowerment

    [root@nfsgw ~]# mkdir /var/nfstmp
    [root@nfsgw ~]# chown nfsuser:nfsuser /var/nfstmp

4)to/usr/local/hadoop/logs Empowerment (in NFSGW Operation on the host)

    [root@nfsgw ~]# setfacl -m user:nfsuser:rwx /usr/local/hadoop/logs

5)Can be created and deleted

    [root@nfsgw ~]# su - nfs
    [nfs@nfsgw ~]$ cd /var/nfstmp/
    [nfs@nfsgw nfstmp]$ touch 1
    [nfs@nfsgw nfstmp]$ ls
    1
    [nfs@nfsgw nfstmp]$ rm -rf 1
    [nfs@nfsgw nfstmp]$ ls
    [nfs@nfsgw nfstmp]$ cd /usr/local/hadoop/logs/
    [nfs@nfsgw logs]$ touch 1
    [nfs@nfsgw logs]$ ls
    1 hadoop-root-secondarynamenode-nn01.log    yarn-root-resourcemanager-nn01.log
    hadoop-root-namenode-nn01.log hadoop-root-secondarynamenode-nn01.out    yarn-root-resourcemanager-nn01.out
    hadoop-root-namenode-nn01.out    hadoop-root-secondarynamenode-nn01.out.1
    hadoop-root-namenode-nn01.out.1  SecurityAuth-root.audit
    [nfs@nfsgw logs]$ rm -rf 1
    [nfs@nfsgw logs]$ ls

6)Startup service

    [root@nfsgw ~]# /Usr / local / Hadoop / SBIN / hadoop-day.sh -- script. / bin / HDFS start portmap / / portmap service can only be started by root user
    starting portmap, logging to /usr/local/hadoop/logs/hadoop-root-portmap-nfsgw.out
    [root@nfsgw ~]# jps
    23714 Jps
    23670 Portmap
    [root@nfsgw ~]# su - nfs
    Last login: Mon Sep 10 12:31:58 CST 2018 on pts/0
    [nfs@nfsgw ~]$ cd /usr/local/hadoop/
    [nfs@nfsgw hadoop]$ ./sbin/hadoop-daemon.sh  --script ./bin/hdfs start nfs3  
    //nfs3 can only be started with a proxy user
    starting nfs3, logging to /usr/local/hadoop/logs/hadoop-nfs-nfs3-nfsgw.out
    [nfs@nfsgw hadoop]$ jps                    
    1362 Jps
    1309 Nfs3 
    [root@nfsgw hadoop]# JPS / / root user execution can see portmap and nfs3
    1216 Portmap
    1309 Nfs3
    1374 Jps

7)To mount the client (the client can use node4 This host)

    [root@node4 ~]# rm -rf /usr/local/hadoop
    [root@node4 ~]# yum -y install nfs-utils
    [root@node4 ~]# mount -t nfs -o \
    vers=3,proto=tcp,nolock,noatime,sync,noacl 192.168.1.64:/  /mnt/  //mount
    [root@node4 ~]# cd /mnt/
    [root@node4 mnt]# ls
    aaa  bbb  fa  system  tmp
    [root@node4 mnt]# touch a
    [root@node4 mnt]# ls
    a  aaa  bbb  fa  system  tmp
    [root@node4 mnt]# rm -rf a
    [root@node4 mnt]# ls
    aaa  bbb  fa  system  tmp

8)Realize automatic mount after power on

    [root@node4 ~]# vim /etc/fstab
    192.168.1.64:/  /mnt/ nfs  vers=3,proto=tcp,nolock,noatime,sync,noacl,_netdev 0 0 
    [root@node4 ~]# mount -a
    [root@node4 ~]# df -h
    192.168.1.26:/   64G  6.2G   58G  10% /mnt
    [root@node4 ~]# rpcinfo -p 192.168.1.64
       program vers proto   port  service
        100005    3   udp   4242  mountd
        100005    1   tcp   4242  mountd
        100000    2   udp    111  portmapper
        100000    2   tcp    111  portmapper
        100005    3   tcp   4242  mountd
        100005    2   tcp   4242  mountd
        100003    3   tcp   2049  nfs
        100005    2   udp   4242  mountd
        100005    1   udp   4242  mountd
    [root@nfsgw ~]# vim /usr/local/hadoop/etc/hadoop/hdfs-site.xml
        <property>
            <name>nfs.exports.allowed.hosts</name>
            <value>* rw</value>
        </property>
        <property>
            <name>nfs.dump.dir</name>
            <value>/var/nfstmp</value>
        </property>


5)Can be created and deleted

    [root@nfsgw ~]# su - nfs
    [nfs@nfsgw ~]$ cd /var/nfstmp/
    [nfs@nfsgw nfstmp]$ touch 1
    [nfs@nfsgw nfstmp]$ ls
    1
    [nfs@nfsgw nfstmp]$ rm -rf 1
    [nfs@nfsgw nfstmp]$ ls
    [nfs@nfsgw nfstmp]$ cd /usr/local/hadoop/logs/
    [nfs@nfsgw logs]$ touch 1
    [nfs@nfsgw logs]$ ls
    1 hadoop-root-secondarynamenode-nn01.log    yarn-root-resourcemanager-nn01.log
    hadoop-root-namenode-nn01.log hadoop-root-secondarynamenode-nn01.out    yarn-root-resourcemanager-nn01.out
    hadoop-root-namenode-nn01.out    hadoop-root-secondarynamenode-nn01.out.1
    hadoop-root-namenode-nn01.out.1  SecurityAuth-root.audit
    [nfs@nfsgw logs]$ rm -rf 1
    [nfs@nfsgw logs]$ ls

6)Startup service

    [root@nfsgw ~]# /Usr / local / Hadoop / SBIN / hadoop-day.sh -- script. / bin / HDFS start portmap / / portmap service can only be started by root user
    starting portmap, logging to /usr/local/hadoop/logs/hadoop-root-portmap-nfsgw.out
    [root@nfsgw ~]# jps
    23714 Jps
    23670 Portmap
    [root@nfsgw ~]# su - nfs
    Last login: Mon Sep 10 12:31:58 CST 2018 on pts/0
    [nfs@nfsgw ~]$ cd /usr/local/hadoop/
    [nfs@nfsgw hadoop]$ ./sbin/hadoop-daemon.sh  --script ./bin/hdfs start nfs3  
    //nfs3 can only be started with a proxy user
    starting nfs3, logging to /usr/local/hadoop/logs/hadoop-nfs-nfs3-nfsgw.out
    [nfs@nfsgw hadoop]$ jps                    
    1362 Jps
    1309 Nfs3 
    [root@nfsgw hadoop]# JPS / / root user execution can see portmap and nfs3
    1216 Portmap
    1309 Nfs3
    1374 Jps

7)To mount the client (the client can use node4 This host)

    [root@node4 ~]# rm -rf /usr/local/hadoop
    [root@node4 ~]# yum -y install nfs-utils
    [root@node4 ~]# mount -t nfs -o \
    vers=3,proto=tcp,nolock,noatime,sync,noacl 192.168.1.64:/  /mnt/  //mount
    [root@node4 ~]# cd /mnt/
    [root@node4 mnt]# ls
    aaa  bbb  fa  system  tmp
    [root@node4 mnt]# touch a
    [root@node4 mnt]# ls
    a  aaa  bbb  fa  system  tmp
    [root@node4 mnt]# rm -rf a
    [root@node4 mnt]# ls
    aaa  bbb  fa  system  tmp

8)Realize automatic mount after power on

    [root@node4 ~]# vim /etc/fstab
    192.168.1.64:/  /mnt/ nfs  vers=3,proto=tcp,nolock,noatime,sync,noacl,_netdev 0 0 
    [root@node4 ~]# mount -a
    [root@node4 ~]# df -h
    192.168.1.26:/   64G  6.2G   58G  10% /mnt
    [root@node4 ~]# rpcinfo -p 192.168.1.64
       program vers proto   port  service
        100005    3   udp   4242  mountd
        100005    1   tcp   4242  mountd

5)Can be created and deleted

    [root@nfsgw ~]# su - nfs
    [nfs@nfsgw ~]$ cd /var/nfstmp/
    [nfs@nfsgw nfstmp]$ touch 1
    [nfs@nfsgw nfstmp]$ ls
    1
    [nfs@nfsgw nfstmp]$ rm -rf 1
    [nfs@nfsgw nfstmp]$ ls
    [nfs@nfsgw nfstmp]$ cd /usr/local/hadoop/logs/
    [nfs@nfsgw logs]$ touch 1
    [nfs@nfsgw logs]$ ls
    1 hadoop-root-secondarynamenode-nn01.log    yarn-root-resourcemanager-nn01.log
    hadoop-root-namenode-nn01.log hadoop-root-secondarynamenode-nn01.out    yarn-root-resourcemanager-nn01.out
    hadoop-root-namenode-nn01.out    hadoop-root-secondarynamenode-nn01.out.1
    hadoop-root-namenode-nn01.out.1  SecurityAuth-root.audit
    [nfs@nfsgw logs]$ rm -rf 1
    [nfs@nfsgw logs]$ ls

6)Startup service

    [root@nfsgw ~]# /Usr / local / Hadoop / SBIN / hadoop-day.sh -- script. / bin / HDFS start portmap / / portmap service can only be started by root user
    starting portmap, logging to /usr/local/hadoop/logs/hadoop-root-portmap-nfsgw.out
    [root@nfsgw ~]# jps
    23714 Jps
    23670 Portmap
    [root@nfsgw ~]# su - nfs
    Last login: Mon Sep 10 12:31:58 CST 2018 on pts/0
    [nfs@nfsgw ~]$ cd /usr/local/hadoop/
    [nfs@nfsgw hadoop]$ ./sbin/hadoop-daemon.sh  --script ./bin/hdfs start nfs3  
    //nfs3 can only be started with a proxy user
    starting nfs3, logging to /usr/local/hadoop/logs/hadoop-nfs-nfs3-nfsgw.out
    [nfs@nfsgw hadoop]$ jps                    
    1362 Jps
    1309 Nfs3 
    [root@nfsgw hadoop]# JPS / / root user execution can see portmap and nfs3
    1216 Portmap
    1309 Nfs3
    1374 Jps

7)To mount the client (the client can use node4 This host)

    [root@node4 ~]# rm -rf /usr/local/hadoop
    [root@node4 ~]# yum -y install nfs-utils
    [root@node4 ~]# mount -t nfs -o \
    vers=3,proto=tcp,nolock,noatime,sync,noacl 192.168.1.64:/  /mnt/  //mount
    [root@node4 ~]# cd /mnt/
    [root@node4 mnt]# ls
    aaa  bbb  fa  system  tmp
    [root@node4 mnt]# touch a
    [root@node4 mnt]# ls
    a  aaa  bbb  fa  system  tmp
    [root@node4 mnt]# rm -rf a
    [root@node4 mnt]# ls
    aaa  bbb  fa  system  tmp

8)Realize automatic mount after power on

    [root@node4 ~]# vim /etc/fstab
    192.168.1.64:/  /mnt/ nfs  vers=3,proto=tcp,nolock,noatime,sync,noacl,_netdev 0 0 
    [root@node4 ~]# mount -a
    [root@node4 ~]# df -h
    192.168.1.26:/   64G  6.2G   58G  10% /mnt
    [root@node4 ~]# rpcinfo -p 192.168.1.64
       program vers proto   port  service
        100005    3   udp   4242  mountd
        100005    1   tcp   4242  mountd
        100000    2   udp    111  portmapper
        100000    2   tcp    111  portmapper
        100005    3   tcp   4242  mountd
        100005    2   tcp   4242  mountd
        100003    3   tcp   2049  nfs
        100005    2   udp   4242  mountd
        100005    1   udp   4242  mountd
        100000    2   udp    111  portmapper
        100000    2   tcp    111  portmapper
        100005    3   tcp   4242  mountd
        100005    2   tcp   4242  mountd
        100003    3   tcp   2049  nfs
        100005    2   udp   4242  mountd
        100005    1   udp   4242  mountd


Published 10 original articles, praised 0, visited 48
Private letter follow

Tags: Hadoop NodeManager ssh xml

Posted on Wed, 15 Jan 2020 02:11:18 -0800 by Call-911