Compiling and installing Hadoop 3 in CentOS environment

Based on the environment ：Centos 7.7

Compilation environment software installation directory

mkdir -p /export/server

Hadoop Compilation and installation

• Installation and compilation related dependencies

  1、yum install gcc gcc-c++ -y
  # The following command does not need to be executed Manual installation cmake
  2、yum install make cmake #( here cmake The recommended version is 3.6 Above version , The version is low and the source code cannot be compiled ！ It can be installed manually )
  3、yum install autoconf automake libtool curl -y
  4、yum install lzo-devel zlib-devel openssl openssl-devel ncurses-devel -y
  5、yum install snappy snappy-devel bzip2 bzip2-devel lzo lzo-devel lzop libXtst -y
  

• Manual installation cmake

  #yum Uninstall installed cmake Version low 
  yum erase cmake
  # decompression 
  tar zxvf cmake-3.13.5.tar.gz
  # Compilation and installation 
  cd /export/server/cmake-3.13.5
  ./configure
  make && make install
  # verification 
  [[email protected] ~]# cmake -version 
  cmake version 3.13.5
  # If the version is not displayed correctly Please disconnect SSH Connect Override login 
  

• Manual installation snappy

  # Uninstall the installed 
  cd /usr/local/lib
  rm -rf libsnappy*
  # Upload and unzip 
  tar zxvf snappy-1.1.3.tar.gz
  # Compilation and installation 
  cd /export/server/snappy-1.1.3
  ./configure
  make && make install
  # Verify that... Is installed 
  [[email protected] snappy-1.1.3]# ls -lh /usr/local/lib |grep snappy
  -rw-r--r-- 1 root root 511K Nov 4 17:13 libsnappy.a
  -rwxr-xr-x 1 root root 955 Nov 4 17:13 libsnappy.la
  lrwxrwxrwx 1 root root 18 Nov 4 17:13 libsnappy.so -> libsnappy.so.1.3.0
  lrwxrwxrwx 1 root root 18 Nov 4 17:13 libsnappy.so.1 -> libsnappy.so.1.3.0
  -rwxr-xr-x 1 root root 253K Nov 4 17:13 libsnappy.so.1.3.0
  

• Installation configuration JDK 1.8

  # Unzip the installation package 
  tar zxvf jdk-8u65-linux-x64.tar.gz
  # Configure environment variables 
  vim /etc/profile
  export JAVA_HOME=/export/server/jdk1.8.0_65
  export PATH=$PATH:$JAVA_HOME/bin
  export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
  source /etc/profile
  # Verify that the installation was successful 
  java -version
  java version "1.8.0_65"
  Java(TM) SE Runtime Environment (build 1.8.0_65-b17)
  Java HotSpot(TM) 64-Bit Server VM (build 25.65-b01, mixed mode)
  You have new mail in /var/spool/mail/root
  

• Installation configuration maven

  # Unzip the installation package 
  tar zxvf apache-maven-3.5.4-bin.tar.gz
  # Configure environment variables 
  vim /etc/profile
  export MAVEN_HOME=/export/server/apache-maven-3.5.4
  export MAVEN_OPTS="-Xms4096m -Xmx4096m"
  export PATH=:$MAVEN_HOME/bin:$PATH
  source /etc/profile
  # Verify that the installation was successful 
  [[email protected] ~]# mvn -v
  Apache Maven 3.5.4
  # add to maven Alibaba cloud warehouse address Speed up domestic compilation 
  vim /export/server/apache-maven-3.5.4/conf/settings.xml
  <mirrors>
  <mirror>
  <id>alimaven</id>
  <name>aliyun maven</name>
  <url>http://maven.aliyun.com/nexus/content/groups/public/</url>
  <mirrorOf>central</mirrorOf>
  </mirror>
  </mirrors>
  

• install ProtocolBuffer 2.5.0

  # decompression 
  tar zxvf protobuf-2.5.0.tar.gz
  # Compilation and installation 
  cd /export/server/protobuf-2.5.0
  ./configure
  make && make install
  # Verify that the installation was successful 
  [[email protected] protobuf-2.5.0]# protoc --version
  libprotoc 2.5.0
  

• compile hadoop

  # Upload and unzip the source package 
  tar zxvf hadoop-3.1.4-src.tar.gz
  # compile 
  cd /export/server/hadoop-3.1.4-src
  mvn clean package -Pdist,native -DskipTests -Dtar -Dbundle.snappy -Dsnappy.lib=/usr/local/lib
  # Parameter description ：
  Pdist,native ： Recompile the generated hadoop Dynamic library ;
  DskipTests ： Skip the test
  Dtar ： Finally, put the document in tar pack
  Dbundle.snappy ： add to snappy Compression support 【 Downloading from the default official website is not supported 】
  Dsnappy.lib=/usr/local/lib ： finger snappy The library path installed on the compiling machine
  

• The path of the compiled installation package

  /export/server/hadoop-3.1.4-src/hadoop-dist/target