Hadoop natywne biblioteki nie znaleziono na OS/X

Question

Pobrałem hadoop kodu źródłowego z github i skompilowana z opcją native:

mvn package -Pdist,native -DskipTests -Dtar -Dmaven.javadoc.skip=true 

Następnie skopiowane .dylib pliki do $ HADOOP_HOME/lib

cp -p hadoop-common-project/hadoop-common/target/hadoop-common-2.7.1/lib/native/*.dylib /usr/local/Cellar/hadoop/2.7.2/libexec/share/hadoop/lib 

LD_LIBRARY_PATH została zaktualizowana i HDFS wznowiona:

echo $LD_LIBRARY_PATH 
/usr/local/Cellar/hadoop/2.7.2/libexec/lib: 
/usr/local/Cellar/hadoop/2.7.2/libexec/share/hadoop/common/lib:/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home//jre/lib 

(Uwaga: oznacza to również, że odpowiedź na Hadoop “Unable to load native-hadoop library for your platform” error on docker-spark? nie działa dla mnie ..)

Ale checknative nadal zwraca równomiernie false:

$stop-dfs.sh && start-dfs.sh && hadoop checknative 
16/06/13 16:12:32 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 
Stopping namenodes on [sparkbook] 
sparkbook: stopping namenode 
localhost: stopping datanode 
Stopping secondary namenodes [0.0.0.0] 
0.0.0.0: stopping secondarynamenode 
16/06/13 16:12:50 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 
16/06/13 16:12:50 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 
Starting namenodes on [sparkbook] 
sparkbook: starting namenode, logging to /usr/local/Cellar/hadoop/2.7.2/libexec/logs/hadoop-macuser-namenode-sparkbook.out 
localhost: starting datanode, logging to /usr/local/Cellar/hadoop/2.7.2/libexec/logs/hadoop-macuser-datanode-sparkbook.out 
Starting secondary namenodes [0.0.0.0] 
0.0.0.0: starting secondarynamenode, logging to /usr/local/Cellar/hadoop/2.7.2/libexec/logs/hadoop-macuser-secondarynamenode-sparkbook.out 
16/06/13 16:13:05 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 
16/06/13 16:13:05 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 
Native library checking: 
hadoop: false 
zlib: false 
snappy: false 
lz4:  false 
bzip2: false 
openssl: false 

Answer 1

uzyskać tej pracy na świeża instalacja macOS 10.12, musiałem wykonać następujące czynności:

1. Zainstalować kompilacji zależności używając homebrew:

   brew install cmake maven openssl protobuf@2.5 snappy 
   

2. Wyjazd Hadoop kod źródłowy

   git clone https://github.com/apache/hadoop.git 
   cd hadoop 
   git checkout rel/release-2.7.3 
   

3. Zastosuj poniższą poprawkę do kompilacji:

   diff --git a/hadoop-common-project/hadoop-common/src/CMakeLists.txt b/hadoop-common-project/hadoop-common/src/CMakeLists.txt 
   index 942b19c..8b34881 100644 
   --- a/hadoop-common-project/hadoop-common/src/CMakeLists.txt 
   +++ b/hadoop-common-project/hadoop-common/src/CMakeLists.txt 
   @@ -16,6 +16,8 @@ 
   # limitations under the License. 
   # 
   
   +SET(CUSTOM_OPENSSL_PREFIX /usr/local/opt/openssl) 
   + 
   cmake_minimum_required(VERSION 2.6 FATAL_ERROR) 
   
   # Default to release builds 
   @@ -116,8 +118,8 @@ set(T main/native/src/test/org/apache/hadoop) 
   GET_FILENAME_COMPONENT(HADOOP_ZLIB_LIBRARY ${ZLIB_LIBRARIES} NAME) 
   
   SET(STORED_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES}) 
   -set_find_shared_library_version("1") 
   -find_package(BZip2 QUIET) 
   +set_find_shared_library_version("1.0") 
   +find_package(BZip2 REQUIRED) 
   if (BZIP2_INCLUDE_DIR AND BZIP2_LIBRARIES) 
       GET_FILENAME_COMPONENT(HADOOP_BZIP2_LIBRARY ${BZIP2_LIBRARIES} NAME) 
       set(BZIP2_SOURCE_FILES 
   diff --git a/hadoop-common-project/hadoop-common/src/main/conf/core-site.xml b/hadoop-common-project/hadoop-common/src/main/conf/core-site.xml 
   index d2ddf89..ac8e351 100644 
   --- a/hadoop-common-project/hadoop-common/src/main/conf/core-site.xml 
   +++ b/hadoop-common-project/hadoop-common/src/main/conf/core-site.xml 
   @@ -17,4 +17,8 @@ 
   <!-- Put site-specific property overrides in this file. --> 
   
   <configuration> 
   +<property> 
   +<name>io.compression.codec.bzip2.library</name> 
   +<value>libbz2.dylib</value> 
   +</property> 
   </configuration> 
   diff --git a/hadoop-tools/hadoop-pipes/pom.xml b/hadoop-tools/hadoop-pipes/pom.xml 
   index 34c0110..70f23a4 100644 
   --- a/hadoop-tools/hadoop-pipes/pom.xml 
   +++ b/hadoop-tools/hadoop-pipes/pom.xml 
   @@ -52,7 +52,7 @@ 
           <mkdir dir="${project.build.directory}/native"/> 
           <exec executable="cmake" dir="${project.build.directory}/native" 
            failonerror="true"> 
   -      <arg line="${basedir}/src/ -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model}"/> 
   +      <arg line="${basedir}/src/ -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model} -DOPENSSL_ROOT_DIR=/usr/local/opt/openssl"/> 
           </exec> 
           <exec executable="make" dir="${project.build.directory}/native" failonerror="true"> 
            <arg line="VERBOSE=1"/> 
   

4. Budowa Hadoop z źródło:

   mvn package -Pdist,native -DskipTests -Dtar -Dmaven.javadoc.skip=true 
   

5. Określ JAVA_LIBRARY_PATH gdy uruchomiony Hadoop:

   $ JAVA_LIBRARY_PATH=/usr/local/opt/openssl/lib:/opt/local/lib:/usr/lib hadoop-dist/target/hadoop-2.7.3/bin/hadoop checknative -a 
   16/10/14 20:16:32 INFO bzip2.Bzip2Factory: Successfully loaded & initialized native-bzip2 library libbz2.dylib 
   16/10/14 20:16:32 INFO zlib.ZlibFactory: Successfully loaded & initialized native-zlib library 
   Native library checking: 
   hadoop: true /Users/admin/Desktop/hadoop/hadoop-dist/target/hadoop-2.7.3/lib/native/libhadoop.dylib 
   zlib: true /usr/lib/libz.1.dylib 
   snappy: true /usr/local/lib/libsnappy.1.dylib 
   lz4:  true revision:99 
   bzip2: true /usr/lib/libbz2.1.0.dylib 
   openssl: true /usr/local/opt/openssl/lib/libcrypto.dylib 
   

Answer 2

Niezbędne krokiem jest skopiowanie *.dylib ze źródeł git zbuduj dir w kodzie $HADOOP_HOME/<common dir>lib dla swojej platformy. Dla OS/X zainstalowane poprzez brew to:

cp /git/hadoop/hadoop-dist/target/hadoop-2.7.1/lib/native/ /usr/local/Cellar/hadoop/2.7.2/libexec/share/hadoop/common/ 

Widzimy wymaganych libs tam teraz:

$ll /usr/local/Cellar/hadoop/2.7.2/libexec/share/hadoop/common/*.dylib 
-rwxr-xr-x 1 macuser staff 149100 Jun 13 13:44 /usr/local/Cellar/hadoop/2.7.2/libexec/share/hadoop/common/libhadoop.dylib 
-rwxr-xr-x 1 macuser staff 149100 Jun 13 13:44 /usr/local/Cellar/hadoop/2.7.2/libexec/share/hadoop/common/libhadoop.1.0.0.dylib 

A teraz polecenie działa hadoop checknative:

$hadoop checknative 

6/06/15 09:10:59 INFO zlib.ZlibFactory: Successfully loaded & initialized native-zlib library 
Native library checking: 
hadoop: true /usr/local/Cellar/hadoop/2.7.2/libexec/share/hadoop/common/libhadoop.dylib 
zlib: true /usr/lib/libz.1.dylib 
snappy: false 
lz4:  true revision:99 
bzip2: false 
openssl: false build does not support openssl. 

Answer 3

Istnieje kilka brakujące etapy użytkownika @ andrewdotn odpowiedzi powyżej:

1) dla etapu (3), tworzenie patcha poprzez dodanie tekstu opublikowanego do pliku tekstowego, np "patch.txt", a następnie wykonaj "łatkę git apply".txt”

2) W uzupełnieniu do kopiowania plików w reżyserii javadba, niektóre aplikacje wymagają również, aby ustawić:

export HADOOP_OPTS="-Djava.library.path=${HADOOP_HOME}/lib/native" 
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${HADOOP_HOME}/lib/native 
export JAVA_LIBRARY_PATH=$JAVA_LIBRARY_PATH:${HADOOP_HOME}/lib/native 

Answer 4

jako aktualizacja do @andrewdotn odpowiedź, tutaj jest plik patch.txt być stosowane z Hadoop 2.8.1:

diff --git a/hadoop-common-project/hadoop-common/src/CMakeLists.txt b/hadoop-common-project/hadoop-common/src/CMakeLists.txt 
index c93bfe78546..e8918f9ca29 100644 
--- a/hadoop-common-project/hadoop-common/src/CMakeLists.txt 
+++ b/hadoop-common-project/hadoop-common/src/CMakeLists.txt 
@@ -20,6 +20,8 @@ 
# CMake configuration. 
# 

+SET(CUSTOM_OPENSSL_PREFIX /usr/local/opt/openssl) 
+ 
cmake_minimum_required(VERSION 2.6 FATAL_ERROR) 

list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/..) 
@@ -50,8 +52,8 @@ get_filename_component(HADOOP_ZLIB_LIBRARY ${ZLIB_LIBRARIES} NAME) 

# Look for bzip2. 
set(STORED_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES}) 
-hadoop_set_find_shared_library_version("1") 
-find_package(BZip2 QUIET) 
+hadoop_set_find_shared_library_version("1.0") 
+find_package(BZip2 REQUIRED) 
if(BZIP2_INCLUDE_DIR AND BZIP2_LIBRARIES) 
    get_filename_component(HADOOP_BZIP2_LIBRARY ${BZIP2_LIBRARIES} NAME) 
    set(BZIP2_SOURCE_FILES 
diff --git a/hadoop-common-project/hadoop-common/src/main/conf/core-site.xml b/hadoop-common-project/hadoop-common/src/main/conf/core-site.xml 
index d2ddf893e49..ac8e351f1c8 100644 
--- a/hadoop-common-project/hadoop-common/src/main/conf/core-site.xml 
+++ b/hadoop-common-project/hadoop-common/src/main/conf/core-site.xml 
@@ -17,4 +17,8 @@ 
<!-- Put site-specific property overrides in this file. --> 

<configuration> 
+<property> 
+<name>io.compression.codec.bzip2.library</name> 
+<value>libbz2.dylib</value> 
+</property> 
</configuration> 
diff --git a/hadoop-tools/hadoop-pipes/pom.xml b/hadoop-tools/hadoop-pipes/pom.xml 
index 8aafad0f7eb..d4832542265 100644 
--- a/hadoop-tools/hadoop-pipes/pom.xml 
+++ b/hadoop-tools/hadoop-pipes/pom.xml 
@@ -55,7 +55,7 @@ 
        <mkdir dir="${project.build.directory}/native"/> 
        <exec executable="cmake" dir="${project.build.directory}/native" 
         failonerror="true"> 
-      <arg line="${basedir}/src/ -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model}"/> 
+      <arg line="${basedir}/src/ -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model} -DOPENSSL_ROOT_DIR=/usr/local/opt/openssl"/> 
        </exec> 
        <exec executable="make" dir="${project.build.directory}/native" failonerror="true"> 
         <arg line="VERBOSE=1"/>