diff --git a/Dockerfile b/Dockerfile index 037fe91..7cb5405 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM bde2020/hadoop-base:2.0.0-hadoop2.7.4-java8 +FROM bde2020/hadoop-base:2.0.0-hadoop3.2.1-java8 MAINTAINER Yiannis Mouchakis MAINTAINER Ivan Ermilov @@ -8,7 +8,7 @@ ARG HIVE_VERSION # Set HIVE_VERSION from arg if provided at build, env if provided at run, or default # https://docs.docker.com/engine/reference/builder/#using-arg-variables # https://docs.docker.com/engine/reference/builder/#environment-replacement -ENV HIVE_VERSION=${HIVE_VERSION:-2.3.2} +ENV HIVE_VERSION=${HIVE_VERSION:-3.1.3} ENV HIVE_HOME /opt/hive ENV PATH $HIVE_HOME/bin:$PATH @@ -17,11 +17,20 @@ ENV HADOOP_HOME /opt/hadoop-$HADOOP_VERSION WORKDIR /opt #Install Hive and PostgreSQL JDBC -RUN apt-get update && apt-get install -y wget procps && \ +RUN sed -i 's/deb.debian.org/archive.debian.org/g' /etc/apt/sources.list && \ + sed -i 's/security.debian.org/archive.debian.org/g' /etc/apt/sources.list && \ + sed -i '/stretch-updates/d' /etc/apt/sources.list && \ + apt-get update && apt-get install -y wget procps && \ wget https://archive.apache.org/dist/hive/hive-$HIVE_VERSION/apache-hive-$HIVE_VERSION-bin.tar.gz && \ tar -xzvf apache-hive-$HIVE_VERSION-bin.tar.gz && \ mv apache-hive-$HIVE_VERSION-bin hive && \ - wget https://jdbc.postgresql.org/download/postgresql-9.4.1212.jar -O $HIVE_HOME/lib/postgresql-jdbc.jar && \ + wget https://jdbc.postgresql.org/download/postgresql-42.7.7.jar -O $HIVE_HOME/lib/postgresql-42.7.7.jar && \ + #添加S3依赖 + wget https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.2.1/hadoop-aws-3.2.1.jar -O $HIVE_HOME/lib/hadoop-aws-3.2.1.jar && \ + wget https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/1.11.901/aws-java-sdk-bundle-1.11.901.jar -O $HIVE_HOME/lib/aws-java-sdk-bundle-1.11.901.jar && \ + # 解决Hive与Hadoop之间的guava库兼容性问题 + cp $HADOOP_HOME/share/hadoop/common/lib/guava-*.jar $HIVE_HOME/lib/ && \ + rm $HIVE_HOME/lib/guava-19.0.jar && \ rm apache-hive-$HIVE_VERSION-bin.tar.gz && \ apt-get --purge remove -y wget && \ apt-get clean && \ @@ -40,10 +49,10 @@ ADD conf/hive-log4j2.properties $HIVE_HOME/conf ADD conf/ivysettings.xml $HIVE_HOME/conf ADD conf/llap-daemon-log4j2.properties $HIVE_HOME/conf -COPY startup.sh /usr/local/bin/ +COPY startup.sh /usr/local/bin/startup.sh RUN chmod +x /usr/local/bin/startup.sh -COPY entrypoint.sh /usr/local/bin/ +COPY entrypoint.sh /usr/local/bin/entrypoint.sh RUN chmod +x /usr/local/bin/entrypoint.sh EXPOSE 10000 diff --git a/conf/hive-env.sh b/conf/hive-env.sh index 91a2905..678296b 100644 --- a/conf/hive-env.sh +++ b/conf/hive-env.sh @@ -51,4 +51,4 @@ # export HIVE_CONF_DIR= # Folder containing extra ibraries required for hive compilation/execution can be controlled by: -# export HIVE_AUX_JARS_PATH= +# export HIVE_AUX_JARS_PATH= \ No newline at end of file diff --git a/entrypoint.sh b/entrypoint.sh index baefba5..7679587 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,5 +1,4 @@ #!/bin/bash - # Set some sensible defaults export CORE_CONF_fs_defaultFS=${CORE_CONF_fs_defaultFS:-hdfs://`hostname -f`:8020} @@ -115,4 +114,4 @@ do wait_for_it ${i} done -exec $@ +exec $@ \ No newline at end of file diff --git a/startup.sh b/startup.sh index f73e066..8086481 100644 --- a/startup.sh +++ b/startup.sh @@ -1,9 +1,8 @@ #!/bin/bash - hadoop fs -mkdir /tmp hadoop fs -mkdir -p /user/hive/warehouse hadoop fs -chmod g+w /tmp hadoop fs -chmod g+w /user/hive/warehouse cd $HIVE_HOME/bin -./hiveserver2 --hiveconf hive.server2.enable.doAs=false +./hiveserver2 --hiveconf hive.server2.enable.doAs=false \ No newline at end of file