diff --git a/README.md b/README.md index e64d2da..f651399 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ To deploy in Docker Swarm: docker stack deploy -c docker-compose.yml hive ``` -To run a PrestoDB 0.181 with Hive connector: +To run PrestoSQL 346 with Hive connector: ``` docker-compose up -d presto-coordinator @@ -34,10 +34,10 @@ Load data into Hive: > LOAD DATA LOCAL INPATH '/opt/hive/examples/files/kv1.txt' OVERWRITE INTO TABLE pokes; ``` -Then query it from PrestoDB. You can get [presto.jar](https://prestosql.io/docs/current/installation/cli.html) from PrestoDB website: +Then query it from PrestoDB. You can get [presto.jar](https://prestosql.io/docs/current/installation/cli.html) from PrestoSQL website: ``` - $ wget https://repo1.maven.org/maven2/io/prestosql/presto-cli/308/presto-cli-308-executable.jar - $ mv presto-cli-308-executable.jar presto.jar + $ wget https://repo1.maven.org/maven2/io/prestosql/presto-cli/346/presto-cli-346-executable.jar + $ mv presto-cli-346-executable.jar presto.jar $ chmod +x presto.jar $ ./presto.jar --server localhost:8080 --catalog hive --schema default presto> select * from pokes; @@ -47,3 +47,4 @@ Then query it from PrestoDB. You can get [presto.jar](https://prestosql.io/docs/ * Ivan Ermilov [@earthquakesan](https://github.com/earthquakesan) (maintainer) * Yiannis Mouchakis [@gmouchakis](https://github.com/gmouchakis) * Ke Zhu [@shawnzhu](https://github.com/shawnzhu) +* Eric Semeniuc [@esemeniuc](https://github.com/esemeniuc) diff --git a/docker-compose.yml b/docker-compose.yml index a1bcc3a..972a7eb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -42,7 +42,11 @@ services: hive-metastore-postgresql: image: bde2020/hive-metastore-postgresql:2.3.0 presto-coordinator: - image: shawnzhu/prestodb:0.181 + build: ./presto + depends_on: + - hive-server + - namenode + - datanode ports: - "8080:8080" diff --git a/presto/Dockerfile b/presto/Dockerfile new file mode 100644 index 0000000..b04c7ae --- /dev/null +++ b/presto/Dockerfile @@ -0,0 +1,22 @@ +FROM bde2020/hadoop-base:latest + +ENV PRESTO_VERSION=346 +ENV PRESTO_HOME=/opt/presto +WORKDIR ${PRESTO_HOME} + +#need java 11 (requires backports for stretch) and python for recent presto +RUN echo "deb http://deb.debian.org/debian stretch-backports main" >> /etc/apt/sources.list && \ + apt update && \ + apt install -y openjdk-11-jre python && \ + rm -rf /var/lib/apt/lists/* + +RUN set -xe && curl -L https://repo1.maven.org/maven2/io/prestosql/presto-server/${PRESTO_VERSION}/presto-server-${PRESTO_VERSION}.tar.gz | tar xz && \ + mv presto-server-${PRESTO_VERSION}/* . && \ + rmdir presto-server-${PRESTO_VERSION} && \ + mkdir -p ${PRESTO_HOME}/data + +COPY etc ${PRESTO_HOME}/etc +VOLUME ["${PRESTO_HOME}/etc", "${PRESTO_HOME}/data"] +EXPOSE 8080 + +CMD ["bin/launcher", "run"] diff --git a/presto/etc/catalog/hive.properties b/presto/etc/catalog/hive.properties new file mode 100644 index 0000000..d1ad644 --- /dev/null +++ b/presto/etc/catalog/hive.properties @@ -0,0 +1,5 @@ +connector.name=hive-hadoop2 +hive.metastore.uri=thrift://hive-metastore:9083 +hive.allow-register-partition-procedure=true +hive.allow-drop-table=true +hive.config.resources=/etc/hadoop/core-site.xml,/etc/hadoop/hdfs-site.xml diff --git a/presto/etc/config.properties b/presto/etc/config.properties new file mode 100644 index 0000000..7da4b86 --- /dev/null +++ b/presto/etc/config.properties @@ -0,0 +1,7 @@ +coordinator=true +node-scheduler.include-coordinator=true +http-server.http.port=8080 +query.max-memory=5GB +query.max-memory-per-node=1GB +discovery-server.enabled=true +discovery.uri=http://presto-coordinator:8080 diff --git a/presto/etc/jvm.config b/presto/etc/jvm.config new file mode 100644 index 0000000..e77a70f --- /dev/null +++ b/presto/etc/jvm.config @@ -0,0 +1,13 @@ +-server +-Xmx16G +-XX:-UseBiasedLocking +-XX:+UseG1GC +-XX:G1HeapRegionSize=32M +-XX:+ExplicitGCInvokesConcurrent +-XX:+ExitOnOutOfMemoryError +-XX:+HeapDumpOnOutOfMemoryError +-XX:ReservedCodeCacheSize=512M +-XX:PerMethodRecompilationCutoff=10000 +-XX:PerBytecodeRecompilationCutoff=10000 +-Djdk.attach.allowAttachSelf=true +-Djdk.nio.maxCachedBufferSize=2000000 diff --git a/presto/etc/node.properties b/presto/etc/node.properties new file mode 100644 index 0000000..554164d --- /dev/null +++ b/presto/etc/node.properties @@ -0,0 +1,3 @@ +node.environment=development +node.id=presto-coordinator +node.data-dir=/opt/presto/data