diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9748859..5ba3994 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -71,3 +71,11 @@ jobs: context: . push: true tags: p4lang/third-party:${{ steps.get-tag.outputs.tag }} + - name: Build and push Dockerfile.24 image if needed + if: ${{ github.repository == 'p4lang/third-party' && github.event_name == 'push' }} + uses: docker/build-push-action@v2 + with: + context: . + file: Dockerfile.24 + push: true + tags: p4lang/third-party:${{ steps.get-tag.outputs.tag }}-24 diff --git a/Dockerfile b/Dockerfile index f3f8bf3..bf3ba69 100644 --- a/Dockerfile +++ b/Dockerfile @@ -194,11 +194,8 @@ WORKDIR /grpc/ RUN export PYTHON3_VERSION=`python3 -c 'import sys; version=sys.version_info[:3]; print("python{0}.{1}".format(*version))'` && \ echo "import site; site.addsitedir('/usr/local/lib/$PYTHON3_VERSION/site-packages')" \ > /usr/local/lib/$PYTHON3_VERSION/dist-packages/use_site_packages.pth -# We don't use `--ignore-installed` here because otherwise we won't use the -# installed version of the protobuf python package that we copied from the -# protobuf build image. RUN pip3 install --user -rrequirements.txt -RUN env GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip3 install --user --ignore-installed . +RUN pip3 install --user . # Build libyang FROM base-builder as libyang diff --git a/Dockerfile.24 b/Dockerfile.24 new file mode 100644 index 0000000..6d118ff --- /dev/null +++ b/Dockerfile.24 @@ -0,0 +1,471 @@ +# # Shared Dockerfile for third party dependencies used in projects in the p4lang +# # organization. +# # +# # Best practices for multi-stage builds are available at +# # https://docs.docker.com/develop/develop-images/dockerfile_best-practices/ +# # +# # The approach used in this Dockerfile is as follows: +# # +# # A multi-stage build creates a separate image for each third party dependency and +# # then copies only the binaries to a single output image at the end. +# # +# # If you add a new dependency, please: +# # (1) Create a new build image for it. (It should have its own FROM section.) +# # (2) Ensure that it installs everything that should be included in the final +# # image to `/output/usr/local`. Use DESTDIR and PYTHONUSERBASE for this. +# # Be sure to install python packages and code using +# # `pip install --user --ignore-installed`. If you absolutely *must* use +# # `python setup.py install`, take a look at the protobuf build image and +# # then reconsider. +# # (3) Use COPY to place the contents of the build image's `/output/usr/local` +# # in the final image. +# # (4) If the new dependency requires that certain apt packages be installed at +# # runtime (as opposed to at build time), create a new _DEPS variable and +# # install the new packages with all of the others; look at how the final +# # image is constructed and you'll understand the pattern. +# # +# # In general you don't have to worry about the size of intermediate build images, +# # but please minimize the amount of data and the number of layers that end up in the +# # final image. + +# # Create an image with tools used as a base for the next layers +# FROM ubuntu:24.04 as base-builder +# ARG MAKEFLAGS=-j2 +# ENV DEBIAN_FRONTEND=noninteractive \ +# CFLAGS="-Os" \ +# CXXFLAGS="-Os" \ +# LDFLAGS="-Wl,-s" \ +# PYTHONUSERBASE=/output/usr/local \ +# LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/output/usr/local/lib/ +# RUN apt-get update -qq && apt-get install -qq --no-install-recommends \ +# autoconf \ +# automake \ +# bison \ +# build-essential \ +# ca-certificates \ +# cmake \ +# cython3 \ +# flex \ +# g++ \ +# libavl-dev \ +# libboost-dev \ +# libboost-test-dev \ +# libev-dev \ +# libevent-dev \ +# libffi-dev \ +# libmemcached-dev \ +# libpcap-dev \ +# libpcre3-dev \ +# libprotobuf-c-dev \ +# libssl-dev \ +# libtool \ +# make \ +# pkg-config \ +# protobuf-c-compiler \ +# python3 \ +# python3-dev \ +# python3-pip \ +# python3-setuptools && \ +# ldconfig && \ +# mkdir -p /output/usr/local + +# RUN pip3 install --break-system-packages "Cython<3" + +# # Build ccache. +# FROM base-builder as ccache +# ENV RUN_FROM_BUILD_FARM=yes +# COPY ./ccache /ccache/ +# WORKDIR /ccache/ +# # Tell the ccache build system not to bother with things like documentation. +# RUN ./autogen.sh && \ +# ./configure --enable-memcached && \ +# make && \ +# # `make install` assumes that we *did* build the docs; make it happy. +# touch ccache.1 && \ +# make DESTDIR=/output install + +# # Build PTF. +# FROM base-builder as ptf +# COPY ./ptf /ptf/ +# WORKDIR /ptf/ +# RUN pip3 install --user --ignore-installed wheel pypcap && \ +# pip3 install --user --ignore-installed -rrequirements.txt && \ +# pip3 install --user --ignore-installed . + +# # Build nanomsg. +# FROM base-builder as nanomsg +# COPY ./nanomsg /nanomsg/ +# RUN mkdir -p /nanomsg/build/ +# WORKDIR /nanomsg/build/ +# RUN cmake .. && \ +# make DESTDIR=/output install + +# # Build nnpy. +# FROM base-builder as nnpy +# COPY --from=nanomsg /output/usr/local /usr/local/ +# COPY ./nnpy /nnpy/ +# WORKDIR /nnpy/ +# RUN ldconfig && \ +# pip3 install --user --ignore-installed wheel cffi && \ +# pip3 install --user --ignore-installed . + +# # Build Thrift. +# FROM base-builder as thrift +# COPY ./thrift /thrift/ +# WORKDIR /thrift/ +# RUN ./bootstrap.sh && \ +# ./configure \ +# --with-cpp=yes \ +# --with-python=yes \ +# --with-c_glib=no \ +# --with-java=no \ +# --with-ruby=no \ +# --with-erlang=no \ +# --with-go=no \ +# --with-nodejs=no \ +# --enable-tests=no && \ +# make && \ +# make DESTDIR=/output install-strip +# WORKDIR /thrift/lib/py/ +# RUN pip3 install --user --ignore-installed . + +# # Build Protocol Buffers. +# FROM base-builder as protobuf +# COPY ./protobuf /protobuf/ +# WORKDIR /protobuf/ +# RUN ./autogen.sh && \ +# ./configure && \ +# make && \ +# make DESTDIR=/output install-strip +# WORKDIR /protobuf/python/ +# # Protobuf is using a deprecated technique to install Python package with +# # easy install. This is causing issues since 2021-04-21. (https://discuss.python.org/t/pypi-org-recently-changed/8433) +# # We have to install pip and install six ourselves so we do not trigger the +# # broken protobuf install process. +# RUN pip3 install --user --ignore-installed wheel six && \ +# python3 setup.py install --user --cpp_implementation +# # We'll finish up the process of building protobuf below, but first, a bit of +# # explanation. +# # +# # Since we can't use `pip` with protobuf's `setup.py`, the package gets +# # installed via setuptools in an "egg" - a directory which acts as a python +# # module, but isn't a package and thus isn't automatically part of the package +# # namespace. Eggs need their contents to be added to python's `sys.path` to +# # become visible. Hooks that run early during python startup normally read +# # `setuptools.pth` and other `.pth` files and add all of the directories +# # referenced in those files to the path; egg contents are placed in those files +# # to be made available to the rest of the python installation. +# # +# # In multistage builds this doesn't work so well, because those `.pth` files can +# # easily be overwritten by other versions from a different image, and the eggs +# # we're installing here will no longer appear in `sys.path`. To work around +# # that, we just copy the contents of the existing `.pth` files into a new one +# # with a name we're sure won't be overwritten. Python will read this new `.pth` +# # file at startup and include the eggs in the path. +# # +# # If you're wondering why the `grep -v` is necessary, meditate upon the fact +# # that these files aren't merely metadata but also executable code, and +# # setuptools, by design, uses this feature to inject code into every python +# # program that runs on your system. +# RUN export PYTHON3_VERSION=`python3 -c 'import sys; version=sys.version_info[:3]; print("python{0}.{1}".format(*version))'` && \ +# cd /output/usr/local/lib/$PYTHON3_VERSION/site-packages&& \ +# cat *.pth | grep -v "import sys" | sort -u > docker_protobuf.pth + +# # Build gRPC. +# # The gRPC build system should detect that a version of protobuf is already +# # installed and should not try to install the third-party one included as a +# # submodule in the grpc repository. +# FROM base-builder as grpc +# COPY --from=protobuf /output/usr/local /usr/local/ +# RUN ldconfig +# COPY ./grpc /grpc/ +# # See https://github.com/grpc/grpc/blob/master/BUILDING.md +# RUN mkdir -p /grpc/cmake/build +# WORKDIR /grpc/cmake/build/ +# RUN cmake ../.. \ +# -DgRPC_INSTALL=ON \ +# -DCMAKE_BUILD_TYPE=Release \ +# -DgRPC_PROTOBUF_PROVIDER=package \ +# -DgRPC_SSL_PROVIDER=package && \ +# make DESTDIR=/output install +# WORKDIR /grpc/ +# # `pip install --user` will place things in `site-packages`, but Ubuntu expects +# # `dist-packages` by default, so we need to set configure `site-packages` as an +# # additional "site-specific directory". +# # Without this, our earlier installation of Protobuf will be ignored. +# RUN export PYTHON3_VERSION=`python3 -c 'import sys; version=sys.version_info[:3]; print("python{0}.{1}".format(*version))'` && \ +# echo "import site; site.addsitedir('/usr/local/lib/$PYTHON3_VERSION/site-packages')" \ +# > /usr/local/lib/$PYTHON3_VERSION/dist-packages/use_site_packages.pth +# # We don't use `--ignore-installed` here because otherwise we won't use the +# # installed version of the protobuf python package that we copied from the +# # protobuf build image. +# # RUN pip3 install --user -rrequirements.txt +# # RUN env GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip3 install --user --ignore-installed . + +# RUN pip3 install --user -rrequirements.txt +# RUN pip3 install --user "Cython==0.29.36" +# RUN env GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip3 install --user --ignore-installed . + +# # Build libyang +# FROM base-builder as libyang +# COPY ./libyang /libyang/ +# RUN mkdir -p /libyang/build/ +# WORKDIR /libyang/build/ +# RUN cmake .. && \ +# make DESTDIR=/output install + +# # Build sysrepo +# FROM base-builder as sysrepo +# COPY --from=libyang /output/usr/local /usr/local/ +# RUN ldconfig +# COPY ./sysrepo /sysrepo/ +# RUN mkdir -p /sysrepo/build/ +# WORKDIR /sysrepo/build/ +# # CALL_TARGET_BINS_DIRECTLY=Off is needed here because of the use of DESTDIR +# # Without it sysrepoctl is executed at install time and assumes YANG files are +# # under /etc/sysrepo/yang +# RUN cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_EXAMPLES=Off -DCALL_TARGET_BINS_DIRECTLY=Off .. && \ +# make DESTDIR=/output install + +# # Construct the final image. +# FROM ubuntu:20.04 +# LABEL maintainer="P4 Developers " +# ARG DEBIAN_FRONTEND=noninteractive +# ARG MAKEFLAGS=-j2 +# RUN CCACHE_RUNTIME_DEPS="libmemcached-dev" && \ +# PTF_RUNTIME_DEPS="libpcap-dev python3-minimal tcpdump" && \ +# NNPY_RUNTIME_DEPS="python3-minimal" && \ +# THRIFT_RUNTIME_DEPS="libssl1.1 python3-minimal" && \ +# GRPC_RUNTIME_DEPS="libssl-dev python3-minimal python3-setuptools" && \ +# SYSREPO_RUNTIME_DEPS="libpcre3 libavl1 libev4 libprotobuf-c1" && \ +# apt-get update && \ +# apt-get install -y --no-install-recommends $CCACHE_RUNTIME_DEPS \ +# $PTF_RUNTIME_DEPS \ +# $NNPY_RUNTIME_DEPS \ +# $THRIFT_RUNTIME_DEPS \ +# $GRPC_RUNTIME_DEPS \ +# $SYSREPO_RUNTIME_DEPS \ +# python-is-python3 && \ +# rm -rf /var/cache/apt/* /var/lib/apt/lists/* +# # Configure ccache so that descendant containers won't need to. +# COPY ./docker/ccache.conf /usr/local/etc/ccache.conf +# # Copy files from the build containers. +# COPY --from=ccache /output/usr/local /usr/local/ +# COPY --from=ptf /output/usr/local /usr/local/ +# COPY --from=nanomsg /output/usr/local /usr/local/ +# COPY --from=nnpy /output/usr/local /usr/local/ +# COPY --from=thrift /output/usr/local /usr/local/ +# COPY --from=protobuf /output/usr/local /usr/local/ +# COPY --from=grpc /output/usr/local /usr/local/ +# COPY --from=libyang /output/usr/local /usr/local/ +# COPY --from=sysrepo /output/usr/local /usr/local/ +# COPY --from=sysrepo /output/etc /etc/ +# # `pip install --user` will place things in `site-packages`, but Ubuntu expects +# # `dist-packages` by default, so we need to set configure `site-packages` as an +# # additional "site-specific directory". +# RUN export PYTHON3_VERSION=`python3 -c 'import sys; version=sys.version_info[:3]; print("python{0}.{1}".format(*version))'` && \ +# echo "import site; site.addsitedir('/usr/local/lib/$PYTHON3_VERSION/site-packages')" \ +# > /usr/local/lib/$PYTHON3_VERSION/dist-packages/use_site_packages.pth + +# RUN ldconfig + + +# =============================== +# Base Builder (Ubuntu 24.04) +# =============================== +FROM ubuntu:24.04 AS base-builder + +ARG MAKEFLAGS=-j2 +ENV DEBIAN_FRONTEND=noninteractive \ + CFLAGS="-Os" \ + CXXFLAGS="-Os" \ + LDFLAGS="-Wl,-s" \ + PYTHONUSERBASE=/output/usr/local + +RUN apt-get update -qq && apt-get install -qq --no-install-recommends \ + autoconf \ + automake \ + bison \ + build-essential \ + gcc-11 \ + g++-11 \ + ca-certificates \ + cmake \ + cython3 \ + flex \ + g++ \ + libavl-dev \ + libboost-dev \ + libboost-test-dev \ + libev-dev \ + libevent-dev \ + libffi-dev \ + libmemcached-dev \ + libpcap-dev \ + libpcre3-dev \ + libprotobuf-c-dev \ + protobuf-c-compiler \ + libprotobuf-dev \ + protobuf-compiler \ + libssl-dev \ + libtool \ + make \ + pkg-config \ + python3 \ + python3-dev \ + python3-pip \ + python3-setuptools && \ + ldconfig && \ + mkdir -p /output/usr/local + + +ENV CC=gcc-11 +ENV CXX=g++-11 + + + +# =============================== +# ccache +# =============================== +FROM base-builder AS ccache +ENV RUN_FROM_BUILD_FARM=yes +COPY ./ccache /ccache/ +WORKDIR /ccache/ +RUN ./autogen.sh && \ + ./configure --enable-memcached && \ + make && \ + touch ccache.1 && \ + make DESTDIR=/output install + +# =============================== +# PTF +# =============================== +FROM base-builder AS ptf +COPY ./ptf /ptf/ +WORKDIR /ptf/ +RUN python3 -m pip install pcapy-ng --break-system-packages && \ + python3 -m pip install -r requirements.txt --break-system-packages && \ + python3 -m pip install . --break-system-packages +# =============================== +# nanomsg +# =============================== +FROM base-builder AS nanomsg +COPY ./nanomsg /nanomsg/ +RUN mkdir -p /nanomsg/build +WORKDIR /nanomsg/build +RUN cmake .. && make DESTDIR=/output install + +# =============================== +# nnpy +# =============================== +FROM base-builder AS nnpy +COPY --from=nanomsg /output/usr/local /usr/local/ +COPY ./nnpy /nnpy/ +WORKDIR /nnpy/ +RUN ldconfig && \ + python3 -m pip install wheel cffi --break-system-packages && \ + python3 -m pip install --no-build-isolation . --break-system-packages + +# =============================== +# Thrift +# =============================== +FROM base-builder AS thrift +COPY ./thrift /thrift/ +WORKDIR /thrift/ +RUN ./bootstrap.sh && \ + ./configure \ + --with-cpp=yes \ + --with-python=yes \ + --with-c_glib=no \ + --with-java=no \ + --with-ruby=no \ + --with-erlang=no \ + --with-go=no \ + --with-nodejs=no \ + --enable-tests=no && \ + make && \ + make DESTDIR=/output install-strip + +WORKDIR /thrift/lib/py/ +RUN python3 -m pip install . --break-system-packages + +# =============================== +# gRPC +# =============================== +FROM base-builder AS grpc + +COPY ./grpc /grpc/ +RUN mkdir -p /grpc/cmake/build +WORKDIR /grpc/cmake/build + +RUN cmake ../.. \ + -DgRPC_INSTALL=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -DgRPC_PROTOBUF_PROVIDER=module \ + -DgRPC_SSL_PROVIDER=package \ + -DCMAKE_CXX_STANDARD=17 \ + -DCMAKE_CXX_STANDARD_REQUIRED=ON && \ + make -j$(nproc) DESTDIR=/output install + +WORKDIR /grpc/ + +RUN python3 -m pip install -r requirements.txt --break-system-packages + +RUN python3 -m pip install . --break-system-packages + +# =============================== +# libyang +# =============================== +FROM base-builder AS libyang +COPY ./libyang /libyang/ +RUN mkdir -p /libyang/build +WORKDIR /libyang/build +RUN cmake .. && make DESTDIR=/output install + +# =============================== +# sysrepo +# =============================== +FROM base-builder AS sysrepo +COPY --from=libyang /output/usr/local /usr/local/ +RUN ldconfig +COPY ./sysrepo /sysrepo/ +RUN mkdir -p /sysrepo/build +WORKDIR /sysrepo/build +RUN cmake -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_EXAMPLES=Off \ + -DCALL_TARGET_BINS_DIRECTLY=Off .. && \ + make DESTDIR=/output install + +# =============================== +# Final Runtime Image (Ubuntu 24.04) +# =============================== +FROM ubuntu:24.04 + +ARG DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y --no-install-recommends \ + libmemcached-dev \ + libpcap-dev \ + python3-minimal \ + tcpdump \ + libssl-dev \ + libpcre3 \ + libavl1 \ + libev4 \ + libprotobuf-c1 \ + python-is-python3 && \ + rm -rf /var/lib/apt/lists/* + +COPY ./docker/ccache.conf /usr/local/etc/ccache.conf + +COPY --from=ccache /output/usr/local /usr/local/ +COPY --from=ptf /output/usr/local /usr/local/ +COPY --from=nanomsg /output/usr/local /usr/local/ +COPY --from=nnpy /output/usr/local /usr/local/ +COPY --from=thrift /output/usr/local /usr/local/ +COPY --from=grpc /output/usr/local /usr/local/ +COPY --from=libyang /output/usr/local /usr/local/ +COPY --from=sysrepo /output/usr/local /usr/local/ +COPY --from=sysrepo /output/etc /etc/ + +RUN ldconfig diff --git a/grpc b/grpc index dc78581..0a82c02 160000 --- a/grpc +++ b/grpc @@ -1 +1 @@ -Subproject commit dc78581af30da834b7b95572f109bf6c708686e0 +Subproject commit 0a82c02a9b817a53574994374dcff53f2e961df2