diff --git a/.gitignore b/.gitignore index 140702aa0..6c42fc23c 100644 --- a/.gitignore +++ b/.gitignore @@ -48,9 +48,12 @@ __pycache__/ GNUmakefile user.props +# Build artifacts +dist/ + # Miscellaneous ###################### .vscode/ .venv/ *.egg-info/ -.coverage \ No newline at end of file +.coverage diff --git a/MANIFEST.in b/MANIFEST.in index 15618d16e..fc6912044 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -24,3 +24,6 @@ recursive-include docs/source/images * # include misc files include pynq/devices/default.xclbin + +# include all deeply nested python and data files in the core module +recursive-include pynq * diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..4bf5f9362 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,59 @@ +# build system requirements +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +# static project meta data +[project] +name = "pynq" +dynamic = ["version"] +description = "(PY)thon productivity for zy(NQ)" +readme = "README.md" +authors = [ + {name = "Xilinx PYNQ Development Team", email = "pynq_support@xilinx.com"} +] +license = {text = "BSD 3-Clause"} +requires-python = ">=3.5.2" +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: BSD License", + "Operating System :: POSIX :: Linux", +] +dependencies = [ + "pynqmetadata", + "pynqutils", + "setuptools>=24.2.0", + "cffi", + "numpy<2.0", + "nest_asyncio", + "grpcio==1.64.0", + "grpcio-tools==1.64.0", +] + +# project urls +[project.urls] +Homepage = "https://github.com/Xilinx/PYNQ" +Download = "https://github.com/Xilinx/PYNQ" + +# cli commands +[project.scripts] +pynq = "pynq._cli.cmd:main" +pynq-get-notebooks = "pynq._cli.get_notebooks:main" + +# custom build commands +[project.entry-points."distutils.commands"] +download_overlays = "pynqutils.setup_utils:du_download_overlays" + +# setuptools specific configurations +[tool.setuptools] +zip-safe = false +include-package-data = true + +# map the dynamic version to the init file +[tool.setuptools.dynamic] +version = {attr = "pynq.__version__"} + +# define package boundaries +[tool.setuptools.packages.find] +where = ["."] +include = ["pynq*"] diff --git a/setup.py b/setup.py index 6bdebbb7c..cb6c21e58 100644 --- a/setup.py +++ b/setup.py @@ -9,50 +9,36 @@ import subprocess import warnings from datetime import datetime -from distutils.dir_util import copy_tree -from distutils.file_util import copy_file, move_file from shutil import rmtree -from setuptools import Distribution, Extension, find_packages, setup +from setuptools import Distribution, Extension, setup from setuptools.command.build_ext import build_ext -# Requirement -required = [ - 'pynqmetadata', - 'pynqutils', - "setuptools>=24.2.0", - "cffi", - "numpy<2.0", - "nest_asyncio", - 'grpcio==1.64.0', - 'grpcio-tools==1.64.0', -] - +# Get remote install flag from environment REMOTE_INSTALL = os.environ.get("PYNQ_REMOTE", False) # Device family constants -ZYNQ_ARCH = "armv7l" -ZU_ARCH = "aarch64" +ZYNQ_ARCH = "armv7l" # 32-bit Zynq-7000 devices +ZU_ARCH = "aarch64" # 64-bit Zynq UltraScale+ / Versal devices + +# Allow overriding the detected architecture if "PYNQ_BUILD_ARCH" in os.environ: CPU_ARCH = os.environ["PYNQ_BUILD_ARCH"] else: CPU_ARCH = platform.machine() -CPU_ARCH_IS_SUPPORTED = CPU_ARCH in [ZYNQ_ARCH, ZU_ARCH] -# Parse version number -def find_version(file_path): - with open(file_path, "r") as fp: - version_file = fp.read() - version_match = re.search( - r"^__version__ = ['\"]([^'\"]*)['\"]", version_file, re.M - ) - if version_match: - return version_match.group(1) - raise NameError("Version string must be defined in {}.".format(file_path)) +# True only when building directly on/for supported Zynq hardware, otherwise False (e.g. dev machines) +CPU_ARCH_IS_SUPPORTED = CPU_ARCH in [ZYNQ_ARCH, ZU_ARCH] -# Exclude specified file or folder when installing overlay def exclude_file_or_folder(exclude, path): + """ + Exclude specified file or folder from the target path when installing overlay. + + Args: + exclude (str): name of the specific file or folder to delete + path (str): directory path to search within + """ for f in os.listdir(path): if f == exclude: if os.path.isdir(os.path.join(path, f)): @@ -61,8 +47,16 @@ def exclude_file_or_folder(exclude, path): os.remove(os.path.join(path, f)) -# Locate overlays in the BOARD folder def find_overlays(path): + """ + Locate and return a list of overlay directories containing (.bit) bitstreams. + + Args: + path (str): directory path to search for board overlays + + Returns: + list[str]: list representing the names of valid overlay directories + """ if os.path.isdir(path): return [ f @@ -74,40 +68,19 @@ def find_overlays(path): return [] -# Extend pynq package files by directory or by file -def extend_pynq_package(data_list): - for data in data_list: - if os.path.isdir(data): - pynq_package_files.extend( - [ - os.path.join("..", root, f) - for root, _, files in os.walk(data) - for f in files - ] - ) - elif os.path.isfile(data): - pynq_package_files.append(os.path.join("..", data)) - - # Enforce platform-dependent distribution class BinaryDistribution(Distribution): + """ + Enforce a platform-dependent distribution for the compiled package. + """ def has_ext_modules(self): + """ + return true to indicate the presence of c-extensions. + """ return True -# Extend pynq package files with Microblaze C BSPs and libraries -pynq_package_files = [] -extend_pynq_package( - [ - "pynq/lib/pynqmicroblaze", - "pynq/lib/arduino", - "pynq/lib/pmod", - "pynq/lib/rpi", - "pynq/lib/logictools", - "pynq/pl_server/default.xclbin", - ] -) - +# Native C extension source lists (video/HDMI capture + display pipeline) # Video source files _video_src = [ "pynq/lib/_pynq/_video/_video.c", @@ -135,6 +108,8 @@ def has_ext_modules(self): _common_src = ["pynq/lib/_pynq/common/xil_stubs.c"] +# Xilinx standalone BSP headers, plus the arch-specific processor headers +# (Cortex-A9 for Zynq-7000, Cortex-A53 64-bit for Zynq UltraScale+/Versal) _bsp_includes = [ "pynq/lib/_pynq/embeddedsw/lib/bsp/standalone/src/common", "pynq/lib/_pynq/embeddedsw/lib/bsp/standalone/src/arm/common", @@ -150,6 +125,7 @@ def has_ext_modules(self): "pynq/lib/_pynq/embeddedsw/lib/bsp/standalone/src/arm/cortexa53/64bit" ) +# The three notebooks copied into every board's "getting_started" folder getting_started_notebooks = [ "jupyter_notebooks.ipynb", "python_environment.ipynb", @@ -157,6 +133,7 @@ def has_ext_modules(self): ] # Merge BSP src to _video src +# (Combine all video-related sources into a single Extension() source list) video = [] video.extend(_video_gpio) video.extend(_video_vtc) @@ -164,8 +141,13 @@ def has_ext_modules(self): video.extend(_common_src) -# Copy notebooks in pynq/notebooks def copy_common_notebooks(staging_notebooks_dir): + """ + Copy the architecture-agnostic jupyter notebooks into the staging directory. + + Args: + staging_notebooks_dir (str): path indicating where to copy the notebooks + """ common_folders_files = [f for f in os.listdir("pynq/notebooks/")] for basename in common_folders_files: if basename != "arch": @@ -173,9 +155,11 @@ def copy_common_notebooks(staging_notebooks_dir): src_folder_file = os.path.join("pynq/notebooks/", basename) if os.path.isdir(src_folder_file): - copy_tree(src_folder_file, dst_folder_file) + shutil.copytree(src_folder_file, dst_folder_file, dirs_exist_ok=True) elif os.path.isfile(src_folder_file): - copy_file(src_folder_file, dst_folder_file) + shutil.copy(src_folder_file, dst_folder_file) + + # Layer in any architecture-specific notebook variants for this build if os.path.exists(os.path.join("pynq/notebooks/arch", CPU_ARCH)): dir_fd = os.open(os.path.join("pynq/notebooks/arch", CPU_ARCH), os.O_RDONLY) dirs = os.fwalk(dir_fd=dir_fd) @@ -183,24 +167,36 @@ def copy_common_notebooks(staging_notebooks_dir): if not os.path.exists(os.path.join(staging_notebooks_dir, dir)): os.mkdir(os.path.join(staging_notebooks_dir, dir)) for f in files: - copy_file( + shutil.copy( os.path.join("pynq/notebooks/arch", CPU_ARCH, dir, f), os.path.join(staging_notebooks_dir, dir, f), ) os.close(dir_fd) -# Copy notebooks in boards/BOARD/notebooks def copy_board_notebooks(staging_notebooks_dir, board): + """ + Copy notebooks specific to boards//notebooks, if present. + + Args: + staging_notebooks_dir (str): path indicating where to copy the notebooks + board (str): name of the target hardware board + """ board_folder = "boards/{}".format(board) src_folder = os.path.join(board_folder, "notebooks") dst_folder = staging_notebooks_dir if os.path.isdir(src_folder): - copy_tree(src_folder, dst_folder) + shutil.copytree(src_folder, dst_folder, dirs_exist_ok=True) -# Copy notebooks in boards/BOARD/OVERLAY/notebooks def copy_overlay_notebooks(staging_notebooks_dir, board): + """ + Download and copy the hardware overlay notebooks for the specified board. + + Args: + staging_notebooks_dir (str): path indicating where to copy the notebooks + board (str): name of the target hardware board + """ from pynqutils.setup_utils import download_overlays board_folder = "boards/{}".format(board) @@ -210,11 +206,16 @@ def copy_overlay_notebooks(staging_notebooks_dir, board): src_folder = os.path.join(board_folder, overlay, "notebooks") dst_folder = os.path.join(staging_notebooks_dir, overlay) if os.path.isdir(src_folder): - copy_tree(src_folder, dst_folder) + shutil.copytree(src_folder, dst_folder, dirs_exist_ok=True) -# Copy documentation files in docs/source and docs/source/images def copy_documentation_files(staging_notebooks_dir): + """ + Copy the "Getting Started" notebooks + images out of docs/source. + + Args: + staging_notebooks_dir (str): path indicating where to copy the files + """ doc_files = list() notebooks_getting_started_dst_dir = os.path.join( staging_notebooks_dir, "getting_started" @@ -250,7 +251,7 @@ def copy_documentation_files(staging_notebooks_dir): os.makedirs(notebooks_getting_started_dst_img_dir) for dst, files in doc_files: for f in files: - copy_file(f, dst) + shutil.copy(f, dst) if os.path.splitext(f)[1] == ".ipynb": dest_nb = os.path.join(dst, os.path.split(f)[1]) # rewrite image paths in notebooks @@ -262,8 +263,13 @@ def copy_documentation_files(staging_notebooks_dir): nb.write(text) -# Rename and copy getting started notebooks def rename_notebooks(staging_notebooks_dir): + """ + Prepend a numerical index to the getting started notebooks for ordered display. + + Args: + staging_notebooks_dir (str): path containing the unindexed notebooks + """ notebooks_getting_started_dst_dir = os.path.join( staging_notebooks_dir, "getting_started" ) @@ -273,11 +279,16 @@ def rename_notebooks(staging_notebooks_dir): dst_file = os.path.join(notebooks_getting_started_dst_dir, new_nb_name) if os.path.exists(dst_file): os.remove(dst_file) - move_file(src_file, dst_file) + shutil.move(src_file, dst_file) -# Get environment variables def check_env(): + """ + Validate and return the board and jupyter notebook environment variables. + + Returns: + tuple: contains the board name (str) and notebooks directory path (str) + """ board = None if "BOARD" not in os.environ: warnings.warn( @@ -301,25 +312,37 @@ def check_env(): return board, notebooks_dir -# Backup notebooks def backup_notebooks(notebooks_dir): + """ + Create a timestamped backup of the existing jupyter notebook directory + + Args: + notebooks_dir (str): path of the notebook directory to backup + """ if os.path.isdir(notebooks_dir): notebooks_dir_backup = "{}_{}".format( notebooks_dir, datetime.now().strftime("%Y_%m_%d_%H_%M_%S") ) - copy_tree(notebooks_dir, notebooks_dir_backup) + shutil.copy(notebooks_dir, notebooks_dir_backup, dirs_exist_ok=True) else: os.makedirs(notebooks_dir, exist_ok=True) -# Change ownership of the notebook folder def change_ownership(notebooks_dir): + """ + Grant read, write, and execute permissions to the notebook folder on unix systems. + + Args: + notebooks_dir (str): path of the notebook directory to modify + """ if os.name != 'nt': # Skip on Windows subprocess.run(["chmod", "-R", "a+rwX", notebooks_dir]) -# Copy all the notebooks def copy_notebooks(): + """ + Orchestrate the backup, copying, and renaming of all repository notebooks. + """ board, notebooks_dir = check_env() if notebooks_dir: backup_notebooks(notebooks_dir) @@ -332,16 +355,30 @@ def copy_notebooks(): change_ownership(notebooks_dir) -# Build extension includes Jupyter notebooks, in addition to C bindings class BuildExtension(build_ext): + """ + Custom build extension to compile c bindings and stage jupyter notebooks. + """ + def run_make(self, src_path, dst_path, output_lib): + """ + Invoke the hardware-specific makefile to compile a shared object library. + + Args: + src_path (str): directory path containing the source files and makefile + dst_path (str): destination directory path for the compiled library + output_lib (str): name of the generated shared object file + """ self.spawn(["make", "PYNQ_BUILD_ARCH={}".format(CPU_ARCH), "-C", src_path]) os.makedirs(os.path.join(self.build_lib, dst_path), exist_ok=True) - copy_file( + shutil.copy( src_path + output_lib, os.path.join(self.build_lib, dst_path, output_lib) ) def install_overlays(self): + """ + Copy the compiled bitstreams and overlays into the final build library. + """ board, _ = check_env() if not REMOTE_INSTALL: board_folder = "boards/{}".format(board) @@ -355,6 +392,9 @@ def install_overlays(self): ) def run(self): + """ + Execute the custom compilation steps and stage the non-python assets. + """ if not REMOTE_INSTALL: if CPU_ARCH == ZYNQ_ARCH: self.run_make("pynq/lib/_pynq/_audio/", "pynq/lib/", "libaudio.so") @@ -376,48 +416,6 @@ def run(self): self.install_overlays() -pynq_version = find_version("pynq/__init__.py") -with open("README.md", encoding="utf-8") as fh: - readme_lines = fh.readlines()[:] -long_description = "".join(readme_lines) -extend_pynq_package( - [ - "pynq/lib/_pynq/embeddedsw/XilinxProcessorIPLib/drivers/v_hdmi_common/src", - "pynq/lib/_pynq/embeddedsw/XilinxProcessorIPLib/drivers/v_hdmirxss/src", - "pynq/lib/_pynq/embeddedsw/XilinxProcessorIPLib/drivers/v_hdmirx/src", - "pynq/lib/_pynq/embeddedsw/XilinxProcessorIPLib/drivers/v_hdmitxss/src", - "pynq/lib/_pynq/embeddedsw/XilinxProcessorIPLib/drivers/v_hdmitx/src", - "pynq/lib/_pynq/embeddedsw/XilinxProcessorIPLib/drivers/video_common/src", - "pynq/lib/_pynq/embeddedsw/XilinxProcessorIPLib/drivers/vphy/src", - "pynq/lib/_pynq/embeddedsw/XilinxProcessorIPLib/drivers/vtc/src", - "pynq/lib/_pynq/embeddedsw/XilinxProcessorIPLib/drivers/iic/src", - "pynq/lib/_pynq/embeddedsw/XilinxProcessorIPLib/drivers/gpio/src", - "pynq/lib/_pynq/embeddedsw/XilinxProcessorIPLib/drivers/iicps/src", - "pynq/lib/_pynq/embeddedsw/XilinxProcessorIPLib/drivers/scugic/src", - "pynq/lib/_pynq/embeddedsw/XilinxProcessorIPLib/drivers/axivdma/src", - "pynq/lib/_pynq/embeddedsw/XilinxProcessorIPLib/drivers/mipicsiss/src", - "pynq/lib/_pynq/embeddedsw/XilinxProcessorIPLib/drivers/csi/src", - "pynq/lib/_pynq/embeddedsw/XilinxProcessorIPLib/drivers/dphy/src", - "pynq/lib/_pynq/embeddedsw/lib/bsp/standalone/src", - "pynq/lib/_pynq/embeddedsw_lib.mk", - "pynq/lib/_pynq/common", - "pynq/lib/_pynq/_audio", - "pynq/lib/_pynq/_video", - "pynq/lib/_pynq/_video/bsp/vtc", - "pynq/lib/_pynq/_video/bsp/gpio", - "pynq/lib/_pynq/_displayport", - "pynq/lib/_pynq/_xhdmi", - "pynq/lib/_pynq/_xiic", - "pynq/lib/_pynq/_pcam5c", - "pynq/notebooks", - "pynq/tests", - "pynq/metadata", - "pynq/devices", - "pynq/lib/tests", - "pynq/remote", - ] -) - if REMOTE_INSTALL: ext_modules = [] # no extension modules for remote install else: @@ -438,39 +436,10 @@ def run(self): else: ext_modules = [] - setup( - name="pynq", - version=pynq_version, - description="(PY)thon productivity for zy(NQ)", - long_description=long_description, - long_description_content_type="text/markdown", - author="Xilinx PYNQ Development Team", - author_email="pynq_support@xilinx.com", - url="https://github.com/Xilinx/PYNQ", - packages=find_packages(), cmdclass={ "build_ext": BuildExtension, }, distclass=BinaryDistribution, - python_requires=">=3.5.2", - install_requires=required, - download_url="https://github.com/Xilinx/PYNQ", - package_data={ - "pynq": pynq_package_files, - }, - entry_points={ - "console_scripts": [ - "pynq = pynq._cli.cmd:main", - "pynq-get-notebooks = pynq._cli.get_notebooks:main", - ], - "distutils.commands": [ - "download_overlays = pynqutils.setup_utils:du_download_overlays" - ], - }, ext_modules=ext_modules, - zip_safe=False, - license="BSD 3-Clause", ) - -