diff --git a/0001-no-forced-glibc.patch b/0001-no-forced-glibc.patch new file mode 100644 index 000000000000..b4b9d8a4936c --- /dev/null +++ b/0001-no-forced-glibc.patch @@ -0,0 +1,46 @@ +diff --git a/truffle/mx.truffle/suite.py b/truffle/mx.truffle/suite.py +index 6948225d6bd..ecf970ce7c2 100644 +--- a/truffle/mx.truffle/suite.py ++++ b/truffle/mx.truffle/suite.py +@@ -1943,18 +1943,8 @@ suite = { + "windows-amd64", + "windows-aarch64", + ], +- "os": { +- "linux": { +- "layout": { +- # only glibc. +- "META-INF/resources/nfi-native/libnfi///bin/" : "dependency:com.oracle.truffle.nfi.native/linux-*/glibc/*" +- }, +- }, +- "": { +- "layout": { +- "META-INF/resources/nfi-native/libnfi///bin/" : "dependency:com.oracle.truffle.nfi.native/*/*/*", +- }, +- }, ++ "layout": { ++ "META-INF/resources/nfi-native/libnfi///bin/" : "dependency:com.oracle.truffle.nfi.native/*/*/*", + }, + "description" : "Contains the native library needed by the libffi NFI backend.", + "maven": False, +@@ -2300,18 +2290,8 @@ suite = { + "native" : True, + "platformDependent" : True, + "description" : "Truffle NFI support distribution for the GraalVM", +- "os": { +- "linux": { +- "layout": { +- # only glibc. +- "./" : "dependency:com.oracle.truffle.nfi.native/linux-*/glibc/*" +- }, +- }, +- "": { +- "layout": { +- "./" : "dependency:com.oracle.truffle.nfi.native/*/*/*", +- }, +- }, ++ "layout": { ++ "./" : "dependency:com.oracle.truffle.nfi.native/*/*/*", + }, + "maven" : False, + "graalCompilerSourceEdition": "ignore", diff --git a/0002-zig.patch b/0002-zig.patch new file mode 100644 index 000000000000..9361d978b79c --- /dev/null +++ b/0002-zig.patch @@ -0,0 +1,338 @@ +diff --git a/sdk/mx.sdk/mx_sdk_toolchain.py b/sdk/mx.sdk/mx_sdk_toolchain.py +index 79c1a1daa61..87be371fb16 100644 +--- a/sdk/mx.sdk/mx_sdk_toolchain.py ++++ b/sdk/mx.sdk/mx_sdk_toolchain.py +@@ -98,9 +98,8 @@ class ToolchainTestBuildTask(mx.BuildTask): + def build(self): + mx_util.ensure_dir_exists(self.subject.get_output_root()) + +- toolchainPath = mx.distribution('LLVM_TOOLCHAIN').output +- clang = os.path.join(toolchainPath, 'bin', 'clang') +- clangxx = os.path.join(toolchainPath, 'bin', 'clang++') ++ toolchainPath = mx.distribution('ZIG_DIR').output ++ zig = os.path.join(toolchainPath, 'zig') + + def runTest(cmd, onError=None, rerunOnFailure=False): + out = mx.OutputCapture() +@@ -117,11 +116,7 @@ class ToolchainTestBuildTask(mx.BuildTask): + def runCompile(compiler, src, binary, onError): + sourceFile = self.subject.getSource(src) + binFile = self.subject.getOutput(binary) +- if mx.is_darwin(): +- compileCmd = ["xcrun", compiler] +- else: +- compileCmd = [compiler, "-fuse-ld=lld"] +- runTest(compileCmd + [sourceFile, '-o', binFile], onError=onError, rerunOnFailure=True) ++ runTest([zig, compiler, sourceFile, '-o', binFile], onError=onError, rerunOnFailure=True) + out = runTest([binFile], onError=lambda status: + mx.abort(f"{os.path.basename(compiler)} could compile {src}, but the result doesn't work. It returned with exit code {status}.") + ) +@@ -130,11 +125,11 @@ class ToolchainTestBuildTask(mx.BuildTask): + if result != expected: + mx.abort(f"{os.path.basename(compiler)} could compile {src}, but the result does not match (expected: \"{expected}\", got: \"{result}\").") + +- runCompile(clang, "hello.c", "hello", onError=lambda status: ++ runCompile("cc", "hello.c", "hello", onError=lambda status: + mx.abort("The LLVM toolchain does not work. Do you have development packages installed?") + ) + +- runCompile(clangxx, "hello.cc", "hello-cxx", onError=lambda status: check_multiple_gcc_issue(clang)) ++ runCompile("c++", "hello.cc", "hello-cxx", onError=lambda status: check_multiple_gcc_issue("c++")) + + def clean(self, forBuild=False): + if os.path.exists(self.subject.get_output_root()): +diff --git a/sdk/mx.sdk/mx_sdk_vm_impl.py b/sdk/mx.sdk/mx_sdk_vm_impl.py +index cdd7909fcba..dcae2aa589b 100644 +--- a/sdk/mx.sdk/mx_sdk_vm_impl.py ++++ b/sdk/mx.sdk/mx_sdk_vm_impl.py +@@ -2677,9 +2677,8 @@ class NativeLibraryLauncherProject(mx_native.DefaultNativeProject): + self.jvm_launcher = _skip_libraries(self.language_library_config) or not _get_svm_support().is_supported() + _dependencies = [] if (self.jvm_launcher) else [GraalVmNativeImage.project_name(self.language_library_config)] + self.jre_base = get_final_graalvm_distribution().path_substitutions.substitute('') +- # We use our LLVM toolchain on Linux because we want to statically link the C++ standard library, +- # and the system toolchain rarely has libstdc++.a installed (it would be an extra CI & dev dependency). +- toolchain = 'sdk:LLVM_NINJA_TOOLCHAIN' if mx.is_linux() else 'mx:DEFAULT_NINJA_TOOLCHAIN' ++ ++ toolchain = 'mx:DEFAULT_NINJA_TOOLCHAIN' + super(NativeLibraryLauncherProject, self).__init__( + _suite, + NativeLibraryLauncherProject.library_launcher_project_name(self.language_library_config), +diff --git a/sdk/mx.sdk/mx_sdk_vm_ng.py b/sdk/mx.sdk/mx_sdk_vm_ng.py +index 21476855589..632a3945f6f 100644 +--- a/sdk/mx.sdk/mx_sdk_vm_ng.py ++++ b/sdk/mx.sdk/mx_sdk_vm_ng.py +@@ -565,9 +565,8 @@ class ThinLauncherProject(mx_native.DefaultNativeProject): + self.relative_extracted_lib_paths = {k: v.replace('/', os.sep) for k, v in kw_args.pop('relative_extracted_lib_paths', {}).items()} + self.liblang_relpath = _pop_path(kw_args, 'relative_liblang_path', None) + self.setup_relative_resources = kw_args.pop('setup_relative_resources', None) +- # We use our LLVM toolchain on Linux because we want to statically link the C++ standard library, +- # and the system toolchain rarely has libstdc++.a installed (it would be an extra CI & dev dependency). +- toolchain = 'sdk:LLVM_NINJA_TOOLCHAIN' if mx.is_linux() else 'mx:DEFAULT_NINJA_TOOLCHAIN' ++ ++ toolchain = 'mx:DEFAULT_NINJA_TOOLCHAIN' + super().__init__( + suite, + name, +diff --git a/sdk/mx.sdk/suite.py b/sdk/mx.sdk/suite.py +index 3922b2820a6..f05c9cd40d1 100644 +--- a/sdk/mx.sdk/suite.py ++++ b/sdk/mx.sdk/suite.py +@@ -223,29 +223,6 @@ suite = { + "digest" : "sha512:9c83d01eb6745a3e20e3426d8334728a977da11528819cdc835bb2d62925b4d15cd167a549e08c4f263d15461771be2bc92d1ea36c1fd99ab7888740733c534c", + "license" : "Apache-2.0-LLVM", + }, +- "MUSL_GCC_TOOLCHAIN" : { +- "packedResource": True, +- "os_arch": { +- "linux": { +- "amd64": { +- "urls" : ["https://lafo.ssw.uni-linz.ac.at/pub/toolchain-gcc-musl/toolchain-gcc-10.2.1-musl-1.2.2-linux-amd64.tar.gz"], +- "digest" : "sha512:8f49b04d4826c560c791e5223f504046fa0daa6b79e581ea1781a2d01f4efe2de4a0fb6771dc1b07318ab0109a61ea3b04255eadf36191a76687f873931eb283", +- }, +- "aarch64": { +- "urls" : ["https://lafo.ssw.uni-linz.ac.at/pub/toolchain-gcc-musl/toolchain-gcc-10.2.1-musl-1.2.2-linux-aarch64.tar.gz"], +- "digest" : "sha512:f5545f6b36c2306861c026895d437a57357515e8dfefb0e8419413f61b146f42dc072f8a8a7a9f4885d6448396d656f59264e61e3f5eedd278486228aa58904e", +- }, +- "": { +- "optional": True, +- }, +- }, +- "": { +- "": { +- "optional": True, +- } +- } +- }, +- }, + "TIKA_1.0.11": { + "urls": ["https://lafo.ssw.uni-linz.ac.at/pub/graal-external-deps/tika-1.0.11.zip"], + "digest": "sha512:0d45935c428bb3226b0d22b8ea8a8f2eb9e145a72225c6320eeb7c7d1cebc644921f6b77d379a9356bfcbdb66e9daf351d2aef18ff47fa67dad2d5cb15def875", +@@ -671,9 +648,8 @@ suite = { + }, + "linux": { + "": { +- "toolchain": "sdk:LLVM_NINJA_TOOLCHAIN", +- "cflags": ["-std=c++17", "-g", "-Wall", "-Werror", "-D_GNU_SOURCE", "-stdlib=libc++"], +- "ldlibs": ["-ldl", "-pthread", "-stdlib=libc++", "-static-libstdc++", "-l:libc++abi.a"], ++ "cflags": ["-std=c++17", "-g", "-Wall", "-Werror", "-D_GNU_SOURCE"], ++ "ldlibs": ["-ldl", "-pthread"], + }, + }, + "darwin": { +@@ -1538,190 +1514,5 @@ suite = { + "defaultBuild" : False, + "graalCompilerSourceEdition": "ignore", + }, +- "LLVM_NINJA_TOOLCHAIN": { +- "native": True, +- "platformDependent": True, +- "os": { +- "linux": { +- "layout": { +- "toolchain.ninja": { +- "source_type": "string", +- "value": ''' +-include +-CC=/bin/clang +-CXX=/bin/clang++ +-AR=/bin/llvm-ar +-CFLAGS= +-CXXFLAGS= +-LDFLAGS=-fuse-ld=lld +-''' +- }, +- }, +- "dependencies": [ +- "mx:GCC_NINJA_TOOLCHAIN", +- ], +- }, +- "darwin": { +- "layout": { +- "toolchain.ninja": { +- "source_type": "string", +- "value": ''' +-include +-CC=xcrun /bin/clang +-CXX=xcrun /bin/clang++ +-AR=xcrun /bin/llvm-ar +-CFLAGS= +-CXXFLAGS= +-LDFLAGS=-fuse-ld=lld +-''' +- }, +- }, +- "dependencies": [ +- "mx:GCC_NINJA_TOOLCHAIN", +- ], +- }, +- "windows": { +- "layout": { +- "toolchain.ninja": { +- "source_type": "string", +- "value": ''' +-include +-CL=\\bin\\clang-cl +-LINK=\\bin\\lld-link +-LIB=\\bin\\llvm-lib +-ML=\\bin\\llvm-ml +-CFLAGS= +-CXXFLAGS= +-LDFLAGS= +-''' +- }, +- }, +- "dependencies": [ +- "mx:MSVC_NINJA_TOOLCHAIN", +- ], +- }, +- }, +- "dependencies": [ +- "LLVM_TOOLCHAIN", +- "org.graalvm.toolchain.test", +- ], +- "graalCompilerSourceEdition": "ignore", +- }, +- "MUSL_NINJA_TOOLCHAIN" : { +- "native" : True, +- "platformDependent" : True, +- "native_toolchain" : { +- "kind": "ninja", +- "compiler": "gcc", +- "target": { +- # host os/arch +- "libc": "musl", +- }, +- }, +- "os_arch": { +- "linux": { +- "amd64": { +- "layout" : { +- "toolchain.ninja" : { +- "source_type": "string", +- "value": ''' +-include +-CC=/x86_64-linux-musl-native/bin/gcc +-CXX=/x86_64-linux-musl-native/bin/g++ +-AR=/x86_64-linux-musl-native/bin/ar +-CFLAGS= +-CXXFLAGS= +-LDFLAGS= +-''' +- }, +- }, +- "dependencies": [ +- "MUSL_GCC_TOOLCHAIN", +- "mx:GCC_NINJA_TOOLCHAIN", +- ], +- }, +- "aarch64": { +- "layout" : { +- "toolchain.ninja" : { +- "source_type": "string", +- "value": ''' +-include +-CC=/aarch64-linux-musl-native/bin/gcc +-CXX=/aarch64-linux-musl-native/bin/g++ +-AR=/aarch64-linux-musl-native/bin/ar +-CFLAGS= +-CXXFLAGS= +-LDFLAGS= +-''' +- }, +- }, +- "dependencies": [ +- "MUSL_GCC_TOOLCHAIN", +- "mx:GCC_NINJA_TOOLCHAIN", +- ], +- }, +- }, +- "": { +- "": { +- "optional": True, +- } +- }, +- }, +- "maven" : False, +- "graalCompilerSourceEdition": "ignore", +- }, +- "MUSL_CMAKE_TOOLCHAIN" : { +- "native" : True, +- "platformDependent" : True, +- "native_toolchain" : { +- "kind": "cmake", +- "compiler": "gcc", +- "target": { +- # host os/arch +- "libc": "musl", +- }, +- }, +- "os_arch": { +- "linux": { +- "amd64": { +- "layout" : { +- "toolchain.cmake" : { +- "source_type": "string", +- "value": ''' +-set(CMAKE_C_COMPILER /x86_64-linux-musl-native/bin/gcc) +-set(CMAKE_CXX_COMPILER /x86_64-linux-musl-native/bin/g++) +-set(CMAKE_AR /x86_64-linux-musl-native/bin/ar) +-''' +- }, +- }, +- "dependencies": [ +- "MUSL_GCC_TOOLCHAIN", +- ], +- }, +- "aarch64": { +- "layout" : { +- "toolchain.cmake" : { +- "source_type": "string", +- "value": ''' +-set(CMAKE_C_COMPILER /aarch64-linux-musl-native/bin/gcc) +-set(CMAKE_CXX_COMPILER /aarch64-linux-musl-native/bin/g++) +-set(CMAKE_AR /aarch64-linux-musl-native/bin/ar) +-''' +- }, +- }, +- "dependencies": [ +- "MUSL_GCC_TOOLCHAIN", +- ], +- }, +- }, +- "": { +- "": { +- "optional": True, +- } +- }, +- }, +- "maven" : False, +- "graalCompilerSourceEdition": "ignore", +- }, + }, + } +diff --git a/truffle/mx.truffle/mx_truffle.py b/truffle/mx.truffle/mx_truffle.py +index 7851eac0ecb..0e4d6a17a5c 100644 +--- a/truffle/mx.truffle/mx_truffle.py ++++ b/truffle/mx.truffle/mx_truffle.py +@@ -1836,8 +1836,7 @@ class LibffiBuilderProject(mx_native.MultitargetProject): + configure_libc = {'glibc': 'gnu', 'musl': 'musl'}.get(toolchain.spec.target.libc) + assert configure_libc, "translation to configure style libc is not supported yet for" + str(toolchain.spec.target.libc) + +- configure_args += ['--host={}-pc-linux-{}'.format(configure_arch, configure_libc)] +- ++ # Pass '-target', ... if needed + configure_args += [' CFLAGS="{}"'.format(' '.join(['-g', '-O3', '-fvisibility=hidden'] + (['-m64'] if mx.get_os() == 'solaris' else []))), + 'CPPFLAGS="-DNO_JAVA_RAW_API"'] + +diff --git a/truffle/mx.truffle/suite.py b/truffle/mx.truffle/suite.py +index ecf970ce7c2..25993d67b71 100644 +--- a/truffle/mx.truffle/suite.py ++++ b/truffle/mx.truffle/suite.py +@@ -1078,7 +1078,6 @@ suite = { + "com.oracle.truffle.nfi.test.native" : { + "subDir" : "src", + "native" : "shared_lib", +- "toolchain" : "sdk:LLVM_NINJA_TOOLCHAIN", + "deliverable" : "nativetest", + "buildDependencies" : [ + "com.oracle.truffle.nfi.native", diff --git a/0003-embed-zig.patch b/0003-embed-zig.patch new file mode 100644 index 000000000000..ecab45190772 --- /dev/null +++ b/0003-embed-zig.patch @@ -0,0 +1,635 @@ +diff --git a/sdk/mx.sdk/mx_sdk.py b/sdk/mx.sdk/mx_sdk.py +index b7b810dbaa5..3c0d9442d0d 100644 +--- a/sdk/mx.sdk/mx_sdk.py ++++ b/sdk/mx.sdk/mx_sdk.py +@@ -161,6 +161,8 @@ graalvm_sdk_native_image_component = mx_sdk_vm.GraalVmJreComponent( + license_files=[], + third_party_license_files=[], + dependencies=['sdkc'], ++ # TODO(@izaakschroeder): Better way of doing this? ++ support_distributions=["mx:ZIG_DIR", "substratevm:NATIVE_IMAGE_ZLIB_SUPPORT"], + jar_distributions=[], + boot_jars=['sdk:NATIVEIMAGE', 'sdk:NATIVEIMAGE_LIBGRAAL', 'sdk:WEBIMAGE_PREVIEW'], + stability="supported", +diff --git a/substratevm/3p/zlib/build.zig b/substratevm/3p/zlib/build.zig +new file mode 100644 +index 00000000000..8773d0c142a +--- /dev/null ++++ b/substratevm/3p/zlib/build.zig +@@ -0,0 +1,44 @@ ++const std = @import("std"); ++ ++pub fn build(b: *std.Build) !void { ++ const target = b.standardTargetOptions(.{}); ++ const optimize = b.standardOptimizeOption(.{}); ++ ++ const lib = b.addStaticLibrary(.{ ++ .name = "z", ++ .target = target, ++ .optimize = optimize, ++ }); ++ lib.linkLibC(); ++ lib.addIncludePath(b.path("src")); ++ lib.installHeadersDirectory(b.path("src"), "", .{}); ++ ++ var flags = std.ArrayList([]const u8).init(b.allocator); ++ defer flags.deinit(); ++ try flags.appendSlice(&.{ ++ "-DHAVE_SYS_TYPES_H", ++ "-DHAVE_STDINT_H", ++ "-DHAVE_STDDEF_H", ++ "-DZ_HAVE_UNISTD_H", ++ }); ++ lib.addCSourceFiles(.{ .files = srcs, .flags = flags.items }); ++ b.installArtifact(lib); ++} ++ ++const srcs = &.{ ++ "src/adler32.c", ++ "src/compress.c", ++ "src/crc32.c", ++ "src/deflate.c", ++ "src/gzclose.c", ++ "src/gzlib.c", ++ "src/gzread.c", ++ "src/gzwrite.c", ++ "src/inflate.c", ++ "src/infback.c", ++ "src/inftrees.c", ++ "src/inffast.c", ++ "src/trees.c", ++ "src/uncompr.c", ++ "src/zutil.c", ++}; +diff --git a/substratevm/mx.substratevm/suite.py b/substratevm/mx.substratevm/suite.py +index 5de8076ee0e..e0be4455694 100644 +--- a/substratevm/mx.substratevm/suite.py ++++ b/substratevm/mx.substratevm/suite.py +@@ -237,6 +237,13 @@ suite = { + "version" : "0.1.16", + }, + }, ++ ++ "ZLIB_SRC" : { ++ "version" : "1.3.1", ++ "digest": "sha512:580677aad97093829090d4b605ac81c50327e74a6c2de0b85dd2e8525553f3ddde17556ea46f8f007f89e435493c9a20bc997d1ef1c1c2c23274528e3c46b94f", ++ "urls" : ["https://github.com/madler/zlib/releases/download/v1.3.1/zlib-1.3.1.tar.gz"], ++ "license" : "MIT", ++ }, + }, + + "projects": { +@@ -2529,6 +2536,16 @@ suite = { + }, + }, + ++ "NATIVE_IMAGE_ZLIB_SUPPORT" : { ++ "native" : True, ++ "platformDependent" : False, ++ "description" : "Native Image zlib distribution for the GraalVM", ++ "layout" : { ++ "./lib/libz/build.zig": "file:3p/zlib/build.zig", ++ "./lib/libz/src/" : "extracted-dependency:ZLIB_SRC/zlib-*/**", ++ }, ++ }, ++ + "NATIVE_IMAGE_LICENSE_GRAALVM_SUPPORT" : { + "native" : True, + "platformDependent" : False, +diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/JNIPlatformNativeLibrarySupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/JNIPlatformNativeLibrarySupport.java +index f74a997579f..07a22890027 100644 +--- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/JNIPlatformNativeLibrarySupport.java ++++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/JNIPlatformNativeLibrarySupport.java +@@ -111,4 +111,5 @@ final class Target_java_io_FileOutputStream_JNI { + @Platforms({Platform.DARWIN.class, Platform.LINUX.class}) + @CLibrary("z") + class ZLib { ++ + } +diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java +index 32e85f41a04..8a26fea2587 100644 +--- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java ++++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java +@@ -652,6 +652,11 @@ public class NativeImageGenerator { + NativeImageCodeCache codeCache; + CompileQueue compileQueue; + try (StopTimer t = TimerCollection.createTimerAndStart(TimerCollection.Registry.COMPILE_TOTAL)) { ++ if (ImageSingletons.contains(CCompilerInvoker.class)) { ++ CCompilerInvoker compilerInvoker = ImageSingletons.lookup(CCompilerInvoker.class); ++ compilerInvoker.beforeCompilation(); ++ } ++ + compileQueue = HostedConfiguration.instance().createCompileQueue(debug, featureHandler, hUniverse, runtimeConfiguration, DeoptTester.enabled()); + if (ImageSingletons.contains(RuntimeCompilationCallbacks.class)) { + ImageSingletons.lookup(RuntimeCompilationCallbacks.class).onCompileQueueCreation(bb, hUniverse, compileQueue); +diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/c/codegen/CCompilerInvoker.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/c/codegen/CCompilerInvoker.java +index 8d0cf363317..766d9f2b8ba 100644 +--- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/c/codegen/CCompilerInvoker.java ++++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/c/codegen/CCompilerInvoker.java +@@ -47,10 +47,14 @@ import com.oracle.svm.core.OS; + import com.oracle.svm.core.SubstrateOptions; + import com.oracle.svm.core.SubstrateTargetDescription; + import com.oracle.svm.core.SubstrateUtil; ++import com.oracle.svm.core.c.libc.LibCBase; ++import com.oracle.svm.core.c.libc.GLibC; ++import com.oracle.svm.core.c.libc.MuslLibC; + import com.oracle.svm.core.option.SubstrateOptionsParser; + import com.oracle.svm.core.util.InterruptImageBuilding; + import com.oracle.svm.core.util.UserError; + import com.oracle.svm.core.util.VMError; ++import com.oracle.svm.hosted.c.NativeLibraries; + import com.oracle.svm.hosted.c.libc.HostedLibCBase; + import com.oracle.svm.hosted.c.util.FileUtils; + import com.oracle.svm.util.ClassUtil; +@@ -79,6 +83,10 @@ public abstract class CCompilerInvoker { + } + + public static CCompilerInvoker create(Path tempDirectory) { ++ String userDefinedPath = SubstrateOptions.CCompilerPath.getValue(); ++ if (userDefinedPath == null) { ++ return new ZigCCompilerInvoker(tempDirectory); ++ } + OS hostOS = OS.getCurrent(); + switch (hostOS) { + case LINUX: +@@ -88,7 +96,10 @@ public abstract class CCompilerInvoker { + case WINDOWS: + return new WindowsCCompilerInvoker(tempDirectory); + default: +- throw UserError.abort("No CCompilerInvoker for operating system %s", hostOS.name()); ++ throw UserError.abort( ++ "No CCompilerInvoker for operating system %s", ++ hostOS.name() ++ ); + } + } + +@@ -102,6 +113,10 @@ public abstract class CCompilerInvoker { + } + } + ++ public void beforeCompilation() { ++ // Can be overridden. ++ } ++ + private static UserError.UserException addSkipCheckingInfo(UserError.UserException err) { + List messages = new ArrayList<>(); + err.getMessages().forEach(messages::add); +@@ -109,6 +124,200 @@ public abstract class CCompilerInvoker { + return UserError.abort(messages); + } + ++ public static class ZigCCompilerInvoker extends CCompilerInvoker { ++ ++ ZigCCompilerInvoker(Path tempDirectory) { ++ super(tempDirectory); ++ } ++ ++ @Override ++ public List createCompilerCommand( ++ List options, ++ Path target, ++ Path... input ++ ) { ++ List wrappedOptions = new ArrayList(options); ++ wrappedOptions.addFirst(getTarget()); ++ wrappedOptions.addFirst("-target"); ++ wrappedOptions.addFirst("cc"); ++ return super.createCompilerCommand( ++ compilerInfo.compilerPath, ++ wrappedOptions, ++ target, ++ input ++ ); ++ } ++ ++ protected String getArch() { ++ if (Platform.includedIn(Platform.AARCH64.class)) { ++ return "aarch64"; ++ } ++ if (Platform.includedIn(Platform.AMD64.class)) { ++ return "x86_64"; ++ } ++ if (Platform.includedIn(Platform.RISCV64.class)) { ++ return "riscv64"; ++ } ++ throw UserError.abort("Unsupported platform"); ++ } ++ ++ protected String getOs() { ++ if (Platform.includedIn(Platform.LINUX.class)) { ++ return "linux"; ++ } ++ if (Platform.includedIn(Platform.WINDOWS.class)) { ++ return "windows"; ++ } ++ if (Platform.includedIn(Platform.MACOS.class)) { ++ return "macos"; ++ } ++ throw UserError.abort("Unsupported platform"); ++ } ++ ++ protected String getLibc() { ++ if (Platform.includedIn(Platform.MACOS.class)) { ++ return "none"; ++ } ++ if (Platform.includedIn(Platform.WINDOWS.class)) { ++ return "gnu"; ++ } ++ if (Platform.includedIn(Platform.LINUX.class)) { ++ if (LibCBase.targetLibCIs(MuslLibC.class)) { ++ return "musl"; ++ } ++ if (LibCBase.targetLibCIs(GLibC.class)) { ++ return "gnu"; ++ } ++ } ++ throw UserError.abort("Unsupported platform"); ++ } ++ ++ protected String getTarget() { ++ return getArch() + "-" + getOs() + "-" + getLibc(); ++ } ++ ++ @Override ++ protected void verify() { ++ // No-op ++ } ++ ++ @Override ++ public void beforeCompilation() { ++ NativeLibraries nativeLibraries = NativeLibraries.singleton(); ++ if (nativeLibraries == null) { ++ return; ++ } ++ File prefixDir = tempDirectory.resolve("root").toAbsolutePath().toFile(); ++ prefixDir.mkdirs(); ++ String prefix = prefixDir.toString(); ++ for (String lib : nativeLibraries.getLibraries()) { ++ this.compileLib(prefix, lib); ++ } ++ } ++ ++ protected Path getLibPath(String name) { ++ String libDir = System.getProperty("sun.boot.library.path"); ++ if (libDir != null) { ++ Path libPath = Paths.get(libDir, "graalvm", "lib", "lib"+name); ++ if (Files.exists(libPath)) { ++ return libPath; ++ } ++ return null; ++ } ++ return null; ++ } ++ ++ protected void compileLib(String prefix, String name) { ++ Path libPath = getLibPath(name); ++ if (libPath == null) { ++ return; ++ } ++ Process compilerProcess = null; ++ ++ List compilerCommand = Arrays.asList( ++ getDefaultCompiler(), ++ "build", ++ "--release=fast", ++ "--search-prefix", prefix, ++ "--prefix", prefix, ++ "-Dtarget="+getTarget() ++ ); ++ try { ++ ProcessBuilder processBuilder = FileUtils.prepareCommand(compilerCommand, libPath); ++ processBuilder.redirectErrorStream(true); ++ processBuilder.environment().put("LC_ALL", "C"); ++ FileUtils.traceCommand(processBuilder); ++ compilerProcess = processBuilder.start(); ++ ++ List lines; ++ try (InputStream inputStream = compilerProcess.getInputStream()) { ++ lines = FileUtils.readAllLines(inputStream); ++ FileUtils.traceCommandOutput(lines); ++ } ++ compilerProcess.waitFor(); ++ ++ if (compilerProcess.exitValue() != 0) { ++ String errorMessage = "Unable to compile %s library %s:%n%s"; ++ throw UserError.abort(errorMessage, name, SubstrateUtil.getShellCommandString(compilerCommand, false), lines.stream().map(str -> " " + str).collect(Collectors.joining(System.lineSeparator()))); ++ } ++ return; ++ } catch (InterruptedException ex) { ++ throw new InterruptImageBuilding("Interrupted during building library " + name); ++ } catch (IOException e) { ++ throw UserError.abort(e, "Building library %s failed: %s", name, SubstrateUtil.getShellCommandString(compilerCommand, false)); ++ } finally { ++ if (compilerProcess != null) { ++ compilerProcess.destroy(); ++ } ++ } ++ ++ ++ } ++ ++ @Override ++ protected String getDefaultCompiler() { ++ // sun.boot.library.path ++ String libDir = System.getProperty("sun.boot.library.path"); ++ if (libDir != null) { ++ Path binPath = Paths.get(libDir, "graalvm", "zig"); ++ return binPath.toAbsolutePath().toString(); ++ } ++ return "zig"; ++ } ++ ++ @Override ++ protected CompilerInfo createCompilerInfo( ++ Path compilerPath, ++ Scanner scanner ++ ) { ++ try { ++ ++ scanner.useDelimiter("[. -]"); ++ scanner.skip("\\p{Zs}*"); ++ int major = scanner.nextInt(); ++ int minor0 = scanner.nextInt(); ++ int minor1 = scanner.nextInt(); ++ return new CompilerInfo( ++ compilerPath, ++ "zig", ++ "Zig Compiler", ++ "zig", ++ major, ++ minor0, ++ minor1, ++ getTarget() ++ ); ++ } catch (NoSuchElementException e) { ++ return null; ++ } ++ } ++ ++ @Override ++ protected List getVersionInfoOptions() { ++ return List.of("version"); ++ } ++ } ++ + private static class WindowsCCompilerInvoker extends CCompilerInvoker { + + WindowsCCompilerInvoker(Path tempDirectory) { +@@ -487,6 +696,7 @@ public abstract class CCompilerInvoker { + try (InputStream compilerErrors = getCompilerErrorStream(compilingProcess)) { + lines = FileUtils.readAllLines(compilerErrors); + FileUtils.traceCommandOutput(lines); ++ + } + boolean errorReported = false; + for (String line : lines) { +@@ -546,12 +756,15 @@ public abstract class CCompilerInvoker { + if (userDefinedPath != null) { + compilerPath = Paths.get(userDefinedPath); + } else { +- String executableName = asExecutableName(getDefaultCompiler()); +- Optional optCompilerPath = lookupSearchPath(executableName); ++ compilerPath = Paths.get(asExecutableName(getDefaultCompiler())); ++ ++ } ++ if (!compilerPath.isAbsolute()) { ++ Optional optCompilerPath = lookupSearchPath(compilerPath.getFileName().toString()); + if (optCompilerPath.isPresent()) { + compilerPath = optCompilerPath.get(); + } else { +- throw UserError.abort("Default native-compiler executable '%s' not found via environment variable PATH", executableName); ++ throw UserError.abort("Default native-compiler executable '%s' not found via environment variable PATH", compilerPath); + } + } + if (Files.isDirectory(compilerPath) || !Files.isExecutable(compilerPath)) { +diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/CCLinkerInvocation.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/CCLinkerInvocation.java +index 82331bbef00..d907448960e 100644 +--- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/CCLinkerInvocation.java ++++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/CCLinkerInvocation.java +@@ -27,6 +27,7 @@ package com.oracle.svm.hosted.image; + import static com.oracle.svm.core.SubstrateOptions.SpawnIsolates; + + import java.io.IOException; ++import java.io.InputStream; + import java.nio.file.Files; + import java.nio.file.Path; + import java.util.ArrayList; +@@ -56,6 +57,7 @@ import com.oracle.svm.core.util.VMError; + import com.oracle.svm.hosted.c.CGlobalDataFeature; + import com.oracle.svm.hosted.c.NativeLibraries; + import com.oracle.svm.hosted.c.codegen.CCompilerInvoker; ++import com.oracle.svm.hosted.c.codegen.CCompilerInvoker.ZigCCompilerInvoker; + import com.oracle.svm.hosted.c.libc.HostedLibCBase; + import com.oracle.svm.hosted.imagelayer.HostedDynamicLayerInfo; + import com.oracle.svm.hosted.jdk.JNIRegistrationSupport; +@@ -65,10 +67,11 @@ import jdk.graal.compiler.options.OptionStability; + + public abstract class CCLinkerInvocation implements LinkerInvocation { + +- protected CCLinkerInvocation(AbstractImage.NativeImageKind imageKind, NativeLibraries nativeLibs, List imageSymbols) { ++ protected CCLinkerInvocation(AbstractImage.NativeImageKind imageKind, NativeLibraries nativeLibs, List imageSymbols, Path tempDirectory) { + this.imageKind = imageKind; + this.nativeLibs = nativeLibs; + this.imageSymbols = imageSymbols; ++ this.tempDirectory = tempDirectory; + } + + public static class Options { +@@ -166,10 +169,6 @@ public abstract class CCLinkerInvocation implements LinkerInvocation { + outputFile = out; + } + +- public void setTempDirectory(Path tempDirectory) { +- this.tempDirectory = tempDirectory; +- } +- + @Override + public Path getTempDirectory() { + return tempDirectory; +@@ -251,14 +250,120 @@ public abstract class CCLinkerInvocation implements LinkerInvocation { + .collect(Collectors.toList()); + } + ++ private static class ZigCCLinkerInvocation extends CCLinkerInvocation { ++ ++ ++ ZigCCLinkerInvocation(AbstractImage.NativeImageKind imageKind, NativeLibraries nativeLibs, List symbols, Path tempDirectory) { ++ super(imageKind, nativeLibs, symbols, tempDirectory); ++ ++ Path prefix = tempDirectory.resolve("root"); ++ additionalPreOptions.add("-L" + prefix.resolve("lib").toAbsolutePath().toString()); ++ additionalPreOptions.add("-isystem" + prefix.resolve("include").toAbsolutePath().toString()); ++ ++ additionalPreOptions.add("-Wa,--noexecstack"); ++ additionalPreOptions.add("-Wl,-z,noexecstack"); ++ ++ // The linker should fail if DT_TEXTREL is needed, otherwise the image won't work on ++ // SELinux. If SpawnIsolates are disabled, this won't work as dynamic relocations ++ // are needed for heap access. ++ if (!SpawnIsolates.getValue()) { ++ additionalPreOptions.add("-Wl,-z,notext"); ++ } ++ ++ if (SubstrateOptions.RemoveUnusedSymbols.getValue()) { ++ /* Perform garbage collection of unused input sections. */ ++ additionalPreOptions.add("-Wl,--gc-sections"); ++ } ++ ++ if (imageKind.isImageLayer) { ++ /* ++ * We do not want interposition to affect the resolution of symbols we define and ++ * reference within this library. ++ */ ++ additionalPreOptions.add("-Wl,-Bsymbolic"); ++ } ++ ++ /* Use --version-script to control the visibility of image symbols. */ ++ try { ++ StringBuilder exportedSymbols = new StringBuilder(); ++ exportedSymbols.append("{\n"); ++ /* Only exported symbols are global ... */ ++ Set globalSymbols = Stream.concat(getImageSymbols(true).stream(), JNIRegistrationSupport.getShimLibrarySymbols()).collect(Collectors.toSet()); ++ if (!globalSymbols.isEmpty()) { ++ exportedSymbols.append("global:\n"); ++ globalSymbols.forEach(symbol -> exportedSymbols.append('\"').append(symbol).append("\";\n")); ++ } ++ /* ... everything else is local. */ ++ exportedSymbols.append("local: *;\n"); ++ exportedSymbols.append("};"); ++ ++ Path exportedSymbolsPath = nativeLibs.tempDirectory.resolve("exported_symbols.list"); ++ Files.write(exportedSymbolsPath, Collections.singleton(exportedSymbols.toString())); ++ additionalPreOptions.add("-Wl,--version-script," + exportedSymbolsPath.toAbsolutePath()); ++ } catch (IOException e) { ++ VMError.shouldNotReachHere(e); ++ } ++ ++ additionalPreOptions.addAll(HostedLibCBase.singleton().getAdditionalLinkerOptions(imageKind)); ++ ++ if (SubstrateOptions.DeleteLocalSymbols.getValue() && !SubstrateOptions.StripDebugInfo.getValue()) { ++ additionalPreOptions.add("-Wl,-x"); ++ } ++ } ++ ++ @Override ++ String getSymbolName(ObjectFile.Symbol symbol) { ++ return symbol.getName(); ++ } ++ ++ @Override ++ protected void setOutputKind(List cmd) { ++ switch (imageKind) { ++ case EXECUTABLE: ++ /* Export global symbols. */ ++ cmd.add("-Wl,--export-dynamic"); ++ break; ++ case STATIC_EXECUTABLE: ++ cmd.add("-static"); ++ break; ++ case IMAGE_LAYER: ++ case SHARED_LIBRARY: ++ cmd.add("-shared"); ++ break; ++ default: ++ VMError.shouldNotReachHereUnexpectedInput(imageKind); // ExcludeFromJacocoGeneratedReport ++ } ++ } ++ ++ @Override ++ protected List getLibrariesCommand() { ++ List cmd = new ArrayList<>(); ++ String previousLayerLib = null; ++ for (String lib : libs) { ++ String linkingMode = null; ++ if (ImageLayerBuildingSupport.buildingExtensionLayer() && HostedDynamicLayerInfo.singleton().isImageLayerLib(lib)) { ++ VMError.guarantee(!lib.isEmpty()); ++ VMError.guarantee(previousLayerLib == null, "We currently only support one previous layer."); // GR-58631 ++ previousLayerLib = lib; ++ } ++ cmd.add("-l" + lib); ++ } ++ ++ if (previousLayerLib != null) { ++ cmd.add("-l" + previousLayerLib); ++ } ++ return cmd; ++ } ++ } ++ + private static class BinutilsCCLinkerInvocation extends CCLinkerInvocation { + + private final boolean dynamicLibC = SubstrateOptions.StaticExecutableWithDynamicLibC.getValue(); + private final boolean staticLibCpp = SubstrateOptions.StaticLibStdCpp.getValue(); + private final boolean customStaticLibs = dynamicLibC || staticLibCpp; + +- BinutilsCCLinkerInvocation(AbstractImage.NativeImageKind imageKind, NativeLibraries nativeLibs, List symbols) { +- super(imageKind, nativeLibs, symbols); ++ BinutilsCCLinkerInvocation(AbstractImage.NativeImageKind imageKind, NativeLibraries nativeLibs, List symbols, Path tempDirectory) { ++ super(imageKind, nativeLibs, symbols, tempDirectory); + additionalPreOptions.add("-z"); + additionalPreOptions.add("noexecstack"); + +@@ -383,9 +488,9 @@ public abstract class CCLinkerInvocation implements LinkerInvocation { + + private static class DarwinCCLinkerInvocation extends CCLinkerInvocation { + +- DarwinCCLinkerInvocation(AbstractImage.NativeImageKind imageKind, NativeLibraries nativeLibs, List symbols) { ++ DarwinCCLinkerInvocation(AbstractImage.NativeImageKind imageKind, NativeLibraries nativeLibs, List symbols, Path tempDirectory) { + // Workaround building images with older Xcode with new libraries +- super(imageKind, nativeLibs, symbols); ++ super(imageKind, nativeLibs, symbols, tempDirectory); + setLinkerFlags(nativeLibs, false); + } + +@@ -486,8 +591,8 @@ public abstract class CCLinkerInvocation implements LinkerInvocation { + + private final String imageName; + +- WindowsCCLinkerInvocation(AbstractImage.NativeImageKind imageKind, NativeLibraries nativeLibs, List symbols, String imageName) { +- super(imageKind, nativeLibs, symbols); ++ WindowsCCLinkerInvocation(AbstractImage.NativeImageKind imageKind, NativeLibraries nativeLibs, List symbols, String imageName, Path tempDirectory) { ++ super(imageKind, nativeLibs, symbols, tempDirectory); + this.imageName = imageName; + } + +@@ -590,23 +695,27 @@ public abstract class CCLinkerInvocation implements LinkerInvocation { + Path outputDirectory, Path tempDirectory, String imageName, List symbols) { + CCLinkerInvocation inv; + +- switch (ObjectFile.getNativeFormat()) { +- case MACH_O: +- inv = new DarwinCCLinkerInvocation(imageKind, nativeLibs, symbols); +- break; +- case PECOFF: +- inv = new WindowsCCLinkerInvocation(imageKind, nativeLibs, symbols, imageName); +- break; +- case ELF: +- default: +- inv = new BinutilsCCLinkerInvocation(imageKind, nativeLibs, symbols); +- break; ++ if (ImageSingletons.lookup(CCompilerInvoker.class) instanceof CCompilerInvoker.ZigCCompilerInvoker) { ++ inv = new ZigCCLinkerInvocation(imageKind, nativeLibs, symbols, tempDirectory); ++ } else { ++ switch (ObjectFile.getNativeFormat()) { ++ case MACH_O: ++ inv = new DarwinCCLinkerInvocation(imageKind, nativeLibs, symbols, tempDirectory); ++ break; ++ case PECOFF: ++ inv = new WindowsCCLinkerInvocation(imageKind, nativeLibs, symbols, imageName, tempDirectory); ++ break; ++ case ELF: ++ default: ++ inv = new BinutilsCCLinkerInvocation(imageKind, nativeLibs, symbols, tempDirectory); ++ break; ++ } + } ++ + + Path outputFile = outputDirectory.resolve(imageKind.getOutputFilename(imageName)); + UserError.guarantee(!Files.isDirectory(outputFile), "Cannot write image to %s. Path exists as directory (use '-o /path/to/image').", outputFile); + inv.setOutputFile(outputFile); +- inv.setTempDirectory(tempDirectory); + + inv.addLibPath(tempDirectory.toString()); + for (String libraryPath : nativeLibs.getLibraryPaths()) { diff --git a/sdk/mx.sdk/mx_sdk.py b/sdk/mx.sdk/mx_sdk.py index 5465c3659ff8..a4117e43721d 100644 --- a/sdk/mx.sdk/mx_sdk.py +++ b/sdk/mx.sdk/mx_sdk.py @@ -162,6 +162,8 @@ def upx(args): license_files=[], third_party_license_files=[], dependencies=['sdkc'], + # TODO(@izaakschroeder): Better way of doing this? + support_distributions=["mx:ZIG_DIR", "substratevm:NATIVE_IMAGE_ZLIB_SUPPORT"], jar_distributions=[], boot_jars=['sdk:NATIVEIMAGE', 'sdk:NATIVEIMAGE_LIBGRAAL', 'sdk:WEBIMAGE_PREVIEW'], stability="supported", diff --git a/sdk/mx.sdk/mx_sdk_toolchain.py b/sdk/mx.sdk/mx_sdk_toolchain.py index 5f9fa425a430..729ea5bac452 100644 --- a/sdk/mx.sdk/mx_sdk_toolchain.py +++ b/sdk/mx.sdk/mx_sdk_toolchain.py @@ -97,9 +97,8 @@ def cleanForbidden(self): def build(self): mx_util.ensure_dir_exists(self.subject.get_output_root()) - toolchainPath = mx.distribution('LLVM_TOOLCHAIN').output - clang = os.path.join(toolchainPath, 'bin', 'clang') - clangxx = os.path.join(toolchainPath, 'bin', 'clang++') + toolchainPath = mx.distribution('ZIG_DIR').output + zig = os.path.join(toolchainPath, 'zig') def runTest(cmd, onError=None, rerunOnFailure=False): out = mx.OutputCapture() @@ -116,11 +115,7 @@ def runTest(cmd, onError=None, rerunOnFailure=False): def runCompile(compiler, src, binary, onError): sourceFile = self.subject.getSource(src) binFile = self.subject.getOutput(binary) - if mx.is_darwin(): - compileCmd = ["xcrun", compiler] - else: - compileCmd = [compiler, "-fuse-ld=lld"] - runTest(compileCmd + [sourceFile, '-o', binFile], onError=onError, rerunOnFailure=True) + runTest([zig, compiler, sourceFile, '-o', binFile], onError=onError, rerunOnFailure=True) out = runTest([binFile], onError=lambda status: mx.abort(f"{os.path.basename(compiler)} could compile {src}, but the result doesn't work. It returned with exit code {status}.") ) @@ -129,11 +124,11 @@ def runCompile(compiler, src, binary, onError): if result != expected: mx.abort(f"{os.path.basename(compiler)} could compile {src}, but the result does not match (expected: \"{expected}\", got: \"{result}\").") - runCompile(clang, "hello.c", "hello", onError=lambda status: + runCompile("cc", "hello.c", "hello", onError=lambda status: mx.abort("The LLVM toolchain does not work. Do you have development packages installed?") ) - runCompile(clangxx, "hello.cc", "hello-cxx", onError=lambda status: check_multiple_gcc_issue(clang)) + runCompile("c++", "hello.cc", "hello-cxx", onError=lambda status: check_multiple_gcc_issue("c++")) def clean(self, forBuild=False): if os.path.exists(self.subject.get_output_root()): diff --git a/sdk/mx.sdk/mx_sdk_vm_ng.py b/sdk/mx.sdk/mx_sdk_vm_ng.py index f28087106f92..a3eb67a35063 100644 --- a/sdk/mx.sdk/mx_sdk_vm_ng.py +++ b/sdk/mx.sdk/mx_sdk_vm_ng.py @@ -591,6 +591,7 @@ def __init__(self, suite, name, deps, workingSets, theLicense=None, **kw_args): self.liblang_relpath = _pop_path(kw_args, 'relative_liblang_path', None) self.setup_relative_resources = kw_args.pop('setup_relative_resources', None) + # toolchain = 'mx:DEFAULT_NINJA_TOOLCHAIN' if not kw_args.get('multitarget'): # We use our LLVM toolchain on Linux by default because we want to statically link the C++ standard library, # and the system toolchain rarely has libstdc++.a installed (it would be an extra CI & dev dependency). diff --git a/sdk/mx.sdk/suite.py b/sdk/mx.sdk/suite.py index bba938446162..c6bae6b71f28 100644 --- a/sdk/mx.sdk/suite.py +++ b/sdk/mx.sdk/suite.py @@ -686,9 +686,8 @@ }, "linux": { "": { - "toolchain": "sdk:LLVM_NINJA_TOOLCHAIN", - "cflags": ["-std=c++17", "-g", "-Wall", "-Werror", "-D_GNU_SOURCE", "-stdlib=libc++"], - "ldlibs": ["-ldl", "-pthread", "-stdlib=libc++", "-static-libstdc++", "-l:libc++abi.a"], + "cflags": ["-std=c++17", "-g", "-Wall", "-Werror", "-D_GNU_SOURCE"], + "ldlibs": ["-ldl", "-pthread"], }, }, "darwin": { diff --git a/substratevm/3p/zlib/build.zig b/substratevm/3p/zlib/build.zig new file mode 100644 index 000000000000..8773d0c142a3 --- /dev/null +++ b/substratevm/3p/zlib/build.zig @@ -0,0 +1,44 @@ +const std = @import("std"); + +pub fn build(b: *std.Build) !void { + const target = b.standardTargetOptions(.{}); + const optimize = b.standardOptimizeOption(.{}); + + const lib = b.addStaticLibrary(.{ + .name = "z", + .target = target, + .optimize = optimize, + }); + lib.linkLibC(); + lib.addIncludePath(b.path("src")); + lib.installHeadersDirectory(b.path("src"), "", .{}); + + var flags = std.ArrayList([]const u8).init(b.allocator); + defer flags.deinit(); + try flags.appendSlice(&.{ + "-DHAVE_SYS_TYPES_H", + "-DHAVE_STDINT_H", + "-DHAVE_STDDEF_H", + "-DZ_HAVE_UNISTD_H", + }); + lib.addCSourceFiles(.{ .files = srcs, .flags = flags.items }); + b.installArtifact(lib); +} + +const srcs = &.{ + "src/adler32.c", + "src/compress.c", + "src/crc32.c", + "src/deflate.c", + "src/gzclose.c", + "src/gzlib.c", + "src/gzread.c", + "src/gzwrite.c", + "src/inflate.c", + "src/infback.c", + "src/inftrees.c", + "src/inffast.c", + "src/trees.c", + "src/uncompr.c", + "src/zutil.c", +}; diff --git a/substratevm/mx.substratevm/suite.py b/substratevm/mx.substratevm/suite.py index 980ceb2f26a0..b33333092f53 100644 --- a/substratevm/mx.substratevm/suite.py +++ b/substratevm/mx.substratevm/suite.py @@ -216,6 +216,13 @@ "version" : "0.1.16", }, }, + + "ZLIB_SRC" : { + "version" : "1.3.1", + "digest": "sha512:580677aad97093829090d4b605ac81c50327e74a6c2de0b85dd2e8525553f3ddde17556ea46f8f007f89e435493c9a20bc997d1ef1c1c2c23274528e3c46b94f", + "urls" : ["https://github.com/madler/zlib/releases/download/v1.3.1/zlib-1.3.1.tar.gz"], + "license" : "MIT", + }, }, "projects": { @@ -2871,6 +2878,16 @@ }, }, + "NATIVE_IMAGE_ZLIB_SUPPORT" : { + "native" : True, + "platformDependent" : False, + "description" : "Native Image zlib distribution for the GraalVM", + "layout" : { + "./lib/libz/build.zig": "file:3p/zlib/build.zig", + "./lib/libz/src/" : "extracted-dependency:ZLIB_SRC/zlib-*/**", + }, + }, + "NATIVE_IMAGE_LICENSE_GRAALVM_SUPPORT" : { "native" : True, "platformDependent" : False, @@ -2943,8 +2960,7 @@ "description" : "LLVM backend for Native Image", "dependencies" : ["com.oracle.svm.core.graal.llvm"], "distDependencies" : [ - "SVM", - "sdk:LLVM_TOOLCHAIN" + "SVM" ], "javaProperties": { "llvm.bin.dir": "/bin/", diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/JNIPlatformNativeLibrarySupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/JNIPlatformNativeLibrarySupport.java index f7f06d4558f8..9c207cb58e59 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/JNIPlatformNativeLibrarySupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/JNIPlatformNativeLibrarySupport.java @@ -111,4 +111,5 @@ final class Target_java_io_FileOutputStream_JNI { @Platforms({Platform.DARWIN.class, Platform.LINUX.class}) @CLibrary("z") class ZLib { + } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/c/codegen/CCompilerInvoker.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/c/codegen/CCompilerInvoker.java index 8311ace63158..370932ced57d 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/c/codegen/CCompilerInvoker.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/c/codegen/CCompilerInvoker.java @@ -82,6 +82,10 @@ protected CCompilerInvoker(Path tempDirectory) { } public static CCompilerInvoker create(Path tempDirectory) { + String userDefinedPath = SubstrateOptions.CCompilerPath.getValue(); + if (userDefinedPath == null) { + return new ZigCCompilerInvoker(tempDirectory); + } OS hostOS = OS.getCurrent(); switch (hostOS) { case LINUX: @@ -112,6 +116,207 @@ private static UserError.UserException addSkipCheckingInfo(UserError.UserExcepti return UserError.abort(messages); } + @SingletonTraits(access = BuildtimeAccessOnly.class, layeredCallbacks = NoLayeredCallbacks.class, other = Disallowed.class) + public static class ZigCCompilerInvoker extends CCompilerInvoker { + + ZigCCompilerInvoker(Path tempDirectory) { + super(tempDirectory); + } + + @Override + public List createCompilerCommand( + List options, + Path target, + Path... input + ) { + List wrappedOptions = new ArrayList(options); + wrappedOptions.addFirst(getTarget()); + wrappedOptions.addFirst("-target"); + wrappedOptions.addFirst("cc"); + return super.createCompilerCommand( + compilerInfo.compilerPath, + wrappedOptions, + target, + input + ); + } + + protected String getArch() { + if (Platform.includedIn(Platform.AARCH64.class)) { + return "aarch64"; + } + if (Platform.includedIn(Platform.AMD64.class)) { + return "x86_64"; + } + if (Platform.includedIn(Platform.RISCV64.class)) { + return "riscv64"; + } + throw UserError.abort("Unsupported platform"); + } + + protected String getOs() { + if (Platform.includedIn(Platform.LINUX.class)) { + return "linux"; + } + if (Platform.includedIn(Platform.WINDOWS.class)) { + return "windows"; + } + if (Platform.includedIn(Platform.MACOS.class)) { + return "macos"; + } + throw UserError.abort("Unsupported platform"); + } + + protected String getLibc() { + if (Platform.includedIn(Platform.MACOS.class)) { + return "none"; + } + if (Platform.includedIn(Platform.WINDOWS.class)) { + return "gnu"; + } + if (Platform.includedIn(Platform.LINUX.class)) { + if (LibCBase.targetLibCIs(MuslLibC.class)) { + return "musl"; + } + if (LibCBase.targetLibCIs(GLibC.class)) { + return "gnu"; + } + } + throw UserError.abort("Unsupported platform"); + } + + protected String getTarget() { + return getArch() + "-" + getOs() + "-" + getLibc(); + } + + @Override + protected void verify() { + // No-op + } + + @Override + public void onCompileQueueCreation(BigBang bb, HostedUniverse hUniverse, CompileQueue compileQueue) { + NativeLibraries nativeLibraries = NativeLibraries.singleton(); + if (nativeLibraries == null) { + return; + } + File prefixDir = tempDirectory.resolve("root").toAbsolutePath().toFile(); + prefixDir.mkdirs(); + String prefix = prefixDir.toString(); + for (String lib : nativeLibraries.getLibraries()) { + this.compileLib(prefix, lib); + } + } + + protected Path getLibPath(String name) { + String libDir = System.getProperty("sun.boot.library.path"); + if (libDir != null) { + Path libPath = Paths.get(libDir, "graalvm", "lib", "lib"+name); + if (Files.exists(libPath)) { + return libPath; + } + return null; + } + return null; + } + + protected void compileLib(String prefix, String name) { + Path libPath = getLibPath(name); + if (libPath == null) { + return; + } + Process compilerProcess = null; + + // FIXME(@izaakschroeder): Add null check + String libDir = System.getProperty("sun.boot.library.path"); + String jdkDir = Paths.get(libDir, "..").resolve().toAbsolutePath().toString(); + + + List compilerCommand = Arrays.asList( + getDefaultCompiler(), + "build", + "--release=fast", + "--search-prefix", jdkDir, + "--search-prefix", prefix, + "--prefix", prefix, + "-Dtarget="+getTarget() + ); + try { + ProcessBuilder processBuilder = FileUtils.prepareCommand(compilerCommand, libPath); + processBuilder.redirectErrorStream(true); + processBuilder.environment().put("LC_ALL", "C"); + FileUtils.traceCommand(processBuilder); + compilerProcess = processBuilder.start(); + + List lines; + try (InputStream inputStream = compilerProcess.getInputStream()) { + lines = FileUtils.readAllLines(inputStream); + FileUtils.traceCommandOutput(lines); + } + compilerProcess.waitFor(); + + if (compilerProcess.exitValue() != 0) { + String errorMessage = "Unable to compile %s library %s:%n%s"; + throw UserError.abort(errorMessage, name, SubstrateUtil.getShellCommandString(compilerCommand, false), lines.stream().map(str -> " " + str).collect(Collectors.joining(System.lineSeparator()))); + } + return; + } catch (InterruptedException ex) { + throw new InterruptImageBuilding("Interrupted during building library " + name); + } catch (IOException e) { + throw UserError.abort(e, "Building library %s failed: %s", name, SubstrateUtil.getShellCommandString(compilerCommand, false)); + } finally { + if (compilerProcess != null) { + compilerProcess.destroy(); + } + } + + + } + + @Override + protected String getDefaultCompiler() { + // sun.boot.library.path + String libDir = System.getProperty("sun.boot.library.path"); + if (libDir != null) { + Path binPath = Paths.get(libDir, "graalvm", "zig"); + return binPath.toAbsolutePath().toString(); + } + return "zig"; + } + + @Override + protected CompilerInfo createCompilerInfo( + Path compilerPath, + Scanner scanner + ) { + try { + + scanner.useDelimiter("[. -]"); + scanner.skip("\\p{Zs}*"); + int major = scanner.nextInt(); + int minor0 = scanner.nextInt(); + int minor1 = scanner.nextInt(); + return new CompilerInfo( + compilerPath, + "zig", + "Zig Compiler", + "zig", + major, + minor0, + minor1, + getTarget() + ); + } catch (NoSuchElementException e) { + return null; + } + } + + @Override + protected List getVersionInfoOptions() { + return List.of("version"); + } + } + @SingletonTraits(access = BuildtimeAccessOnly.class, layeredCallbacks = NoLayeredCallbacks.class, other = Disallowed.class) private static class WindowsCCompilerInvoker extends CCompilerInvoker { @@ -493,6 +698,7 @@ public void compileAndParseError(boolean strict, List compileOptions, Pa try (InputStream compilerErrors = getCompilerErrorStream(compilingProcess)) { lines = FileUtils.readAllLines(compilerErrors); FileUtils.traceCommandOutput(lines); + } boolean errorReported = false; for (String line : lines) { @@ -552,12 +758,15 @@ public Path getCCompilerPath() { if (userDefinedPath != null) { compilerPath = Paths.get(userDefinedPath); } else { - String executableName = asExecutableName(getDefaultCompiler()); - Optional optCompilerPath = lookupSearchPath(executableName); + compilerPath = Paths.get(asExecutableName(getDefaultCompiler())); + + } + if (!compilerPath.isAbsolute()) { + Optional optCompilerPath = lookupSearchPath(compilerPath.getFileName().toString()); if (optCompilerPath.isPresent()) { compilerPath = optCompilerPath.get(); } else { - throw UserError.abort("Default native-compiler executable '%s' not found via environment variable PATH", executableName); + throw UserError.abort("Default native-compiler executable '%s' not found via environment variable PATH", compilerPath); } } if (Files.isDirectory(compilerPath) || !Files.isExecutable(compilerPath)) { diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/CCLinkerInvocation.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/CCLinkerInvocation.java index e25f8a43148c..c5f965550ee7 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/CCLinkerInvocation.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/CCLinkerInvocation.java @@ -27,6 +27,7 @@ import static com.oracle.svm.core.SubstrateOptions.SpawnIsolates; import java.io.IOException; +import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; @@ -54,6 +55,7 @@ import com.oracle.svm.hosted.c.CGlobalDataFeature; import com.oracle.svm.hosted.c.NativeLibraries; import com.oracle.svm.hosted.c.codegen.CCompilerInvoker; +import com.oracle.svm.hosted.c.codegen.CCompilerInvoker.ZigCCompilerInvoker; import com.oracle.svm.hosted.c.libc.HostedLibCBase; import com.oracle.svm.hosted.imagelayer.HostedDynamicLayerInfo; import com.oracle.svm.hosted.jdk.JNIRegistrationSupport; @@ -263,6 +265,112 @@ protected List getNativeLinkerOptions() { .collect(Collectors.toList()); } + private static class ZigCCLinkerInvocation extends CCLinkerInvocation { + + + ZigCCLinkerInvocation(AbstractImage.NativeImageKind imageKind, NativeLibraries nativeLibs, List symbols) { + super(imageKind, nativeLibs, symbols); + + Path prefix = tempDirectory.resolve("root"); + additionalPreOptions.add("-L" + prefix.resolve("lib").toAbsolutePath().toString()); + additionalPreOptions.add("-isystem" + prefix.resolve("include").toAbsolutePath().toString()); + + additionalPreOptions.add("-Wa,--noexecstack"); + additionalPreOptions.add("-Wl,-z,noexecstack"); + + // The linker should fail if DT_TEXTREL is needed, otherwise the image won't work on + // SELinux. If SpawnIsolates are disabled, this won't work as dynamic relocations + // are needed for heap access. + if (!SpawnIsolates.getValue()) { + additionalPreOptions.add("-Wl,-z,notext"); + } + + if (SubstrateOptions.RemoveUnusedSymbols.getValue()) { + /* Perform garbage collection of unused input sections. */ + additionalPreOptions.add("-Wl,--gc-sections"); + } + + if (imageKind.isImageLayer) { + /* + * We do not want interposition to affect the resolution of symbols we define and + * reference within this library. + */ + additionalPreOptions.add("-Wl,-Bsymbolic"); + } + + /* Use --version-script to control the visibility of image symbols. */ + try { + StringBuilder exportedSymbols = new StringBuilder(); + exportedSymbols.append("{\n"); + /* Only exported symbols are global ... */ + Set globalSymbols = Stream.concat(getImageSymbols(true).stream(), JNIRegistrationSupport.getShimLibrarySymbols()).collect(Collectors.toSet()); + if (!globalSymbols.isEmpty()) { + exportedSymbols.append("global:\n"); + globalSymbols.forEach(symbol -> exportedSymbols.append('\"').append(symbol).append("\";\n")); + } + /* ... everything else is local. */ + exportedSymbols.append("local: *;\n"); + exportedSymbols.append("};"); + + Path exportedSymbolsPath = nativeLibs.tempDirectory.resolve("exported_symbols.list"); + Files.write(exportedSymbolsPath, Collections.singleton(exportedSymbols.toString())); + additionalPreOptions.add("-Wl,--version-script," + exportedSymbolsPath.toAbsolutePath()); + } catch (IOException e) { + VMError.shouldNotReachHere(e); + } + + additionalPreOptions.addAll(HostedLibCBase.singleton().getAdditionalLinkerOptions(imageKind)); + + if (SubstrateOptions.DeleteLocalSymbols.getValue() && !SubstrateOptions.StripDebugInfo.getValue()) { + additionalPreOptions.add("-Wl,-x"); + } + } + + @Override + String getSymbolName(ObjectFile.Symbol symbol) { + return symbol.getName(); + } + + @Override + protected void setOutputKind(List cmd) { + switch (imageKind) { + case EXECUTABLE: + /* Export global symbols. */ + cmd.add("-Wl,--export-dynamic"); + break; + case STATIC_EXECUTABLE: + cmd.add("-static"); + break; + case IMAGE_LAYER: + case SHARED_LIBRARY: + cmd.add("-shared"); + break; + default: + VMError.shouldNotReachHereUnexpectedInput(imageKind); // ExcludeFromJacocoGeneratedReport + } + } + + @Override + protected List getLibrariesCommand() { + List cmd = new ArrayList<>(); + String previousLayerLib = null; + for (String lib : libs) { + String linkingMode = null; + if (ImageLayerBuildingSupport.buildingExtensionLayer() && HostedDynamicLayerInfo.singleton().isImageLayerLib(lib)) { + VMError.guarantee(!lib.isEmpty()); + VMError.guarantee(previousLayerLib == null, "We currently only support one previous layer."); // GR-58631 + previousLayerLib = lib; + } + cmd.add("-l" + lib); + } + + if (previousLayerLib != null) { + cmd.add("-l" + previousLayerLib); + } + return cmd; + } + } + private static class BinutilsCCLinkerInvocation extends CCLinkerInvocation { private final boolean dynamicLibC = SubstrateOptions.StaticExecutableWithDynamicLibC.getValue(); @@ -691,23 +799,27 @@ static LinkerInvocation getLinkerInvocation(AbstractImage.NativeImageKind imageK Path outputDirectory, Path tempDirectory, String imageName, List symbols) { CCLinkerInvocation inv; - switch (ObjectFile.getNativeFormat()) { - case MACH_O: - inv = new DarwinCCLinkerInvocation(imageKind, nativeLibs, symbols); - break; - case PECOFF: - inv = new WindowsCCLinkerInvocation(imageKind, nativeLibs, symbols, imageName); - break; - case ELF: - default: - inv = new BinutilsCCLinkerInvocation(imageKind, nativeLibs, symbols); - break; + if (ImageSingletons.lookup(CCompilerInvoker.class) instanceof CCompilerInvoker.ZigCCompilerInvoker) { + inv = new ZigCCLinkerInvocation(imageKind, nativeLibs, symbols); + } else { + switch (ObjectFile.getNativeFormat()) { + case MACH_O: + inv = new DarwinCCLinkerInvocation(imageKind, nativeLibs, symbols); + break; + case PECOFF: + inv = new WindowsCCLinkerInvocation(imageKind, nativeLibs, symbols, imageName); + break; + case ELF: + default: + inv = new BinutilsCCLinkerInvocation(imageKind, nativeLibs, symbols); + break; + } } + Path outputFile = outputDirectory.resolve(imageKind.getOutputFilename(imageName)); UserError.guarantee(!Files.isDirectory(outputFile), "Cannot write image to %s. Path exists as directory (use '-o /path/to/image').", outputFile); inv.setOutputFile(outputFile); - inv.setTempDirectory(tempDirectory); inv.addLibPath(tempDirectory.toString()); for (String libraryPath : nativeLibs.getLibraryPaths()) { diff --git a/truffle/mx.truffle/suite.py b/truffle/mx.truffle/suite.py index 5eada8e8a3f9..8a3a576d9e02 100644 --- a/truffle/mx.truffle/suite.py +++ b/truffle/mx.truffle/suite.py @@ -1148,7 +1148,6 @@ "com.oracle.truffle.nfi.test.native" : { "subDir" : "src", "native" : "shared_lib", - "toolchain" : "sdk:LLVM_NINJA_TOOLCHAIN", "deliverable" : "nativetest", "buildDependencies" : [ "com.oracle.truffle.nfi.native",