From c885b175e2195eb892fb8f30bfb36153ac35a4d7 Mon Sep 17 00:00:00 2001 From: Mayank Ramnani Date: Fri, 20 Sep 2024 20:20:48 -0400 Subject: [PATCH] New langauge design --- .circleci/config.yml | 39 +++ CMakeLists.txt | 3 + new-language/zlib-test/main.cpp | 113 ++++++++ new-language/zlib-test/manifest.h | 431 ++++++++++++++++++++++++++++++ 4 files changed, 586 insertions(+) create mode 100644 .circleci/config.yml create mode 100644 new-language/zlib-test/main.cpp create mode 100644 new-language/zlib-test/manifest.h diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000000..7de10cbffd --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,39 @@ +# Use the latest 2.1 version of CircleCI pipeline process engine. +# See: https://circleci.com/docs/configuration-reference +version: 2.1 + +# Define a job to be invoked later in a workflow. +# See: https://circleci.com/docs/jobs-steps/#jobs-overview & https://circleci.com/docs/configuration-reference/#jobs +jobs: + ninja: + docker: + - image: cimg/base:current + steps: + - checkout + - run: + name: Install ctest, clang-tidy and compiler + command: | + sudo apt-get update + sudo apt-get install -y cmake g++ clang-tidy + - run: + name: Copy clang-tidy config + command: | + cp .clang-tidy /tmp/.clang-tidy + - run: + name: Run clang-tidy, build ninja and run ctest + command: | + mkdir build + cd build + cmake .. + cmake --build . + make + ctest + + +# Orchestrate jobs using workflows +# See: https://circleci.com/docs/workflows/ & https://circleci.com/docs/configuration-reference/#workflows +workflows: + build-and-test: # This is the name of the workflow, feel free to change it to better match your workflow. + # Inside the workflow, you define the jobs you want to run. + jobs: + - ninja diff --git a/CMakeLists.txt b/CMakeLists.txt index 47b1f9c117..c0a7f0a5f6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -123,6 +123,9 @@ set(NINJA_PYTHON "python" CACHE STRING "Python interpreter to use for the browse check_platform_supports_browse_mode(platform_supports_ninja_browse) +# run clang-tidy before source code is built +set(CMAKE_CXX_CLANG_TIDY "clang-tidy;--config-file=/tmp/.clang-tidy") + # Core source files all build into ninja library. add_library(libninja OBJECT src/build_log.cc diff --git a/new-language/zlib-test/main.cpp b/new-language/zlib-test/main.cpp new file mode 100644 index 0000000000..f34664d423 --- /dev/null +++ b/new-language/zlib-test/main.cpp @@ -0,0 +1,113 @@ +#include "manifest.h" +#include +#include +#include +#include +#include +#include + +int executeCommand(const std::string& command) { + std::cout << "Executing command: " << command << std::endl; + auto start_time = std::chrono::steady_clock::now(); + + int result = std::system(command.c_str()); + + auto end_time = std::chrono::steady_clock::now(); + auto duration = std::chrono::duration_cast(end_time - start_time).count(); + std::cout << "Command completed in " << duration << " seconds with result: " << result << std::endl; + return result; +} + +class BuildTarget { +public: + std::string obj_name; + std::string src_path; + + BuildTarget(const std::string& obj, const std::string& src) + : obj_name(obj), src_path(src) {} +}; + +int main() { + shadowdash::ShadowDash shadowDash; + + // Create build directories + executeCommand("mkdir -p CMakeFiles\\zlib.dir"); + + // Set compiler paths (without quotes, they'll be added in the command) + VAR(CC, "C:\\msys64\\mingw64\\bin\\gcc.exe"); + VAR(WINDRES, "C:\\msys64\\mingw64\\bin\\windres.exe"); + + // Define pools + POOL("compile_pool", 8); + + // Set build directory + BUILDDIR("."); + + // Define all variables without extra spaces + VAR(DEFINES, "-DZLIB_DLL -D_LARGEFILE64_SOURCE=1"); + VAR(INCLUDES, "-I./build -IE:/nyu/senior/opensource/zlib"); + VAR(CFLAGS, "-MD -O2"); + VAR(RC_FLAGS, "-D GCC_WINDRES -I E:/nyu/senior/opensource/zlib -I ./build"); + + // Resource compiler rule + RULE(WINDRES_COMPILER, + shadowdash::variable("WINDRES", 7), + shadowdash::variable("RC_FLAGS", 8), + shadowdash::constant("-o", 2), + shadowdash::variable(shadowdash::out, 3), + shadowdash::constant("-i", 2), + shadowdash::variable(shadowdash::in, 2) + ); + + // Compiler rule without extra quotes and spaces + RULE_WITH_POOL(C_COMPILER_zlib, + { + shadowdash::variable("CC", 2), + shadowdash::variable("DEFINES", 7), + shadowdash::variable("INCLUDES", 8), + shadowdash::variable("CFLAGS", 6), + shadowdash::constant("-c", 2), + shadowdash::variable(shadowdash::in, 2), + shadowdash::constant("-o", 2), + shadowdash::variable(shadowdash::out, 3) + }, + "compile_pool", + 1 + ); + + // Build resource file + std::cout << "\nBuilding resource file...\n"; + BUILD("zlib1rc.obj", "WINDRES_COMPILER", {"./win32/zlib1.rc"}); + + // Create stable build targets + std::vector targets; + const std::vector source_files = { + "adler32.c" // Let's test with just one file first + }; + + // Create stable build configurations + for (const auto& src : source_files) { + std::string obj_name = "CMakeFiles/zlib.dir/" + src + ".obj"; + std::string src_path = "./" + src; + targets.emplace_back(obj_name, src_path); + } + + // Set up builds with debug output + std::cout << "\nSetting up builds...\n"; + for (const auto& target : targets) { + std::cout << "Creating build for: " << target.obj_name << " from " << target.src_path << std::endl; + BUILD(target.obj_name.c_str(), "C_COMPILER_zlib", {target.src_path.c_str()}); + } + + // Execute build with verbose output + try { + std::cout << "\nStarting build process...\n"; + shadowDash.executeBuild(); + std::cout << "Build completed successfully!\n"; + } catch (const std::exception& e) { + std::cerr << "Build failed: " << e.what() << std::endl; + return 1; + } + + return 0; +} \ No newline at end of file diff --git a/new-language/zlib-test/manifest.h b/new-language/zlib-test/manifest.h new file mode 100644 index 0000000000..a35c70ea17 --- /dev/null +++ b/new-language/zlib-test/manifest.h @@ -0,0 +1,431 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace shadowdash { + inline int executeCommand(const std::string& command) { + std::cout << "Executing: " << command << std::endl; + return std::system(command.c_str()); + } + inline bool file_exists(const std::string& filename) { + struct stat buffer; + return ::stat(filename.c_str(), &buffer) == 0; + } + + class Expression { + public: + enum class Type { CONSTANT, VARIABLE, PATH }; + + constexpr Expression(Type type, std::string_view content) + : type_(type), content_(content) {} + + Type type_; + std::string_view content_; + }; + + constexpr Expression constant(const char* value, std::size_t len) { + return Expression(Expression::Type::CONSTANT, {value, len}); + } + + constexpr Expression variable(const char* value, std::size_t len) { + return Expression(Expression::Type::VARIABLE, {value, len}); + } + + class Command { + public: + Command(std::initializer_list parts) + : parts_(std::vector(parts)) {} + std::vector parts_; + }; + + class Pool { + public: + Pool(std::string name, int depth) + : name_(std::move(name)), depth_(depth), current_jobs_(0) {} + + Pool(const Pool& other) + : name_(other.name_), depth_(other.depth_), current_jobs_(0) {} + + Pool& operator=(const Pool& other) { + if (this != &other) { + name_ = other.name_; + depth_ = other.depth_; + current_jobs_ = 0; + } + return *this; + } + + bool canStartJob() const { + return current_jobs_ < depth_; + } + + void startJob() { + if (!canStartJob()) { + throw std::runtime_error("Pool " + name_ + " is full"); + } + current_jobs_++; + } + + void finishJob() { + if (current_jobs_ > 0) { + current_jobs_--; + } + } + + private: + std::string name_; + int depth_; + std::atomic current_jobs_; + }; + + class Rule { + public: + Rule(Command command) : command_(std::move(command)) {} + + Rule(Command command, std::string_view pool, int jobs = 1) + : command_(std::move(command)), pool_(pool), jobs_(jobs) {} + + Command command_; + std::optional description_; + std::optional depfile_; + std::optional deps_; + std::optional generator_{false}; + std::optional restat_{false}; + std::optional rspfile_; + std::optional rspfile_content_; + std::optional pool_; + std::optional jobs_{1}; + + bool hasGenerator() const { return generator_ && *generator_; } + bool needsRestat() const { return restat_ && *restat_; } + }; + + class Build { + public: + // Constructor for initializer_list + Build(std::string_view output, + std::string_view rule, + std::initializer_list inputs = {}, + std::initializer_list implicit_inputs = {}, + std::initializer_list order_only_inputs = {}, + std::initializer_list implicit_outputs = {}, + bool is_phony = false) + : output_(output), rule_(rule), + inputs_(inputs.begin(), inputs.end()), + implicit_inputs_(implicit_inputs.begin(), implicit_inputs.end()), + order_only_inputs_(order_only_inputs.begin(), order_only_inputs.end()), + implicit_outputs_(implicit_outputs.begin(), implicit_outputs.end()), + is_phony_(is_phony) {} + + // Constructor for vectors + Build(std::string_view output, + std::string_view rule, + const std::vector& inputs, + const std::vector& implicit_inputs = {}, + const std::vector& order_only_inputs = {}, + const std::vector& implicit_outputs = {}, + bool is_phony = false) + : output_(output), rule_(rule), + inputs_(inputs), + implicit_inputs_(implicit_inputs), + order_only_inputs_(order_only_inputs), + implicit_outputs_(implicit_outputs), + is_phony_(is_phony) {} + + bool needsRebuild() const { + if (is_phony_) return true; + + struct stat output_stat; + if (::stat(std::string(output_).c_str(), &output_stat) != 0) return true; + + for (const auto& input : inputs_) { + struct stat input_stat; + if (::stat(std::string(input).c_str(), &input_stat) != 0) { + return true; + } + if (input_stat.st_mtime > output_stat.st_mtime) { + return true; + } + } + return false; + } + + bool is_phony_; + std::string_view output_; + std::string_view rule_; + std::vector inputs_; + std::vector implicit_inputs_; + std::vector order_only_inputs_; + std::vector implicit_outputs_; + std::unordered_map variables_; + }; + + class ShadowDash { + private: + std::unordered_map rules_; + std::vector builds_; + std::unordered_map variables_; + std::vector defaults_; + std::optional builddir_; + std::unordered_map pools_; + + + std::string resolveVariable(const std::string_view& var, const Build& build) const { + try { + if (var == "in") { + if (build.inputs_.empty()) { + throw std::runtime_error("No input files specified"); + } + return std::string(build.inputs_[0]); // For now, just use first input + } else if (var == "out") { + return std::string(build.output_); + } else { + auto it = variables_.find(var); + if (it != variables_.end()) { + return std::string(it->second); + } + throw std::runtime_error("Undefined variable: " + std::string(var)); + } + } catch (const std::exception& e) { + throw std::runtime_error("Variable resolution failed: " + std::string(var)); + } + } + + std::string constructCommand(const Build& build) const { + if (rules_.find(build.rule_) == rules_.end()) { + throw std::runtime_error("Unknown rule: " + std::string(build.rule_)); + } + + std::stringstream cmd; + const auto& rule = rules_.at(build.rule_); + + // Better command construction without extra spaces + bool first = true; + for (const auto& part : rule.command_.parts_) { + if (!first) { + cmd << " "; // Add single space between parts + } + first = false; + + if (part.type_ == Expression::Type::CONSTANT) { + cmd << part.content_; + } else if (part.type_ == Expression::Type::VARIABLE) { + try { + std::string resolved = resolveVariable(part.content_, build); + if (!resolved.empty()) { + cmd << resolved; + } + } catch (const std::exception& e) { + throw std::runtime_error("Variable resolution failed for '" + + std::string(part.content_) + "': " + e.what()); + } + } + } + return cmd.str(); + } + + + // finds a build given the output name + const Build* findBuildByOutput(std::string_view output) const { + for (const auto& build : builds_) { + if (build.output_ == output) { + return &build; + } + } + return nullptr; + } + + void buildDependencies(const Build& build) const { + for (const auto& input : build.inputs_) { + std::cout << "Building dependency " << input << std::endl; + if (!file_exists(std::string(input))) { + // try to find a build that produces this input, and execute the build + if (const Build* dep_build = findBuildByOutput(input)) { + executeSingleBuild(*dep_build); + } else { + throw std::runtime_error("Input file missing and no rule to build it: " + std::string(input)); + } + } + } + + // do same thing for implicit inputs + for (const auto& input : build.implicit_inputs_) { + if (!file_exists(std::string(input))) { + // try to find a build that produces this input, and execute the build + if (const Build* dep_build = findBuildByOutput(input)) { + executeSingleBuild(*dep_build); + } else { + throw std::runtime_error("Implicit input file missing and no rule to build it: " + std::string(input)); + } + } + } + } + + public: + void defineRule(std::string_view name, Rule rule) { + rules_.emplace(name, std::move(rule)); + } + + void defineBuild(Build build) { + builds_.push_back(std::move(build)); + } + + void defineVariable(std::string_view name, std::string_view value) { + variables_[name] = value; + } + + void definePool(std::string name, int depth) { + pools_.emplace(name, Pool(name, depth)); + } + + void addDefault(std::string_view target) { + defaults_.push_back(target); + } + + void setBuildDir(std::string_view dir) { + builddir_ = dir; + if (!file_exists(std::string(dir))) { + if (!createDirectory(std::string(dir))) { + throw std::runtime_error("Failed to create build directory: " + std::string(dir)); + } + } + } + + void executeBuild() const { + try { + // build everything if no defaults + if (defaults_.empty()) { + for (const auto& build : builds_) { + executeSingleBuild(build); + } + } else { + // build only defaults + for (const auto& default_target : defaults_) { + bool found = false; + for (const auto& build : builds_) { + if (build.output_ == default_target) { + executeSingleBuild(build); + found = true; + break; + } + } + if (!found) { + throw std::runtime_error("Default target not found: " + std::string(default_target)); + } + } + } + } catch (const std::exception& e) { + std::cerr << "Build failed: " << e.what() << "\n"; + throw; + } + } + + void executeSingleBuild(const Build& build) const { + std::cout << "\nStarting build for target: " << build.output_ << std::endl; + + if (!build.needsRebuild()) { + std::cout << "Skipping up-to-date target: " << build.output_ << "\n"; + return; + } + + std::cout << (build.is_phony_ ? "Executing phony target: " : "Building: ") << build.output_ << "\n"; + + // build all its dependencies first + buildDependencies(build); + + for (const auto& dep : build.implicit_inputs_) { + if (!file_exists(std::string(dep))) { + throw std::runtime_error("Missing dependency: " + std::string(dep)); + } + } + + std::string command = constructCommand(build); + std::cout << "Executing: " << command << "\n"; + + // Get the rule and check for pool + const auto& rule = rules_.at(build.rule_); + Pool* pool = nullptr; + if (rule.pool_) { + auto pool_it = pools_.find(std::string(*rule.pool_)); + if (pool_it == pools_.end()) { + throw std::runtime_error("Unknown pool: " + std::string(*rule.pool_)); + } + pool = const_cast(&pool_it->second); + + // Wait until we can acquire a slot in the pool + while (!pool->canStartJob()) { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + pool->startJob(); + } + + try { + if (int result = std::system(command.c_str()); result != 0) { + throw std::runtime_error("Command failed with code " + + std::to_string(result) + ": " + command); + } + } catch (...) { + if (pool) { + pool->finishJob(); + } + throw; + } + + if (pool) { + pool->finishJob(); + } + std::cout << "\n"; + } + }; + inline bool createDirectory(const std::string& path) { + #ifdef _WIN32 + return _mkdir(path.c_str()) == 0 || errno == EEXIST; + #else + return mkdir(path.c_str(), 0755) == 0 || errno == EEXIST; + #endif + } + + static constexpr auto in = "in"; + static constexpr auto out = "out"; + +} // namespace shadowdash + +#define RULE(name, ...) \ + shadowDash.defineRule(#name, shadowdash::Rule{ \ + shadowdash::Command{ __VA_ARGS__ } \ + }) + +#define RULE_WITH_POOL(name, cmd, pool, ...) \ + shadowDash.defineRule(#name, shadowdash::Rule{ \ + cmd, pool, ##__VA_ARGS__ \ + }) + + +#define BUILD(...) \ + shadowDash.defineBuild(shadowdash::Build{ __VA_ARGS__ }) + +#define VAR(name, value) \ + shadowDash.defineVariable(#name, value) + +#define DEFAULT(target) \ + shadowDash.addDefault(target) + +#define BUILDDIR(dir) \ + shadowDash.setBuildDir(dir) + +#define PHONY(name, rule, ...) \ + shadowDash.defineBuild(shadowdash::Build{#name, rule, {__VA_ARGS__}, {}, {}, {}, true}) + +#define POOL(name, depth) \ + shadowDash.definePool(name, depth)