diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000000..7de10cbffd --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,39 @@ +# Use the latest 2.1 version of CircleCI pipeline process engine. +# See: https://circleci.com/docs/configuration-reference +version: 2.1 + +# Define a job to be invoked later in a workflow. +# See: https://circleci.com/docs/jobs-steps/#jobs-overview & https://circleci.com/docs/configuration-reference/#jobs +jobs: + ninja: + docker: + - image: cimg/base:current + steps: + - checkout + - run: + name: Install ctest, clang-tidy and compiler + command: | + sudo apt-get update + sudo apt-get install -y cmake g++ clang-tidy + - run: + name: Copy clang-tidy config + command: | + cp .clang-tidy /tmp/.clang-tidy + - run: + name: Run clang-tidy, build ninja and run ctest + command: | + mkdir build + cd build + cmake .. + cmake --build . + make + ctest + + +# Orchestrate jobs using workflows +# See: https://circleci.com/docs/workflows/ & https://circleci.com/docs/configuration-reference/#workflows +workflows: + build-and-test: # This is the name of the workflow, feel free to change it to better match your workflow. + # Inside the workflow, you define the jobs you want to run. + jobs: + - ninja diff --git a/CMakeLists.txt b/CMakeLists.txt index b8fdee7d3a..c0a7f0a5f6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -123,6 +123,9 @@ set(NINJA_PYTHON "python" CACHE STRING "Python interpreter to use for the browse check_platform_supports_browse_mode(platform_supports_ninja_browse) +# run clang-tidy before source code is built +set(CMAKE_CXX_CLANG_TIDY "clang-tidy;--config-file=/tmp/.clang-tidy") + # Core source files all build into ninja library. add_library(libninja OBJECT src/build_log.cc @@ -135,6 +138,7 @@ add_library(libninja OBJECT src/deps_log.cc src/disk_interface.cc src/edit_distance.cc + src/elide_middle.cc src/eval_env.cc src/graph.cc src/graphviz.cc @@ -265,6 +269,7 @@ if(BUILD_TESTING) src/disk_interface_test.cc src/dyndep_parser_test.cc src/edit_distance_test.cc + src/elide_middle_test.cc src/explanations_test.cc src/graph_test.cc src/json_test.cc @@ -295,6 +300,7 @@ if(BUILD_TESTING) canon_perftest clparser_perftest depfile_parser_perftest + elide_middle_perftest hash_collision_bench manifest_parser_perftest ) diff --git a/README.md b/README.md index 9312d935f3..d9f883a5a8 100644 --- a/README.md +++ b/README.md @@ -1,100 +1,19 @@ -# Ninja - -Ninja is a small build system with a focus on speed. -https://ninja-build.org/ - -See [the manual](https://ninja-build.org/manual.html) or -`doc/manual.asciidoc` included in the distribution for background -and more details. - -Binaries for Linux, Mac and Windows are available on - [GitHub](https://github.com/ninja-build/ninja/releases). -Run `./ninja -h` for Ninja help. - -Installation is not necessary because the only required file is the -resulting ninja binary. However, to enable features like Bash -completion and Emacs and Vim editing modes, some files in misc/ must be -copied to appropriate locations. - -If you're interested in making changes to Ninja, read -[CONTRIBUTING.md](CONTRIBUTING.md) first. - -## Building Ninja itself - -You can either build Ninja via the custom generator script written in Python or -via CMake. For more details see -[the wiki](https://github.com/ninja-build/ninja/wiki). - -### Python - -``` -./configure.py --bootstrap +# Shadowdash Converter +## Build +```sh +./build.sh ``` +This will create the converter in the `debug-build` directory with debug symbols. -This will generate the `ninja` binary and a `build.ninja` file you can now use -to build Ninja with itself. - -If you have a GoogleTest source directory, you can build the tests -by passing its path with `--gtest-source-dir=PATH` option, or the -`GTEST_SOURCE_DIR` environment variable, e.g.: - -``` -./configure.py --bootstrap --gtest-source-dir=/path/to/googletest -./ninja all # build ninja_test and other auxiliary binaries -./ninja_test` # run the unit-test suite. -``` - -Use the CMake build below if you want to use a preinstalled binary -version of the library. - -### CMake - -``` -cmake -Bbuild-cmake -cmake --build build-cmake +## Run +```sh +./debug-build/ninja ``` +Run the above in a directory where a `build.ninja` file is present -The `ninja` binary will now be inside the `build-cmake` directory (you can -choose any other name you like). - -To run the unit tests: - +## Testing +```sh +cd converter/testing/ +../../debug-build/ninja ``` -./build-cmake/ninja_test -``` - -## Generating documentation - -### Ninja Manual - -You must have `asciidoc` and `xsltproc` in your PATH, then do: - -``` -./configure.py -ninja manual doc/manual.pdf -``` - -Which will generate `doc/manual.html`. - -To generate the PDF version of the manual, you must have `dblatext` in your PATH then do: - -``` -./configure.py # only if you didn't do it previously. -ninja doc/manual.pdf -``` - -Which will generate `doc/manual.pdf`. - -### Doxygen documentation - -If you have `doxygen` installed, you can build documentation extracted from C++ -declarations and comments to help you navigate the code. Note that Ninja is a standalone -executable, not a library, so there is no public API, all details exposed here are -internal. - -``` -./configure.py # if needed -ninja doxygen -``` - -Then open `doc/doxygen/html/index.html` in a browser to look at it. +This will run the converter on the hello world example `build.ninja` file and produce `output.cc` as a result. diff --git a/build.sh b/build.sh new file mode 100755 index 0000000000..b09867ae76 --- /dev/null +++ b/build.sh @@ -0,0 +1,2 @@ +cmake -DCMAKE_BUILD_TYPE=Debug -B debug-build +cmake --build debug-build --parallel --config Debug --target ninja diff --git a/configure.py b/configure.py index c88daad508..7d3ecb6d38 100755 --- a/configure.py +++ b/configure.py @@ -539,6 +539,7 @@ def has_re2c() -> bool: 'dyndep', 'dyndep_parser', 'edit_distance', + 'elide_middle', 'eval_env', 'graph', 'graphviz', @@ -638,6 +639,7 @@ def has_re2c() -> bool: 'disk_interface_test', 'dyndep_parser_test', 'edit_distance_test', + 'elide_middle_test', 'explanations_test', 'graph_test', 'json_test', @@ -683,6 +685,7 @@ def has_re2c() -> bool: for name in ['build_log_perftest', 'canon_perftest', + 'elide_middle_perftest', 'depfile_parser_perftest', 'hash_collision_bench', 'manifest_parser_perftest', diff --git a/converter/archive/convert.py b/converter/archive/convert.py new file mode 100644 index 0000000000..0dbbc98e27 --- /dev/null +++ b/converter/archive/convert.py @@ -0,0 +1,98 @@ +import argparse + +def parse_ninja_file(ninja_file_content): + rules = {} + builds = [] + current_rule = None + default_target = None + + lines = ninja_file_content.splitlines() + + for line in lines: + line = line.strip() + if line.startswith("rule"): + current_rule = line.split()[1] + rules[current_rule] = {"command": None, "flags": "-O3"} + elif line.startswith("command ="): + if current_rule: + rules[current_rule]["command"] = line.split("=", 1)[1].strip() + elif line.startswith("flags ="): + if current_rule: + rules[current_rule]["flags"] = line.split("=", 1)[1].strip() + elif line.startswith("build"): + parts = line.split() + output, rule, inputs = parts[1], parts[2], parts[3:] + builds.append((output, rule, inputs)) + elif line.startswith("default"): + default_target = line.split()[1] + + return rules, builds, default_target + +def generate_manifest_cc(rules, builds, default_target): + cc_code = '#include "../manifest.h"\n\nusing namespace shadowdash;\n\nvoid manifest() {\n' + cc_code += f' let(flags, "{rules["compile"]["flags"]}");\n\n' + + for rule, details in rules.items(): + cc_code += f' auto {rule} = rule{{ {{\n' + command = details["command"].replace("$flags", '"flags"_v').replace("$in", "in").replace("$out", "out") + cc_code += f' bind(command, {command}), //\n' + cc_code += f' }} }};\n\n' + + for build in builds: + output, rule, inputs = build + inputs_str = ", ".join([f'str{{ "{inp}" }}' for inp in inputs]) + if rule == "compile" and "flags" in rules["compile"] and rules["compile"]["flags"] != "-O3": + cc_code += f' build(list{{ str{{ "{output}" }} }}, //\n' + cc_code += f' {{}}, //\n' + cc_code += f' {rule}, //\n' + cc_code += f' list{{ {inputs_str} }}, //\n' + cc_code += f' {{}}, //\n' + cc_code += f' {{}}, //\n' + cc_code += f' {{ bind(flags, "{rules["compile"]["flags"]}") }} //\n' + cc_code += f' );\n\n' + else: + cc_code += f' build(list{{ str{{ "{output}" }} }}, //\n' + cc_code += f' {{}}, //\n' + cc_code += f' {rule}, //\n' + cc_code += f' list{{ {inputs_str} }}, //\n' + cc_code += f' {{}}, //\n' + cc_code += f' {{}}, //\n' + cc_code += f' {{}} //\n' + cc_code += f' );\n\n' + + cc_code += f' build(list{{ str{{ "{default_target}" }} }}, //\n' + cc_code += f' {{}}, //\n' + cc_code += f' link, //\n' + cc_code += f' list{{ str{{ "{default_target}.o" }} }}, //\n' + cc_code += f' {{}}, //\n' + cc_code += f' {{}}, //\n' + cc_code += f' {{}} //\n' + cc_code += f' );\n' + + cc_code += '}\n' + return cc_code + +def main(): + # argument parser for input and output file names + parser = argparse.ArgumentParser(description="Convert a build.ninja file to a build.ninja.cc file.") + parser.add_argument('input_file', help="Path to the input build.ninja file") + parser.add_argument('output_file', help="Path to the output build.ninja.cc file") + + args = parser.parse_args() + + # read input file + with open(args.input_file, 'r') as f: + ninja_file_content = f.read() + + # parse the ninja file + rules, builds, default_target = parse_ninja_file(ninja_file_content) + + # generate the manifest.cc content + manifest_cc_content = generate_manifest_cc(rules, builds, default_target) + + # write the output to the file + with open(args.output_file, 'w') as f: + f.write(manifest_cc_content) + +if __name__ == "__main__": + main() diff --git a/converter/src/hooks.cpp b/converter/src/hooks.cpp new file mode 100644 index 0000000000..741b1a4a01 --- /dev/null +++ b/converter/src/hooks.cpp @@ -0,0 +1,21 @@ + +void HandleRule(Rule* rule) +{ + // output string + std::string output = "auto " + rule.name_ + " = rule{{bind(command, \"g++\", \"flags\"_v, \"-c\", "; + + // Iterate over the parsed_ vector in the "command" binding + for (const auto& entry : rule.bindings_["command"].parsed_) { + if (entry.second == RAW) { + output += "\"" + entry.first + "\", "; // Add quotes for raw strings + } else if (entry.second == SPECIAL) { + output += entry.first + ", "; // No quotes for special variables + } + } + + // Finish the string + output += ")}};"; + + // Print the result + std::cout << output << std::endl; +} diff --git a/converter/src/main.cpp b/converter/src/main.cpp new file mode 100644 index 0000000000..be5f3603d4 --- /dev/null +++ b/converter/src/main.cpp @@ -0,0 +1,55 @@ +#include +#include +#include +#include + +// Assuming you have these constants +enum EvalStringType { RAW, SPECIAL }; + +// Example structures based on your description +struct EvalString { + std::string first; + EvalStringType second; +}; + +struct Binding { + std::vector parsed_; +}; + +struct Rule { + std::string name_; + std::map bindings_; +}; + +int main() { + // Simulate your object + Rule rule; + rule.name_ = "compile"; + rule.bindings_["command"].parsed_ = { + {"g++ -c ", RAW}, + {"in", SPECIAL}, + {" -o ", RAW}, + {"out", SPECIAL}, + {" ", RAW} + }; + + // Begin the output string + std::string output = "auto " + rule.name_ + " = rule{{bind(command, \"g++\", \"flags\"_v, \"-c\", "; + + // Iterate over the parsed_ vector in the "command" binding + for (const auto& entry : rule.bindings_["command"].parsed_) { + if (entry.second == RAW) { + output += "\"" + entry.first + "\", "; // Add quotes for raw strings + } else if (entry.second == SPECIAL) { + output += entry.first + ", "; // No quotes for special variables + } + } + + // Finish the string + output += ")}};"; + + // Print the result + std::cout << output << std::endl; + + return 0; +} diff --git a/converter/src/manifest.h b/converter/src/manifest.h new file mode 100644 index 0000000000..81765814da --- /dev/null +++ b/converter/src/manifest.h @@ -0,0 +1,86 @@ +#pragma once + +#include +#include + +namespace shadowdash { + +class Token { + public: + enum Type { LITERAL, VAR }; + + constexpr Token(Type type, std::string_view value) + : type_(type), value_(value) {} + + constexpr Token(std::string_view value) : Token(Token::LITERAL, value) {} + + constexpr Token(const char* value) : Token(Token::LITERAL, value) {} + + Type type_; + std::string_view value_; +}; + +constexpr Token operator"" _l(const char* value, std::size_t len) { + return Token(Token::Type::LITERAL, { value, len }); +} + +constexpr Token operator"" _v(const char* value, std::size_t len) { + return Token(Token::Type::VAR, { value, len }); +} + +class str { + public: + str(std::initializer_list tokens) : tokens_(tokens) {} + std::initializer_list tokens_; +}; + +using binding = std::pair; +using map = std::initializer_list; + +class list { + public: + list(std::initializer_list values) : values_(values) {} + std::initializer_list values_; +}; + +class var { + public: + var(const char* name, str value) {} +}; + +class rule { + public: + rule(map bindings) {} +}; + +class build { + public: + build( // + list outputs, // + list implicit_outputs, // + rule& rule, // + list inputs, // + list implicit_inputs, // + list order_only_inputs, // + map bindings) // + {} +}; + +static constexpr auto in = "in"_v; +static constexpr auto out = "out"_v; + +} // namespace shadowdash + +#define let(name, ...) \ + var name { \ + #name, str { \ + __VA_ARGS__ \ + } \ + } + +#define bind(name, ...) \ + { \ + #name, str { \ + __VA_ARGS__ \ + } \ + } diff --git a/converter/testing/build.ninja b/converter/testing/build.ninja new file mode 100644 index 0000000000..0fe588dc65 --- /dev/null +++ b/converter/testing/build.ninja @@ -0,0 +1,13 @@ +flags = -O3 +rule compile + command = g++ $flags -c $in -o $out + +rule link + command = g++ $in -o $out + +build hello.o: compile hello.cpp + flags = -O2 + +build hello: link hello.o + +default hello diff --git a/converter/testing/build.ninja.cc b/converter/testing/build.ninja.cc new file mode 100644 index 0000000000..5ba0658574 --- /dev/null +++ b/converter/testing/build.ninja.cc @@ -0,0 +1,36 @@ +#include "../manifest.h" + +using namespace shadowdash; + +void manifest() { + + let(flags, "-O3"); + + auto compile = rule{ { + bind(command, "g++", "flags"_v, "-c", "in"_v, "-o", "out"_v), // + } }; + + auto link = rule{ { + bind(command, "g++", "in"_v, "-o", "out"_v), // + } }; + + build(list{ str{ "hello.o" } }, // + {}, // + compile, // + list{ str{ "hello.cc" } }, // + {}, // + {}, // + { bind(flags, "-O2") } // + ); + + build(list{ str{ "hello" } }, // + {}, // + link, // + list{ str{ "hello.o" } }, // + {}, // + {}, // + {} // + ); + + default(str("hello")); +} diff --git a/converter/testing/build.sh b/converter/testing/build.sh new file mode 100755 index 0000000000..9f8a962ab7 --- /dev/null +++ b/converter/testing/build.sh @@ -0,0 +1 @@ +g++ -fPIC -shared -o libmanifest.so build.ninja.cc diff --git a/converter/testing/hello.cpp b/converter/testing/hello.cpp new file mode 100644 index 0000000000..e9753d4812 --- /dev/null +++ b/converter/testing/hello.cpp @@ -0,0 +1,6 @@ +#include + +auto main() -> int{ + std::cout << "Hell, Ninja!" << '\n'; + return 0; +} diff --git a/misc/output_test.py b/misc/output_test.py index cba3ff02fa..f48968bc9a 100755 --- a/misc/output_test.py +++ b/misc/output_test.py @@ -178,6 +178,21 @@ def test_issue_2048(self) -> None: except subprocess.CalledProcessError as err: self.fail("non-zero exit code with: " + err.output) + def test_depfile_directory_creation(self) -> None: + b = BuildDir('''\ + rule touch + command = touch $out && echo "$out: extra" > $depfile + + build somewhere/out: touch + depfile = somewhere_else/out.d + ''') + with b: + self.assertEqual(b.run('', pipe=True), dedent('''\ + [1/1] touch somewhere/out && echo "somewhere/out: extra" > somewhere_else/out.d + ''')) + self.assertTrue(os.path.isfile(os.path.join(b.d.name, "somewhere", "out"))) + self.assertTrue(os.path.isfile(os.path.join(b.d.name, "somewhere_else", "out.d"))) + def test_status(self) -> None: self.assertEqual(run(''), 'ninja: no work to do.\n') self.assertEqual(run('', pipe=True), 'ninja: no work to do.\n') @@ -213,6 +228,71 @@ def test_tool_inputs(self) -> None: out2 ''') + self.assertEqual(run(plan, flags='-t inputs --dependency-order out3'), +'''in2 +in1 +out1 +out2 +implicit +order_only +''') + + # Verify that results are shell-escaped by default, unless --no-shell-escape + # is used. Also verify that phony outputs are never part of the results. + quote = '"' if platform.system() == "Windows" else "'" + + plan = ''' +rule cat + command = cat $in $out +build out1 : cat in1 +build out$ 2 : cat out1 +build out$ 3 : phony out$ 2 +build all: phony out$ 3 +''' + + # Quoting changes the order of results when sorting alphabetically. + self.assertEqual(run(plan, flags='-t inputs all'), +f'''{quote}out 2{quote} +in1 +out1 +''') + + self.assertEqual(run(plan, flags='-t inputs --no-shell-escape all'), +'''in1 +out 2 +out1 +''') + + # But not when doing dependency order. + self.assertEqual( + run( + plan, + flags='-t inputs --dependency-order all' + ), + f'''in1 +out1 +{quote}out 2{quote} +''') + + self.assertEqual( + run( + plan, + flags='-t inputs --dependency-order --no-shell-escape all' + ), + f'''in1 +out1 +out 2 +''') + + self.assertEqual( + run( + plan, + flags='-t inputs --dependency-order --no-shell-escape --print0 all' + ), + f'''in1\0out1\0out 2\0''' + ) + + def test_explain_output(self): b = BuildDir('''\ build .FORCE: phony diff --git a/src/build.cc b/src/build.cc index deb8f04c8b..01f31a9faf 100644 --- a/src/build.cc +++ b/src/build.cc @@ -48,12 +48,10 @@ namespace { /// A CommandRunner that doesn't actually run the commands. struct DryRunCommandRunner : public CommandRunner { - virtual ~DryRunCommandRunner() {} - // Overridden from CommandRunner: - virtual size_t CanRunMore() const; - virtual bool StartCommand(Edge* edge); - virtual bool WaitForCommand(Result* result); + size_t CanRunMore() const override; + bool StartCommand(Edge* edge) override; + bool WaitForCommand(Result* result) override; private: queue finished_; @@ -528,7 +526,7 @@ void Plan::ComputeCriticalPath() { for (Edge* edge : sorted_edges) edge->set_critical_path_weight(EdgeWeightHeuristic(edge)); - // Second propagate / increment weidghts from + // Second propagate / increment weights from // children to parents. Scan the list // in reverse order to do so. for (auto reverse_it = sorted_edges.rbegin(); @@ -595,12 +593,11 @@ void Plan::Dump() const { struct RealCommandRunner : public CommandRunner { explicit RealCommandRunner(const BuildConfig& config) : config_(config) {} - virtual ~RealCommandRunner() {} - virtual size_t CanRunMore() const; - virtual bool StartCommand(Edge* edge); - virtual bool WaitForCommand(Result* result); - virtual vector GetActiveEdges(); - virtual void Abort(); + size_t CanRunMore() const override; + bool StartCommand(Edge* edge) override; + bool WaitForCommand(Result* result) override; + vector GetActiveEdges() override; + void Abort() override; const BuildConfig& config_; SubprocessSet subprocs_; @@ -906,6 +903,12 @@ bool Builder::StartEdge(Edge* edge, string* err) { edge->command_start_time_ = build_start; + // Create depfile directory if needed. + // XXX: this may also block; do we care? + std::string depfile = edge->GetUnescapedDepfile(); + if (!depfile.empty() && !disk_interface_->MakeDirs(depfile)) + return false; + // Create response file, if needed // XXX: this may also block; do we care? string rspfile = edge->GetUnescapedRspfile(); diff --git a/src/build_test.cc b/src/build_test.cc index c84190a040..7675aceecf 100644 --- a/src/build_test.cc +++ b/src/build_test.cc @@ -1852,7 +1852,7 @@ TEST_F(BuildWithLogTest, RestatSingleDependentOutputDirty) { // out2 and out3 will be built even though "in" is not touched when built. // Then, since out2 is rebuilt, out4 should be rebuilt -- the restat on the // "true" rule should not lead to the "touch" edge writing out2 and out3 being - // cleard. + // cleared. command_runner_.commands_ran_.clear(); state_.Reset(); EXPECT_TRUE(builder_.AddTarget("out4", &err)); @@ -3261,7 +3261,7 @@ TEST_F(BuildWithDepsLogTest, RestatMissingDepfileDepslog) { // Touch 'header.in', blank dependencies log (create a different one). // Building header.h triggers 'restat' outputs cleanup. - // Validate that out is rebuilt netherless, as deps are missing. + // Validate that out is rebuilt nevertheless, as deps are missing. fs_.Tick(); fs_.Create("header.in", ""); diff --git a/src/elide_middle.cc b/src/elide_middle.cc new file mode 100644 index 0000000000..cf17da5ba4 --- /dev/null +++ b/src/elide_middle.cc @@ -0,0 +1,276 @@ +// Copyright 2024 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "elide_middle.h" + +#include +#include + +// Convenience class used to iterate over the ANSI color sequences +// of an input string. Note that this ignores non-color related +// ANSI sequences. Usage is: +// +// - Create instance, passing the input string to the constructor. +// - Loop over each sequence with: +// +// AnsiColorSequenceIterator iter; +// while (iter.HasSequence()) { +// .. use iter.SequenceStart() and iter.SequenceEnd() +// iter.NextSequence(); +// } +// +struct AnsiColorSequenceIterator { + // Constructor takes input string . + AnsiColorSequenceIterator(const std::string& input) + : input_(input.data()), input_end_(input_ + input.size()) { + FindNextSequenceFrom(input_); + } + + // Return true if an ANSI sequence was found. + bool HasSequence() const { return cur_end_ != 0; } + + // Start of the current sequence. + size_t SequenceStart() const { return cur_start_; } + + // End of the current sequence (index of the first character + // following the sequence). + size_t SequenceEnd() const { return cur_end_; } + + // Size of the current sequence in characters. + size_t SequenceSize() const { return cur_end_ - cur_start_; } + + // Returns true if |input_index| belongs to the current sequence. + bool SequenceContains(size_t input_index) const { + return (input_index >= cur_start_ && input_index < cur_end_); + } + + // Find the next sequence, if any, from the input. + // Returns false is there is no more sequence. + bool NextSequence() { + if (FindNextSequenceFrom(input_ + cur_end_)) + return true; + + cur_start_ = 0; + cur_end_ = 0; + return false; + } + + // Reset iterator to start of input. + void Reset() { + cur_start_ = cur_end_ = 0; + FindNextSequenceFrom(input_); + } + + private: + // Find the next sequence from the input, |from| being the starting position + // for the search, and must be in the [input_, input_end_] interval. On + // success, returns true after setting cur_start_ and cur_end_, on failure, + // return false. + bool FindNextSequenceFrom(const char* from) { + assert(from >= input_ && from <= input_end_); + auto* seq = + static_cast(::memchr(from, '\x1b', input_end_ - from)); + if (!seq) + return false; + + // The smallest possible color sequence if '\x1c[0m` and has four + // characters. + if (seq + 4 > input_end_) + return false; + + if (seq[1] != '[') + return FindNextSequenceFrom(seq + 1); + + // Skip parameters (digits + ; separator) + auto is_parameter_char = [](char ch) -> bool { + return (ch >= '0' && ch <= '9') || ch == ';'; + }; + + const char* end = seq + 2; + while (is_parameter_char(end[0])) { + if (++end == input_end_) + return false; // Incomplete sequence (no command). + } + + if (*end++ != 'm') { + // Not a color sequence. Restart the search after the first + // character following the [, in case this was a 3-char ANSI + // sequence (which is ignored here). + return FindNextSequenceFrom(seq + 3); + } + + // Found it! + cur_start_ = seq - input_; + cur_end_ = end - input_; + return true; + } + + size_t cur_start_ = 0; + size_t cur_end_ = 0; + const char* input_; + const char* input_end_; +}; + +// A class used to iterate over all characters of an input string, +// and return its visible position in the terminal, and whether that +// specific character is visible (or otherwise part of an ANSI color sequence). +// +// Example sequence and iterations, where 'ANSI' represents an ANSI Color +// sequence, and | is used to express concatenation +// +// |abcd|ANSI|efgh|ANSI|ijk| input string +// +// 11 1111 111 +// 0123 4567 8901 2345 678 input indices +// +// 1 +// 0123 4444 4567 8888 890 visible positions +// +// TTTT FFFF TTTT FFFF TTT is_visible +// +// Usage is: +// +// VisibleInputCharsIterator iter(input); +// while (iter.HasChar()) { +// ... use iter.InputIndex() to get input index of current char. +// ... use iter.VisiblePosition() to get its visible position. +// ... use iter.IsVisible() to check whether the current char is visible. +// +// NextChar(); +// } +// +struct VisibleInputCharsIterator { + VisibleInputCharsIterator(const std::string& input) + : input_size_(input.size()), ansi_iter_(input) {} + + // Return true if there is a character in the sequence. + bool HasChar() const { return input_index_ < input_size_; } + + // Return current input index. + size_t InputIndex() const { return input_index_; } + + // Return current visible position. + size_t VisiblePosition() const { return visible_pos_; } + + // Return true if the current input character is visible + // (i.e. not part of an ANSI color sequence). + bool IsVisible() const { return !ansi_iter_.SequenceContains(input_index_); } + + // Find next character from the input. + void NextChar() { + visible_pos_ += IsVisible(); + if (++input_index_ == ansi_iter_.SequenceEnd()) { + ansi_iter_.NextSequence(); + } + } + + private: + size_t input_size_; + size_t input_index_ = 0; + size_t visible_pos_ = 0; + AnsiColorSequenceIterator ansi_iter_; +}; + +void ElideMiddleInPlace(std::string& str, size_t max_width) { + if (str.size() <= max_width) { + return; + } + // Look for an ESC character. If there is none, use a fast path + // that avoids any intermediate allocations. + if (str.find('\x1b') == std::string::npos) { + const int ellipsis_width = 3; // Space for "...". + + // If max width is too small, do not keep anything from the input. + if (max_width <= ellipsis_width) { + str.assign("...", max_width); + return; + } + + // Keep only |max_width - ellipsis_size| visible characters from the input + // which will be split into two spans separated by "...". + const size_t remaining_size = max_width - ellipsis_width; + const size_t left_span_size = remaining_size / 2; + const size_t right_span_size = remaining_size - left_span_size; + + // Replace the gap in the input between the spans with "..." + const size_t gap_start = left_span_size; + const size_t gap_end = str.size() - right_span_size; + str.replace(gap_start, gap_end - gap_start, "..."); + return; + } + + // Compute visible width. + size_t visible_width = str.size(); + for (AnsiColorSequenceIterator ansi(str); ansi.HasSequence(); + ansi.NextSequence()) { + visible_width -= ansi.SequenceSize(); + } + + if (visible_width <= max_width) + return; + + // Compute the widths of the ellipsis, left span and right span + // visible space. + const size_t ellipsis_width = max_width < 3 ? max_width : 3; + const size_t visible_left_span_size = (max_width - ellipsis_width) / 2; + const size_t visible_right_span_size = + (max_width - ellipsis_width) - visible_left_span_size; + + // Compute the gap of visible characters that will be replaced by + // the ellipsis in visible space. + const size_t visible_gap_start = visible_left_span_size; + const size_t visible_gap_end = visible_width - visible_right_span_size; + + std::string result; + result.reserve(str.size()); + + // Parse the input chars info to: + // + // 1) Append any characters belonging to the left span (visible or not). + // + // 2) Add the ellipsis ("..." truncated to ellipsis_width). + // Note that its color is inherited from the left span chars + // which will never end with an ANSI sequence. + // + // 3) Append any ANSI sequence that appears inside the gap. This + // ensures the characters after the ellipsis appear with + // the right color, + // + // 4) Append any remaining characters (visible or not) to the result. + // + VisibleInputCharsIterator iter(str); + + // Step 1 - determine left span length in input chars. + for (; iter.HasChar(); iter.NextChar()) { + if (iter.VisiblePosition() == visible_gap_start) + break; + } + result.append(str.begin(), str.begin() + iter.InputIndex()); + + // Step 2 - Append the possibly-truncated ellipsis. + result.append("...", ellipsis_width); + + // Step 3 - Append elided ANSI sequences to the result. + for (; iter.HasChar(); iter.NextChar()) { + if (iter.VisiblePosition() == visible_gap_end) + break; + if (!iter.IsVisible()) + result.push_back(str[iter.InputIndex()]); + } + + // Step 4 - Append anything else. + result.append(str.begin() + iter.InputIndex(), str.end()); + + str = std::move(result); +} diff --git a/src/elide_middle.h b/src/elide_middle.h new file mode 100644 index 0000000000..128a9974e4 --- /dev/null +++ b/src/elide_middle.h @@ -0,0 +1,27 @@ +// Copyright 2024 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef NINJA_ELIDE_MIDDLE_H_ +#define NINJA_ELIDE_MIDDLE_H_ + +#include +#include + +/// Elide the given string @a str with '...' in the middle if the length +/// exceeds @a max_width. Note that this handles ANSI color sequences +/// properly (non-color related sequences are ignored, but using them +/// would wreak the cursor position or terminal state anyway). +void ElideMiddleInPlace(std::string& str, size_t max_width); + +#endif // NINJA_ELIDE_MIDDLE_H_ diff --git a/src/elide_middle_perftest.cc b/src/elide_middle_perftest.cc new file mode 100644 index 0000000000..94a8ccb944 --- /dev/null +++ b/src/elide_middle_perftest.cc @@ -0,0 +1,72 @@ +// Copyright 2024 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include + +#include + +#include "elide_middle.h" +#include "metrics.h" + +static const char* kTestInputs[] = { + "01234567890123456789", + "012345\x1B[0;35m67890123456789", + "abcd\x1b[1;31mefg\x1b[0mhlkmnopqrstuvwxyz", +}; + +int main() { + std::vector times; + + int64_t kMaxTimeMillis = 5 * 1000; + int64_t base_time = GetTimeMillis(); + + const int kRuns = 100; + for (int j = 0; j < kRuns; ++j) { + int64_t start = GetTimeMillis(); + if (start >= base_time + kMaxTimeMillis) + break; + + const int kNumRepetitions = 2000; + for (int count = kNumRepetitions; count > 0; --count) { + for (const char* input : kTestInputs) { + size_t input_len = ::strlen(input); + for (size_t max_width = input_len; max_width > 0; --max_width) { + std::string str(input, input_len); + ElideMiddleInPlace(str, max_width); + } + } + } + + int delta = (int)(GetTimeMillis() - start); + times.push_back(delta); + } + + int min = times[0]; + int max = times[0]; + float total = 0; + for (size_t i = 0; i < times.size(); ++i) { + total += times[i]; + if (times[i] < min) + min = times[i]; + else if (times[i] > max) + max = times[i]; + } + + printf("min %dms max %dms avg %.1fms\n", min, max, total / times.size()); + + return 0; +} diff --git a/src/elide_middle_test.cc b/src/elide_middle_test.cc new file mode 100644 index 0000000000..ed80e4eb73 --- /dev/null +++ b/src/elide_middle_test.cc @@ -0,0 +1,101 @@ +// Copyright 2024 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "elide_middle.h" + +#include "test.h" + +namespace { + +std::string ElideMiddle(const std::string& str, size_t width) { + std::string result = str; + ElideMiddleInPlace(result, width); + return result; +} + +} // namespace + + +TEST(ElideMiddle, NothingToElide) { + std::string input = "Nothing to elide in this short string."; + EXPECT_EQ(input, ElideMiddle(input, 80)); + EXPECT_EQ(input, ElideMiddle(input, 38)); + EXPECT_EQ("", ElideMiddle(input, 0)); + EXPECT_EQ(".", ElideMiddle(input, 1)); + EXPECT_EQ("..", ElideMiddle(input, 2)); + EXPECT_EQ("...", ElideMiddle(input, 3)); +} + +TEST(ElideMiddle, ElideInTheMiddle) { + std::string input = "01234567890123456789"; + EXPECT_EQ("...9", ElideMiddle(input, 4)); + EXPECT_EQ("0...9", ElideMiddle(input, 5)); + EXPECT_EQ("012...789", ElideMiddle(input, 9)); + EXPECT_EQ("012...6789", ElideMiddle(input, 10)); + EXPECT_EQ("0123...6789", ElideMiddle(input, 11)); + EXPECT_EQ("01234567...23456789", ElideMiddle(input, 19)); + EXPECT_EQ("01234567890123456789", ElideMiddle(input, 20)); +} + +// A few ANSI escape sequences. These macros make the following +// test easier to read and understand. +#define MAGENTA "\x1B[0;35m" +#define NOTHING "\33[m" +#define RED "\x1b[1;31m" +#define RESET "\x1b[0m" + +TEST(ElideMiddle, ElideAnsiEscapeCodes) { + std::string input = "012345" MAGENTA "67890123456789"; + EXPECT_EQ("012..." MAGENTA "6789", ElideMiddle(input, 10)); + EXPECT_EQ("012345" MAGENTA "67...23456789", ElideMiddle(input, 19)); + + EXPECT_EQ("Nothing " NOTHING " string.", + ElideMiddle("Nothing " NOTHING " string.", 18)); + EXPECT_EQ("0" NOTHING "12...6789", + ElideMiddle("0" NOTHING "1234567890123456789", 10)); + + input = "abcd" RED "efg" RESET "hlkmnopqrstuvwxyz"; + EXPECT_EQ("" RED RESET, ElideMiddle(input, 0)); + EXPECT_EQ("." RED RESET, ElideMiddle(input, 1)); + EXPECT_EQ(".." RED RESET, ElideMiddle(input, 2)); + EXPECT_EQ("..." RED RESET, ElideMiddle(input, 3)); + EXPECT_EQ("..." RED RESET "z", ElideMiddle(input, 4)); + EXPECT_EQ("a..." RED RESET "z", ElideMiddle(input, 5)); + EXPECT_EQ("a..." RED RESET "yz", ElideMiddle(input, 6)); + EXPECT_EQ("ab..." RED RESET "yz", ElideMiddle(input, 7)); + EXPECT_EQ("ab..." RED RESET "xyz", ElideMiddle(input, 8)); + EXPECT_EQ("abc..." RED RESET "xyz", ElideMiddle(input, 9)); + EXPECT_EQ("abc..." RED RESET "wxyz", ElideMiddle(input, 10)); + EXPECT_EQ("abcd..." RED RESET "wxyz", ElideMiddle(input, 11)); + EXPECT_EQ("abcd..." RED RESET "vwxyz", ElideMiddle(input, 12)); + + EXPECT_EQ("abcd" RED "ef..." RESET "uvwxyz", ElideMiddle(input, 15)); + EXPECT_EQ("abcd" RED "ef..." RESET "tuvwxyz", ElideMiddle(input, 16)); + EXPECT_EQ("abcd" RED "efg..." RESET "tuvwxyz", ElideMiddle(input, 17)); + EXPECT_EQ("abcd" RED "efg..." RESET "stuvwxyz", ElideMiddle(input, 18)); + EXPECT_EQ("abcd" RED "efg" RESET "h...stuvwxyz", ElideMiddle(input, 19)); + + input = "abcdef" RED "A" RESET "BC"; + EXPECT_EQ("..." RED RESET "C", ElideMiddle(input, 4)); + EXPECT_EQ("a..." RED RESET "C", ElideMiddle(input, 5)); + EXPECT_EQ("a..." RED RESET "BC", ElideMiddle(input, 6)); + EXPECT_EQ("ab..." RED RESET "BC", ElideMiddle(input, 7)); + EXPECT_EQ("ab..." RED "A" RESET "BC", ElideMiddle(input, 8)); + EXPECT_EQ("abcdef" RED "A" RESET "BC", ElideMiddle(input, 9)); +} + +#undef RESET +#undef RED +#undef NOTHING +#undef MAGENTA diff --git a/src/explanations.h b/src/explanations.h index babebd415d..375b29f283 100644 --- a/src/explanations.h +++ b/src/explanations.h @@ -53,7 +53,6 @@ struct Explanations { } private: - bool enabled_ = false; std::unordered_map> map_; }; diff --git a/src/graph.cc b/src/graph.cc index 143eabdfb4..f04ffb47c8 100644 --- a/src/graph.cc +++ b/src/graph.cc @@ -496,28 +496,6 @@ std::string EdgeEnv::MakePathList(const Node* const* const span, return result; } -void Edge::CollectInputs(bool shell_escape, - std::vector* out) const { - for (std::vector::const_iterator it = inputs_.begin(); - it != inputs_.end(); ++it) { - std::string path = (*it)->PathDecanonicalized(); - if (shell_escape) { - std::string unescaped; - unescaped.swap(path); -#ifdef _WIN32 - GetWin32EscapedString(unescaped, &path); -#else - GetShellEscapedString(unescaped, &path); -#endif - } -#if __cplusplus >= 201103L - out->push_back(std::move(path)); -#else - out->push_back(path); -#endif - } -} - std::string Edge::EvaluateCommand(const bool incl_rsp_file) const { string command = GetBinding("command"); if (incl_rsp_file) { @@ -779,3 +757,47 @@ vector::iterator ImplicitDepLoader::PreallocateSpace(Edge* edge, edge->implicit_deps_ += count; return edge->inputs_.end() - edge->order_only_deps_ - count; } + +void InputsCollector::VisitNode(const Node* node) { + const Edge* edge = node->in_edge(); + + if (!edge) // A source file. + return; + + // Add inputs of the producing edge to the result, + // except if they are themselves produced by a phony + // edge. + for (const Node* input : edge->inputs_) { + if (!visited_nodes_.insert(input).second) + continue; + + VisitNode(input); + + const Edge* input_edge = input->in_edge(); + if (!(input_edge && input_edge->is_phony())) { + inputs_.push_back(input); + } + } +} + +std::vector InputsCollector::GetInputsAsStrings( + bool shell_escape) const { + std::vector result; + result.reserve(inputs_.size()); + + for (const Node* input : inputs_) { + std::string unescaped = input->PathDecanonicalized(); + if (shell_escape) { + std::string path; +#ifdef _WIN32 + GetWin32EscapedString(unescaped, &path); +#else + GetShellEscapedString(unescaped, &path); +#endif + result.push_back(std::move(path)); + } else { + result.push_back(std::move(unescaped)); + } + } + return result; +} diff --git a/src/graph.h b/src/graph.h index 314c44296a..806260e5d7 100644 --- a/src/graph.h +++ b/src/graph.h @@ -201,9 +201,6 @@ struct Edge { void Dump(const char* prefix="") const; - // Append all edge explicit inputs to |*out|. Possibly with shell escaping. - void CollectInputs(bool shell_escape, std::vector* out) const; - // critical_path_weight is the priority during build scheduling. The // "critical path" between this edge's inputs and any target node is // the path which maximises the sum oof weights along that path. @@ -425,4 +422,41 @@ class EdgePriorityQueue: } }; +/// A class used to collect the transitive set of inputs from a given set +/// of starting nodes. Used to implement the `inputs` tool. +/// +/// When collecting inputs, the outputs of phony edges are always ignored +/// from the result, but are followed by the dependency walk. +/// +/// Usage is: +/// - Create instance. +/// - Call VisitNode() for each root node to collect inputs from. +/// - Call inputs() to retrieve the list of input node pointers. +/// - Call GetInputsAsStrings() to retrieve the list of inputs as a string +/// vector. +/// +struct InputsCollector { + /// Visit a single @arg node during this collection. + void VisitNode(const Node* node); + + /// Retrieve list of visited input nodes. A dependency always appears + /// before its dependents in the result, but final order depends on the + /// order of the VisitNode() calls performed before this. + const std::vector& inputs() const { return inputs_; } + + /// Same as inputs(), but returns the list of visited nodes as a list of + /// strings, with optional shell escaping. + std::vector GetInputsAsStrings(bool shell_escape = false) const; + + /// Reset collector state. + void Reset() { + inputs_.clear(); + visited_nodes_.clear(); + } + + private: + std::vector inputs_; + std::set visited_nodes_; +}; + #endif // NINJA_GRAPH_H_ diff --git a/src/graph_test.cc b/src/graph_test.cc index f909b906fd..6c654eeb32 100644 --- a/src/graph_test.cc +++ b/src/graph_test.cc @@ -215,28 +215,90 @@ TEST_F(GraphTest, RootNodes) { } } -TEST_F(GraphTest, CollectInputs) { +TEST_F(GraphTest, InputsCollector) { + // Build plan for the following graph: + // + // in1 + // |___________ + // | | + // === === + // | | + // out1 mid1 + // | ____|_____ + // | | | + // | === ======= + // | | | | + // | out2 out3 out4 + // | | | + // =======phony====== + // | + // all + // + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, + "build out1: cat in1\n" + "build mid1: cat in1\n" + "build out2: cat mid1\n" + "build out3 out4: cat mid1\n" + "build all: phony out1 out2 out3\n")); + + InputsCollector collector; + + // Start visit from out1, this should add in1 to the inputs. + collector.Reset(); + collector.VisitNode(GetNode("out1")); + auto inputs = collector.GetInputsAsStrings(); + ASSERT_EQ(1u, inputs.size()); + EXPECT_EQ("in1", inputs[0]); + + // Add a visit from out2, this should add mid1. + collector.VisitNode(GetNode("out2")); + inputs = collector.GetInputsAsStrings(); + ASSERT_EQ(2u, inputs.size()); + EXPECT_EQ("in1", inputs[0]); + EXPECT_EQ("mid1", inputs[1]); + + // Another visit from all, this should add out1, out2 and out3, + // but not out4. + collector.VisitNode(GetNode("all")); + inputs = collector.GetInputsAsStrings(); + ASSERT_EQ(5u, inputs.size()); + EXPECT_EQ("in1", inputs[0]); + EXPECT_EQ("mid1", inputs[1]); + EXPECT_EQ("out1", inputs[2]); + EXPECT_EQ("out2", inputs[3]); + EXPECT_EQ("out3", inputs[4]); + + collector.Reset(); + + // Starting directly from all, will add out1 before mid1 compared + // to the previous example above. + collector.VisitNode(GetNode("all")); + inputs = collector.GetInputsAsStrings(); + ASSERT_EQ(5u, inputs.size()); + EXPECT_EQ("in1", inputs[0]); + EXPECT_EQ("out1", inputs[1]); + EXPECT_EQ("mid1", inputs[2]); + EXPECT_EQ("out2", inputs[3]); + EXPECT_EQ("out3", inputs[4]); +} + +TEST_F(GraphTest, InputsCollectorWithEscapes) { ASSERT_NO_FATAL_FAILURE(AssertParse( &state_, "build out$ 1: cat in1 in2 in$ with$ space | implicit || order_only\n")); - std::vector inputs; - Edge* edge = GetNode("out 1")->in_edge(); - - // Test without shell escaping. - inputs.clear(); - edge->CollectInputs(false, &inputs); - EXPECT_EQ(5u, inputs.size()); + InputsCollector collector; + collector.VisitNode(GetNode("out 1")); + auto inputs = collector.GetInputsAsStrings(); + ASSERT_EQ(5u, inputs.size()); EXPECT_EQ("in1", inputs[0]); EXPECT_EQ("in2", inputs[1]); EXPECT_EQ("in with space", inputs[2]); EXPECT_EQ("implicit", inputs[3]); EXPECT_EQ("order_only", inputs[4]); - // Test with shell escaping. - inputs.clear(); - edge->CollectInputs(true, &inputs); - EXPECT_EQ(5u, inputs.size()); + inputs = collector.GetInputsAsStrings(true); + ASSERT_EQ(5u, inputs.size()); EXPECT_EQ("in1", inputs[0]); EXPECT_EQ("in2", inputs[1]); #ifdef _WIN32 diff --git a/src/json.h b/src/json.h index f39c759236..3e5cf7817c 100644 --- a/src/json.h +++ b/src/json.h @@ -17,7 +17,7 @@ #include -// Encode a string in JSON format without encolsing quotes +// Encode a string in JSON format without enclosing quotes std::string EncodeJSONString(const std::string& in); // Print a string in JSON format to stdout without enclosing quotes diff --git a/src/line_printer.cc b/src/line_printer.cc index 12e82b3f80..4a7b0bbf70 100644 --- a/src/line_printer.cc +++ b/src/line_printer.cc @@ -28,6 +28,7 @@ #include #endif +#include "elide_middle.h" #include "util.h" using namespace std; @@ -81,7 +82,7 @@ void LinePrinter::Print(string to_print, LineType type) { CONSOLE_SCREEN_BUFFER_INFO csbi; GetConsoleScreenBufferInfo(console_, &csbi); - to_print = ElideMiddle(to_print, static_cast(csbi.dwSize.X)); + ElideMiddleInPlace(to_print, static_cast(csbi.dwSize.X)); if (supports_color_) { // this means ENABLE_VIRTUAL_TERMINAL_PROCESSING // succeeded printf("%s\x1B[K", to_print.c_str()); // Clear to end of line. @@ -108,7 +109,7 @@ void LinePrinter::Print(string to_print, LineType type) { // line-wrapping. winsize size; if ((ioctl(STDOUT_FILENO, TIOCGWINSZ, &size) == 0) && size.ws_col) { - to_print = ElideMiddle(to_print, size.ws_col); + ElideMiddleInPlace(to_print, size.ws_col); } printf("%s", to_print.c_str()); printf("\x1B[K"); // Clear to end of line. diff --git a/src/manifest_parser.cc b/src/manifest_parser.cc index c4b2980164..505499f0a9 100644 --- a/src/manifest_parser.cc +++ b/src/manifest_parser.cc @@ -69,6 +69,13 @@ bool ManifestParser::Parse(const string& filename, const string& input, if (name == "ninja_required_version") CheckNinjaVersion(value); env_->AddBinding(name, value); + + // handle conversion of let + g_output_ss << "\n\tlet("; + g_output_ss << name; + g_output_ss << ", \""; + g_output_ss << value; // parent function gets value this way + g_output_ss << "\");\n"; break; } case Lexer::INCLUDE: @@ -131,6 +138,41 @@ bool ManifestParser::ParsePool(string* err) { return true; } +std::vector split_by_spaces(const std::string& line) { + std::vector tokens; + std::istringstream iss(line); + std::string token; + + while (iss >> token) { + tokens.push_back(token); // Extract token and push to vector + } + + return tokens; +} + +std::string extract_variable(const std::string& token) { + // Check if the token starts with '$' and contains both '${' and '}' + if (token[0] == '$' && token.find("${") == 0 && token.find('}') != std::string::npos) { + // Extract the content between '${' and '}' + size_t start = token.find("{") + 1; + size_t end = token.find("}"); + return token.substr(start, end - start); + } + + // Return an empty string if it's not a valid ${} expression + return ""; +} + +std::string extract_token(const std::string& token){ + std::string output = extract_variable(token); + if (output.empty()){ + return "\"" + token + "\""; + } + else { + return "\"" + output + "\"_v"; + } +} + bool ManifestParser::ParseRule(string* err) { string name; @@ -170,6 +212,21 @@ bool ManifestParser::ParseRule(string* err) { return lexer_.Error("expected 'command =' line", err); env_->AddRule(rule); + // add rule here to the stringstream + g_output_ss << "\n\tauto " << rule->name_ << " = rule{ {\n" + << " "; + g_output_ss << "\tbind(command, "; + + std::vector tokens = split_by_spaces(rule->bindings_.at("command").Unparse()); + + for (size_t i=0; i < tokens.size(); ++i) { + g_output_ss << extract_token(tokens[i]); + if (i != tokens.size() - 1) { + g_output_ss << ","; + } + } + g_output_ss << ")\n" << "\t} };\n"; + return true; } @@ -180,6 +237,7 @@ bool ManifestParser::ParseLet(string* key, EvalString* value, string* err) { return false; if (!lexer_.ReadVarValue(value, err)) return false; + return true; } @@ -200,6 +258,10 @@ bool ManifestParser::ParseDefault(string* err) { if (!state_->AddDefault(path, &default_err)) return lexer_.Error(default_err, err); +// add to converter output for each default: default(str("hello")); + g_output_ss << "\n\tdefault(str(\""; + g_output_ss << path << "\"));\n"; + eval.Clear(); if (!lexer_.ReadPath(&eval, err)) return false; @@ -308,6 +370,8 @@ bool ManifestParser::ParseEdge(string* err) { // Bindings on edges are rare, so allocate per-edge envs only when needed. bool has_indent_token = lexer_.PeekToken(Lexer::INDENT); BindingEnv* env = has_indent_token ? new BindingEnv(env_) : env_; + + std::map savedBindings; while (has_indent_token) { string key; EvalString val; @@ -315,6 +379,10 @@ bool ManifestParser::ParseEdge(string* err) { return false; env->AddBinding(key, val.Evaluate(env_)); + + // mr: save bindings for adding later + savedBindings[key] = val.Evaluate(env_); + has_indent_token = lexer_.PeekToken(Lexer::INDENT); } @@ -409,6 +477,30 @@ bool ManifestParser::ParseEdge(string* err) { } assert(!edge->dyndep_->generated_by_dep_loader()); } + + // add edge here to the stringstream + g_output_ss << "\n\tbuild("; + + g_output_ss << "list{ str{ "; + g_output_ss << "\"" << outs[0].Evaluate(env) << "\""; // todo: make this a loop + g_output_ss << " } },\n"; + + g_output_ss << "\t\t{},\n"; + + g_output_ss << "\t\t" << rule->name_ << ",\n"; + + g_output_ss << "\t\tlist{ str{ "; + g_output_ss << "\"" << ins[0].Evaluate(env) << "\""; // todo: make this a loop + g_output_ss << " } },\n"; + + g_output_ss << "\t\t{},\n"; + g_output_ss << "\t\t{},\n"; + + for (const auto& pair : savedBindings) { + g_output_ss << "\t\t{ bind(" << pair.first << ", \"" << pair.second << "\") }\n"; + } + + g_output_ss << "\t);\n"; return true; } diff --git a/src/metrics.cc b/src/metrics.cc index 632ae43c50..e7cb4d1444 100644 --- a/src/metrics.cc +++ b/src/metrics.cc @@ -37,15 +37,6 @@ int64_t HighResTimer() { .count(); } -constexpr int64_t GetFrequency() { - // If numerator isn't 1 then we lose precision and that will need to be - // assessed. - static_assert(std::chrono::steady_clock::period::num == 1, - "Numerator must be 1"); - return std::chrono::steady_clock::period::den / - std::chrono::steady_clock::period::num; -} - int64_t TimerToMicros(int64_t dt) { // dt is in ticks. We want microseconds. return chrono::duration_cast( diff --git a/src/ninja.cc b/src/ninja.cc index 2902359f15..808380d482 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -21,6 +21,8 @@ #include #include +#include + #ifdef _WIN32 #include "getopt.h" #include @@ -52,6 +54,9 @@ #include "util.h" #include "version.h" +// mr: +#include + using namespace std; #ifdef _WIN32 @@ -154,7 +159,7 @@ struct NinjaMain : public BuildLogUser { bool RebuildManifest(const char* input_file, string* err, Status* status); /// For each edge, lookup in build log how long it took last time, - /// and record that in the edge itself. It will be used for ETA predicton. + /// and record that in the edge itself. It will be used for ETA prediction. void ParsePreviousElapsedTimes(); /// Build the targets listed on the command line. @@ -761,43 +766,50 @@ int NinjaMain::ToolCommands(const Options* options, int argc, char* argv[]) { return 0; } -void CollectInputs(Edge* edge, std::set* seen, - std::vector* result) { - if (!edge) - return; - if (!seen->insert(edge).second) - return; - - for (vector::iterator in = edge->inputs_.begin(); - in != edge->inputs_.end(); ++in) - CollectInputs((*in)->in_edge(), seen, result); - - if (!edge->is_phony()) { - edge->CollectInputs(true, result); - } -} - int NinjaMain::ToolInputs(const Options* options, int argc, char* argv[]) { // The inputs tool uses getopt, and expects argv[0] to contain the name of // the tool, i.e. "inputs". argc++; argv--; + + bool print0 = false; + bool shell_escape = true; + bool dependency_order = false; + optind = 1; int opt; const option kLongOptions[] = { { "help", no_argument, NULL, 'h' }, + { "no-shell-escape", no_argument, NULL, 'E' }, + { "print0", no_argument, NULL, '0' }, + { "dependency-order", no_argument, NULL, + 'd' }, { NULL, 0, NULL, 0 } }; - while ((opt = getopt_long(argc, argv, "h", kLongOptions, NULL)) != -1) { + while ((opt = getopt_long(argc, argv, "h0Ed", kLongOptions, NULL)) != -1) { switch (opt) { + case 'd': + dependency_order = true; + break; + case 'E': + shell_escape = false; + break; + case '0': + print0 = true; + break; case 'h': default: // clang-format off printf( "Usage '-t inputs [options] [targets]\n" "\n" -"List all inputs used for a set of targets. Note that this includes\n" -"explicit, implicit and order-only inputs, but not validation ones.\n\n" +"List all inputs used for a set of targets, sorted in dependency order.\n" +"Note that by default, results are shell escaped, and sorted alphabetically,\n" +"and never include validation target paths.\n\n" "Options:\n" -" -h, --help Print this message.\n"); +" -h, --help Print this message.\n" +" -0, --print0 Use \\0, instead of \\n as a line terminator.\n" +" -E, --no-shell-escape Do not shell escape the result.\n" +" -d, --dependency-order Sort results by dependency order.\n" + ); // clang-format on return 1; } @@ -805,25 +817,31 @@ int NinjaMain::ToolInputs(const Options* options, int argc, char* argv[]) { argv += optind; argc -= optind; - vector nodes; - string err; + std::vector nodes; + std::string err; if (!CollectTargetsFromArgs(argc, argv, &nodes, &err)) { Error("%s", err.c_str()); return 1; } - std::set seen; - std::vector result; - for (vector::iterator in = nodes.begin(); in != nodes.end(); ++in) - CollectInputs((*in)->in_edge(), &seen, &result); - - // Make output deterministic by sorting then removing duplicates. - std::sort(result.begin(), result.end()); - result.erase(std::unique(result.begin(), result.end()), result.end()); + InputsCollector collector; + for (const Node* node : nodes) + collector.VisitNode(node); - for (size_t n = 0; n < result.size(); ++n) - puts(result[n].c_str()); + std::vector inputs = collector.GetInputsAsStrings(shell_escape); + if (!dependency_order) + std::sort(inputs.begin(), inputs.end()); + if (print0) { + for (const std::string& input : inputs) { + fwrite(input.c_str(), input.size(), 1, stdout); + fputc('\0', stdout); + } + fflush(stdout); + } else { + for (const std::string& input : inputs) + puts(input.c_str()); + } return 0; } @@ -1562,9 +1580,18 @@ NORETURN void real_main(int argc, char** argv) { exit((ninja.*options.tool->func)(&options, argc, argv)); } + // add preamble + g_output_ss << "#include \"manifest.h\"\n\n"; + g_output_ss << "using namespace shadowdash;\n\n"; + g_output_ss << "void manifest() {\n"; // start manifest() function + + /* // Limit number of rebuilds, to prevent infinite loops. const int kCycleLimit = 100; for (int cycle = 1; cycle <= kCycleLimit; ++cycle) { + } +} +*/ NinjaMain ninja(ninja_command, config); ManifestParserOptions parser_opts; @@ -1577,7 +1604,27 @@ NORETURN void real_main(int argc, char** argv) { status->Error("%s", err.c_str()); exit(1); } + + g_output_ss << "\n}"; // exit manifest() function + + std::string result = g_output_ss.str(); + std::cout << result << std::endl; // write to stdout for now, change it to write to file later + + // write to file + std::ofstream outFile("output.cc"); + + // Check if the file opened successfully + if (outFile.is_open()) { + outFile << result; + outFile.close(); + } else { + std::cerr << "Error opening file for writing.\n"; + } + + exit(0); // exit(1) was suggested above by ninja to exit out; crashes otherwise +} +/* don't need anything other than parsing if (options.tool && options.tool->when == Tool::RUN_AFTER_LOAD) exit((ninja.*options.tool->func)(&options, argc, argv)); @@ -1615,6 +1662,7 @@ NORETURN void real_main(int argc, char** argv) { options.input_file, kCycleLimit); exit(1); } +*/ } // anonymous namespace diff --git a/src/status_printer.cc b/src/status_printer.cc index ed48a5c77b..62c7d7a9d2 100644 --- a/src/status_printer.cc +++ b/src/status_printer.cc @@ -216,37 +216,34 @@ void StatusPrinter::BuildEdgeFinished(Edge* edge, int64_t start_time_millis, printer_.PrintOnNewLine(edge->EvaluateCommand() + "\n"); } - if (!output.empty()) { - // ninja sets stdout and stderr of subprocesses to a pipe, to be able to - // check if the output is empty. Some compilers, e.g. clang, check - // isatty(stderr) to decide if they should print colored output. - // To make it possible to use colored output with ninja, subprocesses should - // be run with a flag that forces them to always print color escape codes. - // To make sure these escape codes don't show up in a file if ninja's output - // is piped to a file, ninja strips ansi escape codes again if it's not - // writing to a |smart_terminal_|. - // (Launching subprocesses in pseudo ttys doesn't work because there are - // only a few hundred available on some systems, and ninja can launch - // thousands of parallel compile commands.) - string final_output; - if (!printer_.supports_color()) - final_output = StripAnsiEscapeCodes(output); - else - final_output = output; - #ifdef _WIN32 - // Fix extra CR being added on Windows, writing out CR CR LF (#773) - fflush(stdout); // Begin Windows extra CR fix - _setmode(_fileno(stdout), _O_BINARY); + // Fix extra CR being added on Windows, writing out CR CR LF (#773) + fflush(stdout); // Begin Windows extra CR fix + _setmode(_fileno(stdout), _O_BINARY); #endif + // ninja sets stdout and stderr of subprocesses to a pipe, to be able to + // check if the output is empty. Some compilers, e.g. clang, check + // isatty(stderr) to decide if they should print colored output. + // To make it possible to use colored output with ninja, subprocesses should + // be run with a flag that forces them to always print color escape codes. + // To make sure these escape codes don't show up in a file if ninja's output + // is piped to a file, ninja strips ansi escape codes again if it's not + // writing to a |smart_terminal_|. + // (Launching subprocesses in pseudo ttys doesn't work because there are + // only a few hundred available on some systems, and ninja can launch + // thousands of parallel compile commands.) + if (printer_.supports_color() || output.find('\x1b') == std::string::npos) { + printer_.PrintOnNewLine(output); + } else { + std::string final_output = StripAnsiEscapeCodes(output); printer_.PrintOnNewLine(final_output); + } #ifdef _WIN32 - fflush(stdout); - _setmode(_fileno(stdout), _O_TEXT); // End Windows extra CR fix + fflush(stdout); + _setmode(_fileno(stdout), _O_TEXT); // End Windows extra CR fix #endif - } } void StatusPrinter::BuildStarted() { diff --git a/src/status_printer.h b/src/status_printer.h index 2f72ae69e6..08a8d1a93d 100644 --- a/src/status_printer.h +++ b/src/status_printer.h @@ -26,21 +26,19 @@ struct StatusPrinter : Status { explicit StatusPrinter(const BuildConfig& config); /// Callbacks for the Plan to notify us about adding/removing Edge's. - virtual void EdgeAddedToPlan(const Edge* edge); - virtual void EdgeRemovedFromPlan(const Edge* edge); + void EdgeAddedToPlan(const Edge* edge) override; + void EdgeRemovedFromPlan(const Edge* edge) override; - virtual void BuildEdgeStarted(const Edge* edge, int64_t start_time_millis); - virtual void BuildEdgeFinished(Edge* edge, int64_t start_time_millis, + void BuildEdgeStarted(const Edge* edge, int64_t start_time_millis) override; + void BuildEdgeFinished(Edge* edge, int64_t start_time_millis, int64_t end_time_millis, bool success, - const std::string& output); - virtual void BuildStarted(); - virtual void BuildFinished(); + const std::string& output) override; + void BuildStarted() override; + void BuildFinished() override; - virtual void Info(const char* msg, ...); - virtual void Warning(const char* msg, ...); - virtual void Error(const char* msg, ...); - - virtual ~StatusPrinter() {} + void Info(const char* msg, ...) override; + void Warning(const char* msg, ...) override; + void Error(const char* msg, ...) override; /// Format the progress status string by replacing the placeholders. /// See the user manual for more information about the available diff --git a/src/util.cc b/src/util.cc index 6f7e2a4385..4b1e70e040 100644 --- a/src/util.cc +++ b/src/util.cc @@ -26,7 +26,6 @@ #include #include #include -#include #include #include #include @@ -61,6 +60,9 @@ #include "edit_distance.h" +// my globals +std::stringstream g_output_ss; + using namespace std; void Fatal(const char* msg, ...) { @@ -918,53 +920,6 @@ double GetLoadAverage() { } #endif // _WIN32 -string ElideMiddle(const string& str, size_t width) { - switch (width) { - case 0: return ""; - case 1: return "."; - case 2: return ".."; - case 3: return "..."; - } - const int kMargin = 3; // Space for "...". - const static std::regex ansi_escape("\\x1b[^m]*m"); - std::string result = std::regex_replace(str, ansi_escape, ""); - if (result.size() <= width) { - return str; - } - int32_t elide_size = (width - kMargin) / 2; - - std::vector> escapes; - size_t added_len = 0; // total number of characters - - std::sregex_iterator it(str.begin(), str.end(), ansi_escape); - std::sregex_iterator end; - while (it != end) { - escapes.emplace_back(it->position() - added_len, it->str()); - added_len += it->str().size(); - ++it; - } - - std::string new_status = - result.substr(0, elide_size) + "..." + - result.substr(result.size() - elide_size - ((width - kMargin) % 2)); - - added_len = 0; - // We need to put all ANSI escape codes back in: - for (const auto& escape : escapes) { - int32_t pos = escape.first; - if (pos > elide_size) { - pos -= result.size() - width; - if (pos < static_cast(width) - elide_size) { - pos = width - elide_size - (width % 2 == 0 ? 1 : 0); - } - } - pos += added_len; - new_status.insert(pos, escape.second); - added_len += escape.second.size(); - } - return new_status; -} - bool Truncate(const string& path, size_t size, string* err) { #ifdef _WIN32 int fh = _sopen(path.c_str(), _O_RDWR | _O_CREAT, _SH_DENYNO, diff --git a/src/util.h b/src/util.h index 4ab887dfaf..8189bfa5b3 100644 --- a/src/util.h +++ b/src/util.h @@ -48,6 +48,10 @@ NORETURN void Fatal(const char* msg, ...); # define NINJA_FALLTHROUGH // nothing #endif +// my globals +#include +extern std::stringstream g_output_ss; + /// Log a warning message. void Warning(const char* msg, ...); void Warning(const char* msg, va_list ap); @@ -102,10 +106,6 @@ int GetProcessorCount(); /// on error. double GetLoadAverage(); -/// Elide the given string @a str with '...' in the middle if the length -/// exceeds @a width. -std::string ElideMiddle(const std::string& str, size_t width); - /// Truncates a file to the given size. bool Truncate(const std::string& path, size_t size, std::string* err); diff --git a/src/util_test.cc b/src/util_test.cc index 0033a875a5..9b620e895c 100644 --- a/src/util_test.cc +++ b/src/util_test.cc @@ -502,62 +502,3 @@ TEST(StripAnsiEscapeCodes, StripColors) { EXPECT_EQ("affixmgr.cxx:286:15: warning: using the result... [-Wparentheses]", stripped); } - -TEST(ElideMiddle, NothingToElide) { - string input = "Nothing to elide in this short string."; - EXPECT_EQ(input, ElideMiddle(input, 80)); - EXPECT_EQ(input, ElideMiddle(input, 38)); - EXPECT_EQ("", ElideMiddle(input, 0)); - EXPECT_EQ(".", ElideMiddle(input, 1)); - EXPECT_EQ("..", ElideMiddle(input, 2)); - EXPECT_EQ("...", ElideMiddle(input, 3)); -} - -TEST(ElideMiddle, ElideInTheMiddle) { - string input = "01234567890123456789"; - EXPECT_EQ("...9", ElideMiddle(input, 4)); - EXPECT_EQ("0...9", ElideMiddle(input, 5)); - EXPECT_EQ("012...789", ElideMiddle(input, 9)); - EXPECT_EQ("012...6789", ElideMiddle(input, 10)); - EXPECT_EQ("0123...6789", ElideMiddle(input, 11)); - EXPECT_EQ("01234567...23456789", ElideMiddle(input, 19)); - EXPECT_EQ("01234567890123456789", ElideMiddle(input, 20)); -} - -TEST(ElideMiddle, ElideAnsiEscapeCodes) { - std::string input = "012345\x1B[0;35m67890123456789"; - EXPECT_EQ("012...\x1B[0;35m6789", ElideMiddle(input, 10)); - EXPECT_EQ("012345\x1B[0;35m67...23456789", ElideMiddle(input, 19)); - - EXPECT_EQ("Nothing \33[m string.", ElideMiddle("Nothing \33[m string.", 18)); - EXPECT_EQ("0\33[m12...6789", ElideMiddle("0\33[m1234567890123456789", 10)); - - input = "abcd\x1b[1;31mefg\x1b[0mhlkmnopqrstuvwxyz"; - EXPECT_EQ("", ElideMiddle(input, 0)); - EXPECT_EQ(".", ElideMiddle(input, 1)); - EXPECT_EQ("..", ElideMiddle(input, 2)); - EXPECT_EQ("...", ElideMiddle(input, 3)); - EXPECT_EQ("...\x1B[1;31m\x1B[0mz", ElideMiddle(input, 4)); - EXPECT_EQ("a...\x1B[1;31m\x1B[0mz", ElideMiddle(input, 5)); - EXPECT_EQ("a...\x1B[1;31m\x1B[0myz", ElideMiddle(input, 6)); - EXPECT_EQ("ab...\x1B[1;31m\x1B[0myz", ElideMiddle(input, 7)); - EXPECT_EQ("ab...\x1B[1;31m\x1B[0mxyz", ElideMiddle(input, 8)); - EXPECT_EQ("abc...\x1B[1;31m\x1B[0mxyz", ElideMiddle(input, 9)); - EXPECT_EQ("abc...\x1B[1;31m\x1B[0mwxyz", ElideMiddle(input, 10)); - EXPECT_EQ("abcd\x1B[1;31m...\x1B[0mwxyz", ElideMiddle(input, 11)); - EXPECT_EQ("abcd\x1B[1;31m...\x1B[0mvwxyz", ElideMiddle(input, 12)); - - EXPECT_EQ("abcd\x1B[1;31mef...\x1B[0muvwxyz", ElideMiddle(input, 15)); - EXPECT_EQ("abcd\x1B[1;31mef...\x1B[0mtuvwxyz", ElideMiddle(input, 16)); - EXPECT_EQ("abcd\x1B[1;31mefg\x1B[0m...tuvwxyz", ElideMiddle(input, 17)); - EXPECT_EQ("abcd\x1B[1;31mefg\x1B[0m...stuvwxyz", ElideMiddle(input, 18)); - EXPECT_EQ("abcd\x1B[1;31mefg\x1B[0mh...stuvwxyz", ElideMiddle(input, 19)); - - input = "abcdef\x1b[31mA\x1b[0mBC"; - EXPECT_EQ("...\x1B[31m\x1B[0mC", ElideMiddle(input, 4)); - EXPECT_EQ("a...\x1B[31m\x1B[0mC", ElideMiddle(input, 5)); - EXPECT_EQ("a...\x1B[31m\x1B[0mBC", ElideMiddle(input, 6)); - EXPECT_EQ("ab...\x1B[31m\x1B[0mBC", ElideMiddle(input, 7)); - EXPECT_EQ("ab...\x1B[31mA\x1B[0mBC", ElideMiddle(input, 8)); - EXPECT_EQ("abcdef\x1b[31mA\x1b[0mBC", ElideMiddle(input, 9)); -}