diff --git a/converter/.gitignore b/converter/.gitignore new file mode 100644 index 0000000000..c5cb1afac3 --- /dev/null +++ b/converter/.gitignore @@ -0,0 +1 @@ +.coverage \ No newline at end of file diff --git a/converter/docs/build.ninja b/converter/docs/build.ninja new file mode 100644 index 0000000000..e0d6c29ec5 --- /dev/null +++ b/converter/docs/build.ninja @@ -0,0 +1,24 @@ +flags = -O3 + +pool_depth = 4 + +pool heavy_object_pool + depth = pool_depth + +rule compile + command = g++ $flags -c $in -o $out + pool = heavy_object_pool + +rule link + command = g++ $in -o $out + +build hello.o: compile hello.cc + flags = -O2 + pool = console + +build hello: link hello.o + +build dummy: phony + +default hello +default foo1 foo2 \ No newline at end of file diff --git a/converter/docs/build.ninja.cc b/converter/docs/build.ninja.cc new file mode 100644 index 0000000000..a83a4b1091 --- /dev/null +++ b/converter/docs/build.ninja.cc @@ -0,0 +1,51 @@ +#include "manifest.h" + +using namespace shadowdash; + +void manifest() { + let(flags, "-O3"); + + let(pool_depth, "4"); + + auto heavy_object_pool = pool_(bind(depth, "pool_depth"_v)); + + auto compile = rule( { + bind(command, "g++", "flags"_v, "-c", in, "-o", out), + bind(pool, "heavy_object_pool"_v) + } ); + + auto link = rule( { + bind(command, "g++", in, "-o", out), + } ); + + auto build_c = build(list{ str{ "hello.o" } }, + {}, + compile, + list{ str{ "hello.cc" } }, + {}, + {}, + { bind(flags, "-O2"), + bind(pool, "console"_v) } + ); + + auto build_l = build(list{ str{ "hello" } }, + {}, + link, + list{ str{ "hello.o" } }, + {}, + {}, + {} + ); + + auto build_p = build(list{ str{ "dummy" } }, + {}, + phony, + {}, + {}, + {}, + {} + ); + + default_(str{ "hello" }); + default_(list{ str{ "foo1" }, str{ "foo2" } }); +} diff --git a/converter/src/README.md b/converter/src/README.md new file mode 100644 index 0000000000..43a81ed2a4 --- /dev/null +++ b/converter/src/README.md @@ -0,0 +1,61 @@ +# Usage +``` +usage: converter.py [-h] input_file output_file + +Converts ninja files to ShadowDash manifest files. + +positional arguments: + input_file Path to the input file to process + output_file Path to the output file + +options: + -h, --help show this help message and exit +``` + +You can call it using the following command: +```bash +python3 converter.py input_file output_file +``` + +# Prerequisites +- Python 3.6 or higher +- Testing + - pytest + - pytest-mock + - pytest-cov (optional) + +# Test + +## Unit Tests + +The unit tests are located in the `tests` directory. +It requires the `pytest` and `pytest-mock` packages to run the tests. + +To run the unit tests, you can use the following command: +```bash +# from the converter directory +# ./converter +pytest +``` + +Or with coverage report (requires the `pytest-cov` package): +```bash +# from the converter directory +# ./converter +pytest --cov=src --cov-report=term +``` + +## Factual Tests + +To test the program we can use the files in the `docs` folder. The following command will convert the `build.ninja` file to a `build.ninja.cc` file: +```bash +# from the converter directory +# ./converter +python3 -m src.converter ./docs/build.ninja build.ninja.cc +``` +Aside from formatting, the file `build.ninja.cc` should be identical to the `build.ninja.cc` file in the docs directory. + + +# Implementation + +The program is implemeted to match the languages developed by the `ninja` build system and Team 4 `new-language` subteam. diff --git a/converter/src/__init__.py b/converter/src/__init__.py new file mode 100644 index 0000000000..0a5a53bcce --- /dev/null +++ b/converter/src/__init__.py @@ -0,0 +1 @@ +# this file is intentionally left blank to mark the directory as a package diff --git a/converter/src/builder.py b/converter/src/builder.py new file mode 100644 index 0000000000..530686df4d --- /dev/null +++ b/converter/src/builder.py @@ -0,0 +1,25 @@ +from typing import List +from .token import Token + +class Builder: + """ + Builder class to generate the content of a ShadowDash manifest file from tokens. + """ + + @staticmethod + def get_file_content(tokens: List[Token]) -> str: + """ + Generates the content of the ShadowDash manifest file. + + Args: + tokens (List[Token]): List of tokens to be converted. + + Returns: + str: Content of the ShadowDash manifest file. + """ + # Start with the necessary includes and namespace + c = '#include "manifest.h"\n\nusing namespace shadowdash;\n\nvoid manifest() {\n\n' + # Convert each token to its string representation and join them + c += "\n\n".join([token.get_string() for token in tokens]) + c += "\n\n}" + return c \ No newline at end of file diff --git a/converter/src/converter.py b/converter/src/converter.py new file mode 100644 index 0000000000..7661c696ad --- /dev/null +++ b/converter/src/converter.py @@ -0,0 +1,38 @@ +import argparse +import os +from typing import List +from .builder import Builder +from .parser import Parser +from .token import Token + +def main() -> None: + """ + Main function to convert a ninja build file to a ShadowDash manifest file. + """ + parser = argparse.ArgumentParser(description='Converts ninja files to ShadowDash manifest files.') + + # Define input and output file arguments + parser.add_argument('input_file', + help='Path to the input file to process') + parser.add_argument('output_file', + help='Path to the output file') + + args = parser.parse_args() + + # Check if the input file exists + if not os.path.isfile(args.input_file): + print(f"Error: The file {args.input_file} does not exist.") + exit(1) + + # Tokenize the input file + tokenslist: List[Token] = Parser.tokenize(args.input_file) + + # Write the converted content to the output file + with open(args.output_file, 'w') as outfile: + content: str = Builder.get_file_content(tokenslist) + outfile.write(content) + + print(f"File {args.input_file} converted to {args.output_file}") + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/converter/src/parser.py b/converter/src/parser.py new file mode 100644 index 0000000000..fcb2393045 --- /dev/null +++ b/converter/src/parser.py @@ -0,0 +1,52 @@ +import re +from typing import List +from .token import Token, Rule, Pool, Bind, Assign, Build, Default + +class Parser: + """ + Parser class to tokenize a ninja build file. + """ + + @staticmethod + def tokenize(path: str) -> List[Token]: + """ + Tokenizes the given ninja build file. + + Args: + path (str): Path to the ninja build file. + + Returns: + List[Token]: List of tokens parsed from the file. + """ + tokenlist = [] + # Regex pattern to match lines that are not indented and their subsequent indented lines + pattern = re.compile(r"(?P^[^\t\s].*(?:\n[ \t]+.*)*)", re.MULTILINE) + with open(path, "r") as ninjafile: + matches = pattern.findall(ninjafile.read()) + for match in matches: + tokenlist.append(Parser._get_token(match)) + return tokenlist + + @staticmethod + def _get_token(command: str) -> Token: + """ + Converts a command string to a Token object. + + Args: + command (str): Command string from the ninja build file. + + Returns: + Token: Corresponding Token object. + """ + if command.startswith("rule"): + return Rule(command) + elif command.startswith("pool "): + return Pool(command) + elif command.startswith("build "): + return Build(command) + elif command.startswith("bind"): + return Bind(command) + elif command.startswith("default"): + return Default(command) + else: + return Assign(command) diff --git a/converter/src/token.py b/converter/src/token.py new file mode 100644 index 0000000000..291c3ed2d5 --- /dev/null +++ b/converter/src/token.py @@ -0,0 +1,173 @@ +from abc import abstractmethod, ABC + +class Token(ABC): + """ + Abstract base class for all token types. + """ + variables = set() + + @abstractmethod + def get_string(self) -> str: + """ + Return the string representation of the token in C. + """ + raise NotImplementedError + + def __repr__(self): + return self.get_string() + +class Bind(Token): + """ + Token class for 'bind' commands. + """ + def __init__(self, command: str) -> None: + self.__params: list = list(map(lambda x: x.strip(), command.strip().split("="))) + + # Assign the parameters + self.name: str = self.__params[0] + self.values: list = self.__params[1].split() + + def get_string(self) -> str: + """ + Return the string representation of the bind token in C. + """ + result = f"bind({self.name}" + for value in self.values: + if value in ["$in", "$out"]: + result += f", {value.lstrip('$')}" + elif value in Token.variables or value.startswith("$"): + result += f', "{value.lstrip('$')}"_v' + else: + result += f", \"{value}\"" + result += ")" + return result + + +class Rule(Token): + """ + Token class for 'rule' commands. + """ + def __init__(self, command: str) -> None: + self._lines: list = list(map(lambda x: x.strip(), command.strip().split(" "))) + self.rulename: str = self._lines[0].split()[1] + self.subassignments: list = [] + for subassignments in self._lines[1:]: + self.subassignments.append(Bind(subassignments).get_string()) + + def get_string(self) -> str: + """ + Return the string representation of the rule token in C. + """ + subassignmentstring = "\n\t".join([subassign for subassign in self.subassignments]) + + return f"auto {self.rulename} = rule({{\n\t{subassignmentstring}}})" + +class Pool(Token): + """ + Token class for 'pool' commands. + """ + def __init__(self, command: str) -> None: + self._params: list = command.split() + self.poolname: str = self._params[1] + self.depth: str = self._params[2] + self.value: str = self._params[4] + Token.variables.add(self.poolname) + Token.variables.add(self.depth) + self.poolbind: Bind = Bind(self.depth + "=" + self.value) + + def get_string(self) -> str: + """ + Return the string representation of the pool token in C. + """ + return f"auto {self.poolname} = pool_({self.poolbind});" + +class Assign(Token): + """ + Token class for variable assignments. + """ + def __init__(self, command: str) -> None: + self.__params: list = command.split() + + self.variable: str = self.__params[0] + Token.variables.add(self.variable) + self.value: str = self.__params[2] + + def get_string(self) -> str: + """ + Return the string representation of the assignment token in C. + """ + return f'let({self.variable}, "{self.value}");' + + +class Build(Token): + """ + Token class for 'build' commands. + """ + def __init__(self, command: str) -> None: + self.__lines: list = list(map(lambda x: x.strip(), command.strip().split(" "))) + self.__params: list = self.__lines[0].split() + self.in_var: TypeList = TypeList(self.__params[1].rstrip(":")) + self.rule: str = self.__params[2] + self.out_var: TypeList = TypeList(self.__params[3]) if len(self.__params) > 3 else None + + def get_string(self) -> str: + """ + Return the string representation of the build token in C. + """ + result = f"auto build_{self.rule[0]} = build({self.in_var},\n" + result += f"\t{{}},\n" + result += f"\t{self.rule},\n" + result += f"\t{self.out_var if self.out_var else "{}"},\n" + result += f"\t{{}},\n" + result += "\t{" + result += f"{",\n\t".join([Bind(line).get_string() for line in self.__lines[1:]])}" + result += "}\n);" + return result + + +class Default(Token): + """ + Token class for 'default' commands. + """ + def __init__(self, command: str) -> None: + self._params: list = command.split() + if(len(self._params) == 2): # Case where a single variable is given + self.value: TypeString = TypeString(self._params[1]) + else: + self.value: TypeList = TypeList(" ".join(self._params[1:])) + + def get_string(self) -> str: + """ + Return the string representation of the default token in C. + """ + return f"default_({self.value.get_string()});" + +class TypeString(Token): + """ + Token class for string types. + """ + def __init__(self, command: str) -> None: + self.command: str = command + + def get_string(self) -> str: + """ + Return the string representation of the string token in C. + """ + return f"str({self.command})" + + +class TypeList(Token): + """ + Token class for list types. + """ + def __init__(self, command: str) -> None: + self.values: list = command.split() + + def get_string(self) -> str: + """ + Return the string representation of the list token in C. + """ + result = "list(" + result += ", ".join([TypeString(value).get_string() for value in self.values]) + result += ")" + return result \ No newline at end of file diff --git a/converter/tests/__init__.py b/converter/tests/__init__.py new file mode 100644 index 0000000000..0a5a53bcce --- /dev/null +++ b/converter/tests/__init__.py @@ -0,0 +1 @@ +# this file is intentionally left blank to mark the directory as a package diff --git a/converter/tests/conftest.py b/converter/tests/conftest.py new file mode 100644 index 0000000000..30e82e09b5 --- /dev/null +++ b/converter/tests/conftest.py @@ -0,0 +1,6 @@ +import pytest + +@pytest.fixture(autouse=True) +def mock_token_variables(): + from src.token import Token + Token.variables = set() diff --git a/converter/tests/test_builder.py b/converter/tests/test_builder.py new file mode 100644 index 0000000000..e4322aa426 --- /dev/null +++ b/converter/tests/test_builder.py @@ -0,0 +1,13 @@ +import pytest +from src.builder import Builder +from src.token import Token + +class MockToken(Token): + def get_string(self): + return "mock_string" + +def test_get_file_content(): + tokens = [MockToken(), MockToken()] + content = Builder.get_file_content(tokens) + expected_content = '#include "manifest.h"\n\nusing namespace shadowdash;\n\nvoid manifest() {\n\nmock_string\n\nmock_string\n\n}' + assert content == expected_content diff --git a/converter/tests/test_converter.py b/converter/tests/test_converter.py new file mode 100644 index 0000000000..bc7490bbf0 --- /dev/null +++ b/converter/tests/test_converter.py @@ -0,0 +1,54 @@ +import pytest +import os +from src.converter import main +from unittest.mock import patch, mock_open +import argparse + + +def test_main(mocker): + mocker.patch( + "argparse.ArgumentParser.parse_args", + return_value=argparse.Namespace( + input_file="input.ninja", output_file="output.manifest" + ), + ) + mocker.patch("os.path.isfile", return_value=True) + mocker.patch("builtins.open", mocker.mock_open()) + mocker.patch("src.parser.Parser.tokenize", return_value=[]) + mocker.patch("src.builder.Builder.get_file_content", return_value="content") + + main() + + open.assert_called_with("output.manifest", "w") + open().write.assert_called_once_with("content") + + +def test_main_file_not_found(mocker): + mocker.patch( + "argparse.ArgumentParser.parse_args", + return_value=argparse.Namespace( + input_file="nonexistent.ninja", output_file="output.manifest" + ), + ) + mocker.patch("os.path.isfile", return_value=False) + + with pytest.raises(SystemExit): + main() + + +def test_main_no_tokens(mocker): + mocker.patch( + "argparse.ArgumentParser.parse_args", + return_value=argparse.Namespace( + input_file="input.ninja", output_file="output.manifest" + ), + ) + mocker.patch("os.path.isfile", return_value=True) + mocker.patch("builtins.open", mocker.mock_open()) + mocker.patch("src.parser.Parser.tokenize", return_value=[]) + mocker.patch("src.builder.Builder.get_file_content", return_value="") + + main() + + open.assert_called_with("output.manifest", "w") + open().write.assert_called_once_with("") diff --git a/converter/tests/test_parser.py b/converter/tests/test_parser.py new file mode 100644 index 0000000000..7ad20a2f33 --- /dev/null +++ b/converter/tests/test_parser.py @@ -0,0 +1,18 @@ +import pytest +from src.parser import Parser +from src.token import Rule, Pool, Bind, Assign, Build, Default + +def test_tokenize(mocker): + mocker.patch("builtins.open", mocker.mock_open(read_data="rule myrule\n var1 = value1\npool mypool 4 depth = 10\nbuild $in: myrule $out\n var1 = value1\n var2 = value2")) + tokens = Parser.tokenize("dummy_path") + assert isinstance(tokens[0], Rule) + assert isinstance(tokens[1], Pool) + assert isinstance(tokens[2], Build) + +def test_get_token(): + assert isinstance(Parser._get_token("rule myrule\n var1 = value1"), Rule) + assert isinstance(Parser._get_token("pool mypool 4 depth = 10"), Pool) + assert isinstance(Parser._get_token("build $in: myrule $out\n var1 = value1\n var2 = value2"), Build) + assert isinstance(Parser._get_token("bind var1 = value1"), Bind) + assert isinstance(Parser._get_token("default var1 var2"), Default) + assert isinstance(Parser._get_token("var1 = value1"), Assign) diff --git a/converter/tests/test_token.py b/converter/tests/test_token.py new file mode 100644 index 0000000000..9d83d8d3f5 --- /dev/null +++ b/converter/tests/test_token.py @@ -0,0 +1,66 @@ +import pytest +from src.token import Token, Bind, Rule, Pool, Assign, Build, Default, TypeString, TypeList + +def test_bind(): + command = "var1 = value1 $in $out" + bind = Bind(command) + assert bind.get_string() == 'bind(var1, "value1", in, out)' + +def test_rule(): + command = "rule myrule\n var1 = value1\n var2 = value2" + rule = Rule(command) + assert rule.get_string() == 'auto myrule = rule({\n\tbind(var1, "value1")\n\tbind(var2, "value2")})' + +def test_pool(): + command = "pool mypool\n\tdepth = 4" + pool = Pool(command) + assert pool.get_string() == 'auto mypool = pool_(bind(depth, "4"));' + +def test_assign(): + command = "var1 = value1" + assign = Assign(command) + assert assign.get_string() == 'let(var1, "value1");' + +def test_build(): + command = "build $in: myrule $out\n var1 = value1\n var2 = value2" + build = Build(command) + assert build.get_string() == 'auto build_m = build(list(str($in)),\n\t{},\n\tmyrule,\n\tlist(str($out)),\n\t{},\n\t{bind(var1, "value1"),\n\tbind(var2, "value2")}\n);' + +def test_default(): + command = "default var1 var2" + default = Default(command) + assert default.get_string() == 'default_(list(str(var1), str(var2)));' + +def test_typestring(): + command = "value1" + typestring = TypeString(command) + assert typestring.get_string() == 'str(value1)' + +def test_typelist(): + command = "value1 value2" + typelist = TypeList(command) + assert typelist.get_string() == 'list(str(value1), str(value2))' + +def test_token_repr(): + class MockToken(Token): + def get_string(self): + return "mock_string" + + token = MockToken() + assert repr(token) == "mock_string" + +def test_default_single_arg(): + command = "default var1" + default = Default(command) + assert default.get_string() == 'default_(str(var1));' + +def test_token_get_string_not_implemented(): + with pytest.raises(TypeError): + token = Token() + +def test_bind_with_variable(): + Token.variables.add("var2") + command = "var1 = var2 $var3" + bind = Bind(command) + assert bind.get_string() == 'bind(var1, "var2"_v, "var3"_v)' + Token.variables.remove("var2")