From 29e61f70c7ae3fedee2f47d05db147629ece9c6f Mon Sep 17 00:00:00 2001 From: Louis Dionne Date: Fri, 21 Aug 2026 12:22:13 -0400 Subject: [PATCH 1/3] [libc++] Remove the historical benchmarking utilities benchmark-historical, visualize-historical and find-rerun-candidates were built around storing historical benchmark data int a local directory. We now store it in LNT instead. --- libcxx/utils/benchmark-historical | 116 ------------- libcxx/utils/find-rerun-candidates | 242 -------------------------- libcxx/utils/requirements.txt | 3 - libcxx/utils/visualize-historical | 267 ----------------------------- 4 files changed, 628 deletions(-) delete mode 100755 libcxx/utils/benchmark-historical delete mode 100755 libcxx/utils/find-rerun-candidates delete mode 100755 libcxx/utils/visualize-historical diff --git a/libcxx/utils/benchmark-historical b/libcxx/utils/benchmark-historical deleted file mode 100755 index 48946e4bee075..0000000000000 --- a/libcxx/utils/benchmark-historical +++ /dev/null @@ -1,116 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import logging -import os -import pathlib -import subprocess -import sys -import tempfile - -PARENT_DIR = pathlib.Path(os.path.dirname(os.path.abspath(__file__))) - -def directory_path(string): - if os.path.isdir(string): - return pathlib.Path(string) - else: - raise NotADirectoryError(string) - -def whitespace_separated(stream): - """ - Iterate over a stream, yielding whitespace-delimited elements. - """ - for line in stream: - for element in line.split(): - yield element - -def resolve_commit(git_repo, commit): - """ - Resolve the full commit SHA from any tree-ish. - """ - return subprocess.check_output(['git', '-C', git_repo, 'rev-parse', commit], text=True).strip() - - -def main(argv): - parser = argparse.ArgumentParser( - prog='benchmark-historical', - description='Run the libc++ benchmarks against the commits provided on standard input and store the results in ' - 'LNT format in a directory. This makes it easy to generate historical benchmark results of libc++ ' - 'for analysis purposes. This script\'s usage is optimized to be run on a set of commits and then ' - 're-run on a potentially-overlapping set of commits, such as after pulling new commits with Git.') - parser.add_argument('--output', '-o', type=pathlib.Path, required=True, - help='Path to the directory where the resulting .lnt files are stored.') - parser.add_argument('--compiler', type=str, required=True, - help='Path to the compiler to use to build libc++ and run the tests.') - parser.add_argument('--commit-list', type=argparse.FileType('r'), default=sys.stdin, - help='Path to a file containing a whitespace separated list of commits to test. ' - 'By default, this is read from standard input.') - parser.add_argument('--existing', type=str, choices=['skip', 'overwrite', 'append'], default='skip', - help='This option instructs what to do when data for a commit already exists in the output directory. ' - 'Selecting "skip" instructs the tool to skip generating data for a commit that already has data, ' - '"overwrite" will overwrite the existing data with the newly-generated one, and "append" will ' - 'append the new data to the existing one. By default, the tool uses "skip".') - parser.add_argument('lit_options', nargs=argparse.REMAINDER, - help='Optional arguments passed to lit when running the tests. Should be provided last and ' - 'separated from other arguments with a `--`.') - parser.add_argument('--git-repo', type=directory_path, default=pathlib.Path(os.getcwd()), - help='Optional path to the Git repository to use. By default, the current working directory is used.') - parser.add_argument('--dry-run', action='store_true', - help='Do not actually run anything, just print what would be done.') - args = parser.parse_args(argv) - - logging.getLogger().setLevel(logging.INFO) - - # Gather lit options - lit_options = [] - if args.lit_options: - if args.lit_options[0] != '--': - raise ArgumentError('For clarity, Lit options must be separated from other options by --') - lit_options = args.lit_options[1:] - - # Process commits one by one. Commits just need to be whitespace separated: we also handle - # the case where there is more than one commit per line. - for commit in whitespace_separated(args.commit_list): - commit = resolve_commit(args.git_repo, commit) # resolve e.g. HEAD to a real SHA - - output_file = args.output / (commit + '.lnt') - if output_file.exists() and args.existing == 'skip': - logging.info(f'Skipping {commit} which already has data in {output_file}') - continue - else: - logging.info(f'Benchmarking {commit} against test-suite in {args.git_repo}') - - with tempfile.TemporaryDirectory() as libcxx_install_dir: - with tempfile.TemporaryDirectory() as build_dir: - build_cmd = [PARENT_DIR / 'build-at-commit', '--git-repo', args.git_repo, - '--commit', commit, - '--install-dir', libcxx_install_dir, - '--', '-DCMAKE_BUILD_TYPE=RelWithDebInfo', - f'-DCMAKE_CXX_COMPILER={args.compiler}'] - - test_cmd = [PARENT_DIR / 'test-at-commit', '--git-repo', args.git_repo, - '--libcxx-installation', libcxx_install_dir, - '--compiler', args.compiler, - '--build-dir', build_dir] - test_cmd += ['--'] + lit_options - - if args.dry_run: - logging.info(f'Running {" ".join(str(a) for a in build_cmd)}') - logging.info(f'Running {" ".join(str(a) for a in test_cmd)}') - continue - - subprocess.check_call(build_cmd) - subprocess.call(test_cmd) - output_file.parent.mkdir(parents=True, exist_ok=True) - mode = 'a' if args.existing == 'append' else 'w' - if output_file.exists() and args.existing == 'append': - logging.info(f'Appending to existing data for {commit}') - elif output_file.exists() and args.existing == 'overwrite': - logging.info(f'Overwriting existing data for {commit}') - else: - logging.info(f'Writing data for {commit}') - with open(output_file, mode) as out: - subprocess.check_call([(PARENT_DIR / 'consolidate-benchmarks'), build_dir], stdout=out) - -if __name__ == '__main__': - main(sys.argv[1:]) diff --git a/libcxx/utils/find-rerun-candidates b/libcxx/utils/find-rerun-candidates deleted file mode 100755 index 5ac2644005aac..0000000000000 --- a/libcxx/utils/find-rerun-candidates +++ /dev/null @@ -1,242 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import datetime -import functools -import os -import pathlib -import re -import statistics -import subprocess -import sys - -import git -import pandas -import tqdm - -@functools.total_ordering -class Commit: - """ - This class represents a commit inside a given Git repository. - """ - - def __init__(self, git_repo, sha): - self._git_repo = git_repo - self._sha = sha - - def __eq__(self, other): - """ - Return whether two commits refer to the same commit. - - This doesn't take into account the content of the Git tree at those commits, only the - 'identity' of the commits themselves. - """ - return self.fullrev == other.fullrev - - def __lt__(self, other): - """ - Return whether a commit is an ancestor of another commit in the Git repository. - """ - # Is self._sha an ancestor of other._sha? - res = subprocess.run(['git', '-C', self._git_repo, 'merge-base', '--is-ancestor', self._sha, other._sha]) - if res.returncode not in (0, 1): - raise RuntimeError(f'Error when trying to obtain the commit order for {self._sha} and {other._sha}') - return res.returncode == 0 - - def __hash__(self): - """ - Return the full revision for this commit. - """ - return hash(self.fullrev) - - @functools.cache - def show(self, include_diff=False): - """ - Return the commit information equivalent to `git show` associated to this commit. - """ - cmd = ['git', '-C', self._git_repo, 'show', self._sha] - if not include_diff: - cmd.append('--no-patch') - return subprocess.check_output(cmd, text=True) - - @functools.cached_property - def shortrev(self): - """ - Return the shortened version of the given SHA. - """ - return subprocess.check_output(['git', '-C', self._git_repo, 'rev-parse', '--short', self._sha], text=True).strip() - - @functools.cached_property - def fullrev(self): - """ - Return the full SHA associated to this commit. - """ - return subprocess.check_output(['git', '-C', self._git_repo, 'rev-parse', self._sha], text=True).strip() - - @functools.cached_property - def commit_date(self): - """ - Return the date of the commit as a `datetime.datetime` object. - """ - repo = git.Repo(self._git_repo) - return datetime.datetime.fromtimestamp(repo.commit(self._sha).committed_date) - - def prefetch(self): - """ - Prefetch cached properties associated to this commit object. - - This makes it possible to control when time is spent recovering that information from Git for - e.g. better reporting to the user. - """ - self.commit_date - self.fullrev - self.shortrev - self.show() - - def __str__(self): - return self._sha - -def directory_path(string): - if os.path.isdir(string): - return pathlib.Path(string) - else: - raise NotADirectoryError(string) - -def parse_lnt(lines, aggregate=statistics.median): - """ - Parse lines in LNT format and return a list of dictionnaries of the form: - - [ - { - 'benchmark': , - : [float], - : [float], - 'data_points': int, - ... - }, - { - 'benchmark': , - : [float], - : [float], - 'data_points': int, - ... - }, - ... - ] - - If a metric has multiple values associated to it, they are aggregated into a single - value using the provided aggregation function. - """ - results = {} - for line in lines: - line = line.strip() - if not line: - continue - - (identifier, value) = line.split(' ') - (benchmark, metric) = identifier.split('.') - if benchmark not in results: - results[benchmark] = {'benchmark': benchmark} - - entry = results[benchmark] - if metric not in entry: - entry[metric] = [] - entry[metric].append(float(value)) - - for (bm, entry) in results.items(): - metrics = [key for key in entry if isinstance(entry[key], list)] - min_data_points = min(len(entry[metric]) for metric in metrics) - for metric in metrics: - entry[metric] = aggregate(entry[metric]) - entry['data_points'] = min_data_points - - return list(results.values()) - -def sorted_revlist(git_repo, commits): - """ - Return the list of commits sorted by their chronological order (from oldest to newest) in the - provided Git repository. Items earlier in the list are older than items later in the list. - """ - revlist_cmd = ['git', '-C', git_repo, 'rev-list', '--no-walk'] + list(commits) - revlist = subprocess.check_output(revlist_cmd, text=True).strip().splitlines() - return list(reversed(revlist)) - -def main(argv): - parser = argparse.ArgumentParser( - prog='find-rerun-candidates', - description='Find benchmarking data points that are good candidates for additional runs, to reduce noise.') - parser.add_argument('directory', type=directory_path, - help='Path to a valid directory containing benchmark data in LNT format, each file being named .lnt. ' - 'This is also the format generated by the `benchmark-historical` utility.') - parser.add_argument('--metric', type=str, default='execution_time', - help='The metric to analyze. LNT data may contain multiple metrics (e.g. code size, execution time, etc) -- ' - 'this option allows selecting which metric is analyzed for rerun candidates. The default is "execution_time".') - parser.add_argument('--filter', type=str, required=False, - help='An optional regular expression used to filter the benchmarks included in the analysis. ' - 'Only benchmarks whose names match the regular expression will be analyzed.') - parser.add_argument('--outlier-threshold', metavar='FLOAT', type=float, default=0.1, - help='Relative difference from the previous points for considering a data point as an outlier. This threshold is ' - 'expressed as a floating point number, e.g. 0.25 will detect points that differ by more than 25%% from their ' - 'previous result.') - parser.add_argument('--data-points-threshold', type=int, required=False, - help='Number of data points above which an outlier is not considered an outlier. If an outlier has more than ' - 'that number of data points yet its relative difference is above the threshold, it is not considered an ' - 'outlier. This can be used to re-run noisy data points until we have at least N samples, at which point ' - 'we consider the data to be accurate, even if the result is beyond the threshold. By default, there is ' - 'no limit on the number of data points.') - parser.add_argument('--git-repo', type=directory_path, default=pathlib.Path(os.getcwd()), - help='Path to the git repository to use for ordering commits in time. ' - 'By default, the current working directory is used.') - args = parser.parse_args(argv) - - # Extract benchmark data from the directory. - data = {} - files = [f for f in args.directory.glob('*.lnt')] - for file in tqdm.tqdm(files, desc='Parsing LNT files'): - rows = parse_lnt(file.read_text().splitlines()) - (commit, _) = os.path.splitext(os.path.basename(file)) - commit = Commit(args.git_repo, commit) - data[commit] = rows - - # Obtain commit information which is then cached throughout the program. Do this - # eagerly so we can provide a progress bar. - for commit in tqdm.tqdm(data.keys(), desc='Prefetching Git information'): - commit.prefetch() - - # Create a dataframe from the raw data and add some columns to it: - # - 'commit' represents the Commit object associated to the results in that row - # - `revlist_order` represents the order of the commit within the Git repository. - revlist = sorted_revlist(args.git_repo, [c.fullrev for c in data.keys()]) - data = pandas.DataFrame([row | {'commit': c} for (c, rows) in data.items() for row in rows]) - data = data.join(pandas.DataFrame([{'revlist_order': revlist.index(c.fullrev)} for c in data['commit']])) - - # Filter the benchmarks if needed. - if args.filter is not None: - keeplist = [b for b in data['benchmark'] if re.search(args.filter, b) is not None] - data = data[data['benchmark'].isin(keeplist)] - - # Detect outliers by selecting all benchmarks whose change percentage is beyond the threshold. - # If we have a max number of points, also take that into account. - if args.data_points_threshold is not None: - print(f'Generating outliers with more than {args.outlier_threshold * 100}% relative difference and less than {args.data_points_threshold} data points') - else: - print(f'Generating outliers with more than {args.outlier_threshold * 100}% relative difference') - - overall = set() - for (benchmark, series) in data.sort_values(by='revlist_order').groupby('benchmark'): - pct_change = series[args.metric].pct_change() - outliers = series[pct_change.abs() > args.outlier_threshold] - if args.data_points_threshold is not None: - outliers = outliers[outliers['data_points'] < args.data_points_threshold] - outliers = set(outliers['commit']) - overall |= outliers - if len(outliers) > 0: - print(f'{benchmark}: {" ".join(c.shortrev for c in outliers)}') - - if len(overall) > 0: - print(f'Summary: {" ".join(c.shortrev for c in overall)}') - else: - print(f'No outliers') - -if __name__ == '__main__': - main(sys.argv[1:]) diff --git a/libcxx/utils/requirements.txt b/libcxx/utils/requirements.txt index ccccf1b266783..8b9286a158647 100644 --- a/libcxx/utils/requirements.txt +++ b/libcxx/utils/requirements.txt @@ -1,11 +1,8 @@ click -GitPython llvm-lnt numpy pandas plotly PyGithub scipy -statsmodels tabulate -tqdm diff --git a/libcxx/utils/visualize-historical b/libcxx/utils/visualize-historical deleted file mode 100755 index 89c838fad6437..0000000000000 --- a/libcxx/utils/visualize-historical +++ /dev/null @@ -1,267 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import datetime -import functools -import os -import pathlib -import re -import statistics -import subprocess -import sys -import tempfile - -import git -import pandas -import plotly -import plotly.express -import tqdm - -@functools.total_ordering -class Commit: - """ - This class represents a commit inside a given Git repository. - """ - - def __init__(self, git_repo: git.Repo, sha: str): - self._git_repo = git_repo - self._sha = sha - - def __eq__(self, other): - """ - Return whether two commits refer to the same commit. - - This doesn't take into account the content of the Git tree at those commits, only the - 'identity' of the commits themselves. - """ - return self.fullrev == other.fullrev - - def __lt__(self, other): - """ - Return whether a commit is an ancestor of another commit in the Git repository. - """ - # Is self._sha an ancestor of other._sha? - res = subprocess.run(['git', '-C', self._git_repo.git_dir, 'merge-base', '--is-ancestor', self._sha, other._sha]) - if res.returncode not in (0, 1): - raise RuntimeError(f'Error when trying to obtain the commit order for {self._sha} and {other._sha}') - return res.returncode == 0 - - def __hash__(self): - """ - Return the full revision for this commit. - """ - return hash(self.fullrev) - - @functools.cache - def show(self, include_diff=False): - """ - Return the commit information equivalent to `git show` associated to this commit. - """ - cmd = ['git', '-C', self._git_repo.git_dir, 'show', self._sha] - if not include_diff: - cmd.append('--no-patch') - return subprocess.check_output(cmd, text=True) - - @functools.cached_property - def shortrev(self): - """ - Return the shortened version of the given SHA. - """ - return subprocess.check_output(['git', '-C', self._git_repo.git_dir, 'rev-parse', '--short', self._sha], text=True).strip() - - @functools.cached_property - def fullrev(self): - """ - Return the full SHA associated to this commit. - """ - return subprocess.check_output(['git', '-C', self._git_repo.git_dir, 'rev-parse', self._sha], text=True).strip() - - @functools.cached_property - def commit_date(self): - """ - Return the date of the commit as a `datetime.datetime` object. - """ - return datetime.datetime.fromtimestamp(self._git_repo.commit(self._sha).committed_date) - - def prefetch(self): - """ - Prefetch cached properties associated to this commit object. - - This makes it possible to control when time is spent recovering that information from Git for - e.g. better reporting to the user. - """ - self.commit_date - self.fullrev - self.shortrev - self.show() - - def __str__(self): - return self._sha - -def truncate_lines(string, n, marker=None): - """ - Truncate the given string at a certain number of lines. - - Optionally, add a marker on the last line to identify that truncation has happened. - """ - lines = string.splitlines() - truncated = lines[:n] - if marker is not None and len(lines) > len(truncated): - truncated[-1] = marker - assert len(truncated) <= n, "broken post-condition" - return '\n'.join(truncated) - -def create_plot(data, metric, trendline=None, subtitle=None): - """ - Create a plot object showing the evolution of each benchmark throughout the given commits for - the given metric. - """ - data = data.sort_values(by=['revlist_order', 'benchmark']) - revlist = pandas.unique(data['commit']) # list of all commits in chronological order - hover_info = {c: truncate_lines(c.show(), 30, marker='...').replace('\n', '
') for c in revlist} - figure = plotly.express.scatter(data, title=f"{revlist[0].shortrev} to {revlist[-1].shortrev}", - subtitle=subtitle, - x='revlist_order', y=metric, - symbol='benchmark', - color='benchmark', - hover_name=[hover_info[c] for c in data['commit']], - trendline=trendline) - return figure - -def directory_path(string): - if os.path.isdir(string): - return pathlib.Path(string) - else: - raise NotADirectoryError(string) - -def parse_lnt(lines, aggregate=statistics.median): - """ - Parse lines in LNT format and return a list of dictionnaries of the form: - - [ - { - 'benchmark': , - : float, - : float, - ... - }, - { - 'benchmark': , - : float, - : float, - ... - }, - ... - ] - - If a metric has multiple values associated to it, they are aggregated into a single - value using the provided aggregation function. - """ - results = {} - for line in lines: - line = line.strip() - if not line: - continue - - (identifier, value) = line.split(' ') - (benchmark, metric) = identifier.split('.') - if benchmark not in results: - results[benchmark] = {'benchmark': benchmark} - - entry = results[benchmark] - if metric not in entry: - entry[metric] = [] - entry[metric].append(float(value)) - - for (bm, entry) in results.items(): - for metric in entry: - if isinstance(entry[metric], list): - entry[metric] = aggregate(entry[metric]) - - return list(results.values()) - -def sorted_revlist(git_repo, commits): - """ - Return the list of commits sorted by their chronological order (from oldest to newest) in the - provided Git repository. Items earlier in the list are older than items later in the list. - """ - revlist_cmd = ['git', '-C', git_repo, 'rev-list', '--no-walk'] + list(commits) - revlist = subprocess.check_output(revlist_cmd, text=True).strip().splitlines() - return list(reversed(revlist)) - -def main(argv): - parser = argparse.ArgumentParser( - prog='visualize-historical', - description='Visualize historical data in LNT format. This program generates a HTML file that embeds an ' - 'interactive plot with the provided data. The HTML file can then be opened in a browser to ' - 'visualize the data as a chart.', - epilog='This script depends on the modules listed in `libcxx/utils/requirements.txt`.') - parser.add_argument('directory', type=directory_path, - help='Path to a valid directory containing benchmark data in LNT format, each file being named .lnt. ' - 'This is also the format generated by the `benchmark-historical` utility.') - parser.add_argument('--output', '-o', type=pathlib.Path, required=False, - help='Optional path where to output the resulting HTML file. If it already exists, it is overwritten. ' - 'Defaults to a temporary file which is opened automatically once generated, but not removed after ' - 'creation.') - parser.add_argument('--metric', type=str, default='execution_time', - help='The metric to compare. LNT data may contain multiple metrics (e.g. code size, execution time, etc) -- ' - 'this option allows selecting which metric is being visualized. The default is "execution_time".') - parser.add_argument('--filter', type=str, required=False, - help='An optional regular expression used to filter the benchmarks included in the chart. ' - 'Only benchmarks whose names match the regular expression will be included. ' - 'Since the chart is interactive, it generally makes most sense to include all the benchmarks ' - 'and to then filter them in the browser, but in some cases producing a chart with a reduced ' - 'number of data series is useful.') - parser.add_argument('--subtitle', type=str, required=False, - help='Optional subtitle for the chart. This can be used to help identify the contents of the chart.') - parser.add_argument('--git-repo', type=directory_path, default=pathlib.Path(os.getcwd()), - help='Path to the git repository to use for ordering commits in time. ' - 'By default, the current working directory is used.') - parser.add_argument('--open', action='store_true', - help='Whether to automatically open the generated HTML file when finished. If no output file is provided, ' - 'the resulting benchmark is opened automatically by default.') - parser.add_argument('--trendline', type=str, required=False, default=None, choices=('ols', 'lowess', 'expanding'), - help='Optional trendline to add on each series in the chart. See the documentation in ' - 'https://plotly.com/python-api-reference/generated/plotly.express.trendline_functions.html ' - 'details on each option.') - args = parser.parse_args(argv) - repo = git.Repo(args.git_repo) - - # Extract benchmark data from the directory. - data = {} - files = [f for f in args.directory.glob('*.lnt')] - for file in tqdm.tqdm(files, desc='Parsing LNT files'): - rows = parse_lnt(file.read_text().splitlines()) - (commit, _) = os.path.splitext(os.path.basename(file)) - commit = Commit(repo, commit) - data[commit] = rows - - # Obtain commit information which is then cached throughout the program. Do this - # eagerly so we can provide a progress bar. - for commit in tqdm.tqdm(data.keys(), desc='Prefetching Git information'): - commit.prefetch() - - # Create a dataframe from the raw data and add some columns to it: - # - 'commit' represents the Commit object associated to the results in that row - # - `revlist_order` represents the order of the commit within the Git repository. - # - `date` represents the commit date - revlist = sorted_revlist(args.git_repo, [c.fullrev for c in data.keys()]) - data = pandas.DataFrame([row | {'commit': c} for (c, rows) in data.items() for row in rows]) - data = data.join(pandas.DataFrame([{'revlist_order': revlist.index(c.fullrev)} for c in data['commit']])) - data = data.join(pandas.DataFrame([{'date': c.commit_date} for c in data['commit']])) - - # Filter the benchmarks if needed. - if args.filter is not None: - keeplist = [b for b in data['benchmark'] if re.search(args.filter, b) is not None] - data = data[data['benchmark'].isin(keeplist)] - if len(data) == 0: - raise RuntimeError(f'Filter "{args.filter}" resulted in empty data set -- nothing to plot') - - # Plot the data for all the required benchmarks. - figure = create_plot(data, args.metric, trendline=args.trendline, subtitle=args.subtitle) - do_open = args.output is None or args.open - output = args.output if args.output is not None else tempfile.NamedTemporaryFile(suffix='.html').name - plotly.io.write_html(figure, file=output, auto_open=do_open) - -if __name__ == '__main__': - main(sys.argv[1:]) From f079c1373b7e50137777b5f2f508a6f262c783d6 Mon Sep 17 00:00:00 2001 From: Louis Dionne Date: Fri, 21 Aug 2026 13:06:40 -0400 Subject: [PATCH 2/3] Bad merge conflict --- libcxx/utils/requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/libcxx/utils/requirements.txt b/libcxx/utils/requirements.txt index 76d00320ac35a..359fb29b5a1d1 100644 --- a/libcxx/utils/requirements.txt +++ b/libcxx/utils/requirements.txt @@ -5,4 +5,3 @@ numpy pandas plotly scipy -tabulate From 69b82a1222af77b6da879aa6572c53a74a1b900a Mon Sep 17 00:00:00 2001 From: Louis Dionne Date: Fri, 21 Aug 2026 14:48:01 -0400 Subject: [PATCH 3/3] Fix deps --- libcxx/utils/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libcxx/utils/requirements.txt b/libcxx/utils/requirements.txt index 359fb29b5a1d1..5ebc9537ba591 100644 --- a/libcxx/utils/requirements.txt +++ b/libcxx/utils/requirements.txt @@ -1,7 +1,7 @@ -r ci/lnt/requirements.txt click -GitPython numpy pandas plotly scipy +tabulate