From 096063ceca001f58cf7f27a8d95383276524cc32 Mon Sep 17 00:00:00 2001 From: Michael Ernst Date: Thu, 21 Oct 2021 10:28:39 -0700 Subject: [PATCH 01/13] Support JDK 17 rather than JDK 16 --- README.md | 2 +- do_like_javac/tools/check.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index ed49a38..dd1a9ca 100644 --- a/README.md +++ b/README.md @@ -133,7 +133,7 @@ mechanism; from the javac documentation: This tool also supports some other tool-specific optional arguments: * `--stubs /path/to/stubs` tells the checker to run with the specified stub files. * `--ajava /path/to/ajava` tells the checker to run with the specified ajava files. -* `--jdkVersion 8/11` tells the Checker Framework to run using JDK8 or JDK11. +* `--jdkVersion 8/11/17` tells the Checker Framework to run using JDK8, JDK11, or JDK17.. * `--quals /path/to/qual.jar` tells the Checker Framework where to find qualifiers (annotations) to put on the classpath. * `--extraJavacArgs='-AcustomArg1 -AcustomArg2'` passes the given arguments to invocations of `javac` that run a Checker Framework checker as an annotation processor (i.e., the arguments are NOT passed to the `javac` used while diff --git a/do_like_javac/tools/check.py b/do_like_javac/tools/check.py index 7d2bbbd..5d7e460 100644 --- a/do_like_javac/tools/check.py +++ b/do_like_javac/tools/check.py @@ -48,7 +48,7 @@ def getArgumentsByVersion(jdkVersion, other_args=[]): result = [] if version == 8: result += ['-J-Xbootclasspath/p:' + os.environ['CHECKERFRAMEWORK'] + '/checker/dist/javac.jar'] - elif version == 11 or version == 16: + elif version == 11 or version >= 16: release_8 = False for i, str in enumerate(other_args): if str == '--release' and other_args[i+1] == "8": @@ -57,7 +57,7 @@ def getArgumentsByVersion(jdkVersion, other_args=[]): # Avoid javac "error: option --add-opens not allowed with target 1.8" if version == 11: result += ['-J--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED'] - elif version == 16: + elif version >= 16: result += ['-J--add-opens=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED', '-J--add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED', '-J--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED', @@ -69,6 +69,6 @@ def getArgumentsByVersion(jdkVersion, other_args=[]): '-J--add-opens=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED'] else: - raise ValueError("the Checker Framework only supports Java versions 8, 11 and 16") + raise ValueError("the Checker Framework only supports Java versions 8, 11 and 17") return result From ec783a499632773a7e579e739f912bfa3a533861 Mon Sep 17 00:00:00 2001 From: Martin Kellogg Date: Fri, 28 Jan 2022 12:17:04 -0800 Subject: [PATCH 02/13] fix error related to zip files (#54) --- do_like_javac/capture/generic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/do_like_javac/capture/generic.py b/do_like_javac/capture/generic.py index f55f99e..6349a2f 100644 --- a/do_like_javac/capture/generic.py +++ b/do_like_javac/capture/generic.py @@ -12,7 +12,7 @@ def get_entry_point(jar): class_pattern = "Main-Class:" with zipfile.ZipFile(jar, 'r') as zip: - metadata = str.splitlines(zip.read("META-INF/MANIFEST.MF")) + metadata = str.splitlines(zip.read("META-INF/MANIFEST.MF").decode("utf-8")) for line in metadata: if class_pattern in line: content = line[len(class_pattern):].strip() From 18df4466eae932ee2088c20016ace9282b981465 Mon Sep 17 00:00:00 2001 From: Huascar Sanchez Date: Fri, 22 Apr 2022 12:29:51 -0700 Subject: [PATCH 03/13] reuses changes from python3 branch plus ... (#29) * code cleanup * python3 and augmaker graphtool and code cleanup * closes Issue#22 * updates log file name * removes unused imports * more formatting changes and additional comments --- Extending.md | 96 +++++++++++++++++++ README.md | 24 ++--- dljc | 4 +- do_like_javac/__init__.py | 2 +- do_like_javac/__main__.py | 2 +- do_like_javac/arg.py | 13 ++- do_like_javac/cache.py | 1 + do_like_javac/capture/__init__.py | 3 +- do_like_javac/capture/ant.py | 4 +- do_like_javac/capture/generic.py | 26 ++++-- do_like_javac/capture/gradle.py | 5 +- do_like_javac/capture/javac.py | 4 +- do_like_javac/capture/mvn.py | 5 +- do_like_javac/command.py | 15 +-- do_like_javac/log.py | 5 +- do_like_javac/tools/__init__.py | 18 +--- do_like_javac/tools/bixie.py | 5 +- do_like_javac/tools/check.py | 55 ++++++++++- do_like_javac/tools/chicory.py | 5 +- do_like_javac/tools/common.py | 54 ++++++----- do_like_javac/tools/dyntrace.py | 67 ++++++++----- do_like_javac/tools/dyntracecounts.py | 3 +- do_like_javac/tools/graphtools.py | 75 ++++++++++----- do_like_javac/tools/infer.py | 6 +- do_like_javac/tools/jprint.py | 2 +- do_like_javac/tools/jsoninv.py | 10 +- do_like_javac/tools/randoop.py | 5 +- do_like_javac/tools/randoop_old.py | 129 -------------------------- do_like_javac/tools/soot.py | 7 +- 29 files changed, 367 insertions(+), 283 deletions(-) create mode 100644 Extending.md delete mode 100644 do_like_javac/tools/randoop_old.py diff --git a/Extending.md b/Extending.md new file mode 100644 index 0000000..246682b --- /dev/null +++ b/Extending.md @@ -0,0 +1,96 @@ +Extending DLJC +================== + +## Supporting a new build system + +Create a new Python file in `do_like_javac/capture/` named after the build system you +want to support. Start with this template: + +```python +from . import generic + +supported_commands = [...] + +class MyCapture(generic.GenericCapture): + def __init__(self, cmd, args): + super(MyCapture, self).__init__(cmd, args) + self.build_cmd = [...] + + def get_javac_commands(self, verbose_output): + return [] + + get_target_jars(self, verbose_output): + return [] + +def gen_instance(cmd, args): + return MyCapture(cmd, args) +``` + +Where supported\_commands is a list of commands that might be used to invoke the +build system (e.g. "gradle", "gradlew", "ant", "maven"). + +For the constructor, cmd is a sys.argv style list containing the build command +the user supplied at the command line. For example, if they wrote + + dljc -t print -- mybuildsystem -arg1 -arg2 -arg3 + +then cmd would contain `['mybuildsystem', '-arg1', '-arg2', '-arg3']`. You +should set self.build_cmd to be a version of that command that invokes the build +system with flags that give you the verbose output you'll need to parse the +javac commands from. + +`args` is the argparse structure that dljc stores its own arguments in. You can +find more details in `do_like_javac/arg.py`. + +Your job now is to implement `get_javac_commands` and optionally +`get_target_jars`. verbose\_output contains a list of lines of output from +running the build system. If your build system outputs entire javac commands +on a single line, or if you can reconstitute the commands in their entirety, +then there's a convenience method called `javac_parse` defined in the +superclass, which expects a list of words in the command and produces the +output you need. + +If not, the ouput format is a list of javac command dicts, which are of the +form: + +```python +{ + 'java_files': ['Foo.java', 'Bar.java', ...], + 'javac_switches': { + # Switches have the - prefix removed. + # Value is either the argument given to the switch, + # or True for no-argument flags. + 'classpath': ..., + ... + } +} +``` + +`get_target_jars` returns a list of paths to jar files output by the build. + +Finally, add your new module to the `capture_modules` list at the top of +`do_like_javac/capture/__init__.py`. + +## Adding a new analysis tool + +Create a new Python file in `do_like_javac/tools/` named after the tool you +want to add. Use this template: + +```python +argparser = None + +def run(args, javac_commands, jars): + print("My tool results.") +``` + +`args` is the argparse structure created by DLJC. `argparser` is where you can +plug in additional command line options you'd like to be available for your +tool. You can see an example in `do_like_javac/tools/infer.py`. + +`javac_commands` and `jars` are the structures discussed above, consisting of +the javac command used to build the project and the jar files generated (if +that information is available). + +After augmenting `run()` to do what you want, add your tool to the `TOOLS` +dictionary at the top of `do_like_javac/tools/__init__.py`, along with a +`from . import mytoolname` statement. diff --git a/README.md b/README.md index 902da98..0bb8465 100644 --- a/README.md +++ b/README.md @@ -24,18 +24,12 @@ project, selecting "Export" and choosing the "Ant Buildfiles" option. Dependencies ============ -* Python 2.7 - -That's it. No other external dependencies for the core `do-like-javac` scripts. - -Of course, you will also need to have installed: - +* Python 3 * The analysis tool(s) you want to run. * Any build dependencies of the project you're analyzing. -`do-like-javac` was built and tested on Mac OS X and GNU/Linux. It probably also -works on Microsoft Windows, but the method of invocation is probably different and -we provide no support. +`do-like-javac` was built and tested on Mac OS X and GNU/Linux. It may also +work on Microsoft Windows, but we provide no support. Installation ============ @@ -69,15 +63,21 @@ If you're running checking tools, there are a couple more flags that may be helpful. `--quiet` suppresses output from tools, and `--timeout ` kills any tool subcommand that runs longer than ``. +Extending +=========== + +Instructions for adding support for new tools or build systems to DLJC can be +found in [Extending.md](./Extending.md). + Caching ======= `do-like-javac` can only extract data from a full compile of a project. That means if you want to re-run it with new arguments or different analysis tools, you will -have to clean and fully re-compile your project. To save time and shortcut this +have to clean and fully re-compile the project. To save time and shortcut this process, we save a cache of the results in the output directory. If you want `dljc` to use this cache, simply add the `--cache` flag and the cache (if available) will -be used instead of recompiling your project. +be used instead of recompiling the project. **IMPORTANT NOTE**: We don't do any sort of cache invalidation or freshness checking. If you add new files to your project and want `dljc` to pick up on them, you will have @@ -101,7 +101,7 @@ The Bixie tool will run your project through [Bixie](http://sri-csl.github.io/bi Dyntrace --------- -The Dyntrace tool will run your project through Randoop to generate tests, then run those tests with Daikon/Chicory to generate invariants. You will need to provide a library directory with the following jars using the `--lib` option: +The Dyntrace tool will run your project through Randoop to generate tests, then run those tests with Daikon/Chicory to generate likely invariants. You will need to provide a library directory with the following jars using the `--lib` option: * randoop.jar * junit-4.12.jar diff --git a/dljc b/dljc index 893abaa..7f02ceb 100755 --- a/dljc +++ b/dljc @@ -1,5 +1,5 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 import do_like_javac -do_like_javac.run() \ No newline at end of file +do_like_javac.run() diff --git a/do_like_javac/__init__.py b/do_like_javac/__init__.py index bb640a6..6aeb9bd 100644 --- a/do_like_javac/__init__.py +++ b/do_like_javac/__init__.py @@ -1,3 +1,3 @@ -import command +from . import command run = command.main diff --git a/do_like_javac/__main__.py b/do_like_javac/__main__.py index 3654869..af826b1 100644 --- a/do_like_javac/__main__.py +++ b/do_like_javac/__main__.py @@ -1,4 +1,4 @@ -from command import main +from .command import main if __name__ == '__main__': main() diff --git a/do_like_javac/arg.py b/do_like_javac/arg.py index 16c9330..8c83cc7 100644 --- a/do_like_javac/arg.py +++ b/do_like_javac/arg.py @@ -10,8 +10,7 @@ import os import sys -import tools -import capture +from . import capture, tools DEFAULT_OUTPUT_DIRECTORY = os.path.join(os.getcwd(), 'dljc-out') @@ -30,8 +29,10 @@ def __call__(self, parser, namespace, values, option_string=None): default=DEFAULT_OUTPUT_DIRECTORY, dest='output_directory', action=AbsolutePathAction, help='The directory to log results.') + base_group.add_argument('--log_to_stderr', action='store_true', help='''Redirect log messages to stderr instead of log file''') + base_group.add_argument('-t', '--tool', metavar='', action='store',default=None, help='A comma separated list of tools to run. Valid tools: ' + ', '.join(tools.TOOLS)) @@ -51,13 +52,19 @@ def __call__(self, parser, namespace, values, option_string=None): help='''Use the dljc cache (if available)''') base_group.add_argument('-c', '--checker', metavar='', - action='store',default='NullnessChecker', + action='store', + # do not run the NullnessChecker by default + # default='NullnessChecker', help='A checker to check (for checker/inference tools)') base_group.add_argument('-l', '--lib', metavar='', action='store',dest='lib_dir', help='Library directory with JARs for tools that need them.') +base_group.add_argument('--jdkVersion', metavar='', + action='store', + help='Version of the JDK to use with the Checker Framework.') + def split_args_to_parse(): split_index = len(sys.argv) if CMD_MARKER in sys.argv: diff --git a/do_like_javac/cache.py b/do_like_javac/cache.py index 60f057c..0bb4512 100644 --- a/do_like_javac/cache.py +++ b/do_like_javac/cache.py @@ -1,6 +1,7 @@ import os import pickle + def retrieve(cmd, args, capturer): cache_file = os.path.join(args.output_directory, "dljc.cache") diff --git a/do_like_javac/capture/__init__.py b/do_like_javac/capture/__init__.py index a4cf2ac..bd7e9a1 100644 --- a/do_like_javac/capture/__init__.py +++ b/do_like_javac/capture/__init__.py @@ -1,5 +1,4 @@ -import ant, gradle, javac, mvn -import itertools +from . import ant, gradle, javac, mvn capture_modules = [ant, gradle, javac, mvn] diff --git a/do_like_javac/capture/ant.py b/do_like_javac/capture/ant.py index 1cb784f..48eb518 100644 --- a/do_like_javac/capture/ant.py +++ b/do_like_javac/capture/ant.py @@ -6,7 +6,7 @@ # additional grant of patent rights can be found in the PATENTS_Facebook file # in the same directory. -import generic +from . import generic supported_commands = ['ant'] @@ -66,4 +66,4 @@ def get_javac_commands(self, verbose_output): javac_arguments.append(arg) if javac_arguments != []: javac_commands.append(javac_arguments) - return map(self.javac_parse, javac_commands) + return list(map(self.javac_parse, javac_commands)) diff --git a/do_like_javac/capture/generic.py b/do_like_javac/capture/generic.py index 56c50a6..af24750 100644 --- a/do_like_javac/capture/generic.py +++ b/do_like_javac/capture/generic.py @@ -1,16 +1,26 @@ import os -import zipfile import timeit +import zipfile + import do_like_javac.tools.common as cmdtools + def is_switch(s): return s != None and s.startswith('-') +## brought this from github.com/kelloggm/do-like-javac +def is_switch_first_part(s): + return s != None and s.startswith('-') and ("=" not in s) + def get_entry_point(jar): class_pattern = "Main-Class:" with zipfile.ZipFile(jar, 'r') as zip: - metadata = str.splitlines(zip.read("META-INF/MANIFEST.MF")) + metadata = [] + try: + metadata = str.splitlines(zip.read("META-INF/MANIFEST.MF").decode("utf-8")) + except TypeError as e: + print(f"ERROR: unable to read META-INF/MANIFEST.MF. See: {e}") for line in metadata: if class_pattern in line: content = line[len(class_pattern):].strip() @@ -59,10 +69,12 @@ def capture(self): stats = {} start_time = timeit.default_timer() - result = cmdtools.run_cmd(self.build_cmd) - stats['build_time'] = result['time'] + result = cmdtools.run_cmd(self.build_cmd, self.args) + # stats['build_time'] = result['time'] + stats['build_time'] = timeit.default_timer() - start_time - with open(os.path.join(self.args.output_directory, 'build_output.txt'), 'w') as f: + build_out_file = os.path.join(self.args.output_directory, 'build_output.txt') + with open(build_out_file, 'w') as f: f.write(result['output']) if result['return_code'] != 0: @@ -72,7 +84,7 @@ def capture(self): javac_commands = self.get_javac_commands(build_lines) target_jars = self.get_target_jars(build_lines) - jars_with_entry_points = map(get_entry_point, target_jars) + jars_with_entry_points = list(map(get_entry_point, target_jars)) self.record_stats(stats, javac_commands, jars_with_entry_points) @@ -94,7 +106,7 @@ def javac_parse(self, javac_command): files.append(a) possible_switch_arg = False - if is_switch(prev_arg): + if is_switch_first_part(prev_arg): if possible_switch_arg: switches[prev_arg[1:]] = a else: diff --git a/do_like_javac/capture/gradle.py b/do_like_javac/capture/gradle.py index 9590aa3..73af2c2 100644 --- a/do_like_javac/capture/gradle.py +++ b/do_like_javac/capture/gradle.py @@ -6,9 +6,10 @@ # additional grant of patent rights can be found in the PATENTS_Facebook file # in the same directory. -import generic import os +from . import generic + supported_commands = ['gradle', 'gradlew'] def gen_instance(cmd, args): @@ -32,4 +33,4 @@ def get_javac_commands(self, verbose_output): content = line.partition(argument_start_pattern)[2].strip() results.append(content.split(' ')) - return map(self.javac_parse, results) + return list(map(self.javac_parse, results)) diff --git a/do_like_javac/capture/javac.py b/do_like_javac/capture/javac.py index 97901b8..95ffd5f 100644 --- a/do_like_javac/capture/javac.py +++ b/do_like_javac/capture/javac.py @@ -6,7 +6,7 @@ # additional grant of patent rights can be found in the PATENTS_Facebook file # in the same directory. -import generic +from . import generic supported_commands = ['javac'] @@ -20,4 +20,4 @@ def __init__(self, cmd, args): self.cmd = cmd[1:] def get_javac_commands(self, verbose_output): - return map(self.javac_parse, [self.cmd]) + return list(map(self.javac_parse, [self.cmd])) diff --git a/do_like_javac/capture/mvn.py b/do_like_javac/capture/mvn.py index fc13443..46519fd 100644 --- a/do_like_javac/capture/mvn.py +++ b/do_like_javac/capture/mvn.py @@ -7,7 +7,8 @@ # in the same directory. import re -import generic + +from . import generic supported_commands = ['mvn'] @@ -55,4 +56,4 @@ def get_javac_commands(self, verbose_output): if found: files_to_compile.append(found.group(1)) - return map(self.javac_parse, javac_commands) + return list(map(self.javac_parse, javac_commands)) diff --git a/do_like_javac/command.py b/do_like_javac/command.py index 253b027..dc81ab0 100644 --- a/do_like_javac/command.py +++ b/do_like_javac/command.py @@ -1,9 +1,10 @@ +import json +import os import pprint -import arg -import log -import tools -import cache -import os,json,sys +import sys + +from . import arg, cache, log, tools + def output_json(filename, obj): with open(filename, 'w') as f: @@ -20,10 +21,12 @@ def main(): result = cache.retrieve(cmd, args, capturer) if not result: - print "DLJC: Build command failed." + print("DLJC: Build command failed.") sys.exit(1) javac_commands, jars, stats = result + if not javac_commands or len(javac_commands) == 0: + raise ValueError(f"no javac commands found by capturer:\n\tcmd = {cmd}\n\targs = {args}") log.info('Results: %s', pprint.pformat(javac_commands)) output_json(os.path.join(args.output_directory, 'javac.json'), javac_commands) diff --git a/do_like_javac/log.py b/do_like_javac/log.py index ad6fc92..41c60e8 100644 --- a/do_like_javac/log.py +++ b/do_like_javac/log.py @@ -6,11 +6,10 @@ # additional grant of patent rights can be found in the PATENTS_Facebook file # in the same directory. +import logging import os -import sys import platform -import shutil -import logging +import sys FORMAT = '[%(levelname)s] %(message)s' LOG_FILE = 'toplevel.log' diff --git a/do_like_javac/tools/__init__.py b/do_like_javac/tools/__init__.py index 55d45e4..4704b13 100644 --- a/do_like_javac/tools/__init__.py +++ b/do_like_javac/tools/__init__.py @@ -1,15 +1,6 @@ -import jprint -import randoop -import randoop_old -import bixie -import graphtools -import chicory -import dyntrace -import dyntracecounts - # import soot -import check -import infer +from . import (bixie, check, chicory, dyntrace, dyntracecounts, graphtools, + infer, jprint, randoop) TOOLS = { # 'soot' : soot, @@ -17,7 +8,6 @@ 'inference' : infer, 'print' : jprint, 'randoop' : randoop, - 'randoop_old': randoop_old, 'bixie' : bixie, 'graphtool' : graphtools, 'chicory' : chicory, @@ -26,13 +16,13 @@ } def parsers(): - return [mod.argparser for name, mod in TOOLS.iteritems() if mod.argparser] + return [mod.argparser for name, mod in TOOLS.items() if mod.argparser] def check_tool(tool): if tool in TOOLS: return tool else: - print "ERROR: Could not find tool {}".format(tool) + print(f"ERROR: Could not find tool {tool}") return None def parse_tools(tools): diff --git a/do_like_javac/tools/bixie.py b/do_like_javac/tools/bixie.py index b19ed2b..3482873 100644 --- a/do_like_javac/tools/bixie.py +++ b/do_like_javac/tools/bixie.py @@ -1,6 +1,7 @@ -import os -import common import copy +import os + +from . import common argparser = None diff --git a/do_like_javac/tools/check.py b/do_like_javac/tools/check.py index 473dd76..242f64e 100644 --- a/do_like_javac/tools/check.py +++ b/do_like_javac/tools/check.py @@ -1,20 +1,67 @@ -# DEPRECATED -- WILL BE REMOVED IN FUTURE VERSION - -import common import os import pprint +from . import common + argparser = None +## other_args is other command-line arguments to javac. +## brought this from github.com/kelloggm/do-like-javac +def get_arguments_by_version(jdk_version, other_args = None): + if jdk_version is not None: + version = int(jdk_version) + else: + # default jdk version + version = 8 + + other_args = other_args or [] + + # add arguments depending on requested JDK version (default 8) + result = [] + if version == 8: + result += ['-J-Xbootclasspath/p:' + os.environ['CHECKERFRAMEWORK'] + '/checker/dist/javac.jar'] + elif version == 11 or version >= 16: + release_8 = False + for idx, arg_str in enumerate(other_args): + if arg_str == '--release' and other_args[idx + 1] == "8": + release_8 = True + if not release_8: + # Avoid javac "error: option --add-opens not allowed with target 1.8" + if version == 11: + result += ['-J--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED'] + elif version >= 16: + result += ['-J--add-opens=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED', + '-J--add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED', + '-J--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED', + '-J--add-opens=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED', + '-J--add-opens=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED', + '-J--add-opens=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED', + '-J--add-opens=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED', + '-J--add-opens=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED', + '-J--add-opens=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED'] + else: + raise ValueError("the Checker Framework only supports Java versions 8, 11 and 17") + + return result + def run(args, javac_commands, jars): # checker-framework javac. - javacheck = os.environ['JSR308']+"/checker-framework/checker/bin/javac" + javacheck = os.environ['CHECKERFRAMEWORK'] + "/checker/bin/javac" checker_command = [javacheck, "-processor", args.checker] + + checker_command += get_arguments_by_version(args.jdkVersion) for jc in javac_commands: pprint.pformat(jc) javac_switches = jc['javac_switches'] cp = javac_switches['classpath'] + + if args.lib_dir: + cp += args.lib_dir + ':' + java_files = ' '.join(jc['java_files']) cmd = checker_command + ["-classpath", cp, java_files] + + print("Running command", " ".join(cmd)) + common.run_cmd(cmd, args, 'check') diff --git a/do_like_javac/tools/chicory.py b/do_like_javac/tools/chicory.py index efe15f3..7bd87fa 100644 --- a/do_like_javac/tools/chicory.py +++ b/do_like_javac/tools/chicory.py @@ -1,5 +1,6 @@ import os -import dyntrace + +from . import dyntrace argparser = None @@ -9,4 +10,4 @@ def run(args, javac_commands, jars): for jc in javac_commands: dyntrace.dyntrace(args, i, jc, out_dir, args.lib_dir, ['chicory']) - i = i+1 + i = i + 1 diff --git a/do_like_javac/tools/common.py b/do_like_javac/tools/common.py index 66b51a8..1fc0995 100644 --- a/do_like_javac/tools/common.py +++ b/do_like_javac/tools/common.py @@ -1,10 +1,12 @@ -import sys, os, traceback -import subprocess32 as subprocess +import os +import subprocess +import sys import timeit -from threading import Timer +import traceback + def log(args, tool, message): - with open(os.path.join(args.output_directory, '{}.log'.format(tool)), 'a') as f: + with open(os.path.join(args.output_directory, f"{tool}-stdout.log"), 'a') as f: f.write(message) f.flush() @@ -30,7 +32,8 @@ def get_class_files(javac_command): if classdir: for root, dirs, files in os.walk(classdir): - classes.extend([os.path.join(root,file) for file in files if '.class' in file]) + classes.extend([os.path.join(root,file) + for file in files if '.class' in file]) return classes @@ -39,7 +42,8 @@ def class_file_to_class_name(classdir, class_file): return class_file.replace(classdir + "/", '').replace('.class','').replace('/','.') classdir = class_directory(javac_command) - return [class_file_to_class_name(classdir, file) for file in get_class_files(javac_command)] + return [class_file_to_class_name(classdir, file) + for file in get_class_files(javac_command)] def source_path(javac_command): if 'javac_switches' in javac_command: @@ -53,15 +57,15 @@ def source_path(javac_command): def run_cmd(cmd, args=None, tool=None): stats = {'timed_out': False, 'output': ''} - timer = None + # timer = None out = None out_file = None - friendly_cmd = ' '.join(cmd) + friendly_cmd = ' '.join("'" + elt + "'" for elt in cmd) if args and args.verbose and args.log_to_stderr: out = sys.stderr elif tool: - out_file = os.path.join(args.output_directory, tool + ".log") + out_file = os.path.join(args.output_directory, f"{tool}-stdout.log") out = open(out_file, 'a') def output(line): @@ -69,34 +73,28 @@ def output(line): out.write(line) out.flush() - def kill_proc(proc, stats): - output("Timed out after {} seconds on {}\n".format(args.timeout, friendly_cmd)) - stats['timed_out'] = True - proc.kill() - - output("Running {}\n\n".format(friendly_cmd)) + output(f"Running {friendly_cmd}\n\n") try: start_time = timeit.default_timer() - process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + timeout = args and args.timeout - if args and args.timeout: - timer = Timer(args.timeout, kill_proc, [process, stats]) - timer.start() + process = subprocess.run(cmd, timeout=timeout, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) - for line in iter(process.stdout.readline, b''): - stats['output'] = stats['output'] + line - output(line) + stats['output'] = process.stdout.decode('utf-8') + output(stats['output']) - process.stdout.close() - process.wait() stats['time'] = timeit.default_timer() - start_time stats['return_code'] = process.returncode - if timer: - timer.cancel() - except: - output('calling {cmd} failed\n{trace}\n'.format(cmd=friendly_cmd,trace=traceback.format_exc())) + except subprocess.TimeoutExpired as e: + output(f"Timed out after {args.timeout} seconds on {friendly_cmd}\n") + stats['timed_out'] = True + except Exception as e: + print(e) + output(f'calling {friendly_cmd} failed\n{traceback.format_exc()}\n') if out_file: out.close() diff --git a/do_like_javac/tools/dyntrace.py b/do_like_javac/tools/dyntrace.py index 186d6c2..12b2567 100644 --- a/do_like_javac/tools/dyntrace.py +++ b/do_like_javac/tools/dyntrace.py @@ -1,9 +1,8 @@ -import os -import json -import jsoninv -import common -import tempfile import argparse +import json +import os + +from . import common, jsoninv argparser = argparse.ArgumentParser(add_help=False) dyntrace_group = argparser.add_argument_group('dyntrace arguments') @@ -12,6 +11,12 @@ action='store_true', help='Have Daikon emit XML') +# choose error revealing driver +dyntrace_group.add_argument('--error-driver', + action='store_true', + dest='error_driver', + help='Chose Error Revealing Driver') + no_jdk = False no_ternary = False @@ -31,6 +36,8 @@ def lib(jar): classdir = os.path.abspath(common.class_directory(java_command)) randoop_driver = "RegressionTestDriver" + if args.error_driver: + randoop_driver = "ErrorTestDriver" test_src_dir = os.path.join(out_dir, "test-src{}".format(i)) test_class_directory = os.path.join(out_dir, "test-classes{}".format(i)) @@ -52,6 +59,7 @@ def lib(jar): compile_classpath = lib("junit-4.12.jar") + ":" + randoop_classpath chicory_classpath = ':'.join([os.path.abspath(test_class_directory), os.path.join(os.environ.get('DAIKONDIR'), 'daikon.jar'), + os.path.join(os.environ.get('DAIKONDIR'), 'java'), lib("hamcrest-core-1.3.jar"), compile_classpath]) replace_call_classpath = lib('replacecall.jar') @@ -98,7 +106,7 @@ def get_select_list(classdir): return selects def get_special_file(special_type, out_dir, i): - candidate = os.path.join(out_dir, "{}.{}".format(special_type, i)) + candidate = os.path.join(out_dir, f"{special_type}.{i}") if os.path.isfile(candidate): return os.path.normpath(candidate) @@ -129,8 +137,9 @@ def get_omit_list(omit_file_path): def make_class_list(out_dir, classes): with open(os.path.join(out_dir,"classlist.txt"), 'w') as class_file: for c in classes: - class_file.write(c) - class_file.write('\n') + if "package-info" not in c: + class_file.write(c) + class_file.write('\n') class_file.flush() return class_file.name @@ -145,28 +154,30 @@ def generate_tests(args, classpath, class_list_file, test_src_dir, junit_after_p randoop_command = ["java", "-ea", "-classpath", classpath, - "-Xbootclasspath/a:{}".format(rc_classpath), - "-javaagent:{}".format(rc_classpath), + f"-Xbootclasspath/a:{rc_classpath}", + f"-javaagent:{rc_classpath}", "randoop.main.Main", "gentests", - "--classlist={}".format(class_list_file), - "--time-limit={}".format(time_limit), - "--omitmethods={}".format(omitted_methods), + f"--classlist={class_list_file}", + f"--time-limit={time_limit}", + f"--omit-methods={omitted_methods}", "--junit-reflection-allowed=false", "--flaky-test-behavior=DISCARD", "--usethreads=true", "--call-timeout=5", "--silently-ignore-bad-class-names=true", - "--junit-output-dir={}".format(test_src_dir), + f"--junit-output-dir={test_src_dir}", # Uncomment these lines to produce Randoop debugging logs - #"--log={}".format(randoop_log_file), - "--selection-log={}".format(selection_log_file), - "--operation-history-log={}".format(operation_log_file)] + f"--log={randoop_log_file}", + f"--selection-log={selection_log_file}", + f"--operation-history-log={operation_log_file}"] if junit_after_path: - randoop_command.append("--junit-after-all={}".format(junit_after_path)) + randoop_command.append(f"--junit-after-all={junit_after_path}") if output_limit and output_limit > 0: - randoop_command.append('--output-limit={}'.format(output_limit)) + randoop_command.append(f'--output-limit={output_limit}') + + print("Running command", " ".join(randoop_command)) common.run_cmd(randoop_command, args, 'randoop') @@ -191,15 +202,23 @@ def run_chicory(args, classpath, main_class, out_dir, selects=[], omits=[]): chicory_command = ["java", "-Xmx3G", "-classpath", classpath, "daikon.Chicory", - "--output_dir={}".format(out_dir)] + f"--output_dir={out_dir}"] + + randoop_driver = "RegressionTestDriver" + if args.error_driver: + randoop_driver = "ErrorTestDriver" + + decls_dyn_comp_file = f"{randoop_driver}.decls-DynComp" - dc_out_path = os.path.join(out_dir, "RegressionTestDriver.decls-DynComp") + dc_out_path = os.path.join(out_dir, decls_dyn_comp_file) chicory_command.append("--comparability-file={}".format(dc_out_path)) chicory_command.extend(selects) chicory_command.extend(omits) chicory_command.append(main_class) + print("Running command", " ".join(chicory_command)) + common.run_cmd(chicory_command, args, 'chicory') @@ -208,7 +227,7 @@ def run_dyncomp(args, classpath, main_class, out_dir, selects=[], omits=[]): "-classpath", classpath, "daikon.DynComp", "--approximate-omitted-ppts", - "--output-dir={}".format(out_dir)] + f"--output-dir={out_dir}"] if no_jdk: dyncomp_command.append("--rt-file=none") @@ -216,6 +235,8 @@ def run_dyncomp(args, classpath, main_class, out_dir, selects=[], omits=[]): dyncomp_command.extend(omits) dyncomp_command.append(main_class) + print("Running command", " ".join(dyncomp_command)) + common.run_cmd(dyncomp_command, args, 'dyncomp') def run_daikon(args, classpath, out_dir, invcounts): @@ -233,6 +254,8 @@ def run_daikon(args, classpath, out_dir, invcounts): daikon_command.append("daikon.inv.ternary.threeScalar.LinearTernaryFloat.enabled=false") daikon_command.append(os.path.join(out_dir, "RegressionTestDriver.dtrace.gz")) + print("Running command", " ".join(daikon_command)) + common.run_cmd(daikon_command, args, 'daikon') def daikon_print_xml(args, classpath, out_dir): diff --git a/do_like_javac/tools/dyntracecounts.py b/do_like_javac/tools/dyntracecounts.py index 35c5731..c4f6123 100644 --- a/do_like_javac/tools/dyntracecounts.py +++ b/do_like_javac/tools/dyntracecounts.py @@ -1,5 +1,6 @@ import os -import dyntrace + +from . import dyntrace argparser = None diff --git a/do_like_javac/tools/graphtools.py b/do_like_javac/tools/graphtools.py index a83968c..90b69fc 100644 --- a/do_like_javac/tools/graphtools.py +++ b/do_like_javac/tools/graphtools.py @@ -1,42 +1,71 @@ -import os import argparse -import common +import copy +import os + +from . import common argparser = argparse.ArgumentParser(add_help=False) graph_group = argparser.add_argument_group('graphtool arguments') graph_group.add_argument('--graph-jar', metavar='', - action='store',default=None, dest='graph_jar', - help='Path to prog2dfg.jar or apilearner.jar') + action='store', default=None, dest='graph_jar', + help='Path to prog2dfg.jar, apilearner.jar or augmaker.jar') + +# augmaker only arguments +graph_group.add_argument('--batch-size', metavar='', + action='store', default=-1, dest='batches', + help='batch size (augmaker only)') +graph_group.add_argument('--project-file', metavar='', + action='store', default=None, dest='projectfile', + help='list of projects to process (augmaker only)') def run(args, javac_commands, jars): if not args.graph_jar: - print "Could not run graph tool: missing arg --graph-jar" + print("Could not run graph tool: missing arg --graph-jar") return - tool_command = ["java", "-jar", args.graph_jar] - + skip_building = "augmaker.jar" in str(args.graph_jar) + dot_dir = os.path.join(args.output_directory, "dot") if not os.path.isdir(dot_dir): os.makedirs(dot_dir) - for jc in javac_commands: - java_files = jc['java_files'] - java_files_file = os.path.join(os.getcwd(), '__java_file_names.txt') - - class_dir = common.class_directory(jc) + tool_command = ["java", "-jar", args.graph_jar] + if skip_building: + # augmaker.jar does not require projects to be built. + # Additionally, augmaker.jar does not have + # a Main-Class defined in the manifest file. Consequently, + # it must be executed differently, using its cli rather + # than using its jar file. To do this, we must override + # the tool_command variable. + tool_command = ["java", "-cp", f".:{args.graph_jar}:*", 'com.sri.augmake.AugmakerCli'] + cmd = copy.copy(tool_command) + cmd.extend(["--outputdir", dot_dir]) + cmd.extend(["--howmany", args.batches]) + cmd.extend(["--projectfile", args.projectfile]) + cmd.extend(["--dotpretty", "dot"]) + common.run_cmd(cmd, args, 'graphtools') + else: + # prog2dfg.jar and apilearner.jar require projects to be built first + for jc in javac_commands: + java_files = jc['java_files'] + java_files_file = os.path.join(os.getcwd(), '__java_file_names.txt') - with open(java_files_file, 'w') as f: - for s in java_files: - f.write(s) - f.write("\n") + class_dir = common.class_directory(jc) - current_outdir = os.path.join(dot_dir, - class_dir.replace(os.getcwd(),'').replace(os.sep,"_")) + with open(java_files_file, 'w') as f: + for s in java_files: + f.write(s) + f.write("\n") - cmd = tool_command + ["-o", current_outdir, - "-j", class_dir, - "-all", - "-source", java_files_file] + current_outdir = os.path.join(dot_dir, + class_dir.replace(os.getcwd(),'').replace(os.sep,"_")) + + cmd = tool_command + ["-o", current_outdir, + "-j", class_dir, + "-all", + "-source", java_files_file] + + print(f"Running command", " ".join(cmd)) - common.run_cmd(cmd, args, 'graphtools') + common.run_cmd(cmd, args, 'graphtools') diff --git a/do_like_javac/tools/infer.py b/do_like_javac/tools/infer.py index dba8d47..b951655 100644 --- a/do_like_javac/tools/infer.py +++ b/do_like_javac/tools/infer.py @@ -1,6 +1,6 @@ import os import argparse -import common +from . import common argparser = argparse.ArgumentParser(add_help=False) infer_group = argparser.add_argument_group('inference tool arguments') @@ -26,7 +26,7 @@ def run(args, javac_commands, jars): CFI_dist = os.path.join(os.environ['JSR308'], 'checker-framework-inference', 'dist') CFI_command = ['java'] - print os.environ + print(os.environ) for jc in javac_commands: target_cp = jc['javac_switches']['classpath'] + \ @@ -53,5 +53,7 @@ def run(args, javac_commands, jars): '--logLevel=WARNING', '-afud', args.afuOutputDir] cmd.extend(jc['java_files']) + + print(f"Running command", " ".join(cmd)) common.run_cmd(cmd, args, 'infer') diff --git a/do_like_javac/tools/jprint.py b/do_like_javac/tools/jprint.py index 795e14d..887f8a2 100644 --- a/do_like_javac/tools/jprint.py +++ b/do_like_javac/tools/jprint.py @@ -3,7 +3,7 @@ argparser = None def run(args, javac_commands, jars): - print (json.dumps( + print(json.dumps( { "javac_commands": javac_commands, "jars": jars diff --git a/do_like_javac/tools/jsoninv.py b/do_like_javac/tools/jsoninv.py index afba438..6bc409e 100644 --- a/do_like_javac/tools/jsoninv.py +++ b/do_like_javac/tools/jsoninv.py @@ -1,7 +1,9 @@ import os import re import xml.etree.ElementTree as ET -import common + +from . import common + def generate_json_invariants(args, out_dir): filename = os.path.join(out_dir, 'invariants.xml') @@ -11,7 +13,7 @@ def generate_json_invariants(args, out_dir): try: tree = ET.parse(filename) except: - common.log(args, 'jsoninv', 'Failed to parse {}'.format(filename)) + common.log(args, 'jsoninv', f'Failed to parse {filename}') return invariants = tree.getroot() @@ -20,7 +22,7 @@ def generate_json_invariants(args, out_dir): for ppt in invariants: add_ppt(methods, ppt) - js = {"invariants": methods.values()} + js = {"invariants": list(methods.values())} return js @@ -52,7 +54,7 @@ def ppt_info(ppt): return class_name, method_name, args, point def find_method(methods, class_name, method_name, args): - descriptor = "{}.{}({})".format(class_name, method_name, args) + descriptor = f"{class_name}.{method_name}({args})" if descriptor not in methods: methods[descriptor] = {"cls": class_name, "method": method_name, diff --git a/do_like_javac/tools/randoop.py b/do_like_javac/tools/randoop.py index 0b53394..43827d2 100644 --- a/do_like_javac/tools/randoop.py +++ b/do_like_javac/tools/randoop.py @@ -1,5 +1,6 @@ import os -import dyntrace + +from . import dyntrace argparser = None @@ -9,4 +10,4 @@ def run(args, javac_commands, jars): for jc in javac_commands: dyntrace.dyntrace(args, i, jc, out_dir, args.lib_dir, ['randoop']) - i = i+1 + i = i + 1 diff --git a/do_like_javac/tools/randoop_old.py b/do_like_javac/tools/randoop_old.py deleted file mode 100644 index 92ed780..0000000 --- a/do_like_javac/tools/randoop_old.py +++ /dev/null @@ -1,129 +0,0 @@ -import os -import pprint -import shutil -import glob -import urllib - -argparser = None - -def run(args, javac_commands, jars): - pp = pprint.PrettyPrinter(indent=2) - i = 0 - for jc in javac_commands: - javac_switches = jc['javac_switches'] - cp = javac_switches['classpath'] - class_file_dir = javac_switches['d'] - class_files = [y for x in os.walk(class_file_dir) for y in glob.glob(os.path.join(x[0], '*.class'))] - - if len(class_files)==0: - continue - - out_dir_name = "__randoop_%04d" % (i) - if not os.path.exists(out_dir_name): - os.makedirs(out_dir_name) - - class_files_file_name = os.path.join(out_dir_name, 'class_files.txt') - print ("Creating list of files %d in %s." % (len(class_files), os.path.abspath(out_dir_name)) ) - with open(class_files_file_name, mode='w') as myfile: - for class_file_name in class_files: - myfile.write(get_qualified_class_name_from_file(class_file_name, class_file_dir)) - myfile.write(os.linesep) - - (randoop_jar, junit_jar, hamcrest_jar) = find_or_download_jars() - - cp_entries = cp.split(os.pathsep) - clean_cp = list() - clean_cp.append(randoop_jar) - clean_cp.append(class_file_dir) - - lib_dir_name = "__randoop_libs" - if not os.path.exists(lib_dir_name): - os.makedirs(lib_dir_name) - - for cp_entry in cp_entries: - if cp_entry.endswith(".jar"): - #check if the jar is in a sub directory of the project. - #if not copy it into a new subdirectory and add the - #corresponding classpath entry. - if not os.path.realpath(cp_entry).startswith(os.getcwd()): - new_jar_name = os.path.join(lib_dir_name, os.path.basename(cp_entry)) - if not os.path.isfile(new_jar_name): - shutil.copyfile(cp_entry, new_jar_name) - clean_cp.append(new_jar_name) - else: - clean_cp.append(cp_entry) - pass - else: - #todo what happens here? - clean_cp.append(cp_entry) - - randoop_cmd = ['java', '-ea', '-classpath', os.pathsep.join(clean_cp), - "randoop.main.Main", "gentests", "--classlist=%s"%class_files_file_name, - "--timelimit=60", "--silently-ignore-bad-class-names=true", - "--junit-output-dir=%s"%out_dir_name] - - junit_cp = list(clean_cp) - junit_cp.append(junit_jar) - junit_build_cmd = ['javac', '-classpath', os.pathsep.join(junit_cp), os.path.join(out_dir_name, 'RandoopTest*.java'), '-d', out_dir_name] - - junit_run_cp = list(junit_cp) - junit_run_cp.append(hamcrest_jar) - junit_run_cp.append(out_dir_name) - - junit_run_cmd = ['java', '-classpath', os.pathsep.join(junit_run_cp), "org.junit.runner.JUnitCore", 'RandoopTest'] - - bash_script_name = "run_randoop_%04d.sh" % (i) - with open(bash_script_name, mode='w') as myfile: - myfile.write("#!/bin/bash\n") - myfile.write("echo \"Run Randoop\"\n") - myfile.write(" ".join(randoop_cmd)) - myfile.write("\n") - myfile.write("echo \"Build tests\"\n") - myfile.write(" ".join(junit_build_cmd)) - myfile.write("\n") - myfile.write("echo \"Run tests\"\n") - myfile.write(" ".join(junit_run_cmd)) - myfile.write("\n") - print ("Written script to %s" % bash_script_name) - - i += 1 - -def get_qualified_class_name_from_file(class_file_name, class_file_path): - """ terrible hack for now """ - suffix = class_file_name.replace(class_file_path+os.sep, "") - mid_section = suffix.replace(".class", "") - return mid_section.replace(os.sep, ".") - - -def find_or_download_jars(): - ''' - Finds or downloads the randoop, junit, and hamrest jars. - ''' - randoop_jar_dir = os.path.join(os.getcwd(), '__randoop_files') - if not os.path.isdir(randoop_jar_dir): - os.makedirs(randoop_jar_dir) - - randoop_jar = os.path.join(randoop_jar_dir, "randoop-2.0.jar") - if not os.path.isfile(randoop_jar): - print("Downloading randoop to %s" % randoop_jar ) - urllib.urlretrieve ("https://github.com/randoop/randoop/releases/download/v2.0/randoop-2.0.jar", randoop_jar) - - junit_jar = os.path.join(randoop_jar_dir, "junit-4.12.jar") - if not os.path.isfile(junit_jar): - print("Downloading junit to %s" % junit_jar ) - urllib.urlretrieve ("https://github.com/junit-team/junit/releases/download/r4.12/junit-4.12.jar", junit_jar) - - hamcrest_jar = os.path.join(randoop_jar_dir, "hamcrest-core-1.3.jar") - if not os.path.isfile(hamcrest_jar): - print("Downloading hamcrest to %s" % hamcrest_jar ) - urllib.urlretrieve ("http://search.maven.org/remotecontent?filepath=org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar", hamcrest_jar) - - return (randoop_jar, junit_jar, hamcrest_jar) - - - - - - - - diff --git a/do_like_javac/tools/soot.py b/do_like_javac/tools/soot.py index 18c8c50..b6c4a62 100644 --- a/do_like_javac/tools/soot.py +++ b/do_like_javac/tools/soot.py @@ -1,8 +1,7 @@ -# DEPRECATED -- WILL BE REMOVED IN FUTURE VERSION - -import os -import pprint import argparse +import pprint + +from . import common argparser = argparse.ArgumentParser(add_help=False) soot_group = argparser.add_argument_group('soot arguments') From 65fac195db034d999c0f3f2e150b5917466b7abc Mon Sep 17 00:00:00 2001 From: Martin Kellogg Date: Fri, 22 Apr 2022 15:41:19 -0700 Subject: [PATCH 04/13] Only use ajava files produced in last iteration during WPI (#56) --- do_like_javac/tools/wpi.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/do_like_javac/tools/wpi.py b/do_like_javac/tools/wpi.py index 79fe588..e8ae4ed 100644 --- a/do_like_javac/tools/wpi.py +++ b/do_like_javac/tools/wpi.py @@ -155,14 +155,16 @@ def run(args, javac_commands, jars): other_args = [arg for arg in other_args if not arg.startswith("--add-opens")] while diffResult: - iterationCheckerCmd = checker_command + iterationCheckerCmd = checker_command.copy() # TODO: the switch to ajava files instead of stub files should make the separate stubs argument # to dljc unnecessary, as there's no longer any need to combine stub lists. # TODO: do we need to treat the -Aajava argument the same way? I.e., will this work if the user # supplies their own -Aajava= argument as part of the extraJavacArgs argument? if args.stubs: iterationCheckerCmd.append("-Astubs=" + str(args.stubs)) - iterationAjavaDirs = ajavaDirs.copy() + # only use the last element in ajavaDirs, to avoid an ambiguity warning that would be issued + # if two ajava files for the same source file were to be passed + iterationAjavaDirs = [ajavaDirs[-1]] if ajavaDirs else [] if args.ajava: iterationAjavaDirs.append(str(args.ajava)) if iterationAjavaDirs: From d5c9ce7081730f9e09ee12948288826554834e36 Mon Sep 17 00:00:00 2001 From: Martin Kellogg Date: Thu, 5 Jan 2023 10:43:39 -0500 Subject: [PATCH 05/13] remove cruft --- delete-me | 0 do_like_javac/command.py | 1 + do_like_javac/tools/jprint.py | 2 +- do_like_javac/tools/jsoninv.py | 1 + 4 files changed, 3 insertions(+), 1 deletion(-) delete mode 100644 delete-me diff --git a/delete-me b/delete-me deleted file mode 100644 index e69de29..0000000 diff --git a/do_like_javac/command.py b/do_like_javac/command.py index 55188b9..dc81ab0 100644 --- a/do_like_javac/command.py +++ b/do_like_javac/command.py @@ -5,6 +5,7 @@ from . import arg, cache, log, tools + def output_json(filename, obj): with open(filename, 'w') as f: f.write(json.dumps(obj, diff --git a/do_like_javac/tools/jprint.py b/do_like_javac/tools/jprint.py index 5b78cee..887f8a2 100644 --- a/do_like_javac/tools/jprint.py +++ b/do_like_javac/tools/jprint.py @@ -10,4 +10,4 @@ def run(args, javac_commands, jars): }, indent = 2, separators = (',', ': ') - ))); + )); diff --git a/do_like_javac/tools/jsoninv.py b/do_like_javac/tools/jsoninv.py index 9bce7ac..6bc409e 100644 --- a/do_like_javac/tools/jsoninv.py +++ b/do_like_javac/tools/jsoninv.py @@ -4,6 +4,7 @@ from . import common + def generate_json_invariants(args, out_dir): filename = os.path.join(out_dir, 'invariants.xml') if not os.path.exists(filename): From a368d2fc5c60adde0eeddbaf132ff58c791258d2 Mon Sep 17 00:00:00 2001 From: Martin Kellogg Date: Tue, 10 Jan 2023 15:03:55 -0500 Subject: [PATCH 06/13] remove all CF CI logic --- .github/workflows/wpiMany.yml | 11 ----------- Makefile | 10 ---------- run-cf-tests.sh | 30 ------------------------------ 3 files changed, 51 deletions(-) delete mode 100644 .github/workflows/wpiMany.yml delete mode 100644 Makefile delete mode 100755 run-cf-tests.sh diff --git a/.github/workflows/wpiMany.yml b/.github/workflows/wpiMany.yml deleted file mode 100644 index 3e60777..0000000 --- a/.github/workflows/wpiMany.yml +++ /dev/null @@ -1,11 +0,0 @@ -name: run-wpi-many-tests -on: [push, pull_request] - -jobs: - run: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: exec-wpi-many-tests - run: ./run-cf-tests.sh - shell: bash diff --git a/Makefile b/Makefile deleted file mode 100644 index 518a284..0000000 --- a/Makefile +++ /dev/null @@ -1,10 +0,0 @@ -all: shell-script-style python-style - -BASH_SCRIPTS = run-cf-tests.sh -shell-script-style: - shellcheck --format=gcc ${BASH_SCRIPTS} - -PYTHON_FILES=$(shell find . -name '*.py' | grep -v randoop_old.py) -python-style: - black . - pylint -f parseable --disable=W,invalid-name ${PYTHON_FILES} diff --git a/run-cf-tests.sh b/run-cf-tests.sh deleted file mode 100755 index 2e9dc23..0000000 --- a/run-cf-tests.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash - -set -e -set -u -set -x -set -o pipefail - -### This script runs the Checker Framework's tests that depend on -### dljc (i.e. this project's main script). It is intended to be used as part of a CI integration test. -### This script must be run from the top-level directory - -### get plume-scripts so that we can use git-clone-related - -if [ -d /tmp/"$USER"/plume-scripts ] ; then - git -C /tmp/"$USER"/plume-scripts pull -q > /dev/null 2>&1 -else - mkdir -p /tmp/"$USER" && git -C /tmp/"$USER" clone --depth 1 -q https://github.com/plume-lib/plume-scripts.git -fi - -### Clone the CF. If there is a branch with the same name as this branch, git-clone-related will check out that branch. -rm -rf /tmp/"$USER"/checker-framework -/tmp/"$USER"/plume-scripts/git-clone-related typetools checker-framework /tmp/"$USER"/checker-framework -export CHECKERFRAMEWORK=/tmp/"$USER"/checker-framework - -export DLJC=$(pwd)/dljc - -cd /tmp/"$USER"/checker-framework - -### run the CF tests -./gradlew wpiManyTest wpiPlumeLibTest From a1f043331c2de81e82f0e4062447cae6327d13d8 Mon Sep 17 00:00:00 2001 From: Alex Liu Date: Thu, 15 Jun 2023 18:14:35 -0400 Subject: [PATCH 07/13] refine code --- do_like_javac/command.py | 6 ------ do_like_javac/tools/dyntrace.py | 7 ------- 2 files changed, 13 deletions(-) diff --git a/do_like_javac/command.py b/do_like_javac/command.py index d6755a3..dc81ab0 100644 --- a/do_like_javac/command.py +++ b/do_like_javac/command.py @@ -25,14 +25,8 @@ def main(): sys.exit(1) javac_commands, jars, stats = result -<<<<<<< HEAD - - if len(javac_commands) == 0: - raise Exception("command.main: no javac commands found by capturer:\n cmd = {}\n args = {}".format(cmd, args)) -======= if not javac_commands or len(javac_commands) == 0: raise ValueError(f"no javac commands found by capturer:\n\tcmd = {cmd}\n\targs = {args}") ->>>>>>> SRI/master log.info('Results: %s', pprint.pformat(javac_commands)) output_json(os.path.join(args.output_directory, 'javac.json'), javac_commands) diff --git a/do_like_javac/tools/dyntrace.py b/do_like_javac/tools/dyntrace.py index 89b1278..12b2567 100644 --- a/do_like_javac/tools/dyntrace.py +++ b/do_like_javac/tools/dyntrace.py @@ -1,15 +1,8 @@ import argparse import json import os -<<<<<<< HEAD -import tempfile - -from . import jsoninv -from . import common -======= from . import common, jsoninv ->>>>>>> SRI/master argparser = argparse.ArgumentParser(add_help=False) dyntrace_group = argparser.add_argument_group('dyntrace arguments') From 4ed0a7794a2addc84e50bc3446b2d2a95d021746 Mon Sep 17 00:00:00 2001 From: Alex Liu Date: Mon, 19 Jun 2023 09:18:18 -0400 Subject: [PATCH 08/13] remove duplicated import --- do_like_javac/tools/bixie.py | 2 -- do_like_javac/tools/check.py | 11 +++++------ 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/do_like_javac/tools/bixie.py b/do_like_javac/tools/bixie.py index a1bd426..3482873 100644 --- a/do_like_javac/tools/bixie.py +++ b/do_like_javac/tools/bixie.py @@ -3,8 +3,6 @@ from . import common -from . import common - argparser = None def run(args, javac_commands, jars): diff --git a/do_like_javac/tools/check.py b/do_like_javac/tools/check.py index 3cc85d6..866b483 100644 --- a/do_like_javac/tools/check.py +++ b/do_like_javac/tools/check.py @@ -4,8 +4,6 @@ from . import common -from . import common - argparser = None ## other_args is other command-line arguments to javac. @@ -16,9 +14,9 @@ def get_arguments_by_version(jdk_version, other_args = None): else: # default jdk version version = 8 - + other_args = other_args or [] - + # add arguments depending on requested JDK version (default 8) result = [] if version == 8: @@ -44,7 +42,7 @@ def get_arguments_by_version(jdk_version, other_args = None): '-J--add-opens=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED'] else: raise ValueError("the Checker Framework only supports Java versions 8, 11 and 17") - + return result def run(args, javac_commands, jars): @@ -78,6 +76,7 @@ def run(args, javac_commands, jars): pp = javac_switches['processorpath'] + ':' if args.lib_dir: cp += pp + args.lib_dir + ':' + cp = "/Users/alexliu/projects/linear-checker/build/libs/linear-checker.jar:/Users/alexliu/projects/linear-checker/linear-checker-qual/build/libs/linear-checker-qual.jar:/Users/alexliu/projects/linear-checker/linear-checker-qual-android/build/libs/linear-checker-qual-android.jar:/Users/alexliu/.m2/repository/commons-io/commons-io/2.11.0/commons-io-2.11.0.jar" + cp java_files = jc['java_files'] cmd = checker_command + ["-classpath", cp] + java_files common.run_cmd(cmd, args, 'check') @@ -111,7 +110,7 @@ def getArgumentsByVersion(jdkVersion, other_args=[]): '-J--add-opens=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED', '-J--add-opens=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED', '-J--add-opens=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED'] - + else: raise ValueError("the Checker Framework only supports Java versions 8, 11 and 17") From db2d25c2486b3c8e4d7415bdb5af5f1cf6a89d3f Mon Sep 17 00:00:00 2001 From: Alex Liu Date: Fri, 28 Jul 2023 18:50:04 -0400 Subject: [PATCH 09/13] refine code --- do_like_javac/tools/check.py | 1 - 1 file changed, 1 deletion(-) diff --git a/do_like_javac/tools/check.py b/do_like_javac/tools/check.py index 866b483..db42bc3 100644 --- a/do_like_javac/tools/check.py +++ b/do_like_javac/tools/check.py @@ -76,7 +76,6 @@ def run(args, javac_commands, jars): pp = javac_switches['processorpath'] + ':' if args.lib_dir: cp += pp + args.lib_dir + ':' - cp = "/Users/alexliu/projects/linear-checker/build/libs/linear-checker.jar:/Users/alexliu/projects/linear-checker/linear-checker-qual/build/libs/linear-checker-qual.jar:/Users/alexliu/projects/linear-checker/linear-checker-qual-android/build/libs/linear-checker-qual-android.jar:/Users/alexliu/.m2/repository/commons-io/commons-io/2.11.0/commons-io-2.11.0.jar" + cp java_files = jc['java_files'] cmd = checker_command + ["-classpath", cp] + java_files common.run_cmd(cmd, args, 'check') From 2587474bbb0e670b5aa3475ec607a8a47712e6f9 Mon Sep 17 00:00:00 2001 From: Alex Liu Date: Fri, 28 Jul 2023 20:51:24 -0400 Subject: [PATCH 10/13] tests --- do_like_javac/tools/wpi.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/do_like_javac/tools/wpi.py b/do_like_javac/tools/wpi.py index e8ae4ed..2d0b3ac 100644 --- a/do_like_javac/tools/wpi.py +++ b/do_like_javac/tools/wpi.py @@ -141,7 +141,7 @@ def run(args, javac_commands, jars): # can more easily remove key and value for release8, below: if v is not None and v is not True and k.startswith("-add-opens"): other_args.append("-" + k + "=" + v) - else: + else: if v is None or v is not False: other_args.append("-" + k) if v is not None and v is not True: @@ -179,6 +179,7 @@ def run(args, javac_commands, jars): pprint.pformat(jc) cmd = iterationCheckerCmd + ["-classpath", cp] + processorArg + other_args + java_files + print("final cmd ===================================: " + cmd); stats = common.run_cmd(cmd + ["-Ainfer=ajava", "-Awarns"], args, 'wpi') # process outputs From 21b559d8084f4202d80f9e11793ff011dc1716ee Mon Sep 17 00:00:00 2001 From: Alex Liu Date: Fri, 28 Jul 2023 21:12:46 -0400 Subject: [PATCH 11/13] refine code --- do_like_javac/tools/wpi.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/do_like_javac/tools/wpi.py b/do_like_javac/tools/wpi.py index 2d0b3ac..aa6b0bf 100644 --- a/do_like_javac/tools/wpi.py +++ b/do_like_javac/tools/wpi.py @@ -50,6 +50,8 @@ def run(args, javac_commands, jars): # if there is already a WPI directory, delete it and start over if os.path.isdir(wpiDir): shutil.rmtree(wpiDir) + else: + os.mkdir(wpiDir) iteration = 0 diffResult = True @@ -179,7 +181,6 @@ def run(args, javac_commands, jars): pprint.pformat(jc) cmd = iterationCheckerCmd + ["-classpath", cp] + processorArg + other_args + java_files - print("final cmd ===================================: " + cmd); stats = common.run_cmd(cmd + ["-Ainfer=ajava", "-Awarns"], args, 'wpi') # process outputs From 9c1160e4fcde5c20cf3acffbd774b72dea526b50 Mon Sep 17 00:00:00 2001 From: Alex Liu Date: Mon, 31 Jul 2023 00:59:01 -0400 Subject: [PATCH 12/13] refine code --- do_like_javac/tools/infer.py | 43 +++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/do_like_javac/tools/infer.py b/do_like_javac/tools/infer.py index 7ee7062..9428d09 100644 --- a/do_like_javac/tools/infer.py +++ b/do_like_javac/tools/infer.py @@ -77,17 +77,34 @@ def get_tool_command(args, target_classpath, java_files, jaif_file="default.jaif if 'CLASSPATH' in os.environ: cp += ':' + os.environ['CLASSPATH'] - cmd = CFI_command + ['-classpath', cp, - 'checkers.inference.InferenceLauncher', - '--solverArgs', args.solverArgs, - '--cfArgs', args.cfArgs, - '--checker', args.checker, - '--solver', args.solver, - '--mode', args.mode, - '--hacks=true', - '--targetclasspath', target_cp, - '--logLevel=WARNING', - '-afud', args.afuOutputDir] - cmd.extend(jc['java_files']) + # os env classpath must be added to targetclasspath for running CFI in + # typecheck mode + target_classpath += ':' + os.environ['CLASSPATH'] + # TODO: see if this is still needed: + # env_classpath must also have a project's dependent jars + # os.environ['CLASSPATH'] = target_classpath - print(f"Running command", " ".join(cmd)) + CFI_command += [ # '-p', # printCommands before executing + '-classpath', cp, + 'checkers.inference.InferenceLauncher'] + + if not args.cfArgs == "": + CFI_command += [ '--cfArgs', args.cfArgs] + + CFI_command += [ '--checker', args.checker, + '--solver', args.solver, + '--solverArgs', args.solverArgs, + '--mode', args.mode, + '--hacks=true', + '--targetclasspath', target_classpath, + '--logLevel=INFO', + '--jaifFile', jaif_file] + + if args.inPlace: + CFI_command += ['--inPlace=true'] + else: + CFI_command += ['-afud', args.afuOutputDir] + + CFI_command.extend(java_files) + + return CFI_command From 471ab132b370015a80f078cb0ff4378eeb2f5520 Mon Sep 17 00:00:00 2001 From: Alex Liu Date: Tue, 29 Aug 2023 10:10:24 -0600 Subject: [PATCH 13/13] refine code --- do_like_javac/tools/check.py | 39 ------------------------------------ 1 file changed, 39 deletions(-) diff --git a/do_like_javac/tools/check.py b/do_like_javac/tools/check.py index db42bc3..ede7b0d 100644 --- a/do_like_javac/tools/check.py +++ b/do_like_javac/tools/check.py @@ -6,45 +6,6 @@ argparser = None -## other_args is other command-line arguments to javac. -## brought this from github.com/kelloggm/do-like-javac -def get_arguments_by_version(jdk_version, other_args = None): - if jdk_version is not None: - version = int(jdk_version) - else: - # default jdk version - version = 8 - - other_args = other_args or [] - - # add arguments depending on requested JDK version (default 8) - result = [] - if version == 8: - result += ['-J-Xbootclasspath/p:' + os.environ['CHECKERFRAMEWORK'] + '/checker/dist/javac.jar'] - elif version == 11 or version >= 16: - release_8 = False - for idx, arg_str in enumerate(other_args): - if arg_str == '--release' and other_args[idx + 1] == "8": - release_8 = True - if not release_8: - # Avoid javac "error: option --add-opens not allowed with target 1.8" - if version == 11: - result += ['-J--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED'] - elif version >= 16: - result += ['-J--add-opens=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED', - '-J--add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED', - '-J--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED', - '-J--add-opens=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED', - '-J--add-opens=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED', - '-J--add-opens=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED', - '-J--add-opens=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED', - '-J--add-opens=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED', - '-J--add-opens=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED'] - else: - raise ValueError("the Checker Framework only supports Java versions 8, 11 and 17") - - return result - def run(args, javac_commands, jars): # checker-framework javac. javacheck = os.environ['CHECKERFRAMEWORK']+"/checker/bin/javac"