#!/usr/bin/env python
import argparse
import sys
import os
from subprocess import call
SCRIPTS_DIR = os.path.dirname(os.path.realpath(__file__))
PROJECTS_DIR = os.path.join(SCRIPTS_DIR, "projects")
DEFAULT_LLVM_DIR = os.path.realpath(
os.path.join(SCRIPTS_DIR, os.path.pardir, os.path.pardir, os.path.pardir)
)
def add(parser, args):
import SATestAdd
from ProjectMap import ProjectInfo
if args.source == "git" and (args.origin == "" or args.commit == ""):
parser.error("Please provide both --origin and --commit if source is 'git'")
if args.source != "git" and (args.origin != "" or args.commit != ""):
parser.error(
"Options --origin and --commit don't make sense when " "source is not 'git'"
)
project = ProjectInfo(
args.name[0], args.mode, args.source, args.origin, args.commit
)
SATestAdd.add_new_project(project)
def build(parser, args):
import SATestBuild
SATestBuild.VERBOSE = args.verbose
projects = get_projects(parser, args)
tester = SATestBuild.RegressionTester(
args.jobs,
projects,
args.override_compiler,
args.extra_analyzer_config,
args.extra_checkers,
args.regenerate,
args.strictness,
)
tests_passed = tester.test_all()
if not tests_passed:
sys.stderr.write("ERROR: Tests failed.\n")
sys.exit(42)
def compare(parser, args):
import CmpRuns
choices = [
CmpRuns.HistogramType.RELATIVE.value,
CmpRuns.HistogramType.LOG_RELATIVE.value,
CmpRuns.HistogramType.ABSOLUTE.value,
]
if args.histogram is not None and args.histogram not in choices:
parser.error(
"Incorrect histogram type, available choices are {}".format(choices)
)
dir_old = CmpRuns.ResultsDirectory(args.old[0], args.root_old)
dir_new = CmpRuns.ResultsDirectory(args.new[0], args.root_new)
CmpRuns.dump_scan_build_results_diff(
dir_old,
dir_new,
show_stats=args.show_stats,
stats_only=args.stats_only,
histogram=args.histogram,
verbose_log=args.verbose_log,
)
def update(parser, args):
import SATestUpdateDiffs
from ProjectMap import ProjectMap
project_map = ProjectMap()
for project in project_map.projects:
SATestUpdateDiffs.update_reference_results(project, args.git)
def benchmark(parser, args):
from SATestBenchmark import Benchmark
projects = get_projects(parser, args)
benchmark = Benchmark(projects, args.iterations, args.output)
benchmark.run()
def benchmark_compare(parser, args):
import SATestBenchmark
SATestBenchmark.compare(args.old, args.new, args.output)
def get_projects(parser, args):
from ProjectMap import ProjectMap, Size
project_map = ProjectMap()
projects = project_map.projects
def filter_projects(projects, predicate, force=False):
return [
project.with_fields(
enabled=(force or project.enabled) and predicate(project)
)
for project in projects
]
if args.projects:
projects_arg = args.projects.split(",")
available_projects = [project.name for project in projects]
# validate that given projects are present in the project map file
for manual_project in projects_arg:
if manual_project not in available_projects:
parser.error(
"Project '{project}' is not found in "
"the project map file. Available projects are "
"{all}.".format(project=manual_project, all=available_projects)
)
projects = filter_projects(
projects, lambda project: project.name in projects_arg, force=True
)
try:
max_size = Size.from_str(args.max_size)
except ValueError as e:
parser.error("{}".format(e))
projects = filter_projects(projects, lambda project: project.size <= max_size)
return projects
def docker(parser, args):
if len(args.rest) > 0:
if args.rest[0] != "--":
parser.error("REST arguments should start with '--'")
args.rest = args.rest[1:]
if args.build_image:
docker_build_image()
elif args.shell:
docker_shell(args)
else:
sys.exit(docker_run(args, " ".join(args.rest)))
def docker_build_image():
sys.exit(call("docker build --tag satest-image {}".format(SCRIPTS_DIR), shell=True))
def docker_shell(args):
try:
# First we need to start the docker container in a waiting mode,
# so it doesn't do anything, but most importantly keeps working
# while the shell session is in progress.
docker_run(args, "--wait", "--detach")
# Since the docker container is running, we can actually connect to it
call("docker exec -it satest bash", shell=True)
except KeyboardInterrupt:
pass
finally:
docker_cleanup()
def docker_run(args, command, docker_args=""):
try:
return call(
"docker run --rm --name satest "
"-v {llvm}:/llvm-project "
"-v {build}:/build "
"-v {clang}:/analyzer "
"-v {scripts}:/scripts "
"-v {projects}:/projects "
"{docker_args} "
"satest-image:latest {command}".format(
llvm=args.llvm_project_dir,
build=args.build_dir,
clang=args.clang_dir,
scripts=SCRIPTS_DIR,
projects=PROJECTS_DIR,
docker_args=docker_args,
command=command,
),
shell=True,
)
except KeyboardInterrupt:
docker_cleanup()
def docker_cleanup():
print("Please wait for docker to clean up")
call("docker stop satest", shell=True)
def main():
parser = argparse.ArgumentParser()
subparsers = parser.add_subparsers()
# add subcommand
add_parser = subparsers.add_parser(
"add", help="Add a new project for the analyzer testing."
)
# TODO: Add an option not to build.
# TODO: Set the path to the Repository directory.
add_parser.add_argument("name", nargs=1, help="Name of the new project")
add_parser.add_argument(
"--mode",
action="store",
default=1,
type=int,
choices=[0, 1, 2],
help="Build mode: 0 for single file project, "
"1 for scan_build, "
"2 for single file c++11 project",
)
add_parser.add_argument(
"--source",
action="store",
default="script",
choices=["script", "git", "zip"],
help="Source type of the new project: "
"'git' for getting from git "
"(please provide --origin and --commit), "
"'zip' for unpacking source from a zip file, "
"'script' for downloading source by running "
"a custom script",
)
add_parser.add_argument(
"--origin", action="store", default="", help="Origin link for a git repository"
)
add_parser.add_argument(
"--commit", action="store", default="", help="Git hash for a commit to checkout"
)
add_parser.set_defaults(func=add)
# build subcommand
build_parser = subparsers.add_parser(
"build",
help="Build projects from the project map and compare results with "
"the reference.",
)
build_parser.add_argument(
"--strictness",
dest="strictness",
type=int,
default=0,
help="0 to fail on runtime errors, 1 to fail "
"when the number of found bugs are different "
"from the reference, 2 to fail on any "
"difference from the reference. Default is 0.",
)
build_parser.add_argument(
"-r",
dest="regenerate",
action="store_true",
default=False,
help="Regenerate reference output.",
)
build_parser.add_argument(
"--override-compiler",
action="store_true",
default=False,
help="Call scan-build with " "--override-compiler option.",
)
build_parser.add_argument(
"-j",
"--jobs",
dest="jobs",
type=int,
default=0,
help="Number of projects to test concurrently",
)
build_parser.add_argument(
"--extra-analyzer-config",
dest="extra_analyzer_config",
type=str,
default="",
help="Arguments passed to to -analyzer-config",
)
build_parser.add_argument(
"--extra-checkers",
dest="extra_checkers",
type=str,
default="",
help="Extra checkers to enable",
)
build_parser.add_argument(
"--projects",
action="store",
default="",
help="Comma-separated list of projects to test",
)
build_parser.add_argument(
"--max-size",
action="store",
default=None,
help="Maximum size for the projects to test",
)
build_parser.add_argument("-v", "--verbose", action="count", default=0)
build_parser.set_defaults(func=build)
# compare subcommand
cmp_parser = subparsers.add_parser(
"compare",
help="Comparing two static analyzer runs in terms of "
"reported warnings and execution time statistics.",
)
cmp_parser.add_argument(
"--root-old",
dest="root_old",
help="Prefix to ignore on source files for " "OLD directory",
action="store",
type=str,
default="",
)
cmp_parser.add_argument(
"--root-new",
dest="root_new",
help="Prefix to ignore on source files for " "NEW directory",
action="store",
type=str,
default="",
)
cmp_parser.add_argument(
"--verbose-log",
dest="verbose_log",
help="Write additional information to LOG " "[default=None]",
action="store",
type=str,
default=None,
metavar="LOG",
)
cmp_parser.add_argument(
"--stats-only",
action="store_true",
dest="stats_only",
default=False,
help="Only show statistics on reports",
)
cmp_parser.add_argument(
"--show-stats",
action="store_true",
dest="show_stats",
default=False,
help="Show change in statistics",
)
cmp_parser.add_argument(
"--histogram",
action="store",
default=None,
help="Show histogram of paths differences. " "Requires matplotlib",
)
cmp_parser.add_argument("old", nargs=1, help="Directory with old results")
cmp_parser.add_argument("new", nargs=1, help="Directory with new results")
cmp_parser.set_defaults(func=compare)
# update subcommand
upd_parser = subparsers.add_parser(
"update",
help="Update static analyzer reference results based on the previous "
"run of SATest build. Assumes that SATest build was just run.",
)
upd_parser.add_argument(
"--git", action="store_true", help="Stage updated results using git."
)
upd_parser.set_defaults(func=update)
# docker subcommand
dock_parser = subparsers.add_parser(
"docker", help="Run regression system in the docker."
)
dock_parser.add_argument(
"--build-image",
action="store_true",
help="Build docker image for running tests.",
)
dock_parser.add_argument(
"--shell", action="store_true", help="Start a shell on docker."
)
dock_parser.add_argument(
"--llvm-project-dir",
action="store",
default=DEFAULT_LLVM_DIR,
help="Path to LLVM source code. Defaults "
"to the repo where this script is located. ",
)
dock_parser.add_argument(
"--build-dir",
action="store",
default="",
help="Path to a directory where docker should " "build LLVM code.",
)
dock_parser.add_argument(
"--clang-dir",
action="store",
default="",
help="Path to find/install LLVM installation.",
)
dock_parser.add_argument(
"rest",
nargs=argparse.REMAINDER,
default=[],
help="Additional args that will be forwarded " "to the docker's entrypoint.",
)
dock_parser.set_defaults(func=docker)
# benchmark subcommand
bench_parser = subparsers.add_parser(
"benchmark", help="Run benchmarks by building a set of projects multiple times."
)
bench_parser.add_argument(
"-i",
"--iterations",
action="store",
type=int,
default=20,
help="Number of iterations for building each " "project.",
)
bench_parser.add_argument(
"-o",
"--output",
action="store",
default="benchmark.csv",
help="Output csv file for the benchmark results",
)
bench_parser.add_argument(
"--projects",
action="store",
default="",
help="Comma-separated list of projects to test",
)
bench_parser.add_argument(
"--max-size",
action="store",
default=None,
help="Maximum size for the projects to test",
)
bench_parser.set_defaults(func=benchmark)
bench_subparsers = bench_parser.add_subparsers()
bench_compare_parser = bench_subparsers.add_parser(
"compare", help="Compare benchmark runs."
)
bench_compare_parser.add_argument(
"--old",
action="store",
required=True,
help="Benchmark reference results to " "compare agains.",
)
bench_compare_parser.add_argument(
"--new", action="store", required=True, help="New benchmark results to check."
)
bench_compare_parser.add_argument(
"-o", "--output", action="store", required=True, help="Output file for plots."
)
bench_compare_parser.set_defaults(func=benchmark_compare)
args = parser.parse_args()
args.func(parser, args)
if __name__ == "__main__":
main()
Codebase Browser
llvm