Modularize.cpp | Explore in Territory

//===- extra/modularize/Modularize.cpp - Check modularized headers --------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// Introduction
//
// This file implements a tool that checks whether a set of headers provides
// the consistent definitions required to use modules.  It can also check an
// existing module map for full coverage of the headers in a directory tree.
//
// For example, in examining headers, it detects whether the same entity
// (say, a NULL macro or size_t typedef) is defined in multiple headers
// or whether a header produces different definitions under
// different circumstances. These conditions cause modules built from the
// headers to behave poorly, and should be fixed before introducing a module
// map.
//
// Modularize takes as input either one or more module maps (by default,
// "module.modulemap") or one or more text files containing lists of headers
// to check.
//
// In the case of a module map, the module map must be well-formed in
// terms of syntax.  Modularize will extract the header file names
// from the map.  Only normal headers are checked, assuming headers
// marked "private", "textual", or "exclude" are not to be checked
// as a top-level include, assuming they either are included by
// other headers which are checked, or they are not suitable for
// modules.
//
// In the case of a file list, the list is a newline-separated list of headers
// to check with respect to each other.
// Lines beginning with '#' and empty lines are ignored.
// Header file names followed by a colon and other space-separated
// file names will include those extra files as dependencies.
// The file names can be relative or full paths, but must be on the
// same line.
//
// Modularize also accepts regular clang front-end arguments.
//
// Usage:   modularize [(modularize options)]
//   [(include-files_list)|(module map)]+ [(front-end-options) ...]
//
// Options:
//    -prefix=(optional header path prefix)
//          Note that unless a "-prefix (header path)" option is specified,
//          non-absolute file paths in the header list file will be relative
//          to the header list file directory.  Use -prefix to specify a
//          different directory.
//    -module-map-path=(module map)
//          Skip the checks, and instead act as a module.modulemap generation
//          assistant, generating a module map file based on the header list.
//          An optional "-root-module=(rootName)" argument can specify a root
//          module to be created in the generated module.modulemap file.  Note
//          that you will likely need to edit this file to suit the needs of
//          your headers.
//    -problem-files-list=(problem files list file name)
//          For use only with module map assistant.  Input list of files that
//          have problems with respect to modules.  These will still be
//          included in the generated module map, but will be marked as
//          "excluded" headers.
//    -root-module=(root module name)
//          Specifies a root module to be created in the generated
//          module.modulemap file.
//    -block-check-header-list-only
//          Only warn if #include directives are inside extern or namespace
//          blocks if the included header is in the header list.
//    -no-coverage-check
//          Don't do the coverage check.
//    -coverage-check-only
//          Only do the coverage check.
//    -display-file-lists
//          Display lists of good files (no compile errors), problem files,
//          and a combined list with problem files preceded by a '#'.
//          This can be used to quickly determine which files have problems.
//          The latter combined list might be useful in starting to modularize
//          a set of headers.  You can start with a full list of headers,
//          use -display-file-lists option, and then use the combined list as
//          your intermediate list, uncommenting-out headers as you fix them.
//
// Note that by default, the modularize assumes .h files contain C++ source.
// If your .h files in the file list contain another language, you should
// append an appropriate -x option to your command line, i.e.:  -x c
//
// Modularization Issue Checks
//
// In the process of checking headers for modularization issues, modularize
// will do normal parsing, reporting normal errors and warnings,
// but will also report special error messages like the following:
//
//   error: '(symbol)' defined at multiple locations:
//       (file):(row):(column)
//       (file):(row):(column)
//
//   error: header '(file)' has different contents depending on how it was
//     included
//
// The latter might be followed by messages like the following:
//
//   note: '(symbol)' in (file) at (row):(column) not always provided
//
// Checks will also be performed for macro expansions, defined(macro)
// expressions, and preprocessor conditional directives that evaluate
// inconsistently, and can produce error messages like the following:
//
//   (...)/SubHeader.h:11:5:
//   #if SYMBOL == 1
//       ^
//   error: Macro instance 'SYMBOL' has different values in this header,
//          depending on how it was included.
//     'SYMBOL' expanded to: '1' with respect to these inclusion paths:
//       (...)/Header1.h
//         (...)/SubHeader.h
//   (...)/SubHeader.h:3:9:
//   #define SYMBOL 1
//             ^
//   Macro defined here.
//     'SYMBOL' expanded to: '2' with respect to these inclusion paths:
//       (...)/Header2.h
//           (...)/SubHeader.h
//   (...)/SubHeader.h:7:9:
//   #define SYMBOL 2
//             ^
//   Macro defined here.
//
// Checks will also be performed for '#include' directives that are
// nested inside 'extern "C/C++" {}' or 'namespace (name) {}' blocks,
// and can produce error message like the following:
//
// IncludeInExtern.h:2:3
//   #include "Empty.h"
//   ^
// error: Include directive within extern "C" {}.
// IncludeInExtern.h:1:1
// extern "C" {
// ^
// The "extern "C" {}" block is here.
//
// See PreprocessorTracker.cpp for additional details.
//
// Module Map Coverage Check
//
// The coverage check uses the Clang ModuleMap class to read and parse the
// module map file.  Starting at the module map file directory, or just the
// include paths, if specified, it will collect the names of all the files it
// considers headers (no extension, .h, or .inc--if you need more, modify the
// isHeader function).  It then compares the headers against those referenced
// in the module map, either explicitly named, or implicitly named via an
// umbrella directory or umbrella file, as parsed by the ModuleMap object.
// If headers are found which are not referenced or covered by an umbrella
// directory or file, warning messages will be produced, and this program
// will return an error code of 1.  Other errors result in an error code of 2.
// If no problems are found, an error code of 0 is returned.
//
// Note that in the case of umbrella headers, this tool invokes the compiler
// to preprocess the file, and uses a callback to collect the header files
// included by the umbrella header or any of its nested includes.  If any
// front end options are needed for these compiler invocations, these
// can be included on the command line after the module map file argument.
//
// Warning message have the form:
//
//  warning: module.modulemap does not account for file: Level3A.h
//
// Note that for the case of the module map referencing a file that does
// not exist, the module map parser in Clang will (at the time of this
// writing) display an error message.
//
// Module Map Assistant - Module Map Generation
//
// Modularize also has an option ("-module-map-path=module.modulemap") that will
// skip the checks, and instead act as a module.modulemap generation assistant,
// generating a module map file based on the header list.  An optional
// "-root-module=(rootName)" argument can specify a root module to be
// created in the generated module.modulemap file.  Note that you will likely
// need to edit this file to suit the needs of your headers.
//
// An example command line for generating a module.modulemap file:
//
//   modularize -module-map-path=module.modulemap -root-module=myroot \
//      headerlist.txt
//
// Note that if the headers in the header list have partial paths, sub-modules
// will be created for the subdirectories involved, assuming that the
// subdirectories contain headers to be grouped into a module, but still with
// individual modules for the headers in the subdirectory.
//
// See the ModuleAssistant.cpp file comments for additional details about the
// implementation of the assistant mode.
//
// Future directions:
//
// Basically, we want to add new checks for whatever we can check with respect
// to checking headers for module'ability.
//
// Some ideas:
//
// 1. Omit duplicate "not always provided" messages
//
// 2. Add options to disable any of the checks, in case
// there is some problem with them, or the messages get too verbose.
//
// 3. Try to figure out the preprocessor conditional directives that
// contribute to problems and tie them to the inconsistent definitions.
//
// 4. There are some legitimate uses of preprocessor macros that
// modularize will flag as errors, such as repeatedly #include'ing
// a file and using interleaving defined/undefined macros
// to change declarations in the included file.  Is there a way
// to address this?  Maybe have modularize accept a list of macros
// to ignore.  Otherwise you can just exclude the file, after checking
// for legitimate errors.
//
// 5. What else?
//
// General clean-up and refactoring:
//
// 1. The Location class seems to be something that we might
// want to design to be applicable to a wider range of tools, and stick it
// somewhere into Tooling/ in mainline
//
//===----------------------------------------------------------------------===//

#include "Modularize.h"
#include "ModularizeUtilities.h"
#include "PreprocessorTracker.h"
#include "clang/AST/ASTConsumer.h"
#include "clang/AST/ASTContext.h"
#include "clang/AST/RecursiveASTVisitor.h"
#include "clang/Basic/SourceManager.h"
#include "clang/Driver/Options.h"
#include "clang/Frontend/CompilerInstance.h"
#include "clang/Frontend/FrontendAction.h"
#include "clang/Frontend/FrontendActions.h"
#include "clang/Lex/Preprocessor.h"
#include "clang/Tooling/CompilationDatabase.h"
#include "clang/Tooling/Tooling.h"
#include "llvm/Option/Arg.h"
#include "llvm/Option/ArgList.h"
#include "llvm/Option/OptTable.h"
#include "llvm/Option/Option.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/Path.h"
#include <algorithm>
#include <iterator>
#include <map>
#include <string>
#include <vector>

usingnamespaceclang;
usingnamespaceclang::driver;
usingnamespaceclang::driver::options;
usingnamespaceclang::tooling;
usingnamespacellvm;
usingnamespacellvm::opt;
usingnamespaceModularize;

// Option to specify a file name for a list of header files to check.
static cl::list<std::string>
    ListFileNames(cl::Positional, cl::value_desc("list"),
                  cl::desc("<list of one or more header list files>"),
                  cl::CommaSeparated);

// Collect all other arguments, which will be passed to the front end.
static cl::list<std::string>
    CC1Arguments(cl::ConsumeAfter,
                 cl::desc("<arguments to be passed to front end>..."));

// Option to specify a prefix to be prepended to the header names.
static cl::opt<std::string> HeaderPrefix(
    "prefix", cl::init(""),
    cl::desc(
        "Prepend header file paths with this prefix."
        " If not specified,"
        " the files are considered to be relative to the header list file."));

// Option for assistant mode, telling modularize to output a module map
// based on the headers list, and where to put it.
static cl::opt<std::string> ModuleMapPath(
    "module-map-path", cl::init(""),
    cl::desc("Turn on module map output and specify output path or file name."
             " If no path is specified and if prefix option is specified,"
             " use prefix for file path."));

// Option to specify list of problem files for assistant.
// This will cause assistant to exclude these files.
static cl::opt<std::string> ProblemFilesList(
  "problem-files-list", cl::init(""),
  cl::desc(
  "List of files with compilation or modularization problems for"
    " assistant mode.  This will be excluded."));

// Option for assistant mode, telling modularize the name of the root module.
static cl::opt<std::string>
RootModule("root-module", cl::init(""),
           cl::desc("Specify the name of the root module."));

// Option for limiting the #include-inside-extern-or-namespace-block
// check to only those headers explicitly listed in the header list.
// This is a work-around for private includes that purposefully get
// included inside blocks.
static cl::opt<bool>
BlockCheckHeaderListOnly("block-check-header-list-only", cl::init(false),
cl::desc("Only warn if #include directives are inside extern or namespace"
  " blocks if the included header is in the header list."));

// Option for include paths for coverage check.
static cl::list<std::string>
    IncludePaths("I", cl::desc("Include path for coverage check."),
                 cl::value_desc("path"));

// Option for disabling the coverage check.
static cl::opt<bool> NoCoverageCheck("no-coverage-check",
                                     cl::desc("Don't do the coverage check."));

// Option for just doing the coverage check.
static cl::opt<bool>
CoverageCheckOnly("coverage-check-only", cl::init(false),
cl::desc("Only do the coverage check."));

// Option for displaying lists of good, bad, and mixed files.
static cl::opt<bool>
DisplayFileLists("display-file-lists", cl::init(false),
cl::desc("Display lists of good files (no compile errors), problem files,"
  " and a combined list with problem files preceded by a '#'."));

// Save the program name for error messages.
const char *Argv0;
// Save the command line for comments.
std::string CommandLine;

// Helper function for finding the input file in an arguments list.
static std::string findInputFile(const CommandLineArguments &CLArgs) { … }

// This arguments adjuster inserts "-include (file)" arguments for header
// dependencies.  It also inserts a "-w" option and a "-x c++",
// if no other "-x" option is present.
static ArgumentsAdjuster
getModularizeArgumentsAdjuster(DependencyMap &Dependencies) { … }

// FIXME: The Location class seems to be something that we might
// want to design to be applicable to a wider range of tools, and stick it
// somewhere into Tooling/ in mainline
struct Location { … };

struct Entry { … };

// Return a string representing the given kind.
StringRef Entry::getKindName(Entry::EntryKind kind) { … }

struct HeaderEntry { … };

HeaderContents;

class EntityMap : public std::map<std::string, SmallVector<Entry, 2>> { … };

class CollectEntitiesVisitor
    : public RecursiveASTVisitor<CollectEntitiesVisitor> { … };

class CollectEntitiesConsumer : public ASTConsumer { … };

class CollectEntitiesAction : public SyntaxOnlyAction { … };

class ModularizeFrontendActionFactory : public FrontendActionFactory { … };

class CompileCheckVisitor
  : public RecursiveASTVisitor<CompileCheckVisitor> { … };

class CompileCheckConsumer : public ASTConsumer { … };

class CompileCheckAction : public SyntaxOnlyAction { … };

class CompileCheckFrontendActionFactory : public FrontendActionFactory { … };

int main(int Argc, const char **Argv) { … }
llvm/clang-tools-extra/modularize/Modularize.cpp
