git/grep.c

#include "git-compat-util.h"
#include "config.h"
#include "gettext.h"
#include "grep.h"
#include "hex.h"
#include "object-store-ll.h"
#include "pretty.h"
#include "userdiff.h"
#include "xdiff-interface.h"
#include "diff.h"
#include "diffcore.h"
#include "quote.h"
#include "help.h"

static int grep_source_load(struct grep_source *gs);
static int grep_source_is_binary(struct grep_source *gs,
				 struct index_state *istate);

static void std_output(struct grep_opt *opt UNUSED, const void *buf, size_t size)
{ … }

static const char *color_grep_slots[] = …;

static int parse_pattern_type_arg(const char *opt, const char *arg)
{ … }

define_list_config_array_extra(…);

/*
 * Read the configuration file once and store it in
 * the grep_defaults template.
 */
int grep_config(const char *var, const char *value,
		const struct config_context *ctx, void *cb)
{ … }

void grep_init(struct grep_opt *opt, struct repository *repo)
{ … }

static struct grep_pat *create_grep_pat(const char *pat, size_t patlen,
					const char *origin, int no,
					enum grep_pat_token t,
					enum grep_header_field field)
{ … }

static void do_append_grep_pat(struct grep_pat ***tail, struct grep_pat *p)
{ … }

void append_header_grep_pattern(struct grep_opt *opt,
				enum grep_header_field field, const char *pat)
{ … }

void append_grep_pattern(struct grep_opt *opt, const char *pat,
			 const char *origin, int no, enum grep_pat_token t)
{ … }

void append_grep_pat(struct grep_opt *opt, const char *pat, size_t patlen,
		     const char *origin, int no, enum grep_pat_token t)
{ … }

struct grep_opt *grep_opt_dup(const struct grep_opt *opt)
{ … }

static NORETURN void compile_regexp_failed(const struct grep_pat *p,
		const char *error)
{ … }

static int is_fixed(const char *s, size_t len)
{ … }

#ifdef USE_LIBPCRE2
#define GREP_PCRE2_DEBUG_MALLOC …

static void *pcre2_malloc(PCRE2_SIZE size, void *memory_data UNUSED)
{
	void *pointer = malloc(size);
#if GREP_PCRE2_DEBUG_MALLOC
	static int count = 1;
	fprintf(stderr, "PCRE2:%p -> #%02d: alloc(%lu)\n", pointer, count++, size);
#endif
	return pointer;
}

static void pcre2_free(void *pointer, void *memory_data UNUSED)
{
#if GREP_PCRE2_DEBUG_MALLOC
	static int count = 1;
	if (pointer)
		fprintf(stderr, "PCRE2:%p -> #%02d: free()\n", pointer, count++);
#endif
	free(pointer);
}

static int pcre2_jit_functional(void)
{
	static int jit_working = -1;
	pcre2_code *code;
	size_t off;
	int err;

	if (jit_working != -1)
		return jit_working;

	/*
	 * Try to JIT compile a simple pattern to probe if the JIT is
	 * working in general. It might fail for systems where creating
	 * memory mappings for runtime code generation is restricted.
	 */
	code = pcre2_compile((PCRE2_SPTR)".", 1, 0, &err, &off, NULL);
	if (!code)
		return 0;

	jit_working = pcre2_jit_compile(code, PCRE2_JIT_COMPLETE) == 0;
	pcre2_code_free(code);

	return jit_working;
}

static void compile_pcre2_pattern(struct grep_pat *p, const struct grep_opt *opt)
{
	int error;
	PCRE2_UCHAR errbuf[256];
	PCRE2_SIZE erroffset;
	int options = PCRE2_MULTILINE;
	int jitret;
	int patinforet;
	size_t jitsizearg;
	int literal = !opt->ignore_case && (p->fixed || p->is_fixed);

	/*
	 * Call pcre2_general_context_create() before calling any
	 * other pcre2_*(). It sets up our malloc()/free() functions
	 * with which everything else is allocated.
	 */
	p->pcre2_general_context = pcre2_general_context_create(
		pcre2_malloc, pcre2_free, NULL);
	if (!p->pcre2_general_context)
		die("Couldn't allocate PCRE2 general context");

	if (opt->ignore_case) {
		if (!opt->ignore_locale && has_non_ascii(p->pattern)) {
			p->pcre2_tables = pcre2_maketables(p->pcre2_general_context);
			p->pcre2_compile_context = pcre2_compile_context_create(p->pcre2_general_context);
			pcre2_set_character_tables(p->pcre2_compile_context,
							p->pcre2_tables);
		}
		options |= PCRE2_CASELESS;
	}
	if (!opt->ignore_locale && is_utf8_locale() && !literal)
		options |= (PCRE2_UTF | PCRE2_UCP | PCRE2_MATCH_INVALID_UTF);

#ifndef GIT_PCRE2_VERSION_10_35_OR_HIGHER
	/*
	 * Work around a JIT bug related to invalid Unicode character handling
	 * fixed in 10.35:
	 * https://github.com/PCRE2Project/pcre2/commit/c21bd977547d
	 */
	options &= ~PCRE2_UCP;
#endif

#ifndef GIT_PCRE2_VERSION_10_36_OR_HIGHER
	/* Work around https://bugs.exim.org/show_bug.cgi?id=2642 fixed in 10.36 */
	if (PCRE2_MATCH_INVALID_UTF && options & (PCRE2_UTF | PCRE2_CASELESS))
		options |= PCRE2_NO_START_OPTIMIZE;
#endif

	p->pcre2_pattern = pcre2_compile((PCRE2_SPTR)p->pattern,
					 p->patternlen, options, &error, &erroffset,
					 p->pcre2_compile_context);

	if (p->pcre2_pattern) {
		p->pcre2_match_data = pcre2_match_data_create_from_pattern(p->pcre2_pattern, p->pcre2_general_context);
		if (!p->pcre2_match_data)
			die("Couldn't allocate PCRE2 match data");
	} else {
		pcre2_get_error_message(error, errbuf, sizeof(errbuf));
		compile_regexp_failed(p, (const char *)&errbuf);
	}

	pcre2_config(PCRE2_CONFIG_JIT, &p->pcre2_jit_on);
	if (p->pcre2_jit_on) {
		jitret = pcre2_jit_compile(p->pcre2_pattern, PCRE2_JIT_COMPLETE);
		if (jitret == PCRE2_ERROR_NOMEMORY && !pcre2_jit_functional()) {
			/*
			 * Even though pcre2_config(PCRE2_CONFIG_JIT, ...)
			 * indicated JIT support, the library might still
			 * fail to generate JIT code for various reasons,
			 * e.g. when SELinux's 'deny_execmem' or PaX's
			 * MPROTECT prevent creating W|X memory mappings.
			 *
			 * Instead of faling hard, fall back to interpreter
			 * mode, just as if the pattern was prefixed with
			 * '(*NO_JIT)'.
			 */
			p->pcre2_jit_on = 0;
			return;
		} else if (jitret) {
			int need_clip = p->patternlen > 64;
			int clip_len = need_clip ? 64 : p->patternlen;
			die("Couldn't JIT the PCRE2 pattern '%.*s'%s, got '%d'%s",
			    clip_len, p->pattern, need_clip ? "..." : "", jitret,
			    pcre2_jit_functional()
			    ? "\nPerhaps prefix (*NO_JIT) to your pattern?"
			    : "");
		}

		/*
		 * The pcre2_config(PCRE2_CONFIG_JIT, ...) call just
		 * tells us whether the library itself supports JIT,
		 * but to see whether we're going to be actually using
		 * JIT we need to extract PCRE2_INFO_JITSIZE from the
		 * pattern *after* we do pcre2_jit_compile() above.
		 *
		 * This is because if the pattern contains the
		 * (*NO_JIT) verb (see pcre2syntax(3))
		 * pcre2_jit_compile() will exit early with 0. If we
		 * then proceed to call pcre2_jit_match() further down
		 * the line instead of pcre2_match() we'll either
		 * segfault (pre PCRE 10.31) or run into a fatal error
		 * (post PCRE2 10.31)
		 */
		patinforet = pcre2_pattern_info(p->pcre2_pattern, PCRE2_INFO_JITSIZE, &jitsizearg);
		if (patinforet)
			BUG("pcre2_pattern_info() failed: %d", patinforet);
		if (jitsizearg == 0) {
			p->pcre2_jit_on = 0;
			return;
		}
	}
}

static int pcre2match(struct grep_pat *p, const char *line, const char *eol,
		regmatch_t *match, int eflags)
{
	int ret, flags = 0;
	PCRE2_SIZE *ovector;
	PCRE2_UCHAR errbuf[256];

	if (eflags & REG_NOTBOL)
		flags |= PCRE2_NOTBOL;

	if (p->pcre2_jit_on)
		ret = pcre2_jit_match(p->pcre2_pattern, (unsigned char *)line,
				      eol - line, 0, flags, p->pcre2_match_data,
				      NULL);
	else
		ret = pcre2_match(p->pcre2_pattern, (unsigned char *)line,
				  eol - line, 0, flags, p->pcre2_match_data,
				  NULL);

	if (ret < 0 && ret != PCRE2_ERROR_NOMATCH) {
		pcre2_get_error_message(ret, errbuf, sizeof(errbuf));
		die("%s failed with error code %d: %s",
		    (p->pcre2_jit_on ? "pcre2_jit_match" : "pcre2_match"), ret,
		    errbuf);
	}
	if (ret > 0) {
		ovector = pcre2_get_ovector_pointer(p->pcre2_match_data);
		ret = 0;
		match->rm_so = (int)ovector[0];
		match->rm_eo = (int)ovector[1];
	}

	return ret;
}

static void free_pcre2_pattern(struct grep_pat *p)
{
	pcre2_compile_context_free(p->pcre2_compile_context);
	pcre2_code_free(p->pcre2_pattern);
	pcre2_match_data_free(p->pcre2_match_data);
#ifdef GIT_PCRE2_VERSION_10_34_OR_HIGHER
	pcre2_maketables_free(p->pcre2_general_context, p->pcre2_tables);
#else
	free((void *)p->pcre2_tables);
#endif
	pcre2_general_context_free(p->pcre2_general_context);
}
#else /* !USE_LIBPCRE2 */
static void compile_pcre2_pattern(struct grep_pat *p UNUSED,
				  const struct grep_opt *opt UNUSED)
{ … }

static int pcre2match(struct grep_pat *p UNUSED, const char *line UNUSED,
		      const char *eol UNUSED, regmatch_t *match UNUSED,
		      int eflags UNUSED)
{ … }

static void free_pcre2_pattern(struct grep_pat *p UNUSED)
{ … }

static void compile_fixed_regexp(struct grep_pat *p, struct grep_opt *opt)
{ … }
#endif /* !USE_LIBPCRE2 */

static void compile_regexp(struct grep_pat *p, struct grep_opt *opt)
{ … }

static struct grep_expr *grep_not_expr(struct grep_expr *expr)
{ … }

static struct grep_expr *grep_binexp(enum grep_expr_node kind,
				     struct grep_expr *left,
				     struct grep_expr *right)
{ … }

static struct grep_expr *grep_or_expr(struct grep_expr *left, struct grep_expr *right)
{ … }

static struct grep_expr *grep_and_expr(struct grep_expr *left, struct grep_expr *right)
{ … }

static struct grep_expr *compile_pattern_or(struct grep_pat **);
static struct grep_expr *compile_pattern_atom(struct grep_pat **list)
{ … }

static struct grep_expr *compile_pattern_not(struct grep_pat **list)
{ … }

static struct grep_expr *compile_pattern_and(struct grep_pat **list)
{ … }

static struct grep_expr *compile_pattern_or(struct grep_pat **list)
{ … }

static struct grep_expr *compile_pattern_expr(struct grep_pat **list)
{ … }

static struct grep_expr *grep_true_expr(void)
{ … }

static struct grep_expr *prep_header_patterns(struct grep_opt *opt)
{ … }

static struct grep_expr *grep_splice_or(struct grep_expr *x, struct grep_expr *y)
{ … }

void compile_grep_patterns(struct grep_opt *opt)
{ … }

static void free_pattern_expr(struct grep_expr *x)
{ … }

static void free_grep_pat(struct grep_pat *pattern)
{ … }

void free_grep_patterns(struct grep_opt *opt)
{ … }

static const char *end_of_line(const char *cp, unsigned long *left)
{ … }

static int word_char(char ch)
{ … }

static void output_color(struct grep_opt *opt, const void *data, size_t size,
			 const char *color)
{ … }

static void output_sep(struct grep_opt *opt, char sign)
{ … }

static void show_name(struct grep_opt *opt, const char *name)
{ … }

static int patmatch(struct grep_pat *p,
		    const char *line, const char *eol,
		    regmatch_t *match, int eflags)
{ … }

static void strip_timestamp(const char *bol, const char **eol_p)
{ … }

static struct { … } header_field[] = …;

static int headerless_match_one_pattern(struct grep_pat *p,
					const char *bol, const char *eol,
					enum grep_context ctx,
					regmatch_t *pmatch, int eflags)
{ … }

static int match_one_pattern(struct grep_pat *p,
			     const char *bol, const char *eol,
			     enum grep_context ctx, regmatch_t *pmatch,
			     int eflags)
{ … }


static int match_expr_eval(struct grep_opt *opt, struct grep_expr *x,
			   const char *bol, const char *eol,
			   enum grep_context ctx, ssize_t *col,
			   ssize_t *icol, int collect_hits)
{ … }

static int match_expr(struct grep_opt *opt,
		      const char *bol, const char *eol,
		      enum grep_context ctx, ssize_t *col,
		      ssize_t *icol, int collect_hits)
{ … }

static int match_line(struct grep_opt *opt,
		      const char *bol, const char *eol,
		      ssize_t *col, ssize_t *icol,
		      enum grep_context ctx, int collect_hits)
{ … }

static int match_next_pattern(struct grep_pat *p,
			      const char *bol, const char *eol,
			      enum grep_context ctx,
			      regmatch_t *pmatch, int eflags)
{ … }

int grep_next_match(struct grep_opt *opt,
		    const char *bol, const char *eol,
		    enum grep_context ctx, regmatch_t *pmatch,
		    enum grep_header_field field, int eflags)
{ … }

static void show_line_header(struct grep_opt *opt, const char *name,
			     unsigned lno, ssize_t cno, char sign)
{ … }

static void show_line(struct grep_opt *opt,
		      const char *bol, const char *eol,
		      const char *name, unsigned lno, ssize_t cno, char sign)
{ … }

int grep_use_locks;

/*
 * This lock protects access to the gitattributes machinery, which is
 * not thread-safe.
 */
pthread_mutex_t grep_attr_mutex;

static inline void grep_attr_lock(void)
{ … }

static inline void grep_attr_unlock(void)
{ … }

static int match_funcname(struct grep_opt *opt, struct grep_source *gs,
			  const char *bol, const char *eol)
{ … }

static void show_funcname_line(struct grep_opt *opt, struct grep_source *gs,
			       const char *bol, unsigned lno)
{ … }

static int is_empty_line(const char *bol, const char *eol);

static void show_pre_context(struct grep_opt *opt, struct grep_source *gs,
			     const char *bol, const char *end, unsigned lno)
{ … }

static int should_lookahead(struct grep_opt *opt)
{ … }

static int look_ahead(struct grep_opt *opt,
		      unsigned long *left_p,
		      unsigned *lno_p,
		      const char **bol_p)
{ … }

static int fill_textconv_grep(struct repository *r,
			      struct userdiff_driver *driver,
			      struct grep_source *gs)
{ … }

static int is_empty_line(const char *bol, const char *eol)
{ … }

static int grep_source_1(struct grep_opt *opt, struct grep_source *gs, int collect_hits)
{ … }

static void clr_hit_marker(struct grep_expr *x)
{ … }

static int chk_hit_marker(struct grep_expr *x)
{ … }

int grep_source(struct grep_opt *opt, struct grep_source *gs)
{ … }

static void grep_source_init_buf(struct grep_source *gs,
				 const char *buf,
				 unsigned long size)
{ … }

int grep_buffer(struct grep_opt *opt, const char *buf, unsigned long size)
{ … }

void grep_source_init_file(struct grep_source *gs, const char *name,
			   const char *path)
{ … }

void grep_source_init_oid(struct grep_source *gs, const char *name,
			  const char *path, const struct object_id *oid,
			  struct repository *repo)
{ … }

void grep_source_clear(struct grep_source *gs)
{ … }

void grep_source_clear_data(struct grep_source *gs)
{ … }

static int grep_source_load_oid(struct grep_source *gs)
{ … }

static int grep_source_load_file(struct grep_source *gs)
{ … }

static int grep_source_load(struct grep_source *gs)
{ … }

void grep_source_load_driver(struct grep_source *gs,
			     struct index_state *istate)
{ … }

static int grep_source_is_binary(struct grep_source *gs,
				 struct index_state *istate)
{ … }