git/object-file.c

/*
 * GIT - The information manager from hell
 *
 * Copyright (C) Linus Torvalds, 2005
 *
 * This handles basic git object files - packing, unpacking,
 * creation etc.
 */

#define USE_THE_REPOSITORY_VARIABLE

#include "git-compat-util.h"
#include "abspath.h"
#include "config.h"
#include "convert.h"
#include "environment.h"
#include "gettext.h"
#include "hex.h"
#include "string-list.h"
#include "lockfile.h"
#include "pack.h"
#include "commit.h"
#include "run-command.h"
#include "refs.h"
#include "bulk-checkin.h"
#include "repository.h"
#include "replace-object.h"
#include "streaming.h"
#include "dir.h"
#include "list.h"
#include "quote.h"
#include "packfile.h"
#include "object-file.h"
#include "object-store.h"
#include "oidtree.h"
#include "path.h"
#include "promisor-remote.h"
#include "setup.h"
#include "submodule.h"
#include "fsck.h"
#include "loose.h"
#include "object-file-convert.h"

/* The maximum size for an object header. */
#define MAX_HEADER_LEN …


#define EMPTY_TREE_SHA1_BIN_LITERAL …
#define EMPTY_TREE_SHA256_BIN_LITERAL …

#define EMPTY_BLOB_SHA1_BIN_LITERAL …
#define EMPTY_BLOB_SHA256_BIN_LITERAL …

static const struct object_id empty_tree_oid = …;
static const struct object_id empty_blob_oid = …;
static const struct object_id null_oid_sha1 = …;
static const struct object_id empty_tree_oid_sha256 = …;
static const struct object_id empty_blob_oid_sha256 = …;
static const struct object_id null_oid_sha256 = …;

static void git_hash_sha1_init(git_hash_ctx *ctx)
{ … }

static void git_hash_sha1_clone(git_hash_ctx *dst, const git_hash_ctx *src)
{ … }

static void git_hash_sha1_update(git_hash_ctx *ctx, const void *data, size_t len)
{ … }

static void git_hash_sha1_final(unsigned char *hash, git_hash_ctx *ctx)
{ … }

static void git_hash_sha1_final_oid(struct object_id *oid, git_hash_ctx *ctx)
{ … }

static void git_hash_sha1_init_unsafe(git_hash_ctx *ctx)
{ … }

static void git_hash_sha1_clone_unsafe(git_hash_ctx *dst, const git_hash_ctx *src)
{ … }

static void git_hash_sha1_update_unsafe(git_hash_ctx *ctx, const void *data,
				      size_t len)
{ … }

static void git_hash_sha1_final_unsafe(unsigned char *hash, git_hash_ctx *ctx)
{ … }

static void git_hash_sha1_final_oid_unsafe(struct object_id *oid, git_hash_ctx *ctx)
{ … }

static void git_hash_sha256_init(git_hash_ctx *ctx)
{ … }

static void git_hash_sha256_clone(git_hash_ctx *dst, const git_hash_ctx *src)
{ … }

static void git_hash_sha256_update(git_hash_ctx *ctx, const void *data, size_t len)
{ … }

static void git_hash_sha256_final(unsigned char *hash, git_hash_ctx *ctx)
{ … }

static void git_hash_sha256_final_oid(struct object_id *oid, git_hash_ctx *ctx)
{ … }

static void git_hash_unknown_init(git_hash_ctx *ctx UNUSED)
{ … }

static void git_hash_unknown_clone(git_hash_ctx *dst UNUSED,
				   const git_hash_ctx *src UNUSED)
{ … }

static void git_hash_unknown_update(git_hash_ctx *ctx UNUSED,
				    const void *data UNUSED,
				    size_t len UNUSED)
{ … }

static void git_hash_unknown_final(unsigned char *hash UNUSED,
				   git_hash_ctx *ctx UNUSED)
{ … }

static void git_hash_unknown_final_oid(struct object_id *oid UNUSED,
				       git_hash_ctx *ctx UNUSED)
{ … }

const struct git_hash_algo hash_algos[GIT_HASH_NALGOS] = …;

const struct object_id *null_oid(void)
{ … }

const char *empty_tree_oid_hex(const struct git_hash_algo *algop)
{ … }

int hash_algo_by_name(const char *name)
{ … }

int hash_algo_by_id(uint32_t format_id)
{ … }

int hash_algo_by_length(int len)
{ … }

/*
 * This is meant to hold a *small* number of objects that you would
 * want repo_read_object_file() to be able to return, but yet you do not want
 * to write them into the object store (e.g. a browse-only
 * application).
 */
static struct cached_object { … } *cached_objects;
static int cached_object_nr, cached_object_alloc;

static struct cached_object empty_tree = …;

static struct cached_object *find_cached_object(const struct object_id *oid)
{ … }


static int get_conv_flags(unsigned flags)
{ … }


int mkdir_in_gitdir(const char *path)
{ … }

static enum scld_error safe_create_leading_directories_1(char *path, int share)
{ … }

enum scld_error safe_create_leading_directories(char *path)
{ … }

enum scld_error safe_create_leading_directories_no_share(char *path)
{ … }

enum scld_error safe_create_leading_directories_const(const char *path)
{ … }

int odb_mkstemp(struct strbuf *temp_filename, const char *pattern)
{ … }

int odb_pack_keep(const char *name)
{ … }

static void fill_loose_path(struct strbuf *buf, const struct object_id *oid)
{ … }

static const char *odb_loose_path(struct object_directory *odb,
				  struct strbuf *buf,
				  const struct object_id *oid)
{ … }

const char *loose_object_path(struct repository *r, struct strbuf *buf,
			      const struct object_id *oid)
{ … }

/*
 * Return non-zero iff the path is usable as an alternate object database.
 */
static int alt_odb_usable(struct raw_object_store *o,
			  struct strbuf *path,
			  const char *normalized_objdir, khiter_t *pos)
{ … }

/*
 * Prepare alternate object database registry.
 *
 * The variable alt_odb_list points at the list of struct
 * object_directory.  The elements on this list come from
 * non-empty elements from colon separated ALTERNATE_DB_ENVIRONMENT
 * environment variable, and $GIT_OBJECT_DIRECTORY/info/alternates,
 * whose contents is similar to that environment variable but can be
 * LF separated.  Its base points at a statically allocated buffer that
 * contains "/the/directory/corresponding/to/.git/objects/...", while
 * its name points just after the slash at the end of ".git/objects/"
 * in the example above, and has enough space to hold all hex characters
 * of the object ID, an extra slash for the first level indirection, and
 * the terminating NUL.
 */
static void read_info_alternates(struct repository *r,
				 const char *relative_base,
				 int depth);
static int link_alt_odb_entry(struct repository *r, const struct strbuf *entry,
	const char *relative_base, int depth, const char *normalized_objdir)
{ … }

static const char *parse_alt_odb_entry(const char *string,
				       int sep,
				       struct strbuf *out)
{ … }

static void link_alt_odb_entries(struct repository *r, const char *alt,
				 int sep, const char *relative_base, int depth)
{ … }

static void read_info_alternates(struct repository *r,
				 const char *relative_base,
				 int depth)
{ … }

void add_to_alternates_file(const char *reference)
{ … }

void add_to_alternates_memory(const char *reference)
{ … }

struct object_directory *set_temporary_primary_odb(const char *dir, int will_destroy)
{ … }

void restore_primary_odb(struct object_directory *restore_odb, const char *old_path)
{ … }

/*
 * Compute the exact path an alternate is at and returns it. In case of
 * error NULL is returned and the human readable error is added to `err`
 * `path` may be relative and should point to $GIT_DIR.
 * `err` must not be null.
 */
char *compute_alternate_path(const char *path, struct strbuf *err)
{ … }

struct object_directory *find_odb(struct repository *r, const char *obj_dir)
{ … }

static void fill_alternate_refs_command(struct child_process *cmd,
					const char *repo_path)
{ … }

static void read_alternate_refs(const char *path,
				alternate_ref_fn *cb,
				void *data)
{ … }

struct alternate_refs_data { … };

static int refs_from_alternate_cb(struct object_directory *e,
				  void *data)
{ … }

void for_each_alternate_ref(alternate_ref_fn fn, void *data)
{ … }

int foreach_alt_odb(alt_odb_fn fn, void *cb)
{ … }

void prepare_alt_odb(struct repository *r)
{ … }

int has_alt_odb(struct repository *r)
{ … }

/* Returns 1 if we have successfully freshened the file, 0 otherwise. */
static int freshen_file(const char *fn)
{ … }

/*
 * All of the check_and_freshen functions return 1 if the file exists and was
 * freshened (if freshening was requested), 0 otherwise. If they return
 * 0, you should not assume that it is safe to skip a write of the object (it
 * either does not exist on disk, or has a stale mtime and may be subject to
 * pruning).
 */
int check_and_freshen_file(const char *fn, int freshen)
{ … }

static int check_and_freshen_odb(struct object_directory *odb,
				 const struct object_id *oid,
				 int freshen)
{ … }

static int check_and_freshen_local(const struct object_id *oid, int freshen)
{ … }

static int check_and_freshen_nonlocal(const struct object_id *oid, int freshen)
{ … }

static int check_and_freshen(const struct object_id *oid, int freshen)
{ … }

int has_loose_object_nonlocal(const struct object_id *oid)
{ … }

int has_loose_object(const struct object_id *oid)
{ … }

static void mmap_limit_check(size_t length)
{ … }

void *xmmap_gently(void *start, size_t length,
		  int prot, int flags, int fd, off_t offset)
{ … }

const char *mmap_os_err(void)
{ … }

void *xmmap(void *start, size_t length,
	int prot, int flags, int fd, off_t offset)
{ … }

static int format_object_header_literally(char *str, size_t size,
					  const char *type, size_t objsize)
{ … }

int format_object_header(char *str, size_t size, enum object_type type,
			 size_t objsize)
{ … }

int check_object_signature(struct repository *r, const struct object_id *oid,
			   void *buf, unsigned long size,
			   enum object_type type)
{ … }

int stream_object_signature(struct repository *r, const struct object_id *oid)
{ … }

int git_open_cloexec(const char *name, int flags)
{ … }

/*
 * Find "oid" as a loose object in the local repository or in an alternate.
 * Returns 0 on success, negative on failure.
 *
 * The "path" out-parameter will give the path of the object we found (if any).
 * Note that it may point to static storage and is only valid until another
 * call to stat_loose_object().
 */
static int stat_loose_object(struct repository *r, const struct object_id *oid,
			     struct stat *st, const char **path)
{ … }

/*
 * Like stat_loose_object(), but actually open the object and return the
 * descriptor. See the caveats on the "path" parameter above.
 */
static int open_loose_object(struct repository *r,
			     const struct object_id *oid, const char **path)
{ … }

static int quick_has_loose(struct repository *r,
			   const struct object_id *oid)
{ … }

/*
 * Map and close the given loose object fd. The path argument is used for
 * error reporting.
 */
static void *map_fd(int fd, const char *path, unsigned long *size)
{ … }

void *map_loose_object(struct repository *r,
		       const struct object_id *oid,
		       unsigned long *size)
{ … }

enum unpack_loose_header_result unpack_loose_header(git_zstream *stream,
						    unsigned char *map,
						    unsigned long mapsize,
						    void *buffer,
						    unsigned long bufsiz,
						    struct strbuf *header)
{ … }

static void *unpack_loose_rest(git_zstream *stream,
			       void *buffer, unsigned long size,
			       const struct object_id *oid)
{ … }

/*
 * We used to just use "sscanf()", but that's actually way
 * too permissive for what we want to check. So do an anal
 * object header parse by hand.
 */
int parse_loose_header(const char *hdr, struct object_info *oi)
{ … }

static int loose_object_info(struct repository *r,
			     const struct object_id *oid,
			     struct object_info *oi, int flags)
{ … }

int obj_read_use_lock = …;
pthread_mutex_t obj_read_mutex;

void enable_obj_read_lock(void)
{ … }

void disable_obj_read_lock(void)
{ … }

int fetch_if_missing = …;

static int do_oid_object_info_extended(struct repository *r,
				       const struct object_id *oid,
				       struct object_info *oi, unsigned flags)
{ … }

static int oid_object_info_convert(struct repository *r,
				   const struct object_id *input_oid,
				   struct object_info *input_oi, unsigned flags)
{ … }

int oid_object_info_extended(struct repository *r, const struct object_id *oid,
			     struct object_info *oi, unsigned flags)
{ … }


/* returns enum object_type or negative */
int oid_object_info(struct repository *r,
		    const struct object_id *oid,
		    unsigned long *sizep)
{ … }

int pretend_object_file(void *buf, unsigned long len, enum object_type type,
			struct object_id *oid)
{ … }

/*
 * This function dies on corrupt objects; the callers who want to
 * deal with them should arrange to call oid_object_info_extended() and give
 * error messages themselves.
 */
void *repo_read_object_file(struct repository *r,
			    const struct object_id *oid,
			    enum object_type *type,
			    unsigned long *size)
{ … }

void *read_object_with_reference(struct repository *r,
				 const struct object_id *oid,
				 enum object_type required_type,
				 unsigned long *size,
				 struct object_id *actual_oid_return)
{ … }

static void hash_object_body(const struct git_hash_algo *algo, git_hash_ctx *c,
			     const void *buf, unsigned long len,
			     struct object_id *oid,
			     char *hdr, int *hdrlen)
{ … }

static void write_object_file_prepare(const struct git_hash_algo *algo,
				      const void *buf, unsigned long len,
				      enum object_type type, struct object_id *oid,
				      char *hdr, int *hdrlen)
{ … }

static void write_object_file_prepare_literally(const struct git_hash_algo *algo,
				      const void *buf, unsigned long len,
				      const char *type, struct object_id *oid,
				      char *hdr, int *hdrlen)
{ … }

static int check_collision(const char *filename_a, const char *filename_b)
{ … }

/*
 * Move the just written object into its final resting place.
 */
int finalize_object_file(const char *tmpfile, const char *filename)
{ … }

int finalize_object_file_flags(const char *tmpfile, const char *filename,
			       enum finalize_object_file_flags flags)
{ … }

static void hash_object_file_literally(const struct git_hash_algo *algo,
				       const void *buf, unsigned long len,
				       const char *type, struct object_id *oid)
{ … }

void hash_object_file(const struct git_hash_algo *algo, const void *buf,
		      unsigned long len, enum object_type type,
		      struct object_id *oid)
{ … }

/* Finalize a file on disk, and close it. */
static void close_loose_object(int fd, const char *filename)
{ … }

/* Size of directory component, including the ending '/' */
static inline int directory_size(const char *filename)
{ … }

/*
 * This creates a temporary file in the same directory as the final
 * 'filename'
 *
 * We want to avoid cross-directory filename renames, because those
 * can have problems on various filesystems (FAT, NFS, Coda).
 */
static int create_tmpfile(struct strbuf *tmp, const char *filename)
{ … }

/**
 * Common steps for loose object writers to start writing loose
 * objects:
 *
 * - Create tmpfile for the loose object.
 * - Setup zlib stream for compression.
 * - Start to feed header to zlib stream.
 *
 * Returns a "fd", which should later be provided to
 * end_loose_object_common().
 */
static int start_loose_object_common(struct strbuf *tmp_file,
				     const char *filename, unsigned flags,
				     git_zstream *stream,
				     unsigned char *buf, size_t buflen,
				     git_hash_ctx *c, git_hash_ctx *compat_c,
				     char *hdr, int hdrlen)
{ … }

/**
 * Common steps for the inner git_deflate() loop for writing loose
 * objects. Returns what git_deflate() returns.
 */
static int write_loose_object_common(git_hash_ctx *c, git_hash_ctx *compat_c,
				     git_zstream *stream, const int flush,
				     unsigned char *in0, const int fd,
				     unsigned char *compressed,
				     const size_t compressed_len)
{ … }

/**
 * Common steps for loose object writers to end writing loose objects:
 *
 * - End the compression of zlib stream.
 * - Get the calculated oid to "oid".
 */
static int end_loose_object_common(git_hash_ctx *c, git_hash_ctx *compat_c,
				   git_zstream *stream, struct object_id *oid,
				   struct object_id *compat_oid)
{ … }

static int write_loose_object(const struct object_id *oid, char *hdr,
			      int hdrlen, const void *buf, unsigned long len,
			      time_t mtime, unsigned flags)
{ … }

static int freshen_loose_object(const struct object_id *oid)
{ … }

static int freshen_packed_object(const struct object_id *oid)
{ … }

int stream_loose_object(struct input_stream *in_stream, size_t len,
			struct object_id *oid)
{ … }

int write_object_file_flags(const void *buf, unsigned long len,
			    enum object_type type, struct object_id *oid,
			    struct object_id *compat_oid_in, unsigned flags)
{ … }

int write_object_file_literally(const void *buf, unsigned long len,
				const char *type, struct object_id *oid,
				unsigned flags)
{ … }

int force_object_loose(const struct object_id *oid, time_t mtime)
{ … }

int has_object(struct repository *r, const struct object_id *oid,
	       unsigned flags)
{ … }

int repo_has_object_file_with_flags(struct repository *r,
				    const struct object_id *oid, int flags)
{ … }

int repo_has_object_file(struct repository *r,
			 const struct object_id *oid)
{ … }

/*
 * We can't use the normal fsck_error_function() for index_mem(),
 * because we don't yet have a valid oid for it to report. Instead,
 * report the minimal fsck error here, and rely on the caller to
 * give more context.
 */
static int hash_format_check_report(struct fsck_options *opts UNUSED,
				    void *fsck_report UNUSED,
				    enum fsck_msg_type msg_type UNUSED,
				    enum fsck_msg_id msg_id UNUSED,
				    const char *message)
{ … }

static int index_mem(struct index_state *istate,
		     struct object_id *oid,
		     const void *buf, size_t size,
		     enum object_type type,
		     const char *path, unsigned flags)
{ … }

static int index_stream_convert_blob(struct index_state *istate,
				     struct object_id *oid,
				     int fd,
				     const char *path,
				     unsigned flags)
{ … }

static int index_pipe(struct index_state *istate, struct object_id *oid,
		      int fd, enum object_type type,
		      const char *path, unsigned flags)
{ … }

#define SMALL_FILE_SIZE …

static int index_core(struct index_state *istate,
		      struct object_id *oid, int fd, size_t size,
		      enum object_type type, const char *path,
		      unsigned flags)
{ … }

/*
 * This creates one packfile per large blob unless bulk-checkin
 * machinery is "plugged".
 *
 * This also bypasses the usual "convert-to-git" dance, and that is on
 * purpose. We could write a streaming version of the converting
 * functions and insert that before feeding the data to fast-import
 * (or equivalent in-core API described above). However, that is
 * somewhat complicated, as we do not know the size of the filter
 * result, which we need to know beforehand when writing a git object.
 * Since the primary motivation for trying to stream from the working
 * tree file and to avoid mmaping it in core is to deal with large
 * binary blobs, they generally do not want to get any conversion, and
 * callers should avoid this code path when filters are requested.
 */
static int index_blob_stream(struct object_id *oid, int fd, size_t size,
			     const char *path,
			     unsigned flags)
{ … }

int index_fd(struct index_state *istate, struct object_id *oid,
	     int fd, struct stat *st,
	     enum object_type type, const char *path, unsigned flags)
{ … }

int index_path(struct index_state *istate, struct object_id *oid,
	       const char *path, struct stat *st, unsigned flags)
{ … }

int read_pack_header(int fd, struct pack_header *header)
{ … }

void assert_oid_type(const struct object_id *oid, enum object_type expect)
{ … }

int for_each_file_in_obj_subdir(unsigned int subdir_nr,
				struct strbuf *path,
				each_loose_object_fn obj_cb,
				each_loose_cruft_fn cruft_cb,
				each_loose_subdir_fn subdir_cb,
				void *data)
{ … }

int for_each_loose_file_in_objdir_buf(struct strbuf *path,
			    each_loose_object_fn obj_cb,
			    each_loose_cruft_fn cruft_cb,
			    each_loose_subdir_fn subdir_cb,
			    void *data)
{ … }

int for_each_loose_file_in_objdir(const char *path,
				  each_loose_object_fn obj_cb,
				  each_loose_cruft_fn cruft_cb,
				  each_loose_subdir_fn subdir_cb,
				  void *data)
{ … }

int for_each_loose_object(each_loose_object_fn cb, void *data,
			  enum for_each_object_flags flags)
{ … }

static int append_loose_object(const struct object_id *oid,
			       const char *path UNUSED,
			       void *data)
{ … }

struct oidtree *odb_loose_cache(struct object_directory *odb,
				  const struct object_id *oid)
{ … }

void odb_clear_loose_cache(struct object_directory *odb)
{ … }

static int check_stream_oid(git_zstream *stream,
			    const char *hdr,
			    unsigned long size,
			    const char *path,
			    const struct object_id *expected_oid)
{ … }

int read_loose_object(const char *path,
		      const struct object_id *expected_oid,
		      struct object_id *real_oid,
		      void **contents,
		      struct object_info *oi)
{ … }