git/midx-write.c

#define USE_THE_REPOSITORY_VARIABLE

#include "git-compat-util.h"
#include "abspath.h"
#include "config.h"
#include "hex.h"
#include "lockfile.h"
#include "packfile.h"
#include "object-file.h"
#include "hash-lookup.h"
#include "midx.h"
#include "progress.h"
#include "trace2.h"
#include "run-command.h"
#include "chunk-format.h"
#include "pack-bitmap.h"
#include "refs.h"
#include "revision.h"
#include "list-objects.h"
#include "path.h"
#include "pack-revindex.h"

#define PACK_EXPIRED …
#define BITMAP_POS_UNKNOWN …
#define MIDX_CHUNK_FANOUT_SIZE …
#define MIDX_CHUNK_LARGE_OFFSET_WIDTH …

extern int midx_checksum_valid(struct multi_pack_index *m);
extern void clear_midx_files_ext(const char *object_dir, const char *ext,
				 const char *keep_hash);
extern void clear_incremental_midx_files_ext(const char *object_dir,
					     const char *ext,
					     const char **keep_hashes,
					     uint32_t hashes_nr);
extern int cmp_idx_or_pack_name(const char *idx_or_pack_name,
				const char *idx_name);

static size_t write_midx_header(struct hashfile *f,
				unsigned char num_chunks,
				uint32_t num_packs)
{ … }

struct pack_info { … };

static void fill_pack_info(struct pack_info *info,
			   struct packed_git *p, const char *pack_name,
			   uint32_t orig_pack_int_id)
{ … }

static int pack_info_compare(const void *_a, const void *_b)
{ … }

static int idx_or_pack_name_cmp(const void *_va, const void *_vb)
{ … }

struct write_midx_context { … };

static int should_include_pack(const struct write_midx_context *ctx,
			       const char *file_name)
{ … }

static void add_pack_to_midx(const char *full_path, size_t full_path_len,
			     const char *file_name, void *data)
{ … }

struct pack_midx_entry { … };

static int midx_oid_compare(const void *_a, const void *_b)
{ … }

static int nth_midxed_pack_midx_entry(struct multi_pack_index *m,
				      struct pack_midx_entry *e,
				      uint32_t pos)
{ … }

static void fill_pack_entry(uint32_t pack_int_id,
			    struct packed_git *p,
			    uint32_t cur_object,
			    struct pack_midx_entry *entry,
			    int preferred)
{ … }

struct midx_fanout { … };

static void midx_fanout_grow(struct midx_fanout *fanout, size_t nr)
{ … }

static void midx_fanout_sort(struct midx_fanout *fanout)
{ … }

static void midx_fanout_add_midx_fanout(struct midx_fanout *fanout,
					struct multi_pack_index *m,
					uint32_t cur_fanout,
					int preferred_pack)
{ … }

static void midx_fanout_add_pack_fanout(struct midx_fanout *fanout,
					struct pack_info *info,
					uint32_t cur_pack,
					int preferred,
					uint32_t cur_fanout)
{ … }

/*
 * It is possible to artificially get into a state where there are many
 * duplicate copies of objects. That can create high memory pressure if
 * we are to create a list of all objects before de-duplication. To reduce
 * this memory pressure without a significant performance drop, automatically
 * group objects by the first byte of their object id. Use the IDX fanout
 * tables to group the data, copy to a local array, then sort.
 *
 * Copy only the de-duplicated entries (selected by most-recent modified time
 * of a packfile containing the object).
 */
static void compute_sorted_entries(struct write_midx_context *ctx,
				   uint32_t start_pack)
{ … }

static int write_midx_pack_names(struct hashfile *f, void *data)
{ … }

static int write_midx_bitmapped_packs(struct hashfile *f, void *data)
{ … }

static int write_midx_oid_fanout(struct hashfile *f,
				 void *data)
{ … }

static int write_midx_oid_lookup(struct hashfile *f,
				 void *data)
{ … }

static int write_midx_object_offsets(struct hashfile *f,
				     void *data)
{ … }

static int write_midx_large_offsets(struct hashfile *f,
				    void *data)
{ … }

static int write_midx_revindex(struct hashfile *f,
			       void *data)
{ … }

struct midx_pack_order_data { … };

static int midx_pack_order_cmp(const void *va, const void *vb)
{ … }

static uint32_t *midx_pack_order(struct write_midx_context *ctx)
{ … }

static void write_midx_reverse_index(char *midx_name, unsigned char *midx_hash,
				     struct write_midx_context *ctx)
{ … }

static void prepare_midx_packing_data(struct packing_data *pdata,
				      struct write_midx_context *ctx)
{ … }

static int add_ref_to_pending(const char *refname, const char *referent UNUSED,
			      const struct object_id *oid,
			      int flag, void *cb_data)
{ … }

struct bitmap_commit_cb { … };

static const struct object_id *bitmap_oid_access(size_t index,
						 const void *_entries)
{ … }

static void bitmap_show_commit(struct commit *commit, void *_data)
{ … }

static int read_refs_snapshot(const char *refs_snapshot,
			      struct rev_info *revs)
{ … }
static struct commit **find_commits_for_midx_bitmap(uint32_t *indexed_commits_nr_p,
						    const char *refs_snapshot,
						    struct write_midx_context *ctx)
{ … }

static int write_midx_bitmap(const char *midx_name,
			     const unsigned char *midx_hash,
			     struct packing_data *pdata,
			     struct commit **commits,
			     uint32_t commits_nr,
			     uint32_t *pack_order,
			     unsigned flags)
{ … }

static struct multi_pack_index *lookup_multi_pack_index(struct repository *r,
							const char *object_dir)
{ … }

static int fill_packs_from_midx(struct write_midx_context *ctx,
				const char *preferred_pack_name, uint32_t flags)
{ … }

static struct { … } midx_exts[] = …;

static int link_midx_to_chain(struct multi_pack_index *m)
{ … }

static void clear_midx_files(const char *object_dir,
			     const char **hashes,
			     uint32_t hashes_nr,
			     unsigned incremental)
{ … }

static int write_midx_internal(const char *object_dir,
			       struct string_list *packs_to_include,
			       struct string_list *packs_to_drop,
			       const char *preferred_pack_name,
			       const char *refs_snapshot,
			       unsigned flags)
{ … }

int write_midx_file(const char *object_dir,
		    const char *preferred_pack_name,
		    const char *refs_snapshot,
		    unsigned flags)
{ … }

int write_midx_file_only(const char *object_dir,
			 struct string_list *packs_to_include,
			 const char *preferred_pack_name,
			 const char *refs_snapshot,
			 unsigned flags)
{ … }

int expire_midx_packs(struct repository *r, const char *object_dir, unsigned flags)
{ … }

struct repack_info { … };

static int compare_by_mtime(const void *a_, const void *b_)
{ … }

static int want_included_pack(struct repository *r,
			      struct multi_pack_index *m,
			      int pack_kept_objects,
			      uint32_t pack_int_id)
{ … }

static void fill_included_packs_all(struct repository *r,
				    struct multi_pack_index *m,
				    unsigned char *include_pack)
{ … }

static void fill_included_packs_batch(struct repository *r,
				      struct multi_pack_index *m,
				      unsigned char *include_pack,
				      size_t batch_size)
{ … }

int midx_repack(struct repository *r, const char *object_dir, size_t batch_size, unsigned flags)
{ … }