Commit c4584ae3 authored by Junio C Hamano's avatar Junio C Hamano Committed by Linus Torvalds

[PATCH] Remove "delta" object representation.

Packed delta files created by git-pack-objects seems to be the
way to go, and existing "delta" object handling code has exposed
the object representation details to too many places.  Remove it
while we refactor code to come up with a proper interface in
sha1_file.c.
Signed-off-by: default avatarJunio C Hamano <junkio@cox.net>
Signed-off-by: default avatarLinus Torvalds <torvalds@osdl.org>
parent ee85cbc6
......@@ -9,7 +9,7 @@ git-fsck-cache - Verifies the connectivity and validity of the objects in the da
SYNOPSIS
--------
'git-fsck-cache' [--tags] [--root] [--delta-depth] [--unreachable] [--cache] [<object>*]
'git-fsck-cache' [--tags] [--root] [--unreachable] [--cache] [<object>*]
DESCRIPTION
-----------
......@@ -37,9 +37,6 @@ OPTIONS
Consider any object recorded in the cache also as a head node for
an unreachability trace.
--delta-depth::
Report back the length of the longest delta chain found.
It tests SHA1 and general object sanity, and it does full tracking of
the resulting reachability and everything else. It prints out any
corruption it finds (missing or bad objects), and if you use the
......
......@@ -21,13 +21,6 @@ Downloads a remote GIT repository via HTTP.
Get trees associated with the commit objects.
-a::
Get all the objects.
-d::
Do not check for delta base objects (use this option
only when you know the remote repository is not
deltified).
--recover::
Check dependency of deltified object more carefully than
usual, to recover after earlier pull that was interrupted.
-v::
Report what is downloaded.
-w::
......
......@@ -23,13 +23,6 @@ OPTIONS
Get trees associated with the commit objects.
-a::
Get all the objects.
-d::
Do not check for delta base objects (use this option
only when you know the remote repository is not
deltified).
--recover::
Check dependency of deltified object more carefully than
usual, to recover after earlier pull that was interrupted.
-v::
Report what is downloaded.
-w::
......
git-mkdelta(1)
==============
May 2005
NAME
----
git-mkdelta - Creates a delta object
SYNOPSIS
--------
'git-mkdelta' [-v] [-d N | --max-depth=N ] <reference_object> <target_object> [ <next_object> ... ]
DESCRIPTION
-----------
Creates a delta object to replace <reference_object> by using an
ordered list of potential objects to deltafy against earlier objects
in the list.
A cap on the depth of delta references can be provided as well,
otherwise the default is to not have any limit. A limit of 0 will
also undeltafy a given object.
OPTIONS
-------
-v::
Verbose
-d|--max-depth::
limit the number of delta references in a chain
If 0 then all objects are undeltafied.
Author
------
Git is written by Linus Torvalds <torvalds@osdl.org> and the git-list <git@vger.kernel.org>.
Documentation
--------------
Documentation by David Greaves, Junio C Hamano and the git-list <git@vger.kernel.org>.
GIT
---
Part of the link:git.html[git] suite
......@@ -31,13 +31,6 @@ commit-id::
Get trees associated with the commit objects.
-a::
Get all the objects.
-d::
Do not check for delta base objects (use this option
only when you know the remote repository is not
deltified).
--recover::
Check dependency of deltified object more carefully than
usual, to recover after earlier pull that was interrupted.
-v::
Report what is downloaded.
-w::
......
......@@ -28,13 +28,6 @@ commit-id::
Get tree associated with the requested commit object.
-a::
Get all the objects.
-d::
Do not check for delta base objects (use this option
only when you know the local repository is not
deltified).
--recover::
Check dependency of deltified object more carefully than
usual, to recover after earlier push that was interrupted.
-v::
Report what is uploaded.
-w::
......
......@@ -51,9 +51,6 @@ link:git-init-db.html[git-init-db]::
link:git-merge-base.html[git-merge-base]::
Finds as good a common ancestor as possible for a merge
link:git-mkdelta.html[git-mkdelta]::
Creates a delta object
link:git-mktag.html[git-mktag]::
Creates a tag object
......
......@@ -22,7 +22,7 @@ INSTALL=install
SCRIPTS=git git-apply-patch-script git-merge-one-file-script git-prune-script \
git-pull-script git-tag-script git-resolve-script git-whatchanged \
git-deltafy-script git-fetch-script git-status-script git-commit-script \
git-fetch-script git-status-script git-commit-script \
git-log-script git-shortlog git-cvsimport-script git-diff-script \
git-reset-script git-add-script git-checkout-script git-clone-script \
gitk git-cherry git-rebase-script git-relink-script
......@@ -34,7 +34,7 @@ PROG= git-update-cache git-diff-files git-init-db git-write-tree \
git-unpack-file git-export git-diff-cache git-convert-cache \
git-http-pull git-ssh-push git-ssh-pull git-rev-list git-mktag \
git-diff-helper git-tar-tree git-local-pull git-write-blob \
git-get-tar-commit-id git-mkdelta git-apply git-stripspace \
git-get-tar-commit-id git-apply git-stripspace \
git-cvs2git git-diff-stages git-rev-parse git-patch-id \
git-pack-objects git-unpack-objects
......@@ -44,7 +44,7 @@ install: $(PROG) $(SCRIPTS)
$(INSTALL) $(PROG) $(SCRIPTS) $(dest)$(bin)
LIB_OBJS=read-cache.o sha1_file.o usage.o object.o commit.o tree.o blob.o \
tag.o delta.o date.o index.o diff-delta.o patch-delta.o entry.o \
tag.o date.o index.o diff-delta.o patch-delta.o entry.o \
epoch.o refs.o csum-file.o
LIB_FILE=libgit.a
LIB_H=cache.h object.h blob.h tree.h commit.h tag.h delta.h epoch.h csum-file.h
......@@ -117,7 +117,6 @@ git-mktag: mktag.c
git-diff-helper: diff-helper.c
git-tar-tree: tar-tree.c
git-write-blob: write-blob.c
git-mkdelta: mkdelta.c
git-stripspace: stripspace.c
git-cvs2git: cvs2git.c
git-diff-stages: diff-stages.c
......
......@@ -32,8 +32,8 @@ build up a hierarchy of objects.
All objects have a statically determined "type" aka "tag", which is
determined at object creation time, and which identifies the format of
the object (i.e. how it is used, and how it can refer to other
objects). There are currently five different object types: "blob",
"tree", "commit", "tag" and "delta"
objects). There are currently four different object types: "blob",
"tree", "commit" and "tag".
A "blob" object cannot refer to any other object, and is, like the tag
implies, a pure storage object containing some user data. It is used to
......@@ -62,16 +62,12 @@ A "tag" object symbolically identifies and can be used to sign other
objects. It contains the identifier and type of another object, a
symbolic name (of course!) and, optionally, a signature.
A "delta" object is used internally by the object database to minimise
disk usage. Instead of storing the entire contents of a revision, git
can behave in a similar manner to RCS et al and simply store a delta.
Regardless of object type, all objects share the following
characteristics: they are all deflated with zlib, and have a header
that not only specifies their tag, but also provides size information
about the data in the object. It's worth noting that the SHA1 hash
that is used to name the object is the hash of the original data or
the delta. (Historical note: in the dawn of the age of git the hash
that is used to name the object is the hash of the original data.
(Historical note: in the dawn of the age of git the hash
was the sha1 of the _compressed_ object)
As a result, the general consistency of an object can always be tested
......@@ -219,29 +215,6 @@ verification) has to come from outside.
A tag is created with link:git-mktag.html[git-mktag] and
it's data can be accessed by link:git-cat-file.html[git-cat-file]
Delta Object
~~~~~~~~~~~~
The "delta" object is used internally by the object database to
minimise storage usage by using xdeltas (byte level diffs). Deltas can
form chains of arbitrary length as RCS does (although this is
configureable at creation time). Most operations won't see or even be
aware of delta objects as they are automatically 'applied' and appear
as 'real' git objects In other words, if you write your own routines
to look at the contents of the object database then you need to know
about this - otherwise you don't. Actually, that's not quite true -
one important area where deltas are likely to prove very valuable is
in reducing bandwidth loads - so the more sophisticated network tools
for git repositories will be aware of them too.
Finally, git repositories can (and must) be deltafied in the
background - the work to calculate the differences does not take place
automatically at commit time.
A delta can be created (or undeltafied) with
link:git-mkdelta.html[git-mkdelta] it's raw data cannot be accessed at
present.
The "index" aka "Current Directory Cache"
-----------------------------------------
......
......@@ -162,7 +162,6 @@ extern char *sha1_file_name(const unsigned char *sha1);
extern void * map_sha1_file(const unsigned char *sha1, unsigned long *size);
extern int unpack_sha1_header(z_stream *stream, void *map, unsigned long mapsize, void *buffer, unsigned long size);
extern int parse_sha1_header(char *hdr, char *type, unsigned long *sizep);
extern int sha1_delta_base(const unsigned char *, unsigned char *);
extern int sha1_file_size(const unsigned char *, unsigned long *);
extern void * unpack_sha1_file(void *map, unsigned long mapsize, char *type, unsigned long *size);
extern void * read_sha1_file(const unsigned char *sha1, char *type, unsigned long *size);
......
#include "object.h"
#include "blob.h"
#include "tree.h"
#include "commit.h"
#include "tag.h"
#include "delta.h"
#include "cache.h"
/* the delta object definition (it can alias any other object) */
struct delta {
union {
struct object object;
struct blob blob;
struct tree tree;
struct commit commit;
struct tag tag;
} u;
};
struct delta *lookup_delta(const unsigned char *sha1)
{
struct object *obj = lookup_object(sha1);
if (!obj) {
struct delta *ret = xmalloc(sizeof(struct delta));
memset(ret, 0, sizeof(struct delta));
created_object(sha1, &ret->u.object);
return ret;
}
return (struct delta *) obj;
}
int parse_delta_buffer(struct delta *item, void *buffer, unsigned long size)
{
struct object *reference;
struct object_list *p;
if (item->u.object.delta)
return 0;
item->u.object.delta = 1;
if (size <= 20)
return -1;
reference = lookup_object(buffer);
if (!reference) {
struct delta *ref = xmalloc(sizeof(struct delta));
memset(ref, 0, sizeof(struct delta));
created_object(buffer, &ref->u.object);
reference = &ref->u.object;
}
p = xmalloc(sizeof(*p));
p->item = &item->u.object;
p->next = reference->attached_deltas;
reference->attached_deltas = p;
return 0;
}
int process_deltas(void *src, unsigned long src_size, const char *src_type,
struct object_list *delta_list)
{
int deepest = 0;
do {
struct object *obj = delta_list->item;
static char type[10];
void *map, *delta, *buf;
unsigned long map_size, delta_size, buf_size;
map = map_sha1_file(obj->sha1, &map_size);
if (!map)
continue;
delta = unpack_sha1_file(map, map_size, type, &delta_size);
munmap(map, map_size);
if (!delta)
continue;
if (strcmp(type, "delta") || delta_size <= 20) {
free(delta);
continue;
}
buf = patch_delta(src, src_size,
delta+20, delta_size-20,
&buf_size);
free(delta);
if (!buf)
continue;
if (check_sha1_signature(obj->sha1, buf, buf_size, src_type) < 0)
printf("sha1 mismatch for delta %s\n", sha1_to_hex(obj->sha1));
if (obj->type && obj->type != src_type) {
error("got %s when expecting %s for delta %s",
src_type, obj->type, sha1_to_hex(obj->sha1));
free(buf);
continue;
}
obj->type = src_type;
if (src_type == blob_type) {
parse_blob_buffer((struct blob *)obj, buf, buf_size);
} else if (src_type == tree_type) {
parse_tree_buffer((struct tree *)obj, buf, buf_size);
} else if (src_type == commit_type) {
parse_commit_buffer((struct commit *)obj, buf, buf_size);
} else if (src_type == tag_type) {
parse_tag_buffer((struct tag *)obj, buf, buf_size);
} else {
error("unknown object type %s", src_type);
free(buf);
continue;
}
if (obj->attached_deltas) {
int depth = process_deltas(buf, buf_size, src_type,
obj->attached_deltas);
if (deepest < depth)
deepest = depth;
}
free(buf);
} while ((delta_list = delta_list->next));
return deepest + 1;
}
......@@ -9,13 +9,4 @@ extern void *patch_delta(void *src_buf, unsigned long src_size,
void *delta_buf, unsigned long delta_size,
unsigned long *dst_size);
/* handling of delta objects */
struct delta;
struct object_list;
extern struct delta *lookup_delta(const unsigned char *sha1);
extern int parse_delta_buffer(struct delta *item, void *buffer, unsigned long size);
extern int parse_delta(struct delta *item, unsigned char sha1);
extern int process_deltas(void *src, unsigned long src_size,
const char *src_type, struct object_list *delta);
#endif
......@@ -6,46 +6,15 @@
#include "tree.h"
#include "blob.h"
#include "tag.h"
#include "delta.h"
#define REACHABLE 0x0001
static int show_root = 0;
static int show_tags = 0;
static int show_unreachable = 0;
static int show_max_delta_depth = 0;
static int keep_cache_objects = 0;
static unsigned char head_sha1[20];
static void expand_deltas(void)
{
int i, max_depth = 0;
/*
* To be as efficient as possible we look for delta heads and
* recursively process them going backward, and parsing
* resulting objects along the way. This allows for processing
* each delta objects only once regardless of the delta depth.
*/
for (i = 0; i < nr_objs; i++) {
struct object *obj = objs[i];
if (obj->parsed && !obj->delta && obj->attached_deltas) {
int depth = 0;
char type[10];
unsigned long size;
void *buf = read_sha1_file(obj->sha1, type, &size);
if (!buf)
continue;
depth = process_deltas(buf, size, obj->type,
obj->attached_deltas);
if (max_depth < depth)
max_depth = depth;
}
}
if (show_max_delta_depth)
printf("maximum delta depth = %d\n", max_depth);
}
static void check_connectivity(void)
{
int i;
......@@ -56,12 +25,8 @@ static void check_connectivity(void)
struct object_list *refs;
if (!obj->parsed) {
if (obj->delta)
printf("unresolved delta %s\n",
sha1_to_hex(obj->sha1));
else
printf("missing %s %s\n",
obj->type, sha1_to_hex(obj->sha1));
printf("missing %s %s\n",
obj->type, sha1_to_hex(obj->sha1));
continue;
}
......@@ -75,12 +40,8 @@ static void check_connectivity(void)
}
if (show_unreachable && !(obj->flags & REACHABLE)) {
if (obj->attached_deltas)
printf("foreign delta reference %s\n",
sha1_to_hex(obj->sha1));
else
printf("unreachable %s %s\n",
obj->type, sha1_to_hex(obj->sha1));
printf("unreachable %s %s\n",
obj->type, sha1_to_hex(obj->sha1));
continue;
}
......@@ -240,8 +201,6 @@ static int fsck_sha1(unsigned char *sha1)
return fsck_commit((struct commit *) obj);
if (obj->type == tag_type)
return fsck_tag((struct tag *) obj);
if (!obj->type && obj->delta)
return 0;
return -1;
}
......@@ -427,10 +386,6 @@ int main(int argc, char **argv)
show_root = 1;
continue;
}
if (!strcmp(arg, "--delta-depth")) {
show_max_delta_depth = 1;
continue;
}
if (!strcmp(arg, "--cache")) {
keep_cache_objects = 1;
continue;
......@@ -447,8 +402,6 @@ int main(int argc, char **argv)
}
fsck_sha1_list();
expand_deltas();
heads = 0;
for (i = 1; i < argc; i++) {
const char *arg = argv[i];
......
#!/bin/bash
# Example script to deltify an entire GIT repository based on the commit list.
# The most recent version of a file is the reference and previous versions
# are made delta against the best earlier version available. And so on for
# successive versions going back in time. This way the increasing delta
# overhead is pushed towards older versions of any given file.
#
# The -d argument allows to provide a limit on the delta chain depth.
# If 0 is passed then everything is undeltafied. Limiting the delta
# depth is meaningful for subsequent access performance to old revisions.
# A value of 16 might be a good compromize between performance and good
# space saving. Current default is unbounded.
#
# The --max-behind=30 argument is passed to git-mkdelta so to keep
# combinations and memory usage bounded a bit. If you have lots of memory
# and CPU power you may remove it (or set to 0) to let git-mkdelta find the
# best delta match regardless of the number of revisions for a given file.
# You can also make the value smaller to make it faster and less
# memory hungry. A value of 5 ought to still give pretty good results.
# When set to 0 or ommitted then look behind is unbounded. Note that
# git-mkdelta might die with a segmentation fault in that case if it
# runs out of memory. Note that the GIT repository will still be consistent
# even if git-mkdelta dies unexpectedly.
set -e
max_depth=
[ "$1" == "-d" ] && max_depth="--max-depth=$2" && shift 2
overlap=30
max_behind="--max-behind=$overlap"
function process_list() {
if [ "$list" ]; then
echo "Processing $curr_file"
echo "$list" | xargs git-mkdelta $max_depth $max_behind -v
fi
}
rev_list=""
curr_file=""
git-rev-list HEAD |
while true; do
# Let's batch revisions into groups of 1000 to give it a chance to
# scale with repositories containing long revision lists. We also
# overlap with the previous batch the size of mkdelta's look behind
# value in order to account for the processing discontinuity.
rev_list="$(echo -e -n "$rev_list" | tail --lines=$overlap)"
for i in $(seq 1000); do
read rev || break
rev_list="$rev_list$rev\n"
done
echo -e -n "$rev_list" |
git-diff-tree -r -t --stdin |
awk '/^:/ { if ($5 == "M") printf "%s %s\n%s %s\n", $4, $6, $3, $6 }' |
LC_ALL=C sort -s -k 2 | uniq |
while read sha1 file; do
if [ "$file" == "$curr_file" ]; then
list="$list $sha1"
else
process_list
curr_file="$file"
list="$sha1"
fi
done
[ "$rev" ] || break
done
process_list
curr_file="root directory"
list="$(
git-rev-list HEAD |
while read commit; do
git-cat-file commit $commit |
sed -n 's/tree //p;Q'
done
)"
process_list
......@@ -150,10 +150,6 @@ int main(int argc, char **argv)
get_tree = 1;
} else if (argv[arg][1] == 'c') {
get_history = 1;
} else if (argv[arg][1] == 'd') {
get_delta = 0;
} else if (!strcmp(argv[arg], "--recover")) {
get_delta = 2;
} else if (argv[arg][1] == 'a') {
get_all = 1;
get_tree = 1;
......
......@@ -120,10 +120,6 @@ int main(int argc, char **argv)
get_tree = 1;
else if (argv[arg][1] == 'c')
get_history = 1;
else if (argv[arg][1] == 'd')
get_delta = 0;
else if (!strcmp(argv[arg], "--recover"))
get_delta = 2;
else if (argv[arg][1] == 'a') {
get_all = 1;
get_tree = 1;
......
/*
* Deltafication of a GIT database.
*
* (C) 2005 Nicolas Pitre <nico@cam.org>
*
* This code is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*/
#include "cache.h"
#include "delta.h"
static int replace_object(char *buf, unsigned long size, unsigned char *sha1)
{
char tmpfile[PATH_MAX];
int fd;
snprintf(tmpfile, sizeof(tmpfile), "%s/obj_XXXXXX", get_object_directory());
fd = mkstemp(tmpfile);
if (fd < 0)
return error("%s: %s\n", tmpfile, strerror(errno));
if (write(fd, buf, size) != size) {
perror("unable to write file");
close(fd);
unlink(tmpfile);
return -1;
}
fchmod(fd, 0444);
close(fd);
if (rename(tmpfile, sha1_file_name(sha1))) {
perror("unable to replace original object");
unlink(tmpfile);
return -1;
}
return 0;
}
static void *create_object(unsigned char *buf, unsigned long len,
char *hdr, int hdrlen, unsigned long *retsize)
{
unsigned char *compressed;
unsigned long size;
z_stream stream;
/* Set it up */
memset(&stream, 0, sizeof(stream));
deflateInit(&stream, Z_BEST_COMPRESSION);
size = deflateBound(&stream, len+hdrlen);
compressed = xmalloc(size);
/* Compress it */
stream.next_out = compressed;
stream.avail_out = size;
/* First header.. */
stream.next_in = (unsigned char *)hdr;
stream.avail_in = hdrlen;
while (deflate(&stream, 0) == Z_OK)
/* nothing */;
/* Then the data itself.. */
stream.next_in = buf;
stream.avail_in = len;
while (deflate(&stream, Z_FINISH) == Z_OK)
/* nothing */;
deflateEnd(&stream);
*retsize = stream.total_out;
return compressed;
}
static int restore_original_object(unsigned char *buf, unsigned long len,
char *type, unsigned char *sha1)
{
char hdr[50];
int hdrlen, ret;
void *compressed;
unsigned long size;
hdrlen = sprintf(hdr, "%s %lu", type, len)+1;
compressed = create_object(buf, len, hdr, hdrlen, &size);
ret = replace_object(compressed, size, sha1);
free(compressed);
return ret;
}
static void *create_delta_object(unsigned char *buf, unsigned long len,
unsigned char *sha1_ref, unsigned long *size)
{
char hdr[50];
int hdrlen;
/* Generate the header + sha1 of reference for delta */
hdrlen = sprintf(hdr, "delta %lu", len+20)+1;
memcpy(hdr + hdrlen, sha1_ref, 20);
hdrlen += 20;
return create_object(buf, len, hdr, hdrlen, size);
}
static void *get_buffer(unsigned char *sha1, char *type,
unsigned long *size, unsigned long *compsize)
{
unsigned long mapsize;
void *map = map_sha1_file(sha1, &mapsize);
if (map) {
void *buffer = unpack_sha1_file(map, mapsize, type, size);
munmap(map, mapsize);
if (compsize)
*compsize = mapsize;
if (buffer)
return buffer;
}
error("unable to get object %s", sha1_to_hex(sha1));
return NULL;
}
static void *expand_delta(void *delta, unsigned long *size, char *type,
unsigned int *depth, unsigned char **links)
{
void *buf = NULL;
unsigned int level = (*depth)++;
if (*size < 20) {
error("delta object is bad");
free(delta);
} else {
unsigned long ref_size;
void *ref = get_buffer(delta, type, &ref_size, NULL);
if (ref && !strcmp(type, "delta"))
ref = expand_delta(ref, &ref_size, type, depth, links);
else if (ref)
{
*links = xmalloc(*depth * 20);
}
if (ref) {
buf = patch_delta(ref, ref_size, delta+20, *size-20, size);
free(ref);
if (buf)
memcpy(*links + level*20, delta, 20);
else
free(*links);
}
free(delta);
}
return buf;
}
static char *mkdelta_usage =
"mkdelta [--max-depth=N] [--max-behind=N] <reference_sha1> <target_sha1> [<next_sha1> ...]";
struct delta {
unsigned char sha1[20]; /* object sha1 */
unsigned long size; /* object size */
void *buf; /* object content */
unsigned char *links; /* delta reference links */
unsigned int depth; /* delta depth */
};
int main(int argc, char **argv)
{
struct delta *ref, trg;
char ref_type[20], trg_type[20], *skip_reason;
void *best_buf;
unsigned long best_size, orig_size, orig_compsize;
unsigned int r, orig_ref, best_ref, nb_refs, next_ref, max_refs = 0;
unsigned int i, duplicate, skip_lvl, verbose = 0, quiet = 0;
unsigned int max_depth = -1;
for (i = 1; i < argc; i++) {