8000 SHA256 proof of concept by ethomson · Pull Request #6191 · libgit2/libgit2 · GitHub
[go: up one dir, main page]
8000
Skip to content

SHA256 proof of concept #6191

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 21 commits into from
Jul 14, 2022
Merged

SHA256 proof of concept #6191

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
dbc4ac1
oid: `GIT_OID_*SZ` is now `GIT_OID_SHA1_*SIZE`
ethomson Jan 23, 2022
0acaf3a
oid: define GIT_OID_SHA1_ZERO
ethomson Jan 17, 2022
74ffce2
oid: introduce `git_oid_t`
ethomson Dec 11, 2021
0b06821
oid: add functions to inspect oid information
ethomson Dec 11, 2021
c436011
oid: include maximum oid raw/hex sizes
ethomson Dec 12, 2021 8000
b7a46fa
object: move oid header parsing to object
ethomson Jan 23, 2022
6183829
object: move oid header printing to object
ethomson Jan 26, 2022
3fbf580
oid: give oids a type
ethomson Jan 23, 2022
0db1c57
oid: add sha256 typed oids
ethomson Jan 25, 2022
8173241
clar: format sha256 in cl_assert_equal_oid
ethomson Jan 27, 2022
1479806
oid: specify a default implementation
ethomson Jan 26, 2022
c50b280
oid: provide an oid type to hash type map
ethomson Jan 26, 2022
8444b6d
odb_hash*: accept the oid type to hash into
ethomson Jan 26, 2022
3eba918
odb: add git_odb_options
ethomson Jan 26, 2022
dbccfc2
odb: accept an oid type in options
ethomson Jan 26, 2022
4d7ec76
odb: add git_odb_loose_backend_options
ethomson Dec 12, 2021
162c996
oid: add git_oid_fmt_substr
ethomson Jan 25, 2022
04f3468
odb_loose: SHA256 support for loose object storage
ethomson Jan 26, 2022
d103620
meta: generated `features.h` is now `git2_features.h`
ethomson Jun 18, 2022
6c57bac
sha256: make sha256 an experimental optional feature
ethomson Jun 15, 2022
6013b6a
ci: run sha256 builds in ci
ethomson Jun 16, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
odb_loose: SHA256 support for loose object storage 
Teach the loose object database how to cope with SHA256 objects.
  • Loading branch information
@ethomson
ethomson committed Jun 20, 2022
commit 04f34688c43d28d1fa110df7d32e8665232199ff
10 changes: 8 additions & 2 deletions include/git2/odb_backend.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,13 +59,19 @@ typedef struct {

/** Permissions to use creating a file or 0 for defaults */
unsigned int file_mode;

/**
* Type of object IDs to use for this object database, or
* 0 for default (currently SHA1).
*/
git_oid_t oid_type;
} git_odb_backend_loose_options;

/* The current version of the diff options structure */
#define GIT_ODB_BACKEND_LOOSE_OPTIONS_VERSION 1

/* Stack initializer for diff options. Alternatively use
* `git_diff_options_init` programmatic initialization.
/* Stack initializer for odb loose backend options. Alternatively use
* `git_odb_backend_loose_options_init` programmatic initialization.
*/
#define GIT_ODB_BACKEND_LOOSE_OPTIONS_INIT \
{ GIT_ODB_BACKEND_LOOSE_OPTIONS_VERSION, 0, -1 }
Expand Down
2 changes: 2 additions & 0 deletions src/libgit2/odb.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -663,6 +663,8 @@ int git_odb__add_default_backends(
if (db->do_fsync)
loose_opts.flags |= GIT_ODB_BACKEND_LOOSE_FSYNC;

loose_opts.oid_type = db->options.oid_type;

/* add the loose object backend */
if (git_odb_backend_loose(&loose, objects_dir, &loose_opts) < 0 ||
add_backend_internal(db, loose, git_odb__loose_priority, as_alternates, inode) < 0)
Expand Down
101 changes: 57 additions & 44 deletions src/libgit2/odb_loose.c
View file
Open in desktop
< E85D td class="blob-code blob-code-context js-file-line"> /* prealloc memory for OBJ_DIR/xx/xx..38x..xx */
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ typedef struct loose_backend {
git_odb_backend parent;

git_odb_backend_loose_options options;
size_t oid_hexsize;

size_t objects_dirlen;
char objects_dir[GIT_FLEX_ARRAY];
Expand All @@ -56,13 +57,19 @@ typedef struct loose_backend {
* in order to locate objects matching a short oid.
*/
typedef struct {
loose_backend *backend;

size_t dir_len;
unsigned char short_oid[GIT_OID_SHA1_HEXSIZE]; /* hex formatted oid to match */

/* Hex formatted oid to match (and its length) */
unsigned char short_oid[GIT_OID_MAX_HEXSIZE];
size_t short_oid_len;
int found; /* number of matching
* objects already found */
unsigned char res_oid[GIT_OID_SHA1_HEXSIZE]; /* hex formatted oid of
* the object found */

/* Number of matching objects found so far */
int found;

/* Hex formatted oid of the object found */
unsigned char res_oid[GIT_OID_MAX_HEXSIZE];
} loose_locate_object_state;


Expand All @@ -75,20 +82,17 @@ typedef struct {
static int object_file_name(
git_str *name, const loose_backend *be, const git_oid *id)
{
size_t alloclen;

/* expand length for object root + 40 hex sha1 chars + 2 * '/' + '\0' */
GIT_ERROR_CHECK_ALLOC_ADD(&alloclen, be->objects_dirlen, GIT_OID_SHA1_HEXSIZE);
GIT_ERROR_CHECK_ALLOC_ADD(&alloclen, alloclen, 3);
if (git_str_grow(name, alloclen) < 0)
return -1;
/* append loose object filename: aa/aaa... (41 bytes plus NUL) */
size_t path_size = be->oid_hexsize + 1;

git_str_set(name, be->objects_dir, be->objects_dirlen);
git_fs_path_to_dir(name);

/* loose object filename: aa/aaa... (41 bytes) */
if (git_str_grow_by(name, path_size + 1) < 0)
return -1;

git_oid_pathfmt(name->ptr + name->size, id);
name->size += GIT_OID_SHA1_HEXSIZE + 1;
name->size += path_size;
name->ptr[name->size] = '\0';

return 0;
Expand Down Expand Up @@ -460,8 +464,9 @@ static int locate_object(
/* Explore an entry of a directory and see if it matches a short oid */
static int fn_locate_object_short_oid(void *state, git_str *pathbuf) {
loose_locate_object_state *sstate = (loose_locate_object_state *)state;
size_t hex_size = sstate->backend->oid_hexsize;

if (git_str_len(pathbuf) - sstate->dir_len != GIT_OID_SHA1_HEXSIZE - 2) {
if (git_str_len(pathbuf) - sstate->dir_len != hex_size - 2) {
/* Entry cannot be an object. Continue to next entry */
return 0;
}
Expand All @@ -476,7 +481,9 @@ static int fn_locate_object_short_oid(void *state, git_str *pathbuf) {
if (!sstate->found) {
sstate->res_oid[0] = sstate->short_oid[0];
sstate->res_oid[1] = sstate->short_oid[1];
memcpy(sstate->res_oid+2, pathbuf->ptr+sstate->dir_len, GIT_OID_SHA1_HEXSIZE-2);
memcpy(sstate->res_oid + 2,
pathbuf->ptr+sstate->dir_len,
hex_size - 2);
}
sstate->found++;
}
Expand All @@ -502,7 +509,7 @@ static int locate_object_short_oid(
int error;

GIT_ERROR_CHECK_ALLOC_ADD(&alloc_len, dir_len, GIT_OID_SHA1_HEXSIZE);
GIT_ERROR_CHECK_ALLOC_ADD(&alloc_len, dir_len, backend->oid_hexsize);
GIT_ERROR_CHECK_ALLOC_ADD(&alloc_len, alloc_len, 3);
if (git_str_grow(object_location, alloc_len) < 0)
return -1;
Expand All @@ -526,6 +533,7 @@ static int locate_object_short_oid(
return git_odb__error_notfound("no matching loose object for prefix",
short_oid, len);

state.backend = backend;
state.dir_len = git_str_len(object_location);
state.short_oid_len = len;
state.found = 0;
Expand All @@ -544,12 +552,12 @@ static int locate_object_short_oid(
return git_odb__error_ambiguous("multiple matches in loose objects");

/* Convert obtained hex formatted oid to raw */
error = git_oid_fromstr(res_oid, (char *)state.res_oid, GIT_OID_SHA1);
error = git_oid_fromstr(res_oid, (char *)state.res_oid, backend->options.oid_type);
if (error)
return error;

/* Update the location according to the oid obtained */
GIT_ERROR_CHECK_ALLOC_ADD(&alloc_len, dir_len, GIT_OID_SHA1_HEXSIZE);
GIT_ERROR_CHECK_ALLOC_ADD(&alloc_len, dir_len, backend->oid_hexsize);
GIT_ERROR_CHECK_ALLOC_ADD(&alloc_len, alloc_len, 2);

git_str_truncate(object_location, dir_len);
Expand All @@ -558,20 +566,12 @@ static int locate_object_short_oid(

git_oid_pathfmt(object_location->ptr + dir_len, res_oid);

object_location->size += GIT_OID_SHA1_HEXSIZE + 1;
object_location->size += backend->oid_hexsize + 1;
object_location->ptr[object_location->size] = '\0';

return 0;
}









/***********************************************************
*
* LOOSE BACKEND PUBLIC API
Expand All @@ -594,7 +594,7 @@ static int loose_backend__read_header(size_t *len_p, git_object_t *type_p, git_o

if (locate_object(&object_path, (loose_backend *)backend, oid) < 0) {
error = git_odb__error_notfound("no matching loose object",
oid, GIT_OID_SHA1_HEXSIZE);
oid, ((struct loose_backend *)backend)->oid_hexsize);
} else if ((error = read_header_loose(&raw, &object_path)) == 0) {
*len_p = raw.len;
*type_p = raw.type;
Expand All @@ -616,7 +616,7 @@ static int loose_backend__read(void **buffer_p, size_t *len_p, git_object_t *typ

if (locate_object(&object_path, (loose_backend *)backend, oid) < 0) {
error = git_odb__error_notfound("no matching loose object",
oid, GIT_OID_SHA1_HEXSIZE);
oid, ((struct loose_backend *)backend)->oid_hexsize);
} else if ((error = read_loose(&raw, &object_path)) == 0) {
*buffer_p = raw.data;
*len_p = raw.len;
Expand All @@ -633,17 +633,19 @@ static int loose_backend__read_prefix(
void **buffer_p,
size_t *len_p,
git_object_t *type_p,
git_odb_backend *backend,
git_odb_backend *_backend,
const git_oid *short_oid,
size_t len)
{
struct loose_backend *backend = (struct loose_backend *)_backend;
int error = 0;

GIT_ASSERT_ARG(len >= GIT_OID_MINPREFIXLEN && len <= GIT_OID_SHA1_HEXSIZE);
GIT_ASSERT_ARG(len >= GIT_OID_MINPREFIXLEN &&
len <= backend->oid_hexsize);

if (len == GIT_OID_SHA1_HEXSIZE) {
if (len == backend->oid_hexsize) {
/* We can fall back to regular read method */
error = loose_backend__read(buffer_p, len_p, type_p, backend, short_oid);
error = loose_backend__read(buffer_p, len_p, type_p, _backend, short_oid);
if (!error)
git_oid_cpy(out_oid, short_oid);
} else {
Expand Down Expand Up @@ -702,15 +704,18 @@ static int loose_backend__exists_prefix(
}

struct foreach_state {
struct loose_backend *backend;
size_t dir_len;
git_odb_foreach_cb cb;
void *data;
};

GIT_INLINE(int) filename_to_oid(git_oid *oid, const char *ptr)
GIT_INLINE(int) filename_to_oid(struct loose_backend *backend, git_oid *oid, const char *ptr)
{
int v, i = 0;
if (strlen(ptr) != GIT_OID_SHA1_HEXSIZE+1)
int v;
size_t i = 0;

if (strlen(ptr) != backend->oid_hexsize + 1)
return -1;

if (ptr[2] != '/') {
Expand All @@ -724,15 +729,15 @@ GIT_INLINE(int) filename_to_oid(git_oid *oid, const char *ptr)
oid->id[0] = (unsigned char) v;

ptr += 3;
for (i = 0; i < 38; i += 2) {
for (i = 0; i < backend->oid_hexsize - 2; i += 2) {
v = (git__fromhex(ptr[i]) << 4) | git__fromhex(ptr[i + 1]);
if (v < 0)
return -1;

oid->id[1 + i/2] = (unsigned char) v;
}

oid->type = GIT_OID_SHA1;
oid->type = backend->options.oid_type;

return 0;
}
Expand All @@ -742,7 +747,7 @@ static int foreach_object_dir_cb(void *_state, git_str *path)
git_oid oid;
struct foreach_state *state = (struct foreach_state *) _state;

if (filename_to_oid(&oid, path->ptr + state->dir_len) < 0)
if (filename_to_oid(state->backend, &oid, path->ptr + state->dir_len) < 0)
return 0;

return git_error_set_after_callback_function(
Expand Down Expand Up @@ -779,6 +784,7 @@ static int loose_backend__foreach(git_odb_backend *_backend, git_odb_foreach_cb
return -1;

memset(&state, 0, sizeof(state));
state.backend = backend;
state.cb = cb;
state.data = data;
state.dir_len = git_str_len(&buf);
Expand Down Expand Up @@ -999,6 +1005,7 @@ static int loose_backend__readstream(
loose_readstream *stream = NULL;
git_hash_ctx *hash_ctx = NULL;
git_str object_path = GIT_STR_INIT;
git_hash_algorithm_t algorithm;
obj_hdr hdr;
int error = 0;

Expand All @@ -1015,7 +1022,7 @@ static int loose_backend__readstream(

if (locate_object(&object_path, backend, oid) < 0) {
error = git_odb__error_notfound("no matching loose object",
oid, GIT_OID_SHA1_HEXSIZE);
oid, backend->oid_hexsize);
goto done;
}

Expand All @@ -1025,9 +1032,11 @@ static int loose_backend__readstream(
hash_ctx = git__malloc(sizeof(git_hash_ctx));
GIT_ERROR_CHECK_ALLOC(hash_ctx);

if ((error = git_hash_ctx_init(hash_ctx, GIT_HASH_ALGORITHM_SHA1)) < 0 ||
(error = git_futils_mmap_ro_file(&stream->map, object_path.ptr)) < 0 ||
(error = git_zstream_init(&stream->zstream, GIT_ZSTREAM_INFLATE)) < 0)
algorithm = git_oid_algorithm(backend->options.oid_type);

if ((error = git_hash_ctx_init(hash_ctx, algorithm)) < 0 ||
(error = git_futils_mmap_ro_file(&stream->map, object_path.ptr)) < 0 ||
(error = git_zstream_init(&stream->zstream, GIT_ZSTREAM_INFLATE)) < 0)
goto done;

/* check for a packlike loose object */
Expand Down Expand Up @@ -1145,6 +1154,9 @@ static void normalize_options(

if (opts->file_mode == 0)
opts->file_mode = GIT_OBJECT_FILE_MODE;

if (opts->oid_type == 0)
opts->oid_type = GIT_OID_DEFAULT;
}

int git_odb_backend_loose(
Expand Down Expand Up @@ -1173,6 +1185,7 @@ int git_odb_backend_loose(
backend->objects_dir[backend->objects_dirlen++] = '/';

normalize_options(&backend->options, opts);
backend->oid_hexsize = git_oid_hexsize(backend->options.oid_type);

backend->parent.read = &loose_backend__read;
backend->parent.write = &loose_backend__write;
Expand Down
Loading
0