object: lookup sha256 objects

This is much of the plumbing for the object database to support SHA256, and for objects to be able to parse SHA256 versions of themselves.
libgit2 · ethomson · Feb 14, 2023 · Dec 3, 2022 · Sep 19, 2022 · Dec 3, 2022
commit fe2ee3a018286b04cd0c64f84d437c7317c8f138
diff --git a/examples/index-pack.c b/examples/index-pack.c
@@ -28,11 +28,18 @@ int lg2_index_pack(git_repository *repo, int argc, char **argv)
 		return EXIT_FAILURE;
 	}
 
-	if (git_indexer_new(&idx, ".", 0, NULL, NULL) < 0) {
+#ifdef GIT_EXPERIMENTAL_SHA256
+	error = git_indexer_new(&idx, ".", git_repository_oid_type(repo), NULL);
+#else
+	error = git_indexer_new(&idx, ".", 0, NULL, NULL);
+#endif
+
+	if (error < 0) {
 		puts("bad idx");
 		return -1;
 	}
 
+
 	if ((fd = open(argv[1], 0)) < 0) {
 		perror("open");
 		return -1;

diff --git a/fuzzers/objects_fuzzer.c b/fuzzers/objects_fuzzer.c
@@ -39,7 +39,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
 	 * to do.
 	 */
 	for (i = 0; i < ARRAY_SIZE(types); i++) {
-		if (git_object__from_raw(&object, (const char *) data, size, types[i]) < 0)
+		if (git_object__from_raw(&object, (const char *) data, size, types[i], GIT_OID_SHA1) < 0)
 			continue;
 		git_object_free(object);
 		object = NULL;

diff --git a/fuzzers/packfile_fuzzer.c b/fuzzers/packfile_fuzzer.c
@@ -67,6 +67,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
 	git_str path = GIT_STR_INIT;
 	git_oid oid;
 	bool append_hash = false;
+	int error;
 
 	if (size == 0)
 		return 0;
@@ -82,7 +83,13 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
 		abort();
 	}
 
-	if (git_indexer_new(&indexer, ".", 0, odb, NULL) < 0) {
+#ifdef GIT_EXPERIMENTAL_SHA256
+	error = git_indexer_new(&indexer, ".", GIT_OID_SHA1, NULL);
+#else
+	error = git_indexer_new(&indexer, ".", 0, odb, NULL);
+#endif
+
+	if (error < 0) {
 		fprintf(stderr, "Failed to create the indexer: %s\n",
 			git_error_last()->message);
 		abort();

diff --git a/include/git2/indexer.h b/include/git2/indexer.h
@@ -62,6 +62,19 @@ typedef int GIT_CALLBACK(git_indexer_progress_cb)(const git_indexer_progress *st
 typedef struct git_indexer_options {
 	unsigned int version;
 
+#ifdef GIT_EXPERIMENTAL_SHA256
+	/** permissions to use creating packfile or 0 for defaults */
+	unsigned int mode;
+
+	/**
+	 * object database from which to read base objects when
+	 * fixing thin packs. This can be NULL if there are no thin
+	 * packs; if a thin pack is encountered, an error will be
+	 * returned if there are bases missing.
+	 */
+	git_odb *odb;
+#endif
+
 	/** progress_cb function to call with progress information */
 	git_indexer_progress_cb progress_cb;
 
@@ -87,6 +100,21 @@ GIT_EXTERN(int) git_indexer_options_init(
 	git_indexer_options *opts,
 	unsigned int version);
 
+#ifdef GIT_EXPERIMENTAL_SHA256
+/**
+ * Create a new indexer instance
+ *
+ * @param out where to store the indexer instance
+ * @param path to the directory where the packfile should be stored
+ * @param oid_type the oid type to use for objects
+ * @return 0 or an error code.
+ */
+GIT_EXTERN(int) git_indexer_new(
+		git_indexer **out,
+		const char *path,
+		git_oid_t oid_type,
+		git_indexer_options *opts);
+#else
 /**
  * Create a new indexer instance
  *
@@ -106,6 +134,7 @@ GIT_EXTERN(int) git_indexer_new(
 		unsigned int mode,
 		git_odb *odb,
 		git_indexer_options *opts);
+#endif
 
 /**
  * Add data to the indexer

diff --git a/include/git2/object.h b/include/git2/object.h
@@ -225,6 +225,7 @@ GIT_EXTERN(int) git_object_peel(
  */
 GIT_EXTERN(int) git_object_dup(git_object **dest, git_object *source);
 
+#ifdef GIT_EXPERIMENTAL_SHA256
 /**
  * Analyzes a buffer of raw object content and determines its validity.
  * Tree, commit, and tag objects will be parsed and ensured that they
@@ -238,14 +239,39 @@ GIT_EXTERN(int) git_object_dup(git_object **dest, git_object *source);
  * @param valid Output pointer to set with validity of the object content
  * @param buf The contents to validate
  * @param len The length of the buffer
- * @param type The type of the object in the buffer
+ * @param object_type The type of the object in the buffer
+ * @param oid_type The object ID type for the OIDs in the given buffer
  * @return 0 on success or an error code
  */
 GIT_EXTERN(int) git_object_rawcontent_is_valid(
 	int *valid,
 	const char *buf,
 	size_t len,
-	git_object_t type);
+	git_object_t object_type,
+	git_oid_t oid_type);
+#else
+/**
+ * Analyzes a buffer of raw object content and determines its validity.
+ * Tree, commit, and tag objects will be parsed and ensured that they
+ * are valid, parseable content.  (Blobs are always valid by definition.)
+ * An error message will be set with an informative message if the object
+ * is not valid.
+ *
+ * @warning This function is experimental and its signature may change in
+ * the future.
+ *
+ * @param valid Output pointer to set with validity of the object content
+ * @param buf The contents to validate
+ * @param len The length of the buffer
+ * @param object_type The type of the object in the buffer
+ * @return 0 on success or an error code
+ */
+GIT_EXTERN(int) git_object_rawcontent_is_valid(
+	int *valid,
+	const char *buf,
+	size_t len,
+	git_object_t object_type);
+#endif
 
 /** @} */
 GIT_END_DECL

diff --git a/src/cli/cmd_hash_object.c b/src/cli/cmd_hash_object.c
@@ -49,26 +49,37 @@ static void print_help(void)
 	cli_opt_help_fprint(stdout, opts);
 }
 
-static int hash_buf(git_odb *odb, git_str *buf, git_object_t type)
+static int hash_buf(
+	git_odb *odb,
+	git_str *buf,
+	git_object_t object_type,
+	git_oid_t oid_type)
 {
 	git_oid oid;
 
 	if (!literally) {
 		int valid = 0;
 
-		if (git_object_rawcontent_is_valid(&valid, buf->ptr, buf->size, type) < 0 || !valid)
+#ifdef GIT_EXPERIMENTAL_SHA256
+		if (git_object_rawcontent_is_valid(&valid, buf->ptr, buf->size, object_type, oid_type) < 0 || !valid)
+			return cli_error_git();
+#else
+		GIT_UNUSED(oid_type);
+
+		if (git_object_rawcontent_is_valid(&valid, buf->ptr, buf->size, object_type) < 0 || !valid)
 			return cli_error_git();
+#endif
 	}
 
 	if (write_object) {
-		if (git_odb_write(&oid, odb, buf->ptr, buf->size, type) < 0)
+		if (git_odb_write(&oid, odb, buf->ptr, buf->size, object_type) < 0)
 			return cli_error_git();
 	} else {
 #ifdef GIT_EXPERIMENTAL_SHA256
-		if (git_odb_hash(&oid, buf->ptr, buf->size, type, GIT_OID_SHA1) < 0)
+		if (git_odb_hash(&oid, buf->ptr, buf->size, object_type, GIT_OID_SHA1) < 0)
 			return cli_error_git();
 #else
-		if (git_odb_hash(&oid, buf->ptr, buf->size, type) < 0)
+		if (git_odb_hash(&oid, buf->ptr, buf->size, object_type) < 0)
 			return cli_error_git();
 #endif
 	}
@@ -83,9 +94,10 @@ int cmd_hash_object(int argc, char **argv)
 {
 	git_repository *repo = NULL;
 	git_odb *odb = NULL;
+	git_oid_t oid_type;
 	git_str buf = GIT_STR_INIT;
 	cli_opt invalid_opt;
-	git_object_t type = GIT_OBJECT_BLOB;
+	git_object_t object_type = GIT_OBJECT_BLOB;
 	char **filename;
 	int ret = 0;
 
@@ -97,7 +109,7 @@ int cmd_hash_object(int argc, char **argv)
 		return 0;
 	}
 
-	if (type_name && (type = git_object_string2type(type_name)) == GIT_OBJECT_INVALID)
+	if (type_name && (object_type = git_object_string2type(type_name)) == GIT_OBJECT_INVALID)
 		return cli_error_usage("invalid object type '%s'", type_name);
 
 	if (write_object &&
@@ -107,6 +119,8 @@ int cmd_hash_object(int argc, char **argv)
 		goto done;
 	}
 
+	oid_type = git_repository_oid_type(repo);
+
 	/*
 	 * TODO: we're reading blobs, we shouldn't pull them all into main
 	 * memory, we should just stream them into the odb instead.
@@ -118,7 +132,7 @@ int cmd_hash_object(int argc, char **argv)
 			goto done;
 		}
 
-		if ((ret = hash_buf(odb, &buf, type)) != 0)
+		if ((ret = hash_buf(odb, &buf, object_type, oid_type)) != 0)
 			goto done;
 	} else {
 		for (filename = filenames; *filename; filename++) {
@@ -127,7 +141,7 @@ int cmd_hash_object(int argc, char **argv)
 				goto done;
 			}
 
-			if ((ret = hash_buf(odb, &buf, type)) != 0)
+			if ((ret = hash_buf(odb, &buf, object_type, oid_type)) != 0)
 				goto done;
 		}
 	}

diff --git a/src/libgit2/blob.c b/src/libgit2/blob.c
@@ -52,23 +52,25 @@ void git_blob__free(void *_blob)
 	git__free(blob);
 }
 
-int git_blob__parse_raw(void *_blob, const char *data, size_t size)
+int git_blob__parse_raw(void *_blob, const char *data, size_t size, git_oid_t oid_type)
 {
 	git_blob *blob = (git_blob *) _blob;
 
 	GIT_ASSERT_ARG(blob);
+	GIT_UNUSED(oid_type);
 
 	blob->raw = 1;
 	blob->data.raw.data = data;
 	blob->data.raw.size = size;
 	return 0;
 }
 
-int git_blob__parse(void *_blob, git_odb_object *odb_obj)
+int git_blob__parse(void *_blob, git_odb_object *odb_obj, git_oid_t oid_type)
 {
 	git_blob *blob = (git_blob *) _blob;
 
 	GIT_ASSERT_ARG(blob);
+	GIT_UNUSED(oid_type);
 
 	git_cached_obj_incref((git_cached_obj *)odb_obj);
 	blob->raw = 0;

diff --git a/src/libgit2/blob.h b/src/libgit2/blob.h
@@ -36,8 +36,8 @@ struct git_blob {
 	} while(0)
 
 void git_blob__free(void *blob);
-int git_blob__parse(void *blob, git_odb_object *obj);
-int git_blob__parse_raw(void *blob, const char *data, size_t size);
+int git_blob__parse(void *blob, git_odb_object *obj, git_oid_t oid_type);
+int git_blob__parse_raw(void *blob, const char *data, size_t size, git_oid_t oid_type);
 int git_blob__getbuf(git_str *buffer, git_blob *blob);
 
 extern int git_blob__create_from_paths(