From eac773d92bccc7d3fc7ce1b18578d374873e0d7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mart=C3=ADn=20Nieto?= Date: Wed, 14 Jan 2015 15:05:43 +0100 Subject: [PATCH 01/99] config: add parsing and getter for paths --- CHANGELOG.md | 4 ++++ include/git2/config.h | 32 ++++++++++++++++++++++++++++++ src/config.c | 41 ++++++++++++++++++++++++++++++++++++++ tests/config/read.c | 46 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 123 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 96bd9a16e90..892a489a820 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ v0.22 + 1 ### API additions +* Parsing and retrieving a configuration value as a path is exposed + via `git_config_parse_path()` and `git_config_get_path()` + respectively. + ### API removals ### Breaking API changes diff --git a/include/git2/config.h b/include/git2/config.h index e32c614eaea..1ed8d244187 100644 --- a/include/git2/config.h +++ b/include/git2/config.h @@ -319,6 +319,24 @@ GIT_EXTERN(int) git_config_get_int64(int64_t *out, const git_config *cfg, const */ GIT_EXTERN(int) git_config_get_bool(int *out, const git_config *cfg, const char *name); +/** + * Get the value of a path config variable. + * + * A leading '~' will be expanded to the global search path (which + * defaults to the user's home directory but can be overridden via + * `git_libgit2_opts()`. + * + * All config files will be looked into, in the order of their + * defined level. A higher level means a higher priority. The + * first occurrence of the variable will be returned here. + * + * @param out the buffer in which to store the result + * @param cfg where to look for the variable + * @param name the variable's name + * @param 0 or an error code + */ +GIT_EXTERN(int) git_config_get_path(git_buf *out, const git_config *cfg, const char *name); + /** * Get the value of a string config variable. * @@ -615,6 +633,20 @@ GIT_EXTERN(int) git_config_parse_int32(int32_t *out, const char *value); */ GIT_EXTERN(int) git_config_parse_int64(int64_t *out, const char *value); +/** + * Parse a string value as a path. + * + * A leading '~' will be expanded to the global search path (which + * defaults to the user's home directory but can be overridden via + * `git_libgit2_opts()`. + * + * If the value does not begin with a tilde, the input will be + * returned. + * + * @param out placae to store the result of parsing + * @param value the path to evaluate + */ +GIT_EXTERN(int) git_config_parse_path(git_buf *out, const char *value); /** * Perform an operation on each config variable in given config backend diff --git a/src/config.c b/src/config.c index 0f8c2446570..f807701381c 100644 --- a/src/config.c +++ b/src/config.c @@ -785,6 +785,17 @@ int git_config_get_bool(int *out, const git_config *cfg, const char *name) return git_config_parse_bool(out, entry->value); } +int git_config_get_path(git_buf *out, const git_config *cfg, const char *name) +{ + const git_config_entry *entry; + int error; + + if ((error = get_entry(&entry, cfg, name, true, GET_ALL_ERRORS)) < 0) + return error; + + return git_config_parse_path(out, entry->value); +} + int git_config_get_string( const char **out, const git_config *cfg, const char *name) { @@ -1184,6 +1195,36 @@ int git_config_parse_int32(int32_t *out, const char *value) return -1; } +int git_config_parse_path(git_buf *out, const char *value) +{ + int error = 0; + const git_buf *home; + + assert(out && value); + + git_buf_sanitize(out); + + if (value[0] == '~') { + if (value[1] != '\0' && value[1] != '/') { + giterr_set(GITERR_CONFIG, "retrieving a homedir by name is not supported"); + return -1; + } + + if ((error = git_sysdir_get(&home, GIT_SYSDIR_GLOBAL)) < 0) + return error; + + git_buf_sets(out, home->ptr); + git_buf_puts(out, value + 1); + + if (git_buf_oom(out)) + return -1; + + return 0; + } + + return git_buf_sets(out, value); +} + /* Take something the user gave us and make it nice for our hash function */ int git_config__normalize_name(const char *in, char **out) { diff --git a/tests/config/read.c b/tests/config/read.c index 25672729f1d..1799970fbfa 100644 --- a/tests/config/read.c +++ b/tests/config/read.c @@ -1,4 +1,6 @@ #include "clar_libgit2.h" +#include "buffer.h" +#include "path.h" void test_config_read__simple_read(void) { @@ -567,3 +569,47 @@ void test_config_read__override_variable(void) git_config_free(cfg); } + +void test_config_read__path(void) +{ + git_config *cfg; + git_buf path = GIT_BUF_INIT; + git_buf old_path = GIT_BUF_INIT; + git_buf home_path = GIT_BUF_INIT; + git_buf expected_path = GIT_BUF_INIT; + + cl_git_pass(p_mkdir("fakehome", 0777)); + cl_git_pass(git_path_prettify(&home_path, "fakehome", NULL)); + cl_git_pass(git_libgit2_opts(GIT_OPT_GET_SEARCH_PATH, GIT_CONFIG_LEVEL_GLOBAL, &old_path)); + cl_git_pass(git_libgit2_opts(GIT_OPT_SET_SEARCH_PATH, GIT_CONFIG_LEVEL_GLOBAL, home_path.ptr)); + cl_git_mkfile("./testconfig", "[some]\n path = ~/somefile"); + cl_git_pass(git_path_join_unrooted(&expected_path, "somefile", home_path.ptr, NULL)); + + cl_git_pass(git_config_open_ondisk(&cfg, "./testconfig")); + cl_git_pass(git_config_get_path(&path, cfg, "some.path")); + cl_assert_equal_s(expected_path.ptr, path.ptr); + git_buf_free(&path); + + cl_git_mkfile("./testconfig", "[some]\n path = ~/"); + cl_git_pass(git_path_join_unrooted(&expected_path, "", home_path.ptr, NULL)); + + cl_git_pass(git_config_get_path(&path, cfg, "some.path")); + cl_assert_equal_s(expected_path.ptr, path.ptr); + git_buf_free(&path); + + cl_git_mkfile("./testconfig", "[some]\n path = ~"); + cl_git_pass(git_buf_sets(&expected_path, home_path.ptr)); + + cl_git_pass(git_config_get_path(&path, cfg, "some.path")); + cl_assert_equal_s(expected_path.ptr, path.ptr); + git_buf_free(&path); + + cl_git_mkfile("./testconfig", "[some]\n path = ~user/foo"); + cl_git_fail(git_config_get_path(&path, cfg, "some.path")); + + cl_git_pass(git_libgit2_opts(GIT_OPT_SET_SEARCH_PATH, GIT_CONFIG_LEVEL_GLOBAL, old_path.ptr)); + git_buf_free(&old_path); + git_buf_free(&home_path); + git_buf_free(&expected_path); + git_config_free(cfg); +} From 7cfbf3b7704da1763ba9f857e1f421b22c5f683a Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Thu, 15 Jan 2015 16:56:58 -0600 Subject: [PATCH 02/99] README: Point to libgit2 account's AppVeyor badge Microsoft is sponsoring a Pro account at AppVeyor for the libgit2 and LibGit2Sharp projects. Point to that account's badge. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 422377ecbdc..664735b9314 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ libgit2 - the Git linkable library ================================== [![Travis Build Status](https://secure.travis-ci.org/libgit2/libgit2.svg?branch=master)](http://travis-ci.org/libgit2/libgit2) -[![AppVeyor Build Status](https://ci.appveyor.com/api/projects/status/gnjsdi9r48cfoveg/branch/master?svg=true)](https://ci.appveyor.com/project/nulltoken/libgit2/branch/master) +[![AppVeyor Build Status](https://ci.appveyor.com/api/projects/status/xvof5b4t5480a2q3/branch/master?svg=true)](https://ci.appveyor.com/project/libgit2/libgit2/branch/master) [![Coverity Scan Build Status](https://scan.coverity.com/projects/639/badge.svg)](https://scan.coverity.com/projects/639) `libgit2` is a portable, pure C implementation of the Git core methods From 526182d28c42bba3ae6b16940438d800f99285d5 Mon Sep 17 00:00:00 2001 From: Jacques Germishuys Date: Tue, 20 Jan 2015 23:24:32 +0200 Subject: [PATCH 03/99] Remove logically dead code (we're already asserting) --- src/repository.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/repository.c b/src/repository.c index 0cf8eb66b61..f49a25124e3 100644 --- a/src/repository.c +++ b/src/repository.c @@ -1836,7 +1836,7 @@ int git_repository_hashfile( */ error = git_path_join_unrooted( - &full_path, path, repo ? git_repository_workdir(repo) : NULL, NULL); + &full_path, path, git_repository_workdir(repo), NULL); if (error < 0) return error; From 9ae4ad2dafd918b95dc976bed97604a985a2b4dd Mon Sep 17 00:00:00 2001 From: Jacques Germishuys Date: Tue, 20 Jan 2015 23:26:55 +0200 Subject: [PATCH 04/99] Treat a NULL string as if it's empty --- src/pool.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pool.c b/src/pool.c index a5dfa0f4955..7350c04c186 100644 --- a/src/pool.c +++ b/src/pool.c @@ -225,7 +225,7 @@ char *git_pool_strcat(git_pool *pool, const char *a, const char *b) void *ptr; size_t len_a, len_b; - assert(pool && a && b && pool->item_size == sizeof(char)); + assert(pool && pool->item_size == sizeof(char)); len_a = a ? strlen(a) : 0; len_b = b ? strlen(b) : 0; From 1d67e8fd0cccef467f4e87f9931896652185dc72 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Fri, 16 Jan 2015 21:28:25 +0000 Subject: [PATCH 05/99] Windows CI: use 32 and 64 bit for AppVeyor builds Add 64 bit and always build with default calling conventions, to avoid trying to build with stdcall on amd64. --- appveyor.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 5c35385561e..0dcfd4dc049 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -2,6 +2,12 @@ version: '{build}' branches: only: - master +environment: + matrix: + - GENERATOR: "Visual Studio 11" + ARCH: 32 + - GENERATOR: "Visual Studio 11 Win64" + ARCH: 64 build_script: - ps: >- choco install cmake @@ -12,7 +18,7 @@ build_script: cd build - cmake -D ENABLE_TRACE=ON -D BUILD_CLAR=ON -D STDCALL=ON .. -G"Visual Studio 11" + cmake -D ENABLE_TRACE=ON -D BUILD_CLAR=ON .. -G"$env:GENERATOR" cmake --build . --config RelWithDebInfo test_script: From 0ad5c845d2c5fb709cc4eb4ef7f9309684f75934 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Sun, 18 Jan 2015 00:47:59 -0600 Subject: [PATCH 06/99] structinit test: show which byte differs --- tests/structinit/structinit.c | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/tests/structinit/structinit.c b/tests/structinit/structinit.c index 0e00ab5a025..620743b9021 100644 --- a/tests/structinit/structinit.c +++ b/tests/structinit/structinit.c @@ -25,11 +25,8 @@ do { \ structname structname##_func_latest; \ int structname##_curr_ver = structver - 1; \ cl_git_pass(funcinitname(&structname##_func_latest, structver)); \ - cl_check_( \ - memcmp(&structname##_macro_latest, &structname##_func_latest, \ - sizeof(structname)) == 0, \ - "Macro-based and function-based initializer for " STRINGIFY(structname) \ - " are not equivalent."); \ + options_cmp(&structname##_macro_latest, &structname##_func_latest, \ + sizeof(structname), STRINGIFY(structname)); \ \ while (structname##_curr_ver > 0) \ { \ @@ -39,6 +36,24 @@ do { \ }\ } while(0) +static void options_cmp(void *one, void *two, size_t size, const char *name) +{ + size_t i; + + for (i = 0; i < size; i++) { + if (((char *)one)[i] != ((char *)two)[i]) { + char desc[1024]; + + p_snprintf(desc, 1024, "Difference in %s at byte %d: macro=%u / func=%u", + name, i, ((char *)one)[i], ((char *)two)[i]); + clar__fail(__FILE__, __LINE__, + "Difference between macro and function options initializer", + desc, 0); + return; + } + } +} + void test_structinit_structinit__compare(void) { /* blame */ From 7c48508b287d1477089db0400ee8f8165f34b054 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Wed, 21 Jan 2015 12:55:17 -0600 Subject: [PATCH 07/99] structinit test: only run on DEBUG builds The structinit tests don't make sense unless structure padding is uniformly initialized, which is unlikely to happen on release builds. Only enable them for DEBUG builds. Further, rename them to core::structinit. --- tests/{structinit => core}/structinit.c | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) rename tests/{structinit => core}/structinit.c (88%) diff --git a/tests/structinit/structinit.c b/tests/core/structinit.c similarity index 88% rename from tests/structinit/structinit.c rename to tests/core/structinit.c index 620743b9021..d064f348e37 100644 --- a/tests/structinit/structinit.c +++ b/tests/core/structinit.c @@ -24,6 +24,7 @@ do { \ structname structname##_macro_latest = macroinit; \ structname structname##_func_latest; \ int structname##_curr_ver = structver - 1; \ + memset(&structname##_func_latest, 0, sizeof(structname##_func_latest)); \ cl_git_pass(funcinitname(&structname##_func_latest, structver)); \ options_cmp(&structname##_macro_latest, &structname##_func_latest, \ sizeof(structname), STRINGIFY(structname)); \ @@ -54,8 +55,22 @@ static void options_cmp(void *one, void *two, size_t size, const char *name) } } -void test_structinit_structinit__compare(void) +void test_core_structinit__compare(void) { + /* These tests assume that they can memcmp() two structures that were + * initialized with the same static initializer. Eg, + * git_blame_options = GIT_BLAME_OPTIONS_INIT; + * + * This assumption fails when there is padding between structure members, + * which is not guaranteed to be initialized to anything sane at all. + * + * Assume most compilers, in a debug build, will clear that memory for + * us or set it to sentinal markers. Etc. + */ +#if !defined(DEBUG) && !defined(_DEBUG) + clar__skip(); +#endif + /* blame */ CHECK_MACRO_FUNC_INIT_EQUAL( \ git_blame_options, GIT_BLAME_OPTIONS_VERSION, \ From 1188803922ded3a4dded62ae2fac0f3300f6bdd9 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Fri, 23 Jan 2015 00:18:42 +0000 Subject: [PATCH 08/99] checkout: free last created directory --- src/checkout.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/checkout.c b/src/checkout.c index 73750f7f620..52a076da6fa 100644 --- a/src/checkout.c +++ b/src/checkout.c @@ -2212,6 +2212,7 @@ static void checkout_data_clear(checkout_data *data) git__free(data->pfx); data->pfx = NULL; + git_buf_free(&data->last_mkdir); git_buf_free(&data->path); git_buf_free(&data->tmp); From fa8ca519ccc6c0821864fcdb203fac92409ee0ec Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Latour Date: Fri, 23 Jan 2015 15:35:29 -0800 Subject: [PATCH 09/99] Allow passing a NULL index to git_repository_set_index() This is supported by the underlying set_index() implementation and setting the repository index to NULL is recommended by the git_repository_set_bare() documentation. --- src/repository.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/repository.c b/src/repository.c index f49a25124e3..30f0cd0f20a 100644 --- a/src/repository.c +++ b/src/repository.c @@ -776,7 +776,7 @@ int git_repository_index(git_index **out, git_repository *repo) void git_repository_set_index(git_repository *repo, git_index *index) { - assert(repo && index); + assert(repo); set_index(repo, index); } From 22b6a92365a95c4497e53e2e08e360e5b241aa98 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Latour Date: Fri, 23 Jan 2015 15:59:54 -0800 Subject: [PATCH 10/99] Fixed typo in git_repository_reinit_filesystem() documentation --- include/git2/sys/repository.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/git2/sys/repository.h b/include/git2/sys/repository.h index dd7b22e062b..74c3b8a2f50 100644 --- a/include/git2/sys/repository.h +++ b/include/git2/sys/repository.h @@ -53,7 +53,7 @@ GIT_EXTERN(void) git_repository__cleanup(git_repository *repo); * * @param repo A repository object * @param recurse_submodules Should submodules be updated recursively - * @returrn 0 on success, < 0 on error + * @return 0 on success, < 0 on error */ GIT_EXTERN(int) git_repository_reinit_filesystem( git_repository *repo, From 86815dca20604147f81618e54032dc194653a668 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Latour Date: Fri, 23 Jan 2015 16:04:23 -0800 Subject: [PATCH 11/99] Make sure sys/repository.h includes the required headers It was missing "common.h" and "types.h" like other system headers. This generated compilation errors if including it directly. --- include/git2/sys/repository.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/include/git2/sys/repository.h b/include/git2/sys/repository.h index dd7b22e062b..082a844b8f3 100644 --- a/include/git2/sys/repository.h +++ b/include/git2/sys/repository.h @@ -7,6 +7,9 @@ #ifndef INCLUDE_sys_git_repository_h__ #define INCLUDE_sys_git_repository_h__ +#include "git2/common.h" +#include "git2/types.h" + /** * @file git2/sys/repository.h * @brief Git repository custom implementation routines From b2a7bcdb54ef0346a9a595100592a139538b3452 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Latour Date: Fri, 23 Jan 2015 20:57:13 -0800 Subject: [PATCH 12/99] Fixed git_repository_set_bare() not setting "core.bare" correctly --- src/repository.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/repository.c b/src/repository.c index f49a25124e3..daf9ee424a7 100644 --- a/src/repository.c +++ b/src/repository.c @@ -1723,7 +1723,7 @@ int git_repository_set_bare(git_repository *repo) if ((error = git_repository_config__weakptr(&config, repo)) < 0) return error; - if ((error = git_config_set_bool(config, "core.bare", false)) < 0) + if ((error = git_config_set_bool(config, "core.bare", true)) < 0) return error; if ((error = git_config__update_entry(config, "core.worktree", NULL, true, true)) < 0) From 3cda6be76b3c7615d1ff7180825bf5f7b1124268 Mon Sep 17 00:00:00 2001 From: Raphael Kubo da Costa Date: Sat, 24 Jan 2015 16:19:43 +0200 Subject: [PATCH 13/99] openssl: Add all required includes for AF_INET6 and in6_addr. This fixes the build at least on FreeBSD, where those types were not defined indirectly: src/openssl_stream.c:100:18: error: variable has incomplete type 'struct in6_addr' struct in6_addr addr6; ^ src/openssl_stream.c:100:9: note: forward declaration of 'struct in6_addr' struct in6_addr addr6; ^ src/openssl_stream.c:111:18: error: use of undeclared identifier 'AF_INET' if (p_inet_pton(AF_INET, host, &addr4)) { ^ src/unix/posix.h:31:40: note: expanded from macro 'p_inet_pton' ^ src/openssl_stream.c:115:18: error: use of undeclared identifier 'AF_INET6' if(p_inet_pton(AF_INET6, host, &addr6)) { ^ src/unix/posix.h:31:40: note: expanded from macro 'p_inet_pton' ^ --- src/openssl_stream.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/openssl_stream.c b/src/openssl_stream.c index 3a6369dee77..2a17a3abfe1 100644 --- a/src/openssl_stream.c +++ b/src/openssl_stream.c @@ -12,6 +12,9 @@ #include #include +#include +#include +#include #include "global.h" #include "posix.h" From dc5fe00c6562e46c88a5c5b975121c202eae1180 Mon Sep 17 00:00:00 2001 From: Boris Egorov Date: Sun, 25 Jan 2015 00:07:23 +0600 Subject: [PATCH 14/99] pathspec: do not try to dereference NULL pathspec_match_free() should not dereference a NULL passed to it. I found this issue when I tried to run example log program with nonexistent branch: ./example/log help Such call leads to segmentation fault. --- src/pathspec.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/pathspec.c b/src/pathspec.c index 8b469f71794..6aef574c87a 100644 --- a/src/pathspec.c +++ b/src/pathspec.c @@ -296,6 +296,9 @@ int git_pathspec_matches_path( static void pathspec_match_free(git_pathspec_match_list *m) { + if (!m) + return; + git_pathspec_free(m->pathspec); m->pathspec = NULL; From b63b3b0e4df81174a323ff6b1b4450014502e9b7 Mon Sep 17 00:00:00 2001 From: Jacques Germishuys Date: Sun, 25 Jan 2015 14:08:05 +0200 Subject: [PATCH 15/99] Ensure git_index_entry is not NULL before trying to free it --- src/index.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/index.c b/src/index.c index 079b0cb6505..cbace3606c3 100644 --- a/src/index.c +++ b/src/index.c @@ -292,6 +292,9 @@ static void index_entry_reuc_free(git_index_reuc_entry *reuc) static void index_entry_free(git_index_entry *entry) { + if (!entry) + return; + memset(&entry->id, 0, sizeof(entry->id)); git__free(entry); } From 636af2198e7d9128d6bf49cd7f7119de62fd5975 Mon Sep 17 00:00:00 2001 From: Jacques Germishuys Date: Sun, 25 Jan 2015 14:38:10 +0200 Subject: [PATCH 16/99] Return early if allocating a git_pathspec_match_list failed --- src/pathspec.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/pathspec.c b/src/pathspec.c index 8b469f71794..97b5d94dc2a 100644 --- a/src/pathspec.c +++ b/src/pathspec.c @@ -315,6 +315,9 @@ static git_pathspec_match_list *pathspec_match_alloc( m = NULL; } + if (!m) + return NULL; + /* need to keep reference to pathspec and increment refcount because * failures array stores pointers to the pattern strings of the * pathspec that had no matches From bb6aafe88f01d2902085fd78464ae277ebea0a67 Mon Sep 17 00:00:00 2001 From: Jacques Germishuys Date: Sun, 25 Jan 2015 14:40:23 +0200 Subject: [PATCH 17/99] Ensure the diff hunk callback is specified before trying to dereference it --- src/diff_patch.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/diff_patch.c b/src/diff_patch.c index 3129d06e1f2..a151077539d 100644 --- a/src/diff_patch.c +++ b/src/diff_patch.c @@ -823,7 +823,8 @@ int git_patch__invoke_callbacks( for (i = 0; !error && i < git_array_size(patch->hunks); ++i) { diff_patch_hunk *h = git_array_get(patch->hunks, i); - error = hunk_cb(patch->delta, &h->hunk, payload); + if (hunk_cb) + error = hunk_cb(patch->delta, &h->hunk, payload); if (!line_cb) continue; From 36e13399c030227d19523d19e3b75073a72cadb3 Mon Sep 17 00:00:00 2001 From: Boris Egorov Date: Sun, 25 Jan 2015 22:34:46 +0600 Subject: [PATCH 18/99] describe example: fix memory allocation size We need to allocate memory for sizeof(char *) * ncommits, not just for ncommits. Issue detected by GCC's AddressSanitizer. --- examples/describe.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/describe.c b/examples/describe.c index 09a4fd008d6..72d98115343 100644 --- a/examples/describe.c +++ b/examples/describe.c @@ -96,7 +96,8 @@ static void parse_options(describe_options *opts, int argc, char **argv) const char *curr = argv[args.pos]; if (curr[0] != '-') { - opts->commits = (const char **)realloc((void *)opts->commits, ++opts->commit_count); + size_t newsz = ++opts->commit_count * sizeof(opts->commits[0]); + opts->commits = (const char **)realloc((void *)opts->commits, newsz); opts->commits[opts->commit_count - 1] = curr; } else if (!strcmp(curr, "--all")) { opts->describe_options.describe_strategy = GIT_DESCRIBE_ALL; @@ -123,7 +124,8 @@ static void parse_options(describe_options *opts, int argc, char **argv) } else { if (!opts->format_options.dirty_suffix || !opts->format_options.dirty_suffix[0]) { - opts->commits = (const char **)malloc(++opts->commit_count); + size_t sz = ++opts->commit_count * sizeof(opts->commits[0]); + opts->commits = (const char **)malloc(sz); opts->commits[0] = "HEAD"; } } From 461fd5a0c75cbe62dcdd3de23b84e27deb3ded8b Mon Sep 17 00:00:00 2001 From: Raphael Kubo da Costa Date: Sat, 24 Jan 2015 16:15:12 +0200 Subject: [PATCH 19/99] Add libssh2's library and include directories. Follow-up to 4f91bfa ("Find libssh2 via pkg-config"): FindPkgConfig sets _INCLUDE_DIRS, not _INCLUDE_DIR like FindLIBSSH2 did. Additionally, if using only FindPkgConfig to find libssh2, we have to call LINK_DIRECTORIES() as well to pass the appropriate -L entries, otherwise the build will only work if libssh2.so is in a directory searched automatically by the linker. --- CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a0f01bbb5d0..704b4597db7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -218,7 +218,8 @@ IF (USE_SSH) ENDIF() IF (LIBSSH2_FOUND) ADD_DEFINITIONS(-DGIT_SSH) - INCLUDE_DIRECTORIES(${LIBSSH2_INCLUDE_DIR}) + INCLUDE_DIRECTORIES(${LIBSSH2_INCLUDE_DIRS}) + LINK_DIRECTORIES(${LIBSSH2_LIBRARY_DIRS}) SET(LIBGIT2_PC_REQUIRES "${LIBGIT2_PC_REQUIRES} libssh2") SET(SSH_LIBRARIES ${LIBSSH2_LIBRARIES}) ENDIF() From 9a294fd8c5c8e2e7d384e852bc33988962e5156b Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Latour Date: Tue, 27 Jan 2015 08:17:23 -0800 Subject: [PATCH 20/99] Clarified git_repository_is_empty() documentation --- include/git2/repository.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/git2/repository.h b/include/git2/repository.h index 124aa67e6fe..2fb381316f3 100644 --- a/include/git2/repository.h +++ b/include/git2/repository.h @@ -342,8 +342,8 @@ GIT_EXTERN(int) git_repository_head_unborn(git_repository *repo); /** * Check if a repository is empty * - * An empty repository has just been initialized and contains - * no references. + * An empty repository has just been initialized and contains no references + * apart from HEAD, which must be pointing to the unborn master branch. * * @param repo Repo to test * @return 1 if the repository is empty, 0 if it isn't, error code From 9dcc4a36f709f013ed111924754021fddfc933e6 Mon Sep 17 00:00:00 2001 From: Linquize Date: Wed, 28 Jan 2015 23:04:50 +0800 Subject: [PATCH 21/99] Fix test failures when 8.3 is disabled --- src/repository.c | 2 +- tests/checkout/nasty.c | 24 +++++++++++++++++++++++- tests/path/win32.c | 20 +++++++++++++++++--- 3 files changed, 41 insertions(+), 5 deletions(-) diff --git a/src/repository.c b/src/repository.c index d87ce730aa9..f338af76c04 100644 --- a/src/repository.c +++ b/src/repository.c @@ -808,7 +808,7 @@ const char *git_repository__8dot3_name(git_repository *repo) /* We anticipate the 8.3 name is "GIT~1", so use a static for * easy testing in the common case */ - if (strcasecmp(repo->name_8dot3, git_repository__8dot3_default) == 0) + if (repo->name_8dot3 && strcasecmp(repo->name_8dot3, git_repository__8dot3_default) == 0) repo->has_8dot3_default = 1; } #endif diff --git a/tests/checkout/nasty.c b/tests/checkout/nasty.c index bc25a3b5233..5084b21181a 100644 --- a/tests/checkout/nasty.c +++ b/tests/checkout/nasty.c @@ -10,6 +10,25 @@ static const char *repo_name = "nasty"; static git_repository *repo; static git_checkout_options checkout_opts; +#ifdef GIT_WIN32 +static bool is_8dot3_disabled(void) +{ +#define IS_8DOT3_BUF_SIZE (MAX_PATH + 20) + char src[IS_8DOT3_BUF_SIZE]; + wchar_t dest[IS_8DOT3_BUF_SIZE], shortPath[IS_8DOT3_BUF_SIZE]; + FILE *fp; + strcpy_s(src, IS_8DOT3_BUF_SIZE, clar_sandbox_path()); + strcat_s(src, IS_8DOT3_BUF_SIZE, "/longer_than_8dot3"); + git__utf8_to_16(dest, IS_8DOT3_BUF_SIZE, src); + fp = _wfopen(dest, L"w"); + cl_assert(fp); + fclose(fp); + cl_assert(GetShortPathNameW(dest, shortPath, IS_8DOT3_BUF_SIZE) > 0); + DeleteFileW(dest); + return !wcscmp(dest, shortPath); +} +#endif + void test_checkout_nasty__initialize(void) { repo = cl_git_sandbox_init(repo_name); @@ -220,7 +239,10 @@ void test_checkout_nasty__git_custom_shortname(void) cl_must_pass(p_rename("nasty/.git", "nasty/_temp")); cl_git_write2file("nasty/git~1", "", 0, O_RDWR|O_CREAT, 0666); cl_must_pass(p_rename("nasty/_temp", "nasty/.git")); - test_checkout_fails("refs/heads/git_tilde2", ".git/foobar"); + if (is_8dot3_disabled()) + test_checkout_passes("refs/heads/git_tilde2", ".git/foobar"); + else + test_checkout_fails("refs/heads/git_tilde2", ".git/foobar"); #endif } diff --git a/tests/path/win32.c b/tests/path/win32.c index 22742f82d5a..63c19e3e11a 100644 --- a/tests/path/win32.c +++ b/tests/path/win32.c @@ -4,6 +4,18 @@ #ifdef GIT_WIN32 #include "win32/path_w32.h" + +static bool is_8dot3_disabled(void) +{ + wchar_t shortPath[MAX_PATH]; + wchar_t *dest = L"longer_than_8dot3"; + FILE *fp = _wfopen(dest, L"w"); + cl_assert(fp); + fclose(fp); + cl_assert(GetShortPathNameW(dest, shortPath, MAX_PATH) > 0); + DeleteFileW(dest); + return !wcscmp(dest, shortPath); +} #endif void test_utf8_to_utf16(const char *utf8_in, const wchar_t *utf16_expected) @@ -193,9 +205,11 @@ void test_path_win32__8dot3_name(void) { #ifdef GIT_WIN32 char *shortname; + bool disable8dot3 = is_8dot3_disabled(); /* Some guaranteed short names */ - cl_assert_equal_s("PROGRA~1", (shortname = git_win32_path_8dot3_name("C:\\Program Files"))); + shortname = git_win32_path_8dot3_name("C:\\Program Files"); + cl_assert(!shortname || !strcmp(shortname, "PROGRA~1")); // null when 8.3 stripped, otherwise in 8.3 format git__free(shortname); cl_assert_equal_s("WINDOWS", (shortname = git_win32_path_8dot3_name("C:\\WINDOWS"))); @@ -203,12 +217,12 @@ void test_path_win32__8dot3_name(void) /* Create some predictible short names */ cl_must_pass(p_mkdir(".foo", 0777)); - cl_assert_equal_s("FOO~1", (shortname = git_win32_path_8dot3_name(".foo"))); + cl_assert_equal_s(disable8dot3 ? ".foo" : "FOO~1", (shortname = git_win32_path_8dot3_name(".foo"))); git__free(shortname); cl_git_write2file("bar~1", "foobar\n", 7, O_RDWR|O_CREAT, 0666); cl_must_pass(p_mkdir(".bar", 0777)); - cl_assert_equal_s("BAR~2", (shortname = git_win32_path_8dot3_name(".bar"))); + cl_assert_equal_s(disable8dot3 ? ".bar" : "BAR~2", (shortname = git_win32_path_8dot3_name(".bar"))); git__free(shortname); #endif } From f4e3a6ab593b5b4870cab0b358718725644c4b82 Mon Sep 17 00:00:00 2001 From: nulltoken Date: Sat, 31 Jan 2015 22:15:10 +0100 Subject: [PATCH 22/99] AppVeyor: Prettify build script --- appveyor.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 0dcfd4dc049..aa3a70eb51b 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -9,17 +9,12 @@ environment: - GENERATOR: "Visual Studio 11 Win64" ARCH: 64 build_script: -- ps: >- +- ps: | choco install cmake - choco install python2 - mkdir build - cd build - cmake -D ENABLE_TRACE=ON -D BUILD_CLAR=ON .. -G"$env:GENERATOR" - cmake --build . --config RelWithDebInfo test_script: - ps: ctest -V . From 2fbce0bfaca90dd71a2cc10a7fa82088150e04b7 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Mon, 2 Feb 2015 22:18:38 -0500 Subject: [PATCH 23/99] checkout test: ensure .gitattributes lifecycle The .gitattributes cache should not reload .gitattributes in the middle of checking out, only between checkout operations. Otherwise, we'll spend all our time stat'ing and read'ing the gitattributes. --- tests/checkout/tree.c | 50 ++++++++++++++++++ .../6f/d5c7dd2ab27b48c493023f794be09861e9045f | 1 + .../c3/6d8ea75da8cb510fcb0c408c1d7e53f9a99dbe | Bin 0 -> 192 bytes .../e3/6900c3224db4adf4c7f7a09d4ac80247978a13 | Bin 0 -> 59 bytes .../testrepo/.gitted/refs/heads/ident | 1 + 5 files changed, 52 insertions(+) create mode 100644 tests/resources/testrepo/.gitted/objects/6f/d5c7dd2ab27b48c493023f794be09861e9045f create mode 100644 tests/resources/testrepo/.gitted/objects/c3/6d8ea75da8cb510fcb0c408c1d7e53f9a99dbe create mode 100644 tests/resources/testrepo/.gitted/objects/e3/6900c3224db4adf4c7f7a09d4ac80247978a13 create mode 100644 tests/resources/testrepo/.gitted/refs/heads/ident diff --git a/tests/checkout/tree.c b/tests/checkout/tree.c index f37e6d35947..42f32a8d93f 100644 --- a/tests/checkout/tree.c +++ b/tests/checkout/tree.c @@ -1130,3 +1130,53 @@ void test_checkout_tree__can_collect_perfdata(void) git_object_free(obj); } + +void update_attr_callback( + const char *path, + size_t completed_steps, + size_t total_steps, + void *payload) +{ + if (path && strcmp(path, "ident1.txt") == 0) + cl_git_write2file("testrepo/.gitattributes", + "*.txt ident\n", 12, O_RDWR|O_CREAT, 0666); +} + +void test_checkout_tree__caches_attributes_during_checkout(void) +{ + git_checkout_options opts = GIT_CHECKOUT_OPTIONS_INIT; + git_oid oid; + git_object *obj = NULL; + git_buf ident1 = GIT_BUF_INIT, ident2 = GIT_BUF_INIT; + char *ident_paths[] = { "ident1.txt", "ident2.txt" }; + + opts.progress_cb = update_attr_callback; + + assert_on_branch(g_repo, "master"); + opts.checkout_strategy = GIT_CHECKOUT_FORCE; + opts.paths.strings = ident_paths; + opts.paths.count = 2; + + cl_git_pass(git_reference_name_to_id(&oid, g_repo, "refs/heads/ident")); + cl_git_pass(git_object_lookup(&obj, g_repo, &oid, GIT_OBJ_ANY)); + + cl_git_pass(git_checkout_tree(g_repo, obj, &opts)); + + cl_git_pass(git_futils_readbuffer(&ident1, "testrepo/ident1.txt")); + cl_git_pass(git_futils_readbuffer(&ident2, "testrepo/ident2.txt")); + + cl_assert_equal_strn(ident1.ptr, "# $Id$", 6); + cl_assert_equal_strn(ident2.ptr, "# $Id$", 6); + + cl_git_pass(git_checkout_tree(g_repo, obj, &opts)); + + cl_git_pass(git_futils_readbuffer(&ident1, "testrepo/ident1.txt")); + cl_git_pass(git_futils_readbuffer(&ident2, "testrepo/ident2.txt")); + + cl_assert_equal_strn(ident1.ptr, "# $Id: ", 7); + cl_assert_equal_strn(ident2.ptr, "# $Id: ", 7); + + git_buf_free(&ident1); + git_buf_free(&ident2); + git_object_free(obj); +} diff --git a/tests/resources/testrepo/.gitted/objects/6f/d5c7dd2ab27b48c493023f794be09861e9045f b/tests/resources/testrepo/.gitted/objects/6f/d5c7dd2ab27b48c493023f794be09861e9045f new file mode 100644 index 00000000000..7f0c6fe4e5a --- /dev/null +++ b/tests/resources/testrepo/.gitted/objects/6f/d5c7dd2ab27b48c493023f794be09861e9045f @@ -0,0 +1 @@ +xA!D}G Mr\m[F11gȢ(GRr3Co"v^hq<7AY{"&$DSg([B!ΡwƳYgl$%Eֲ'\d_w-[k'1hZt B&;:A"m%V \ No newline at end of file diff --git a/tests/resources/testrepo/.gitted/objects/c3/6d8ea75da8cb510fcb0c408c1d7e53f9a99dbe b/tests/resources/testrepo/.gitted/objects/c3/6d8ea75da8cb510fcb0c408c1d7e53f9a99dbe new file mode 100644 index 0000000000000000000000000000000000000000..0975f7fdf227dbb428c5ce60d76b0195eb30008a GIT binary patch literal 192 zcmV;x06+hD0V^p=O;s>5He@g~FfcPQQ3!H%bn$g%SfOmF@NI2De~WFK%%kl}eMcVO zI|fyeRFs&PoDrXvnUktlQc=QSHvO9SOUI?QUN62aDtz+zSJ(!nGf<_ODXDoShG12X zGZ_vm`EFVJ<@omnbG=S5xlivBMpk8nTa}Rk5Gdqi=4Hp13Wl&a7@k~+SQsz=t2+7DSR!GatNmVG%%*jzmN>#{ANzE$( R36-Q4rKad{0RV?s56uB&8Eyao literal 0 HcmV?d00001 diff --git a/tests/resources/testrepo/.gitted/refs/heads/ident b/tests/resources/testrepo/.gitted/refs/heads/ident new file mode 100644 index 00000000000..2cfd880a3d4 --- /dev/null +++ b/tests/resources/testrepo/.gitted/refs/heads/ident @@ -0,0 +1 @@ +6fd5c7dd2ab27b48c493023f794be09861e9045f From 60561d54468d7097e04466fd8125be5231cea637 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Tue, 3 Feb 2015 03:36:07 +0000 Subject: [PATCH 24/99] tests: update for new test data --- tests/refs/list.c | 4 ++-- tests/repo/iterator.c | 3 ++- tests/revwalk/basic.c | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/refs/list.c b/tests/refs/list.c index de5c0fd3da2..374943b052f 100644 --- a/tests/refs/list.c +++ b/tests/refs/list.c @@ -36,7 +36,7 @@ void test_refs_list__all(void) /* We have exactly 12 refs in total if we include the packed ones: * there is a reference that exists both in the packfile and as * loose, but we only list it once */ - cl_assert_equal_i((int)ref_list.count, 14); + cl_assert_equal_i((int)ref_list.count, 15); git_strarray_free(&ref_list); } @@ -51,7 +51,7 @@ void test_refs_list__do_not_retrieve_references_which_name_end_with_a_lock_exten "144344043ba4d4a405da03de3844aa829ae8be0e\n"); cl_git_pass(git_reference_list(&ref_list, g_repo)); - cl_assert_equal_i((int)ref_list.count, 14); + cl_assert_equal_i((int)ref_list.count, 15); git_strarray_free(&ref_list); } diff --git a/tests/repo/iterator.c b/tests/repo/iterator.c index a1aa4b2c7a7..26e8954fe3a 100644 --- a/tests/repo/iterator.c +++ b/tests/repo/iterator.c @@ -906,6 +906,7 @@ void test_repo_iterator__fs2(void) static const char *expect_base[] = { "heads/br2", "heads/dir", + "heads/ident", "heads/long-file-name", "heads/master", "heads/packed-test", @@ -923,7 +924,7 @@ void test_repo_iterator__fs2(void) cl_git_pass(git_iterator_for_filesystem( &i, "testrepo/.git/refs", 0, NULL, NULL)); - expect_iterator_items(i, 12, expect_base, 12, expect_base); + expect_iterator_items(i, 13, expect_base, 13, expect_base); git_iterator_free(i); } diff --git a/tests/revwalk/basic.c b/tests/revwalk/basic.c index ff05fa2496d..4ae95277542 100644 --- a/tests/revwalk/basic.c +++ b/tests/revwalk/basic.c @@ -177,7 +177,7 @@ void test_revwalk_basic__glob_heads_with_invalid(void) /* walking */; /* git log --branches --oneline | wc -l => 16 */ - cl_assert_equal_i(17, i); + cl_assert_equal_i(18, i); } void test_revwalk_basic__push_head(void) From 9f779aacdd950fd53a407da615ca60d628e31d35 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Thu, 29 Jan 2015 14:40:55 -0600 Subject: [PATCH 25/99] attrcache: don't re-read attrs during checkout During checkout, assume that the .gitattributes files aren't modified during the checkout. Instead, create an "attribute session" during checkout. Assume that attribute data read in the same checkout "session" hasn't been modified since the checkout started. (But allow subsequent checkouts to invalidate the cache.) Further, cache nonexistent git_attr_file data even when .gitattributes files are not found to prevent re-scanning for nonexistent files. --- src/attr.c | 66 +++++++++++++++++++++++++++++++++--------------- src/attr_file.c | 57 ++++++++++++++++++++++++++++------------- src/attr_file.h | 44 ++++++++++++++++++++++++-------- src/attrcache.c | 5 ++-- src/attrcache.h | 1 + src/checkout.c | 13 +++++++--- src/filter.c | 29 +++++++++++++++++---- src/filter.h | 10 ++++++++ src/ignore.c | 4 +-- src/repository.h | 2 ++ 10 files changed, 172 insertions(+), 59 deletions(-) diff --git a/src/attr.c b/src/attr.c index a021726891c..df139e0823f 100644 --- a/src/attr.c +++ b/src/attr.c @@ -29,6 +29,7 @@ git_attr_t git_attr_value(const char *attr) static int collect_attr_files( git_repository *repo, + git_attr_session *attr_session, uint32_t flags, const char *path, git_vector *files); @@ -57,7 +58,7 @@ int git_attr_get( if (git_attr_path__init(&path, pathname, git_repository_workdir(repo)) < 0) return -1; - if ((error = collect_attr_files(repo, flags, pathname, &files)) < 0) + if ((error = collect_attr_files(repo, NULL, flags, pathname, &files)) < 0) goto cleanup; memset(&attr, 0, sizeof(attr)); @@ -90,9 +91,10 @@ typedef struct { git_attr_assignment *found; } attr_get_many_info; -int git_attr_get_many( +int git_attr_get_many_with_session( const char **values, git_repository *repo, + git_attr_session *attr_session, uint32_t flags, const char *pathname, size_t num_attr, @@ -115,7 +117,7 @@ int git_attr_get_many( if (git_attr_path__init(&path, pathname, git_repository_workdir(repo)) < 0) return -1; - if ((error = collect_attr_files(repo, flags, pathname, &files)) < 0) + if ((error = collect_attr_files(repo, attr_session, flags, pathname, &files)) < 0) goto cleanup; info = git__calloc(num_attr, sizeof(attr_get_many_info)); @@ -161,6 +163,17 @@ int git_attr_get_many( return error; } +int git_attr_get_many( + const char **values, + git_repository *repo, + uint32_t flags, + const char *pathname, + size_t num_attr, + const char **names) +{ + return git_attr_get_many_with_session( + values, repo, NULL, flags, pathname, num_attr, names); +} int git_attr_foreach( git_repository *repo, @@ -183,7 +196,7 @@ int git_attr_foreach( if (git_attr_path__init(&path, pathname, git_repository_workdir(repo)) < 0) return -1; - if ((error = collect_attr_files(repo, flags, pathname, &files)) < 0 || + if ((error = collect_attr_files(repo, NULL, flags, pathname, &files)) < 0 || (error = git_strmap_alloc(&seen)) < 0) goto cleanup; @@ -219,6 +232,7 @@ int git_attr_foreach( static int preload_attr_file( git_repository *repo, + git_attr_session *attr_session, git_attr_file_source source, const char *base, const char *file) @@ -229,19 +243,22 @@ static int preload_attr_file( if (!file) return 0; if (!(error = git_attr_cache__get( - &preload, repo, source, base, file, git_attr_file__parse_buffer))) + &preload, repo, attr_session, source, base, file, git_attr_file__parse_buffer))) git_attr_file__free(preload); return error; } -static int attr_setup(git_repository *repo) +static int attr_setup(git_repository *repo, git_attr_session *attr_session) { int error = 0; const char *workdir = git_repository_workdir(repo); git_index *idx = NULL; git_buf sys = GIT_BUF_INIT; + if (attr_session && attr_session->setup) + return 0; + if ((error = git_attr_cache__init(repo)) < 0) return error; @@ -251,7 +268,7 @@ static int attr_setup(git_repository *repo) if (!(error = git_sysdir_find_system_file(&sys, GIT_ATTR_FILE_SYSTEM))) { error = preload_attr_file( - repo, GIT_ATTR_FILE__FROM_FILE, NULL, sys.ptr); + repo, attr_session, GIT_ATTR_FILE__FROM_FILE, NULL, sys.ptr); git_buf_free(&sys); } if (error < 0) { @@ -263,25 +280,28 @@ static int attr_setup(git_repository *repo) } if ((error = preload_attr_file( - repo, GIT_ATTR_FILE__FROM_FILE, + repo, attr_session, GIT_ATTR_FILE__FROM_FILE, NULL, git_repository_attr_cache(repo)->cfg_attr_file)) < 0) return error; if ((error = preload_attr_file( - repo, GIT_ATTR_FILE__FROM_FILE, + repo, attr_session, GIT_ATTR_FILE__FROM_FILE, git_repository_path(repo), GIT_ATTR_FILE_INREPO)) < 0) return error; if (workdir != NULL && (error = preload_attr_file( - repo, GIT_ATTR_FILE__FROM_FILE, workdir, GIT_ATTR_FILE)) < 0) + repo, attr_session, GIT_ATTR_FILE__FROM_FILE, workdir, GIT_ATTR_FILE)) < 0) return error; if ((error = git_repository_index__weakptr(&idx, repo)) < 0 || (error = preload_attr_file( - repo, GIT_ATTR_FILE__FROM_INDEX, NULL, GIT_ATTR_FILE)) < 0) + repo, attr_session, GIT_ATTR_FILE__FROM_INDEX, NULL, GIT_ATTR_FILE)) < 0) return error; + if (attr_session) + attr_session->setup = 1; + return error; } @@ -321,6 +341,7 @@ int git_attr_add_macro( typedef struct { git_repository *repo; + git_attr_session *attr_session; uint32_t flags; const char *workdir; git_index *index; @@ -356,6 +377,7 @@ static int attr_decide_sources( static int push_attr_file( git_repository *repo, + git_attr_session *attr_session, git_vector *list, git_attr_file_source source, const char *base, @@ -364,8 +386,9 @@ static int push_attr_file( int error = 0; git_attr_file *file = NULL; - error = git_attr_cache__get( - &file, repo, source, base, filename, git_attr_file__parse_buffer); + error = git_attr_cache__get(&file, repo, attr_session, + source, base, filename, git_attr_file__parse_buffer); + if (error < 0) return error; @@ -387,8 +410,8 @@ static int push_one_attr(void *ref, const char *path) info->flags, info->workdir != NULL, info->index != NULL, src); for (i = 0; !error && i < n_src; ++i) - error = push_attr_file( - info->repo, info->files, src[i], path, GIT_ATTR_FILE); + error = push_attr_file(info->repo, info->attr_session, + info->files, src[i], path, GIT_ATTR_FILE); return error; } @@ -407,6 +430,7 @@ static void release_attr_files(git_vector *files) static int collect_attr_files( git_repository *repo, + git_attr_session *attr_session, uint32_t flags, const char *path, git_vector *files) @@ -416,7 +440,7 @@ static int collect_attr_files( const char *workdir = git_repository_workdir(repo); attr_walk_up_info info = { NULL }; - if ((error = attr_setup(repo)) < 0) + if ((error = attr_setup(repo, attr_session)) < 0) return error; /* Resolve path in a non-bare repo */ @@ -435,12 +459,13 @@ static int collect_attr_files( */ error = push_attr_file( - repo, files, GIT_ATTR_FILE__FROM_FILE, + repo, attr_session, files, GIT_ATTR_FILE__FROM_FILE, git_repository_path(repo), GIT_ATTR_FILE_INREPO); if (error < 0) goto cleanup; - info.repo = repo; + info.repo = repo; + info.attr_session = attr_session; info.flags = flags; info.workdir = workdir; if (git_repository_index__weakptr(&info.index, repo) < 0) @@ -457,7 +482,7 @@ static int collect_attr_files( if (git_repository_attr_cache(repo)->cfg_attr_file != NULL) { error = push_attr_file( - repo, files, GIT_ATTR_FILE__FROM_FILE, + repo, attr_session, files, GIT_ATTR_FILE__FROM_FILE, NULL, git_repository_attr_cache(repo)->cfg_attr_file); if (error < 0) goto cleanup; @@ -467,7 +492,8 @@ static int collect_attr_files( error = git_sysdir_find_system_file(&dir, GIT_ATTR_FILE_SYSTEM); if (!error) error = push_attr_file( - repo, files, GIT_ATTR_FILE__FROM_FILE, NULL, dir.ptr); + repo, attr_session, files, GIT_ATTR_FILE__FROM_FILE, + NULL, dir.ptr); else if (error == GIT_ENOTFOUND) { giterr_clear(); error = 0; diff --git a/src/attr_file.c b/src/attr_file.c index 5b008b0e32e..9896df4f9b6 100644 --- a/src/attr_file.c +++ b/src/attr_file.c @@ -96,6 +96,7 @@ static int attr_file_oid_from_index( int git_attr_file__load( git_attr_file **out, git_repository *repo, + git_attr_session *attr_session, git_attr_file_entry *entry, git_attr_file_source source, git_attr_file_parser parser) @@ -105,6 +106,7 @@ int git_attr_file__load( git_buf content = GIT_BUF_INIT; git_attr_file *file; struct stat st; + bool nonexistent = false; *out = NULL; @@ -127,22 +129,16 @@ int git_attr_file__load( case GIT_ATTR_FILE__FROM_FILE: { int fd; - if (p_stat(entry->fullpath, &st) < 0) - return git_path_set_error(errno, entry->fullpath, "stat"); - if (S_ISDIR(st.st_mode)) - return GIT_ENOTFOUND; - - /* For open or read errors, return ENOTFOUND to skip item */ + /* For open or read errors, pretend that we got ENOTFOUND. */ /* TODO: issue warning when warning API is available */ - if ((fd = git_futils_open_ro(entry->fullpath)) < 0) - return GIT_ENOTFOUND; - - error = git_futils_readbuffer_fd(&content, fd, (size_t)st.st_size); - p_close(fd); - - if (error < 0) - return GIT_ENOTFOUND; + if (p_stat(entry->fullpath, &st) < 0 || + S_ISDIR(st.st_mode) || + (fd = git_futils_open_ro(entry->fullpath)) < 0 || + (error = git_futils_readbuffer_fd(&content, fd, (size_t)st.st_size)) < 0) + nonexistent = true; + else + p_close(fd); break; } @@ -154,13 +150,21 @@ int git_attr_file__load( if ((error = git_attr_file__new(&file, entry, source)) < 0) goto cleanup; + /* store the key of the attr_reader; don't bother with cache + * invalidation during the same attr reader session. + */ + if (attr_session) + file->session_key = attr_session->key; + if (parser && (error = parser(repo, file, git_buf_cstr(&content))) < 0) { git_attr_file__free(file); goto cleanup; } - /* write cache breaker */ - if (source == GIT_ATTR_FILE__FROM_INDEX) + /* write cache breakers */ + if (nonexistent) + file->nonexistent = 1; + else if (source == GIT_ATTR_FILE__FROM_INDEX) git_oid_cpy(&file->cache_data.oid, git_blob_id(blob)); else if (source == GIT_ATTR_FILE__FROM_FILE) git_futils_filestamp_set_from_stat(&file->cache_data.stamp, &st); @@ -175,11 +179,22 @@ int git_attr_file__load( return error; } -int git_attr_file__out_of_date(git_repository *repo, git_attr_file *file) +int git_attr_file__out_of_date( + git_repository *repo, + git_attr_session *attr_session, + git_attr_file *file) { if (!file) return 1; + /* we are never out of date if we just created this data in the same + * attr_session; otherwise, nonexistent files must be invalidated + */ + if (attr_session && attr_session->key == file->session_key) + return 0; + else if (file->nonexistent) + return 1; + switch (file->source) { case GIT_ATTR_FILE__IN_MEMORY: return 0; @@ -831,3 +846,11 @@ void git_attr_rule__free(git_attr_rule *rule) git__free(rule); } +int git_attr_session__init(git_attr_session *session, git_repository *repo) +{ + assert(repo); + + session->key = git_atomic_inc(&repo->attr_session_key); + + return 0; +} diff --git a/src/attr_file.h b/src/attr_file.h index 93de84d12c1..b8e37256b60 100644 --- a/src/attr_file.h +++ b/src/attr_file.h @@ -38,11 +38,11 @@ GIT_ATTR_FNMATCH_ALLOWMACRO | GIT_ATTR_FNMATCH_NOLEADINGDIR) typedef enum { - GIT_ATTR_FILE__IN_MEMORY = 0, - GIT_ATTR_FILE__FROM_FILE = 1, - GIT_ATTR_FILE__FROM_INDEX = 2, + GIT_ATTR_FILE__IN_MEMORY = 0, + GIT_ATTR_FILE__FROM_FILE = 1, + GIT_ATTR_FILE__FROM_INDEX = 2, - GIT_ATTR_FILE_NUM_SOURCES = 3 + GIT_ATTR_FILE_NUM_SOURCES = 3 } git_attr_file_source; extern const char *git_attr__true; @@ -84,6 +84,8 @@ typedef struct { git_attr_file_source source; git_vector rules; /* vector of or */ git_pool pool; + unsigned int nonexistent:1; + int session_key; union { git_oid oid; git_futils_filestamp stamp; @@ -96,11 +98,6 @@ struct git_attr_file_entry { char fullpath[GIT_FLEX_ARRAY]; }; -typedef int (*git_attr_file_parser)( - git_repository *repo, - git_attr_file *file, - const char *data); - typedef struct { git_buf full; char *path; @@ -108,6 +105,32 @@ typedef struct { int is_dir; } git_attr_path; +/* A git_attr_session can provide an "instance" of reading, to prevent cache + * invalidation during a single operation instance (like checkout). + */ + +typedef struct { + int key; + unsigned int setup; +} git_attr_session; + +extern int git_attr_session__init(git_attr_session *attr_session, git_repository *repo); +extern void git_attr_session__free(git_attr_session *session); + +extern int git_attr_get_many_with_session( + const char **values_out, + git_repository *repo, + git_attr_session *attr_session, + uint32_t flags, + const char *path, + size_t num_attr, + const char **names); + +typedef int (*git_attr_file_parser)( + git_repository *repo, + git_attr_file *file, + const char *data); + /* * git_attr_file API */ @@ -122,6 +145,7 @@ void git_attr_file__free(git_attr_file *file); int git_attr_file__load( git_attr_file **out, git_repository *repo, + git_attr_session *attr_session, git_attr_file_entry *ce, git_attr_file_source source, git_attr_file_parser parser); @@ -130,7 +154,7 @@ int git_attr_file__load_standalone( git_attr_file **out, const char *path); int git_attr_file__out_of_date( - git_repository *repo, git_attr_file *file); + git_repository *repo, git_attr_session *session, git_attr_file *file); int git_attr_file__parse_buffer( git_repository *repo, git_attr_file *attrs, const char *data); diff --git a/src/attrcache.c b/src/attrcache.c index b4579bfc06b..f75edad68ee 100644 --- a/src/attrcache.c +++ b/src/attrcache.c @@ -196,6 +196,7 @@ static int attr_cache_lookup( int git_attr_cache__get( git_attr_file **out, git_repository *repo, + git_attr_session *attr_session, git_attr_file_source source, const char *base, const char *filename, @@ -211,8 +212,8 @@ int git_attr_cache__get( return error; /* load file if we don't have one or if existing one is out of date */ - if (!file || (error = git_attr_file__out_of_date(repo, file)) > 0) - error = git_attr_file__load(&updated, repo, entry, source, parser); + if (!file || (error = git_attr_file__out_of_date(repo, attr_session, file)) > 0) + error = git_attr_file__load(&updated, repo, attr_session, entry, source, parser); /* if we loaded the file, insert into and/or update cache */ if (updated) { diff --git a/src/attrcache.h b/src/attrcache.h index be0a22f5c0d..44e1ffdce28 100644 --- a/src/attrcache.h +++ b/src/attrcache.h @@ -31,6 +31,7 @@ extern int git_attr_cache__do_init(git_repository *repo); extern int git_attr_cache__get( git_attr_file **file, git_repository *repo, + git_attr_session *attr_session, git_attr_file_source source, const char *base, const char *filename, diff --git a/src/checkout.c b/src/checkout.c index 52a076da6fa..04493a3cbb5 100644 --- a/src/checkout.c +++ b/src/checkout.c @@ -28,6 +28,7 @@ #include "buf_text.h" #include "merge_file.h" #include "path.h" +#include "attr.h" /* See docs/checkout-internals.md for more information */ @@ -69,6 +70,7 @@ typedef struct { size_t completed_steps; git_checkout_perfdata perfdata; git_buf last_mkdir; + git_attr_session attr_session; } checkout_data; typedef struct { @@ -1425,8 +1427,8 @@ static int blob_content_to_file( hint_path = path; if (!data->opts.disable_filters) - error = git_filter_list_load( - &fl, git_blob_owner(blob), blob, hint_path, + error = git_filter_list__load_with_attr_session( + &fl, data->repo, &data->attr_session, blob, hint_path, GIT_FILTER_TO_WORKTREE, GIT_FILTER_OPT_DEFAULT); if (!error) @@ -2008,7 +2010,8 @@ static int checkout_write_merge( in_data.ptr = (char *)result.ptr; in_data.size = result.len; - if ((error = git_filter_list_load(&fl, data->repo, NULL, git_buf_cstr(&path_workdir), + if ((error = git_filter_list__load_with_attr_session( + &fl, data->repo, &data->attr_session, NULL, git_buf_cstr(&path_workdir), GIT_FILTER_TO_WORKTREE, GIT_FILTER_OPT_DEFAULT)) < 0 || (error = git_filter_list_apply_to_data(&out_data, fl, &in_data)) < 0) goto done; @@ -2218,6 +2221,8 @@ static void checkout_data_clear(checkout_data *data) git_index_free(data->index); data->index = NULL; + + git_attr_session__free(&data->attr_session); } static int checkout_data_init( @@ -2360,6 +2365,8 @@ static int checkout_data_init( data->workdir_len = git_buf_len(&data->path); + git_attr_session__init(&data->attr_session, data->repo); + cleanup: if (error < 0) checkout_data_clear(data); diff --git a/src/filter.c b/src/filter.c index b5a8bdd660b..22eaf51a28e 100644 --- a/src/filter.c +++ b/src/filter.c @@ -394,15 +394,19 @@ static int filter_list_new( } static int filter_list_check_attributes( - const char ***out, git_filter_def *fdef, const git_filter_source *src) + const char ***out, + git_repository *repo, + git_attr_session *attr_session, + git_filter_def *fdef, + const git_filter_source *src) { int error; size_t i; const char **strs = git__calloc(fdef->nattrs, sizeof(const char *)); GITERR_CHECK_ALLOC(strs); - error = git_attr_get_many( - strs, src->repo, 0, src->path, fdef->nattrs, fdef->attrs); + error = git_attr_get_many_with_session( + strs, repo, attr_session, 0, src->path, fdef->nattrs, fdef->attrs); /* if no values were found but no matches are needed, it's okay! */ if (error == GIT_ENOTFOUND && !fdef->nmatches) { @@ -448,9 +452,10 @@ int git_filter_list_new( return filter_list_new(out, &src); } -int git_filter_list_load( +int git_filter_list__load_with_attr_session( git_filter_list **filters, git_repository *repo, + git_attr_session *attr_session, git_blob *blob, /* can be NULL */ const char *path, git_filter_mode_t mode, @@ -481,7 +486,9 @@ int git_filter_list_load( continue; if (fdef->nattrs > 0) { - error = filter_list_check_attributes(&values, fdef, &src); + error = filter_list_check_attributes( + &values, repo, attr_session, fdef, &src); + if (error == GIT_ENOTFOUND) { error = 0; continue; @@ -523,6 +530,18 @@ int git_filter_list_load( return error; } +int git_filter_list_load( + git_filter_list **filters, + git_repository *repo, + git_blob *blob, /* can be NULL */ + const char *path, + git_filter_mode_t mode, + uint32_t options) +{ + return git_filter_list__load_with_attr_session( + filters, repo, NULL, blob, path, mode, options); +} + void git_filter_list_free(git_filter_list *fl) { uint32_t i; diff --git a/src/filter.h b/src/filter.h index 5a366108b41..390ffebad6a 100644 --- a/src/filter.h +++ b/src/filter.h @@ -8,6 +8,7 @@ #define INCLUDE_filter_h__ #include "common.h" +#include "attr_file.h" #include "git2/filter.h" /* Amount of file to examine for NUL byte when checking binary-ness */ @@ -25,6 +26,15 @@ typedef enum { extern void git_filter_free(git_filter *filter); +extern int git_filter_list__load_with_attr_session( + git_filter_list **filters, + git_repository *repo, + git_attr_session *attr_session, + git_blob *blob, /* can be NULL */ + const char *path, + git_filter_mode_t mode, + uint32_t options); + /* * Available filters */ diff --git a/src/ignore.c b/src/ignore.c index 6e00f7db5f9..dd299f076a3 100644 --- a/src/ignore.c +++ b/src/ignore.c @@ -161,7 +161,7 @@ static int push_ignore_file( git_attr_file *file = NULL; error = git_attr_cache__get( - &file, ignores->repo, GIT_ATTR_FILE__FROM_FILE, + &file, ignores->repo, NULL, GIT_ATTR_FILE__FROM_FILE, base, filename, parse_ignore_file); if (error < 0) return error; @@ -189,7 +189,7 @@ static int get_internal_ignores(git_attr_file **out, git_repository *repo) return error; error = git_attr_cache__get( - out, repo, GIT_ATTR_FILE__IN_MEMORY, NULL, GIT_IGNORE_INTERNAL, NULL); + out, repo, NULL, GIT_ATTR_FILE__IN_MEMORY, NULL, GIT_IGNORE_INTERNAL, NULL); /* if internal rules list is empty, insert default rules */ if (!error && !(*out)->rules.length) diff --git a/src/repository.h b/src/repository.h index 6da8c289b57..dffa9a8aee8 100644 --- a/src/repository.h +++ b/src/repository.h @@ -133,6 +133,8 @@ struct git_repository { has_8dot3_default:1; unsigned int lru_counter; + git_atomic attr_session_key; + git_cvar_value cvar_cache[GIT_CVAR_CACHE_MAX]; }; From 1589a93aa6c7239423201465ceaddf461e98ac26 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Mon, 2 Feb 2015 16:50:10 -0500 Subject: [PATCH 26/99] Fix branch creation when branch name matches namespace of previously deleted branch --- src/refdb_fs.c | 9 +++ tests/refs/branches/create.c | 104 +++++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+) diff --git a/src/refdb_fs.c b/src/refdb_fs.c index fc41a95d780..5cd2112fc0b 100644 --- a/src/refdb_fs.c +++ b/src/refdb_fs.c @@ -1771,6 +1771,15 @@ static int reflog_append(refdb_fs_backend *backend, const git_reference *ref, co goto cleanup; } + /* If the new branch matches part of the namespace of a previously deleted branch, + * there maybe an obsolete/unused directory (or directory hierarchy) in the way. + */ + if (git_path_isdir(git_buf_cstr(&path)) && + (git_futils_rmdir_r(git_buf_cstr(&path), NULL, GIT_RMDIR_SKIP_NONEMPTY) < 0)) { + error = -1; + goto cleanup; + } + error = git_futils_writebuffer(&buf, git_buf_cstr(&path), O_WRONLY|O_CREAT|O_APPEND, GIT_REFLOG_FILE_MODE); cleanup: diff --git a/tests/refs/branches/create.c b/tests/refs/branches/create.c index 3a4f33b6ebf..af9963bde4c 100644 --- a/tests/refs/branches/create.c +++ b/tests/refs/branches/create.c @@ -196,3 +196,107 @@ void test_refs_branches_create__can_create_branch_with_unicode(void) branch = NULL; } } + +/** + * Verify that we can create a branch with a name that matches the + * namespace of a previously delete branch. + * + * git branch level_one/level_two + * git branch -D level_one/level_two + * git branch level_one + * + * We expect the delete to have deleted the files: + * ".git/refs/heads/level_one/level_two" + * ".git/logs/refs/heads/level_one/level_two" + * It may or may not have deleted the (now empty) + * containing directories. To match git.git behavior, + * the second create needs to implicilty delete the + * directories and create the new files. + * "refs/heads/level_one" + * "logs/refs/heads/level_one" + * + * We should not fail to create the branch or its + * reflog because of an obsolete namespace container + * directory. + */ +void test_refs_branches_create__name_vs_namespace(void) +{ + const char * name; + struct item { + const char *first; + const char *second; + }; + static const struct item item[] = { + { "level_one/level_two", "level_one" }, + { "a/b/c/d/e", "a/b/c/d" }, + { "ss/tt/uu/vv/ww", "ss" }, + /* And one test case that is deeper. */ + { "xx1/xx2/xx3/xx4", "xx1/xx2/xx3/xx4/xx5/xx6" }, + { NULL, NULL }, + }; + const struct item *p; + + retrieve_known_commit(&target, repo); + + for (p=item; p->first; p++) { + cl_git_pass(git_branch_create(&branch, repo, p->first, target, 0, NULL, NULL)); + cl_git_pass(git_oid_cmp(git_reference_target(branch), git_commit_id(target))); + cl_git_pass(git_branch_name(&name, branch)); + cl_assert_equal_s(name, p->first); + + cl_git_pass(git_branch_delete(branch)); + git_reference_free(branch); + branch = NULL; + + cl_git_pass(git_branch_create(&branch, repo, p->second, target, 0, NULL, NULL)); + git_reference_free(branch); + branch = NULL; + } +} + +/** + * We still need to fail if part of the namespace is + * still in use. + */ +void test_refs_branches_create__name_vs_namespace_fail(void) +{ + const char * name; + struct item { + const char *first; + const char *first_alternate; + const char *second; + }; + static const struct item item[] = { + { "level_one/level_two", "level_one/alternate", "level_one" }, + { "a/b/c/d/e", "a/b/c/d/alternate", "a/b/c/d" }, + { "ss/tt/uu/vv/ww", "ss/alternate", "ss" }, + { NULL, NULL, NULL }, + }; + const struct item *p; + + retrieve_known_commit(&target, repo); + + for (p=item; p->first; p++) { + cl_git_pass(git_branch_create(&branch, repo, p->first, target, 0, NULL, NULL)); + cl_git_pass(git_oid_cmp(git_reference_target(branch), git_commit_id(target))); + cl_git_pass(git_branch_name(&name, branch)); + cl_assert_equal_s(name, p->first); + + cl_git_pass(git_branch_delete(branch)); + git_reference_free(branch); + branch = NULL; + + cl_git_pass(git_branch_create(&branch, repo, p->first_alternate, target, 0, NULL, NULL)); + cl_git_pass(git_oid_cmp(git_reference_target(branch), git_commit_id(target))); + cl_git_pass(git_branch_name(&name, branch)); + cl_assert_equal_s(name, p->first_alternate); + + /* we do not delete the alternate. */ + git_reference_free(branch); + branch = NULL; + + cl_git_fail(git_branch_create(&branch, repo, p->second, target, 0, NULL, NULL)); + git_reference_free(branch); + branch = NULL; + } +} From 5f28ec84a10b283f79f027f83ba03d7774a987f7 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Tue, 3 Feb 2015 12:16:11 -0500 Subject: [PATCH 27/99] win32: cleanup 8.3 disabled tests --- src/repository.c | 3 ++- tests/checkout/nasty.c | 30 +++++++----------------------- tests/path/win32.c | 23 +++++++---------------- 3 files changed, 16 insertions(+), 40 deletions(-) diff --git a/src/repository.c b/src/repository.c index f338af76c04..433754489f1 100644 --- a/src/repository.c +++ b/src/repository.c @@ -808,7 +808,8 @@ const char *git_repository__8dot3_name(git_repository *repo) /* We anticipate the 8.3 name is "GIT~1", so use a static for * easy testing in the common case */ - if (repo->name_8dot3 && strcasecmp(repo->name_8dot3, git_repository__8dot3_default) == 0) + if (repo->name_8dot3 && + strcasecmp(repo->name_8dot3, git_repository__8dot3_default) == 0) repo->has_8dot3_default = 1; } #endif diff --git a/tests/checkout/nasty.c b/tests/checkout/nasty.c index 5084b21181a..08e7628f5d0 100644 --- a/tests/checkout/nasty.c +++ b/tests/checkout/nasty.c @@ -10,25 +10,6 @@ static const char *repo_name = "nasty"; static git_repository *repo; static git_checkout_options checkout_opts; -#ifdef GIT_WIN32 -static bool is_8dot3_disabled(void) -{ -#define IS_8DOT3_BUF_SIZE (MAX_PATH + 20) - char src[IS_8DOT3_BUF_SIZE]; - wchar_t dest[IS_8DOT3_BUF_SIZE], shortPath[IS_8DOT3_BUF_SIZE]; - FILE *fp; - strcpy_s(src, IS_8DOT3_BUF_SIZE, clar_sandbox_path()); - strcat_s(src, IS_8DOT3_BUF_SIZE, "/longer_than_8dot3"); - git__utf8_to_16(dest, IS_8DOT3_BUF_SIZE, src); - fp = _wfopen(dest, L"w"); - cl_assert(fp); - fclose(fp); - cl_assert(GetShortPathNameW(dest, shortPath, IS_8DOT3_BUF_SIZE) > 0); - DeleteFileW(dest); - return !wcscmp(dest, shortPath); -} -#endif - void test_checkout_nasty__initialize(void) { repo = cl_git_sandbox_init(repo_name); @@ -236,13 +217,16 @@ void test_checkout_nasty__git_tilde1(void) void test_checkout_nasty__git_custom_shortname(void) { #ifdef GIT_WIN32 + char *shortname = git_win32_path_8dot3_name("C:\\Program Files"); + if (shortname == NULL) + clar__skip(); + + git__free(shortname); + cl_must_pass(p_rename("nasty/.git", "nasty/_temp")); cl_git_write2file("nasty/git~1", "", 0, O_RDWR|O_CREAT, 0666); cl_must_pass(p_rename("nasty/_temp", "nasty/.git")); - if (is_8dot3_disabled()) - test_checkout_passes("refs/heads/git_tilde2", ".git/foobar"); - else - test_checkout_fails("refs/heads/git_tilde2", ".git/foobar"); + test_checkout_fails("refs/heads/git_tilde2", ".git/foobar"); #endif } diff --git a/tests/path/win32.c b/tests/path/win32.c index 63c19e3e11a..41831a88c1b 100644 --- a/tests/path/win32.c +++ b/tests/path/win32.c @@ -4,18 +4,6 @@ #ifdef GIT_WIN32 #include "win32/path_w32.h" - -static bool is_8dot3_disabled(void) -{ - wchar_t shortPath[MAX_PATH]; - wchar_t *dest = L"longer_than_8dot3"; - FILE *fp = _wfopen(dest, L"w"); - cl_assert(fp); - fclose(fp); - cl_assert(GetShortPathNameW(dest, shortPath, MAX_PATH) > 0); - DeleteFileW(dest); - return !wcscmp(dest, shortPath); -} #endif void test_utf8_to_utf16(const char *utf8_in, const wchar_t *utf16_expected) @@ -205,11 +193,14 @@ void test_path_win32__8dot3_name(void) { #ifdef GIT_WIN32 char *shortname; - bool disable8dot3 = is_8dot3_disabled(); /* Some guaranteed short names */ shortname = git_win32_path_8dot3_name("C:\\Program Files"); - cl_assert(!shortname || !strcmp(shortname, "PROGRA~1")); // null when 8.3 stripped, otherwise in 8.3 format + + if (shortname == NULL) + clar__skip(); + + cl_assert_equal_s("PROGRA~1", shortname); git__free(shortname); cl_assert_equal_s("WINDOWS", (shortname = git_win32_path_8dot3_name("C:\\WINDOWS"))); @@ -217,12 +208,12 @@ void test_path_win32__8dot3_name(void) /* Create some predictible short names */ cl_must_pass(p_mkdir(".foo", 0777)); - cl_assert_equal_s(disable8dot3 ? ".foo" : "FOO~1", (shortname = git_win32_path_8dot3_name(".foo"))); + cl_assert_equal_s("FOO~1", (shortname = git_win32_path_8dot3_name(".foo"))); git__free(shortname); cl_git_write2file("bar~1", "foobar\n", 7, O_RDWR|O_CREAT, 0666); cl_must_pass(p_mkdir(".bar", 0777)); - cl_assert_equal_s(disable8dot3 ? ".bar" : "BAR~2", (shortname = git_win32_path_8dot3_name(".bar"))); + cl_assert_equal_s("BAR~2", (shortname = git_win32_path_8dot3_name(".bar"))); git__free(shortname); #endif } From 3538f8f131e8ebe597c1172b381bb260ac66f1c0 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Tue, 3 Feb 2015 13:41:35 -0500 Subject: [PATCH 28/99] diff docs: update `git_diff_delta` description --- include/git2/diff.h | 6 +++--- include/git2/patch.h | 9 +++------ 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/include/git2/diff.h b/include/git2/diff.h index 4403944f4f7..9fcc3bb0867 100644 --- a/include/git2/diff.h +++ b/include/git2/diff.h @@ -849,9 +849,9 @@ GIT_EXTERN(size_t) git_diff_num_deltas_of_type( /** * Return the diff delta for an entry in the diff list. * - * The `git_delta` pointer points to internal data and you do not have - * to release it when you are done with it. It will go away when the - * `git_diff` (or any associated `git_patch`) goes away. + * The `git_diff_delta` pointer points to internal data and you do not + * have to release it when you are done with it. It will go away when + * the * `git_diff` (or any associated `git_patch`) goes away. * * Note that the flags on the delta related to whether it has binary * content or not may not be set if there are no attributes set for the diff --git a/include/git2/patch.h b/include/git2/patch.h index 47c3956690a..790cb74fcce 100644 --- a/include/git2/patch.h +++ b/include/git2/patch.h @@ -29,7 +29,7 @@ GIT_BEGIN_DECL typedef struct git_patch git_patch; /** - * Return the diff delta and patch for an entry in the diff list. + * Return a patch for an entry in the diff list. * * The `git_patch` is a newly created object contains the text diffs * for the delta. You have to call `git_patch_free()` when you are @@ -40,10 +40,6 @@ typedef struct git_patch git_patch; * created, the output will be set to NULL, and the `binary` flag will be * set true in the `git_diff_delta` structure. * - * The `git_diff_delta` pointer points to internal data and you do not have - * to release it when you are done with it. It will go away when the - * `git_diff` and `git_patch` go away. - * * It is okay to pass NULL for either of the output parameters; if you pass * NULL for the `git_patch`, then the text diff will not be calculated. * @@ -139,7 +135,8 @@ GIT_EXTERN(int) git_patch_from_buffers( GIT_EXTERN(void) git_patch_free(git_patch *patch); /** - * Get the delta associated with a patch + * Get the delta associated with a patch. This delta points to internal + * data and you do not have to release it when you are done with it. */ GIT_EXTERN(const git_diff_delta *) git_patch_get_delta(const git_patch *patch); From e15a0849ebc9c2f492c043080fd6e50373517cc8 Mon Sep 17 00:00:00 2001 From: "Yury G. Kudryashov" Date: Wed, 4 Feb 2015 00:15:11 +0300 Subject: [PATCH 29/99] Tell user if libssh was not found Fixes #2665 --- CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index a0f01bbb5d0..7db18c78493 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -221,6 +221,8 @@ IF (LIBSSH2_FOUND) INCLUDE_DIRECTORIES(${LIBSSH2_INCLUDE_DIR}) SET(LIBGIT2_PC_REQUIRES "${LIBGIT2_PC_REQUIRES} libssh2") SET(SSH_LIBRARIES ${LIBSSH2_LIBRARIES}) +ELSE() + MESSAGE(STATUS "LIBSSH2 not found. Set CMAKE_PREFIX_PATH if it is installed outside of the default search path.") ENDIF() # Optional external dependency: libgssapi From 710f66af351abb43e43e27ac23cb488babbaa04e Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Tue, 3 Feb 2015 18:41:15 -0500 Subject: [PATCH 30/99] bswap: remove unused bswap --- src/bswap.h | 97 ---------------------------------------------------- src/common.h | 1 - 2 files changed, 98 deletions(-) delete mode 100644 src/bswap.h diff --git a/src/bswap.h b/src/bswap.h deleted file mode 100644 index 486df82f4ce..00000000000 --- a/src/bswap.h +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright (C) the libgit2 contributors. All rights reserved. - * - * This file is part of libgit2, distributed under the GNU GPL v2 with - * a Linking Exception. For full terms see the included COPYING file. - */ - -#include "common.h" - -/* - * Default version that the compiler ought to optimize properly with - * constant values. - */ -GIT_INLINE(uint32_t) default_swab32(uint32_t val) -{ - return (((val & 0xff000000) >> 24) | - ((val & 0x00ff0000) >> 8) | - ((val & 0x0000ff00) << 8) | - ((val & 0x000000ff) << 24)); -} - -#undef bswap32 - -GIT_INLINE(uint16_t) default_swab16(uint16_t val) -{ - return (((val & 0xff00) >> 8) | - ((val & 0x00ff) << 8)); -} - -#undef bswap16 - -#if defined(__GNUC__) && defined(__i386__) - -#define bswap32(x) ({ \ - uint32_t __res; \ - if (__builtin_constant_p(x)) { \ - __res = default_swab32(x); \ - } else { \ - __asm__("bswap %0" : "=r" (__res) : "0" ((uint32_t)(x))); \ - } \ - __res; }) - -#define bswap16(x) ({ \ - uint16_t __res; \ - if (__builtin_constant_p(x)) { \ - __res = default_swab16(x); \ - } else { \ - __asm__("xchgb %b0,%h0" : "=q" (__res) : "0" ((uint16_t)(x))); \ - } \ - __res; }) - -#elif defined(__GNUC__) && defined(__x86_64__) - -#define bswap32(x) ({ \ - uint32_t __res; \ - if (__builtin_constant_p(x)) { \ - __res = default_swab32(x); \ - } else { \ - __asm__("bswapl %0" : "=r" (__res) : "0" ((uint32_t)(x))); \ - } \ - __res; }) - -#define bswap16(x) ({ \ - uint16_t __res; \ - if (__builtin_constant_p(x)) { \ - __res = default_swab16(x); \ - } else { \ - __asm__("xchgb %b0,%h0" : "=Q" (__res) : "0" ((uint16_t)(x))); \ - } \ - __res; }) - -#elif defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_X64)) - -#include - -#define bswap32(x) _byteswap_ulong(x) -#define bswap16(x) _byteswap_ushort(x) - -#endif - -#ifdef bswap32 - -#undef ntohl -#undef htonl -#define ntohl(x) bswap32(x) -#define htonl(x) bswap32(x) - -#endif - -#ifdef bswap16 - -#undef ntohs -#undef htons -#define ntohs(x) bswap16(x) -#define htons(x) bswap16(x) - -#endif diff --git a/src/common.h b/src/common.h index a8f60fe4874..4b4a9977545 100644 --- a/src/common.h +++ b/src/common.h @@ -58,7 +58,6 @@ #include "git2/types.h" #include "git2/errors.h" #include "thread-utils.h" -#include "bswap.h" #include From 07c989e98d013132168326eb7fd6fdf91a2399b9 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Tue, 3 Feb 2015 20:01:24 -0500 Subject: [PATCH 31/99] win32: further cleanups for 8.3 disabling --- tests/checkout/nasty.c | 5 +---- tests/clar_libgit2.c | 22 ++++++++++++++++++++++ tests/clar_libgit2.h | 4 ++++ tests/path/win32.c | 8 +++----- 4 files changed, 30 insertions(+), 9 deletions(-) diff --git a/tests/checkout/nasty.c b/tests/checkout/nasty.c index 08e7628f5d0..952a6a1127a 100644 --- a/tests/checkout/nasty.c +++ b/tests/checkout/nasty.c @@ -217,12 +217,9 @@ void test_checkout_nasty__git_tilde1(void) void test_checkout_nasty__git_custom_shortname(void) { #ifdef GIT_WIN32 - char *shortname = git_win32_path_8dot3_name("C:\\Program Files"); - if (shortname == NULL) + if (!cl_sandbox_supports_8dot3()) clar__skip(); - git__free(shortname); - cl_must_pass(p_rename("nasty/.git", "nasty/_temp")); cl_git_write2file("nasty/git~1", "", 0, O_RDWR|O_CREAT, 0666); cl_must_pass(p_rename("nasty/_temp", "nasty/.git")); diff --git a/tests/clar_libgit2.c b/tests/clar_libgit2.c index a8a8ba6abb6..6087c2a67b9 100644 --- a/tests/clar_libgit2.c +++ b/tests/clar_libgit2.c @@ -538,3 +538,25 @@ void cl_sandbox_set_search_path_defaults(void) GIT_OPT_SET_SEARCH_PATH, GIT_CONFIG_LEVEL_SYSTEM, sandbox_path); } +#ifdef GIT_WIN32 +bool cl_sandbox_supports_8dot3(void) +{ + git_buf longpath = GIT_BUF_INIT; + char *shortname; + bool supported; + + cl_git_pass( + git_buf_joinpath(&longpath, clar_sandbox_path(), "longer_than_8dot3")); + + cl_git_write2file(longpath.ptr, "", 0, O_RDWR|O_CREAT, 0666); + shortname = git_win32_path_8dot3_name(longpath.ptr); + + supported = (shortname != NULL); + + git__free(shortname); + git_buf_free(&longpath); + + return supported; +} +#endif + diff --git a/tests/clar_libgit2.h b/tests/clar_libgit2.h index e1d62c8202e..86c90b04974 100644 --- a/tests/clar_libgit2.h +++ b/tests/clar_libgit2.h @@ -161,4 +161,8 @@ void cl_fake_home_cleanup(void *); void cl_sandbox_set_search_path_defaults(void); +#ifdef GIT_WIN32 +bool cl_sandbox_supports_8dot3(void); +#endif + #endif diff --git a/tests/path/win32.c b/tests/path/win32.c index 41831a88c1b..4ff0397386c 100644 --- a/tests/path/win32.c +++ b/tests/path/win32.c @@ -194,13 +194,11 @@ void test_path_win32__8dot3_name(void) #ifdef GIT_WIN32 char *shortname; - /* Some guaranteed short names */ - shortname = git_win32_path_8dot3_name("C:\\Program Files"); - - if (shortname == NULL) + if (!cl_sandbox_supports_8dot3()) clar__skip(); - cl_assert_equal_s("PROGRA~1", shortname); + /* Some guaranteed short names */ + cl_assert_equal_s("PROGRA~1", (shortname = git_win32_path_8dot3_name("C:\\Program Files"))); git__free(shortname); cl_assert_equal_s("WINDOWS", (shortname = git_win32_path_8dot3_name("C:\\WINDOWS"))); From f45f9b6dbc8f7ee37b354196350d0e575cf7c0fe Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Wed, 4 Feb 2015 03:52:34 +0000 Subject: [PATCH 32/99] structinit test: use %PRIuZ instead of %d for size_t --- tests/core/structinit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/structinit.c b/tests/core/structinit.c index d064f348e37..25254b7139d 100644 --- a/tests/core/structinit.c +++ b/tests/core/structinit.c @@ -45,7 +45,7 @@ static void options_cmp(void *one, void *two, size_t size, const char *name) if (((char *)one)[i] != ((char *)two)[i]) { char desc[1024]; - p_snprintf(desc, 1024, "Difference in %s at byte %d: macro=%u / func=%u", + p_snprintf(desc, 1024, "Difference in %s at byte %" PRIuZ ": macro=%u / func=%u", name, i, ((char *)one)[i], ((char *)two)[i]); clar__fail(__FILE__, __LINE__, "Difference between macro and function options initializer", From 35425b51198b07738e8a5bd773a7711a0c869834 Mon Sep 17 00:00:00 2001 From: Brian Gesiak Date: Wed, 4 Feb 2015 01:06:07 -0500 Subject: [PATCH 33/99] README.md: Add Swift bindings "Gift" --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 664735b9314..b9a42d76d7e 100644 --- a/README.md +++ b/README.md @@ -215,6 +215,8 @@ Here are the bindings to libgit2 that are currently available: * Rugged * Rust * git2-rs +* Swift + * Gift * Vala * libgit2.vapi From d4b1b76701cb0495993b31b1b0e98c0a4b49c0ce Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Tue, 3 Feb 2015 00:03:49 -0500 Subject: [PATCH 34/99] checkout: cache system attributes file location --- src/attr.c | 68 +++++++++++++++++++++++++++++++++++++------------ src/attr_file.c | 10 ++++++++ src/attr_file.h | 4 ++- 3 files changed, 65 insertions(+), 17 deletions(-) diff --git a/src/attr.c b/src/attr.c index df139e0823f..020bb1c8bd5 100644 --- a/src/attr.c +++ b/src/attr.c @@ -249,6 +249,46 @@ static int preload_attr_file( return error; } +static int system_attr_file( + git_buf *out, + git_repository *repo, + git_attr_session *attr_session) +{ + int error; + + if (!attr_session) { + error = git_sysdir_find_system_file(out, GIT_ATTR_FILE_SYSTEM); + + if (error == GIT_ENOTFOUND) + giterr_clear(); + + return error; + } + + if (!attr_session->init_sysdir) { + error = git_sysdir_find_system_file(&attr_session->sysdir, GIT_ATTR_FILE_SYSTEM); + + if (error == GIT_ENOTFOUND) + giterr_clear(); + else if (error) + return error; + + attr_session->init_sysdir = 1; + } + + if (attr_session->sysdir.size == 0) + return GIT_ENOTFOUND; + + /* We can safely provide a git_buf with no allocation (asize == 0) to + * a consumer. This allows them to treat this as a regular `git_buf`, + * but their call to `git_buf_free` will not attempt to free it. + */ + out->ptr = attr_session->sysdir.ptr; + out->size = attr_session->sysdir.size; + out->asize = 0; + return 0; +} + static int attr_setup(git_repository *repo, git_attr_session *attr_session) { int error = 0; @@ -256,7 +296,7 @@ static int attr_setup(git_repository *repo, git_attr_session *attr_session) git_index *idx = NULL; git_buf sys = GIT_BUF_INIT; - if (attr_session && attr_session->setup) + if (attr_session && attr_session->init_setup) return 0; if ((error = git_attr_cache__init(repo)) < 0) @@ -266,18 +306,15 @@ static int attr_setup(git_repository *repo, git_attr_session *attr_session) * definitions will be available for later file parsing */ - if (!(error = git_sysdir_find_system_file(&sys, GIT_ATTR_FILE_SYSTEM))) { + error = system_attr_file(&sys, repo, attr_session); + + if (error == 0) error = preload_attr_file( repo, attr_session, GIT_ATTR_FILE__FROM_FILE, NULL, sys.ptr); - git_buf_free(&sys); - } - if (error < 0) { - if (error == GIT_ENOTFOUND) { - giterr_clear(); - error = 0; - } else - return error; - } + else if (error != GIT_ENOTFOUND) + return error; + + git_buf_free(&sys); if ((error = preload_attr_file( repo, attr_session, GIT_ATTR_FILE__FROM_FILE, @@ -300,7 +337,7 @@ static int attr_setup(git_repository *repo, git_attr_session *attr_session) return error; if (attr_session) - attr_session->setup = 1; + attr_session->init_setup = 1; return error; } @@ -489,15 +526,14 @@ static int collect_attr_files( } if ((flags & GIT_ATTR_CHECK_NO_SYSTEM) == 0) { - error = git_sysdir_find_system_file(&dir, GIT_ATTR_FILE_SYSTEM); + error = system_attr_file(&dir, repo, attr_session); + if (!error) error = push_attr_file( repo, attr_session, files, GIT_ATTR_FILE__FROM_FILE, NULL, dir.ptr); - else if (error == GIT_ENOTFOUND) { - giterr_clear(); + else if (error == GIT_ENOTFOUND) error = 0; - } } cleanup: diff --git a/src/attr_file.c b/src/attr_file.c index 9896df4f9b6..2ebd3b91fe5 100644 --- a/src/attr_file.c +++ b/src/attr_file.c @@ -854,3 +854,13 @@ int git_attr_session__init(git_attr_session *session, git_repository *repo) return 0; } + +void git_attr_session__free(git_attr_session *session) +{ + if (!session) + return; + + git_buf_free(&session->sysdir); + + memset(session, 0, sizeof(git_attr_session)); +} diff --git a/src/attr_file.h b/src/attr_file.h index b8e37256b60..5c64f7cc319 100644 --- a/src/attr_file.h +++ b/src/attr_file.h @@ -111,7 +111,9 @@ typedef struct { typedef struct { int key; - unsigned int setup; + unsigned int init_setup:1, + init_sysdir:1; + git_buf sysdir; } git_attr_session; extern int git_attr_session__init(git_attr_session *attr_session, git_repository *repo); From f58cc280c9ddeff6fd85f56e70073c3ed4fb0650 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Tue, 3 Feb 2015 00:28:32 -0500 Subject: [PATCH 35/99] attr_session: keep a temp buffer --- src/attr_file.c | 1 + src/attr_file.h | 1 + src/attrcache.c | 10 +++++++--- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/attr_file.c b/src/attr_file.c index 2ebd3b91fe5..8997946b96c 100644 --- a/src/attr_file.c +++ b/src/attr_file.c @@ -861,6 +861,7 @@ void git_attr_session__free(git_attr_session *session) return; git_buf_free(&session->sysdir); + git_buf_free(&session->tmp); memset(session, 0, sizeof(git_attr_session)); } diff --git a/src/attr_file.h b/src/attr_file.h index 5c64f7cc319..aa9a16de088 100644 --- a/src/attr_file.h +++ b/src/attr_file.h @@ -114,6 +114,7 @@ typedef struct { unsigned int init_setup:1, init_sysdir:1; git_buf sysdir; + git_buf tmp; } git_attr_session; extern int git_attr_session__init(git_attr_session *attr_session, git_repository *repo); diff --git a/src/attrcache.c b/src/attrcache.c index f75edad68ee..9a9b9d09dd9 100644 --- a/src/attrcache.c +++ b/src/attrcache.c @@ -149,6 +149,7 @@ static int attr_cache_lookup( git_attr_file **out_file, git_attr_file_entry **out_entry, git_repository *repo, + git_attr_session *attr_session, git_attr_file_source source, const char *base, const char *filename) @@ -162,9 +163,12 @@ static int attr_cache_lookup( /* join base and path as needed */ if (base != NULL && git_path_root(filename) < 0) { - if (git_buf_joinpath(&path, base, filename) < 0) + git_buf *p = attr_session ? &attr_session->tmp : &path; + + if (git_buf_joinpath(p, base, filename) < 0) return -1; - filename = path.ptr; + + filename = p->ptr; } relfile = filename; @@ -208,7 +212,7 @@ int git_attr_cache__get( git_attr_file *file = NULL, *updated = NULL; if ((error = attr_cache_lookup( - &file, &entry, repo, source, base, filename)) < 0) + &file, &entry, repo, attr_session, source, base, filename)) < 0) return error; /* load file if we don't have one or if existing one is out of date */ From 500ec5431564d9167985a02909823e1286136d5b Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Tue, 3 Feb 2015 01:46:01 -0500 Subject: [PATCH 36/99] checkout: hold seen dir paths in a map --- src/checkout.c | 77 +++++++++++++++++++++++++------------------------- src/fileops.c | 35 +++++++++++++++++------ src/fileops.h | 13 +++++++-- 3 files changed, 76 insertions(+), 49 deletions(-) diff --git a/src/checkout.c b/src/checkout.c index 04493a3cbb5..3f65a9ed75a 100644 --- a/src/checkout.c +++ b/src/checkout.c @@ -29,6 +29,10 @@ #include "merge_file.h" #include "path.h" #include "attr.h" +#include "pool.h" +#include "strmap.h" + +GIT__USE_STRMAP; /* See docs/checkout-internals.md for more information */ @@ -69,7 +73,7 @@ typedef struct { size_t total_steps; size_t completed_steps; git_checkout_perfdata perfdata; - git_buf last_mkdir; + git_strmap *mkdir_map; git_attr_session attr_session; } checkout_data; @@ -1293,25 +1297,6 @@ static int checkout_get_actions( return error; } -static int checkout_mkdir( - checkout_data *data, - const char *path, - const char *base, - mode_t mode, - unsigned int flags) -{ - struct git_futils_mkdir_perfdata mkdir_perfdata = {0}; - - int error = git_futils_mkdir_withperf( - path, base, mode, flags, &mkdir_perfdata); - - data->perfdata.mkdir_calls += mkdir_perfdata.mkdir_calls; - data->perfdata.stat_calls += mkdir_perfdata.stat_calls; - data->perfdata.chmod_calls += mkdir_perfdata.chmod_calls; - - return error; -} - static bool should_remove_existing(checkout_data *data) { int ignorecase = 0; @@ -1327,31 +1312,43 @@ static bool should_remove_existing(checkout_data *data) #define MKDIR_REMOVE_EXISTING \ MKDIR_NORMAL | GIT_MKDIR_REMOVE_FILES | GIT_MKDIR_REMOVE_SYMLINKS +static int checkout_mkdir( + checkout_data *data, + const char *path, + const char *base, + mode_t mode, + unsigned int flags) +{ + struct git_futils_mkdir_options mkdir_opts = {0}; + int error; + + mkdir_opts.dir_map = data->mkdir_map; + mkdir_opts.pool = &data->pool; + + error = git_futils_mkdir_ext( + path, base, mode, flags, &mkdir_opts); + + data->perfdata.mkdir_calls += mkdir_opts.perfdata.mkdir_calls; + data->perfdata.stat_calls += mkdir_opts.perfdata.stat_calls; + data->perfdata.chmod_calls += mkdir_opts.perfdata.chmod_calls; + + return error; +} + static int mkpath2file( checkout_data *data, const char *path, unsigned int mode) { - git_buf *mkdir_path = &data->tmp; struct stat st; bool remove_existing = should_remove_existing(data); + unsigned int flags = + (remove_existing ? MKDIR_REMOVE_EXISTING : MKDIR_NORMAL) | + GIT_MKDIR_SKIP_LAST; int error; - if ((error = git_buf_sets(mkdir_path, path)) < 0) + if ((error = checkout_mkdir( + data, path, data->opts.target_directory, mode, flags)) < 0) return error; - git_buf_rtruncate_at_char(mkdir_path, '/'); - - if (!data->last_mkdir.size || - data->last_mkdir.size != mkdir_path->size || - memcmp(mkdir_path->ptr, data->last_mkdir.ptr, mkdir_path->size) != 0) { - - if ((error = checkout_mkdir( - data, mkdir_path->ptr, data->opts.target_directory, mode, - remove_existing ? MKDIR_REMOVE_EXISTING : MKDIR_NORMAL)) < 0) - return error; - - git_buf_swap(&data->last_mkdir, mkdir_path); - } - if (remove_existing) { data->perfdata.stat_calls++; @@ -2215,13 +2212,16 @@ static void checkout_data_clear(checkout_data *data) git__free(data->pfx); data->pfx = NULL; - git_buf_free(&data->last_mkdir); + git_strmap_free(data->mkdir_map); + git_buf_free(&data->path); git_buf_free(&data->tmp); git_index_free(data->index); data->index = NULL; + git_strmap_free(data->mkdir_map); + git_attr_session__free(&data->attr_session); } @@ -2360,7 +2360,8 @@ static int checkout_data_init( (error = git_vector_init(&data->update_conflicts, 0, NULL)) < 0 || (error = git_pool_init(&data->pool, 1, 0)) < 0 || (error = git_buf_puts(&data->path, data->opts.target_directory)) < 0 || - (error = git_path_to_dir(&data->path)) < 0) + (error = git_path_to_dir(&data->path)) < 0 || + (error = git_strmap_alloc(&data->mkdir_map)) < 0) goto cleanup; data->workdir_len = git_buf_len(&data->path); diff --git a/src/fileops.c b/src/fileops.c index 2ee9535be84..ff2acfc2bda 100644 --- a/src/fileops.c +++ b/src/fileops.c @@ -7,11 +7,14 @@ #include "common.h" #include "fileops.h" #include "global.h" +#include "strmap.h" #include #if GIT_WIN32 #include "win32/findfile.h" #endif +GIT__USE_STRMAP; + int git_futils_mkpath2file(const char *file_path, const mode_t mode) { return git_futils_mkdir( @@ -321,12 +324,12 @@ GIT_INLINE(int) validate_existing( return 0; } -int git_futils_mkdir_withperf( +int git_futils_mkdir_ext( const char *path, const char *base, mode_t mode, uint32_t flags, - struct git_futils_mkdir_perfdata *perfdata) + struct git_futils_mkdir_options *opts) { int error = -1; git_buf make_path = GIT_BUF_INIT; @@ -394,11 +397,14 @@ int git_futils_mkdir_withperf( *tail = '\0'; st.st_mode = 0; + if (opts->dir_map && git_strmap_exists(opts->dir_map, make_path.ptr)) + continue; + /* See what's going on with this path component */ - perfdata->stat_calls++; + opts->perfdata.stat_calls++; if (p_lstat(make_path.ptr, &st) < 0) { - perfdata->mkdir_calls++; + opts->perfdata.mkdir_calls++; if (errno != ENOENT || p_mkdir(make_path.ptr, mode) < 0) { giterr_set(GITERR_OS, "Failed to make directory '%s'", make_path.ptr); @@ -416,7 +422,7 @@ int git_futils_mkdir_withperf( } if ((error = validate_existing( - make_path.ptr, &st, mode, flags, perfdata)) < 0) + make_path.ptr, &st, mode, flags, &opts->perfdata)) < 0) goto done; } @@ -425,7 +431,7 @@ int git_futils_mkdir_withperf( (lastch == '\0' && (flags & GIT_MKDIR_CHMOD) != 0)) && st.st_mode != mode) { - perfdata->chmod_calls++; + opts->perfdata.chmod_calls++; if ((error = p_chmod(make_path.ptr, mode)) < 0 && lastch == '\0') { @@ -434,6 +440,17 @@ int git_futils_mkdir_withperf( goto done; } } + + if (opts->dir_map && opts->pool) { + char *cache_path = git_pool_malloc(opts->pool, make_path.size + 1); + GITERR_CHECK_ALLOC(cache_path); + + memcpy(cache_path, make_path.ptr, make_path.size + 1); + + git_strmap_insert(opts->dir_map, cache_path, cache_path, error); + if (error < 0) + goto done; + } } error = 0; @@ -441,7 +458,7 @@ int git_futils_mkdir_withperf( /* check that full path really is a directory if requested & needed */ if ((flags & GIT_MKDIR_VERIFY_DIR) != 0 && lastch != '\0') { - perfdata->stat_calls++; + opts->perfdata.stat_calls++; if (p_stat(make_path.ptr, &st) < 0 || !S_ISDIR(st.st_mode)) { giterr_set(GITERR_OS, "Path is not a directory '%s'", @@ -461,8 +478,8 @@ int git_futils_mkdir( mode_t mode, uint32_t flags) { - struct git_futils_mkdir_perfdata perfdata = {0}; - return git_futils_mkdir_withperf(path, base, mode, flags, &perfdata); + struct git_futils_mkdir_options options = {0}; + return git_futils_mkdir_ext(path, base, mode, flags, &options); } int git_futils_mkdir_r(const char *path, const char *base, const mode_t mode) diff --git a/src/fileops.h b/src/fileops.h index 4aaf1781c36..0f6466c59c2 100644 --- a/src/fileops.h +++ b/src/fileops.h @@ -11,6 +11,8 @@ #include "map.h" #include "posix.h" #include "path.h" +#include "pool.h" +#include "strmap.h" /** * Filebuffer methods @@ -95,6 +97,13 @@ struct git_futils_mkdir_perfdata size_t chmod_calls; }; +struct git_futils_mkdir_options +{ + git_strmap *dir_map; + git_pool *pool; + struct git_futils_mkdir_perfdata perfdata; +}; + /** * Create a directory or entire path. * @@ -106,10 +115,10 @@ struct git_futils_mkdir_perfdata * @param base Root for relative path. These directories will never be made. * @param mode The mode to use for created directories. * @param flags Combination of the mkdir flags above. - * @param perfdata Performance data, use `git_futils_mkdir` if you don't want this data. + * @param opts Extended options, use `git_futils_mkdir` if you are not interested. * @return 0 on success, else error code */ -extern int git_futils_mkdir_withperf(const char *path, const char *base, mode_t mode, uint32_t flags, struct git_futils_mkdir_perfdata *perfdata); +extern int git_futils_mkdir_ext(const char *path, const char *base, mode_t mode, uint32_t flags, struct git_futils_mkdir_options *opts); /** * Create a directory or entire path. Similar to `git_futils_mkdir_withperf` From fa89ff20cd777fafc3e971bbf8081d97ce082c15 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Tue, 3 Feb 2015 05:35:24 +0000 Subject: [PATCH 37/99] remove some unused warnings --- src/attr.c | 5 ++--- tests/checkout/tree.c | 4 ++++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/attr.c b/src/attr.c index 020bb1c8bd5..2e1ef7c2a67 100644 --- a/src/attr.c +++ b/src/attr.c @@ -251,7 +251,6 @@ static int preload_attr_file( static int system_attr_file( git_buf *out, - git_repository *repo, git_attr_session *attr_session) { int error; @@ -306,7 +305,7 @@ static int attr_setup(git_repository *repo, git_attr_session *attr_session) * definitions will be available for later file parsing */ - error = system_attr_file(&sys, repo, attr_session); + error = system_attr_file(&sys, attr_session); if (error == 0) error = preload_attr_file( @@ -526,7 +525,7 @@ static int collect_attr_files( } if ((flags & GIT_ATTR_CHECK_NO_SYSTEM) == 0) { - error = system_attr_file(&dir, repo, attr_session); + error = system_attr_file(&dir, attr_session); if (!error) error = push_attr_file( diff --git a/tests/checkout/tree.c b/tests/checkout/tree.c index 42f32a8d93f..239ee55bc5b 100644 --- a/tests/checkout/tree.c +++ b/tests/checkout/tree.c @@ -1137,6 +1137,10 @@ void update_attr_callback( size_t total_steps, void *payload) { + GIT_UNUSED(completed_steps); + GIT_UNUSED(total_steps); + GIT_UNUSED(payload); + if (path && strcmp(path, "ident1.txt") == 0) cl_git_write2file("testrepo/.gitattributes", "*.txt ident\n", 12, O_RDWR|O_CREAT, 0666); From a2e4593e8259c58b60579c5579330f03587a9ce6 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Tue, 3 Feb 2015 18:41:40 -0500 Subject: [PATCH 38/99] hash_generic: __extension__ keyword for pedantry --- src/hash/hash_generic.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hash/hash_generic.c b/src/hash/hash_generic.c index 32fcd869c8d..472a7a6968a 100644 --- a/src/hash/hash_generic.c +++ b/src/hash/hash_generic.c @@ -18,7 +18,7 @@ * rotate with a loop. */ -#define SHA_ASM(op, x, n) ({ unsigned int __res; __asm__(op " %1,%0":"=r" (__res):"i" (n), "0" (x)); __res; }) +#define SHA_ASM(op, x, n) (__extension__ ({ unsigned int __res; __asm__(op " %1,%0":"=r" (__res):"i" (n), "0" (x)); __res; })) #define SHA_ROL(x,n) SHA_ASM("rol", x, n) #define SHA_ROR(x,n) SHA_ASM("ror", x, n) From 2107dac34624421015c3a633975bde0ff797ba53 Mon Sep 17 00:00:00 2001 From: Boris Egorov Date: Wed, 4 Feb 2015 15:14:18 +0600 Subject: [PATCH 39/99] for-each-ref example: init/shutdown libgit2 in main I don't get how it was working without git_libgit2_init() call. I run it and libgit2 throws assertion somewhere in its internals. Now it works. Updated commit with shutdown at the end. --- examples/for-each-ref.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/examples/for-each-ref.c b/examples/for-each-ref.c index d6846bb0d8c..a8ceaaff96b 100644 --- a/examples/for-each-ref.c +++ b/examples/for-each-ref.c @@ -34,6 +34,7 @@ static int show_ref(git_reference *ref, void *data) int main(int argc, char **argv) { git_repository *repo; + git_libgit2_init(); if (argc != 1 || argv[1] /* silence -Wunused-parameter */) fatal("Sorry, no for-each-ref options supported yet", NULL); @@ -42,5 +43,7 @@ int main(int argc, char **argv) "Could not open repository", NULL); check_lg2(git_reference_foreach(repo, show_ref, repo), "Could not iterate over references", NULL); + + git_libgit2_shutdown(); return 0; } From 865baaf928bd29274c753d85db59a324d4bd8b82 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Thu, 5 Feb 2015 11:06:01 -0500 Subject: [PATCH 40/99] repo: ensure we can create repo at filesystem root Test to ensure that we can create a repository at the filesystem root. Introduces a new test environment variable, `GITTEST_INVASIVE_FILESYSTEM` for tests that do terrible things like escaping the clar sandbox and writing to the root directory. It is expected that the CI builds will enable this but that normal people would not want this. --- tests/repo/init.c | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/repo/init.c b/tests/repo/init.c index ed86f6e4f9c..91747c9f5d6 100644 --- a/tests/repo/init.c +++ b/tests/repo/init.c @@ -714,3 +714,29 @@ void test_repo_init__init_with_initial_commit(void) git_index_free(index); } + +void test_repo_init__at_filesystem_root(void) +{ + git_repository *repo; + const char *sandbox = clar_sandbox_path(); + git_buf root = GIT_BUF_INIT; + int root_len; + + if (!cl_getenv("GITTEST_INVASIVE_FILESYSTEM")) + cl_skip(); + + root_len = git_path_root(sandbox); + cl_assert(root_len >= 0); + + git_buf_put(&root, sandbox, root_len+1); + git_buf_joinpath(&root, root.ptr, "libgit2_test_dir"); + + cl_assert(!git_path_exists(root.ptr)); + + cl_git_pass(git_repository_init(&repo, root.ptr, 0)); + cl_assert(git_path_isdir(root.ptr)); + cl_git_pass(git_futils_rmdir_r(root.ptr, NULL, GIT_RMDIR_REMOVE_FILES)); + + git_buf_free(&root); + git_repository_free(repo); +} From 2f4ee00039ebe63e2d954ede118593c279486893 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Thu, 5 Feb 2015 11:55:00 -0500 Subject: [PATCH 41/99] appveyor: include the invasive filesystem tests --- appveyor.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/appveyor.yml b/appveyor.yml index aa3a70eb51b..de633bbee63 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -3,6 +3,8 @@ branches: only: - master environment: + GITTEST_INVASIVE_FILESYSTEM: 1 + matrix: - GENERATOR: "Visual Studio 11" ARCH: 32 From 3c68bfcd088dd5a1ef06b98ea7fd203bb048fa42 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Wed, 4 Feb 2015 18:24:31 -0500 Subject: [PATCH 42/99] stat: don't remove trailing '/' from root on win32 `p_stat` calls `git_win32_path_from_utf8`, which canonicalizes the path. Do not further try to modify the path, else we trim the trailing slash from a root directory and try to access `C:` instead of `C:/`. --- src/win32/posix_w32.c | 8 ++------ tests/core/stat.c | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/src/win32/posix_w32.c b/src/win32/posix_w32.c index e446ccab03d..346f537e40e 100644 --- a/src/win32/posix_w32.c +++ b/src/win32/posix_w32.c @@ -448,12 +448,8 @@ int p_stat(const char* path, struct stat* buf) git_win32_path path_w; int len; - if ((len = git_win32_path_from_utf8(path_w, path)) < 0) - return -1; - - git_win32__path_trim_end(path_w, len); - - if (lstat_w(path_w, buf, false) < 0) + if ((len = git_win32_path_from_utf8(path_w, path)) < 0 || + lstat_w(path_w, buf, false) < 0) return -1; /* The item is a symbolic link or mount point. No need to iterate diff --git a/tests/core/stat.c b/tests/core/stat.c index 2e4abfb796d..bd9b990e324 100644 --- a/tests/core/stat.c +++ b/tests/core/stat.c @@ -95,3 +95,20 @@ void test_core_stat__0(void) cl_assert_error(ENOTDIR); } +void test_core_stat__root(void) +{ + const char *sandbox = clar_sandbox_path(); + git_buf root = GIT_BUF_INIT; + int root_len; + struct stat st; + + root_len = git_path_root(sandbox); + cl_assert(root_len >= 0); + + git_buf_set(&root, sandbox, root_len+1); + + cl_must_pass(p_stat(root.ptr, &st)); + cl_assert(S_ISDIR(st.st_mode)); + + git_buf_free(&root); +} From 9cb5b0f73c610692cc62f6d98f527bf1ab326924 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Wed, 4 Feb 2015 23:45:22 -0500 Subject: [PATCH 43/99] mkdir: respect the root path Don't try to strip trailing paths from the root directory on Windows (trying to create `C:` will fail). --- src/fileops.c | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/src/fileops.c b/src/fileops.c index 2ee9535be84..4a62d210daa 100644 --- a/src/fileops.c +++ b/src/fileops.c @@ -330,7 +330,7 @@ int git_futils_mkdir_withperf( { int error = -1; git_buf make_path = GIT_BUF_INIT; - ssize_t root = 0, min_root_len; + ssize_t root = 0, min_root_len, root_len; char lastch = '/', *tail; struct stat st; @@ -343,22 +343,29 @@ int git_futils_mkdir_withperf( goto done; } - /* remove trailing slashes on path */ - while (make_path.ptr[make_path.size - 1] == '/') { - make_path.size--; - make_path.ptr[make_path.size] = '\0'; - } + /* Trim trailing slashes (except the root) */ + if ((root_len = git_path_root(make_path.ptr)) < 0) + root_len = 0; + else + root_len++; + + while (make_path.size > (size_t)root_len && + make_path.ptr[make_path.size - 1] == '/') + make_path.ptr[--make_path.size] = '\0'; /* if we are not supposed to made the last element, truncate it */ if ((flags & GIT_MKDIR_SKIP_LAST2) != 0) { - git_buf_rtruncate_at_char(&make_path, '/'); + git_path_dirname_r(&make_path, make_path.ptr); flags |= GIT_MKDIR_SKIP_LAST; } - if ((flags & GIT_MKDIR_SKIP_LAST) != 0) - git_buf_rtruncate_at_char(&make_path, '/'); + if ((flags & GIT_MKDIR_SKIP_LAST) != 0) { + git_path_dirname_r(&make_path, make_path.ptr); + } - /* if nothing left after truncation, then we're done! */ - if (!make_path.size) { + /* We were either given the root path (or trimmed it to + * the root), we don't have anything to do. + */ + if (make_path.size <= (size_t)root_len) { error = 0; goto done; } From 83276dccbef0cc1d70ec45c086e38a3e232e2b32 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Thu, 5 Feb 2015 13:24:38 -0500 Subject: [PATCH 44/99] Update clar to d17c7f6 --- tests/clar.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/clar.c b/tests/clar.c index 51f163526f0..17f767a0211 100644 --- a/tests/clar.c +++ b/tests/clar.c @@ -269,6 +269,7 @@ clar_usage(const char *arg) printf(" -sname\tRun only the suite with `name` (can go to individual test name)\n"); printf(" -iname\tInclude the suite with `name`\n"); printf(" -xname\tExclude the suite with `name`\n"); + printf(" -v \tIncrease verbosity (show suite names)\n"); printf(" -q \tOnly report tests that had an error\n"); printf(" -Q \tQuit as soon as a test fails\n"); printf(" -l \tPrint suite names\n"); @@ -347,6 +348,10 @@ clar_parse_args(int argc, char **argv) exit(0); } + case 'v': + _clar.report_suite_names = 1; + break; + default: clar_usage(argv[0]); } From cc36f424d46f7c11ebff1dbf23df3be3d211c9b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mart=C3=ADn=20Nieto?= Date: Fri, 6 Feb 2015 01:09:26 +0100 Subject: [PATCH 45/99] repository: parse init.templatedir as a path This is a path so we must use the path getter so we get the tilde expansion done. --- src/repository.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/repository.c b/src/repository.c index 433754489f1..c9275078fda 100644 --- a/src/repository.c +++ b/src/repository.c @@ -1265,7 +1265,8 @@ static int repo_init_structure( if (opts->template_path) tdir = opts->template_path; else if ((error = git_config_open_default(&cfg)) >= 0) { - error = git_config_get_string(&tdir, cfg, "init.templatedir"); + if (!git_config_get_path(&template_buf, cfg, "init.templatedir")) + tdir = template_buf.ptr; giterr_clear(); } From d97ba7f6744012fb51127d5313543a10c5a60e43 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 6 Feb 2015 15:45:59 -0500 Subject: [PATCH 46/99] Large merge perf test. --- tests/perf/helper__perf__do_merge.c | 78 +++++++++++++++++++++++++++++ tests/perf/helper__perf__do_merge.h | 4 ++ tests/perf/helper__perf__timer.c | 73 +++++++++++++++++++++++++++ tests/perf/helper__perf__timer.h | 27 ++++++++++ tests/perf/merge.c | 44 ++++++++++++++++ 5 files changed, 226 insertions(+) create mode 100644 tests/perf/helper__perf__do_merge.c create mode 100644 tests/perf/helper__perf__do_merge.h create mode 100644 tests/perf/helper__perf__timer.c create mode 100644 tests/perf/helper__perf__timer.h create mode 100644 tests/perf/merge.c diff --git a/tests/perf/helper__perf__do_merge.c b/tests/perf/helper__perf__do_merge.c new file mode 100644 index 00000000000..00221851edb --- /dev/null +++ b/tests/perf/helper__perf__do_merge.c @@ -0,0 +1,78 @@ +#include "clar_libgit2.h" +#include "helper__perf__do_merge.h" +#include "helper__perf__timer.h" + +static git_repository * g_repo; + +void perf__do_merge(const char *fixture, + const char *test_name, + const char *id_a, + const char *id_b) +{ + git_checkout_options checkout_opts = GIT_CHECKOUT_OPTIONS_INIT; + git_clone_options clone_opts = GIT_CLONE_OPTIONS_INIT; + git_merge_options merge_opts = GIT_MERGE_OPTIONS_INIT; + git_oid oid_a; + git_oid oid_b; + git_reference *ref_branch_a = NULL; + git_reference *ref_branch_b = NULL; + git_commit *commit_a = NULL; + git_commit *commit_b = NULL; + git_annotated_commit *annotated_commits[1] = { NULL }; + perf_timer t_total = PERF_TIMER_INIT; + perf_timer t_clone = PERF_TIMER_INIT; + perf_timer t_checkout = PERF_TIMER_INIT; + perf_timer t_merge = PERF_TIMER_INIT; + + perf__timer__start(&t_total); + + checkout_opts.checkout_strategy = GIT_CHECKOUT_SAFE_CREATE; + clone_opts.checkout_opts = checkout_opts; + cl_git_pass(git_signature_now(&clone_opts.signature, "Me", "foo@example.com")); + + perf__timer__start(&t_clone); + cl_git_pass(git_clone(&g_repo, fixture, test_name, &clone_opts)); + perf__timer__stop(&t_clone); + + git_oid_fromstr(&oid_a, id_a); + cl_git_pass(git_commit_lookup(&commit_a, g_repo, &oid_a)); + cl_git_pass(git_branch_create(&ref_branch_a, g_repo, + "A", commit_a, + 0, NULL, NULL)); + + perf__timer__start(&t_checkout); + cl_git_pass(git_checkout_tree(g_repo, (git_object*)commit_a, &checkout_opts)); + perf__timer__stop(&t_checkout); + + cl_git_pass(git_repository_set_head(g_repo, + git_reference_name(ref_branch_a), + NULL, NULL)); + + git_oid_fromstr(&oid_b, id_b); + cl_git_pass(git_commit_lookup(&commit_b, g_repo, &oid_b)); + cl_git_pass(git_branch_create(&ref_branch_b, g_repo, + "B", commit_b, + 0, NULL, NULL)); + + cl_git_pass(git_annotated_commit_lookup(&annotated_commits[0], g_repo, &oid_b)); + + perf__timer__start(&t_merge); + cl_git_pass(git_merge(g_repo, + (const git_annotated_commit **)annotated_commits, 1, + &merge_opts, &checkout_opts)); + perf__timer__stop(&t_merge); + + git_reference_free(ref_branch_a); + git_reference_free(ref_branch_b); + git_commit_free(commit_a); + git_commit_free(commit_b); + git_annotated_commit_free(annotated_commits[0]); + git_repository_free(g_repo); + + perf__timer__stop(&t_total); + + perf__timer__report(&t_clone, "%s: clone", test_name); + perf__timer__report(&t_checkout, "%s: checkout", test_name); + perf__timer__report(&t_merge, "%s: merge", test_name); + perf__timer__report(&t_total, "%s: total", test_name); +} diff --git a/tests/perf/helper__perf__do_merge.h b/tests/perf/helper__perf__do_merge.h new file mode 100644 index 00000000000..4a4723da5fd --- /dev/null +++ b/tests/perf/helper__perf__do_merge.h @@ -0,0 +1,4 @@ +void perf__do_merge(const char *fixture, + const char *test_name, + const char *id_a, + const char *id_b); diff --git a/tests/perf/helper__perf__timer.c b/tests/perf/helper__perf__timer.c new file mode 100644 index 00000000000..8a7ed09e828 --- /dev/null +++ b/tests/perf/helper__perf__timer.c @@ -0,0 +1,73 @@ +#include "clar_libgit2.h" +#include "helper__perf__timer.h" + +#if defined(GIT_WIN32) + +void perf__timer__start(perf_timer *t) +{ + QueryPerformanceCounter(&t->time_started); +} + +void perf__timer__stop(perf_timer *t) +{ + LARGE_INTEGER time_now; + QueryPerformanceCounter(&time_now); + + t->sum.QuadPart += (time_now.QuadPart - t->time_started.QuadPart); +} + +void perf__timer__report(perf_timer *t, const char *fmt, ...) +{ + va_list arglist; + LARGE_INTEGER freq; + double fraction; + + QueryPerformanceFrequency(&freq); + + fraction = ((double)t->sum.QuadPart) / ((double)freq.QuadPart); + + printf("%10.3f: ", fraction); + + va_start(arglist, fmt); + vprintf(fmt, arglist); + va_end(arglist); + + printf("\n"); +} + +#else + +#include + +static uint32_t now_in_ms(void) +{ + struct timeval now; + gettimeofday(&now, NULL); + return (uint32_t)((now.tv_sec * 1000) + (now.tv_usec / 1000)); +} + +void perf__timer__start(perf_timer *t) +{ + t->time_started = now_in_ms(); +} + +void perf__timer__stop(perf_timer *t) +{ + uint32_t now = now_in_ms(); + t->sum += (now - t->time_started); +} + +void perf__timer__report(perf_timer *t, const char *fmt, ...) +{ + va_list arglist; + + printf("%10.3f: ", ((double)t->sum) / 1000); + + va_start(arglist, fmt); + vprintf(fmt, arglist); + va_end(arglist); + + printf("\n"); +} + +#endif diff --git a/tests/perf/helper__perf__timer.h b/tests/perf/helper__perf__timer.h new file mode 100644 index 00000000000..5aff4b1366f --- /dev/null +++ b/tests/perf/helper__perf__timer.h @@ -0,0 +1,27 @@ +#if defined(GIT_WIN32) + +struct perf__timer +{ + LARGE_INTEGER sum; + LARGE_INTEGER time_started; +}; + +#define PERF_TIMER_INIT {0} + +#else + +struct perf__timer +{ + uint32_t sum; + uint32_t time_started; +}; + +#define PERF_TIMER_INIT {0} + +#endif + +typedef struct perf__timer perf_timer; + +void perf__timer__start(perf_timer *t); +void perf__timer__stop(perf_timer *t); +void perf__timer__report(perf_timer *t, const char *fmt, ...); diff --git a/tests/perf/merge.c b/tests/perf/merge.c new file mode 100644 index 00000000000..b2ef082ebcb --- /dev/null +++ b/tests/perf/merge.c @@ -0,0 +1,44 @@ +#include "clar_libgit2.h" +#include "helper__perf__do_merge.h" + +/* This test requires a large repo with many files. + * It doesn't care about the contents, just the size. + * + * For now, we use the LibGit2 repo containing the + * source tree because it is already here. + * + * `find . | wc -l` reports 5128. + * + */ +#define SRC_REPO (cl_fixture("../..")) + +/* We need 2 arbitrary commits within that repo + * that have a large number of changed files. + * Again, we don't care about the actual contents, + * just the size. + * + * For now, we use these public branches: + * maint/v0.21 d853fb9f24e0fe63b3dce9fbc04fd9cfe17a030b Always checkout with case sensitive iterator + * maint/v0.22 1ce9ea3ba9b4fa666602d52a5281d41a482cc58b checkout tests: cleanup realpath impl on Win32 + * + */ +#define ID_BRANCH_A "d853fb9f24e0fe63b3dce9fbc04fd9cfe17a030b" +#define ID_BRANCH_B "1ce9ea3ba9b4fa666602d52a5281d41a482cc58b" + + +void test_perf_merge__initialize(void) +{ +} + +void test_perf_merge__cleanup(void) +{ +} + +void test_perf_merge__m1(void) +{ +#if 1 + cl_skip(); +#else + perf__do_merge(SRC_REPO, "m1", ID_BRANCH_A, ID_BRANCH_B); +#endif +} From 554063a5fe25e7ff9034f50c6bede5573b96dacb Mon Sep 17 00:00:00 2001 From: Arthur Schreiber Date: Fri, 6 Feb 2015 23:12:58 +0100 Subject: [PATCH 47/99] Speed up AppVeyor builds. AppVeyor build machines come with Python 2.7 and CMake 2.8 pre-installed and in the PATH. --- appveyor.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index aa3a70eb51b..d11fec12cbb 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -10,8 +10,6 @@ environment: ARCH: 64 build_script: - ps: | - choco install cmake - choco install python2 mkdir build cd build cmake -D ENABLE_TRACE=ON -D BUILD_CLAR=ON .. -G"$env:GENERATOR" From ec7e1c93ce1c177026073abd16d6a8d2b548d3bd Mon Sep 17 00:00:00 2001 From: Ben Chatelain Date: Tue, 10 Feb 2015 08:31:48 -0700 Subject: [PATCH 48/99] Fix doc comment formatting --- include/git2/annotated_commit.h | 6 +++--- include/git2/cherrypick.h | 4 ++-- include/git2/config.h | 8 ++++---- include/git2/describe.h | 5 +++-- include/git2/oid.h | 2 +- include/git2/rebase.h | 10 +++++----- include/git2/revert.h | 4 ++-- include/git2/transport.h | 2 +- include/git2/types.h | 4 +--- 9 files changed, 22 insertions(+), 23 deletions(-) diff --git a/include/git2/annotated_commit.h b/include/git2/annotated_commit.h index 87e3398a03b..e842d20328c 100644 --- a/include/git2/annotated_commit.h +++ b/include/git2/annotated_commit.h @@ -44,7 +44,7 @@ GIT_EXTERN(int) git_annotated_commit_from_ref( * @param repo repository that contains the given commit * @param branch_name name of the (remote) branch * @param remote_url url of the remote - * @param oid the commit object id of the remote branch + * @param id the commit object id of the remote branch * @return 0 on success or error code */ GIT_EXTERN(int) git_annotated_commit_from_fetchhead( @@ -80,7 +80,7 @@ GIT_EXTERN(int) git_annotated_commit_lookup( /** * Gets the commit ID that the given `git_annotated_commit` refers to. * - * @param head the given annotated commit + * @param commit the given annotated commit * @return commit id */ GIT_EXTERN(const git_oid *) git_annotated_commit_id( @@ -89,7 +89,7 @@ GIT_EXTERN(const git_oid *) git_annotated_commit_id( /** * Frees a `git_annotated_commit`. * - * @param annotated_commit annotated commit to free + * @param commit annotated commit to free */ GIT_EXTERN(void) git_annotated_commit_free( git_annotated_commit *commit); diff --git a/include/git2/cherrypick.h b/include/git2/cherrypick.h index 208166c195f..191170a58bb 100644 --- a/include/git2/cherrypick.h +++ b/include/git2/cherrypick.h @@ -29,8 +29,8 @@ typedef struct { /** For merge commits, the "mainline" is treated as the parent. */ unsigned int mainline; - git_merge_options merge_opts; /*< Options for the merging */ - git_checkout_options checkout_opts; /*< Options for the checkout */ + git_merge_options merge_opts; /* Options for the merging */ + git_checkout_options checkout_opts; /* Options for the checkout */ } git_cherrypick_options; #define GIT_CHERRYPICK_OPTIONS_VERSION 1 diff --git a/include/git2/config.h b/include/git2/config.h index 1ed8d244187..46b285cbc49 100644 --- a/include/git2/config.h +++ b/include/git2/config.h @@ -59,9 +59,9 @@ typedef enum { * An entry in a configuration file */ typedef struct { - const char *name; /*< Name of the entry (normalised) */ - const char *value; /*< String value of the entry */ - git_config_level_t level; /*< Which config file this was found in */ + const char *name; /* Name of the entry (normalised) */ + const char *value; /* String value of the entry */ + git_config_level_t level; /* Which config file this was found in */ } git_config_entry; typedef int (*git_config_foreach_cb)(const git_config_entry *, void *); @@ -333,7 +333,7 @@ GIT_EXTERN(int) git_config_get_bool(int *out, const git_config *cfg, const char * @param out the buffer in which to store the result * @param cfg where to look for the variable * @param name the variable's name - * @param 0 or an error code + * @return 0 or an error code */ GIT_EXTERN(int) git_config_get_path(git_buf *out, const git_config *cfg, const char *name); diff --git a/include/git2/describe.h b/include/git2/describe.h index 66b88c4faca..d01a3f74c81 100644 --- a/include/git2/describe.h +++ b/include/git2/describe.h @@ -129,7 +129,7 @@ GIT_EXTERN(int) git_describe_commit( * worktree. After peforming describe on HEAD, a status is run and the * description is considered to be dirty if there are. * - * @param result pointer to store the result. You must free this once + * @param out pointer to store the result. You must free this once * you're done with it. * @param repo the repository in which to perform the describe * @param opts the lookup options @@ -142,9 +142,10 @@ GIT_EXTERN(int) git_describe_workdir( /** * Print the describe result to a buffer * + * @param out The buffer to store the result * @param result the result from `git_describe_commit()` or * `git_describe_workdir()`. - * @param opt the formatting options + * @param opts the formatting options */ GIT_EXTERN(int) git_describe_format( git_buf *out, diff --git a/include/git2/oid.h b/include/git2/oid.h index db2f3af7015..8ad51c8badf 100644 --- a/include/git2/oid.h +++ b/include/git2/oid.h @@ -123,7 +123,7 @@ GIT_EXTERN(void) git_oid_pathfmt(char *out, const git_oid *id); * will be stored in TLS (i.e. one buffer per thread) to allow for * concurrent calls of the function. * - * @param id the oid structure to format + * @param oid The oid structure to format * @return the c-string */ GIT_EXTERN(char *) git_oid_tostr_s(const git_oid *oid); diff --git a/include/git2/rebase.h b/include/git2/rebase.h index 19246503690..1da3fb5aa6c 100644 --- a/include/git2/rebase.h +++ b/include/git2/rebase.h @@ -156,7 +156,7 @@ GIT_EXTERN(int) git_rebase_init( * invocation of `git_rebase_init` or by another client. * * @param out Pointer to store the rebase object - * @param reop The repository that has a rebase in-progress + * @param repo The repository that has a rebase in-progress * @return Zero on success; -1 on failure. */ GIT_EXTERN(int) git_rebase_open(git_rebase **out, git_repository *repo); @@ -195,8 +195,8 @@ GIT_EXTERN(git_rebase_operation *) git_rebase_operation_byindex( * working directory will be updated with the changes. If there are conflicts, * you will need to address those before committing the changes. * - * @param out Pointer to store the rebase operation that is to be performed next - * @param repo The rebase in progress + * @param operation Pointer to store the rebase operation that is to be performed next + * @param rebase The rebase in progress * @param checkout_opts Options to specify how the patch should be checked out * @return Zero on success; -1 on failure. */ @@ -211,7 +211,7 @@ GIT_EXTERN(int) git_rebase_next( * invocation. * * @param id Pointer in which to store the OID of the newly created commit - * @param repo The rebase that is in-progress + * @param rebase The rebase that is in-progress * @param author The author of the updated commit, or NULL to keep the * author from the original commit * @param committer The committer of the rebase @@ -255,7 +255,7 @@ GIT_EXTERN(int) git_rebase_abort( * @param rebase The rebase that is in-progress * @param signature The identity that is finishing the rebase (optional) * @param opts Options to specify how rebase is finished - * @param Zero on success; -1 on error + * @return Zero on success; -1 on error */ GIT_EXTERN(int) git_rebase_finish( git_rebase *rebase, diff --git a/include/git2/revert.h b/include/git2/revert.h index 4e0ae9ba90d..eb069ae1276 100644 --- a/include/git2/revert.h +++ b/include/git2/revert.h @@ -29,8 +29,8 @@ typedef struct { /** For merge commits, the "mainline" is treated as the parent. */ unsigned int mainline; - git_merge_options merge_opts; /*< Options for the merging */ - git_checkout_options checkout_opts; /*< Options for the checkout */ + git_merge_options merge_opts; /* Options for the merging */ + git_checkout_options checkout_opts; /* Options for the checkout */ } git_revert_options; #define GIT_REVERT_OPTIONS_VERSION 1 diff --git a/include/git2/transport.h b/include/git2/transport.h index 39df479c754..c10907f5f63 100644 --- a/include/git2/transport.h +++ b/include/git2/transport.h @@ -292,7 +292,7 @@ GIT_EXTERN(int) git_cred_username_new(git_cred **cred, const char *username); * * - cred: The newly created credential object. * - url: The resource for which we are demanding a credential. - * - username_from_url: The username that was embedded in a "user@host" + * - username_from_url: The username that was embedded in a "user\@host" * remote url, or NULL if not included. * - allowed_types: A bitmask stating which cred types are OK to return. * - payload: The payload provided when specifying this callback. diff --git a/include/git2/types.h b/include/git2/types.h index aa7a56f569a..35e1573c739 100644 --- a/include/git2/types.h +++ b/include/git2/types.h @@ -302,9 +302,7 @@ typedef struct { /** * Callback for the user's custom certificate checks. * - * @param type The type of certificate or host info, SSH or X.509 - * @param data The data for the certificate or host info - * @param len The size of the certificate or host info + * @param cert The host certificate * @param valid Whether the libgit2 checks (OpenSSL or WinHTTP) think * this certificate is valid * @param host Hostname of the host libgit2 connected to From c03e8c224cf18104bc97a68aed85317820d062ae Mon Sep 17 00:00:00 2001 From: Ben Chatelain Date: Tue, 10 Feb 2015 12:44:05 -0700 Subject: [PATCH 49/99] Use correct Doxygen trailing comment syntax --- include/git2/cherrypick.h | 4 ++-- include/git2/config.h | 6 +++--- include/git2/revert.h | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/include/git2/cherrypick.h b/include/git2/cherrypick.h index 191170a58bb..edec96a944b 100644 --- a/include/git2/cherrypick.h +++ b/include/git2/cherrypick.h @@ -29,8 +29,8 @@ typedef struct { /** For merge commits, the "mainline" is treated as the parent. */ unsigned int mainline; - git_merge_options merge_opts; /* Options for the merging */ - git_checkout_options checkout_opts; /* Options for the checkout */ + git_merge_options merge_opts; /**< Options for the merging */ + git_checkout_options checkout_opts; /**< Options for the checkout */ } git_cherrypick_options; #define GIT_CHERRYPICK_OPTIONS_VERSION 1 diff --git a/include/git2/config.h b/include/git2/config.h index 46b285cbc49..70c8beade64 100644 --- a/include/git2/config.h +++ b/include/git2/config.h @@ -59,9 +59,9 @@ typedef enum { * An entry in a configuration file */ typedef struct { - const char *name; /* Name of the entry (normalised) */ - const char *value; /* String value of the entry */ - git_config_level_t level; /* Which config file this was found in */ + const char *name; /**< Name of the entry (normalised) */ + const char *value; /**< String value of the entry */ + git_config_level_t level; /**< Which config file this was found in */ } git_config_entry; typedef int (*git_config_foreach_cb)(const git_config_entry *, void *); diff --git a/include/git2/revert.h b/include/git2/revert.h index eb069ae1276..2de1942197a 100644 --- a/include/git2/revert.h +++ b/include/git2/revert.h @@ -29,8 +29,8 @@ typedef struct { /** For merge commits, the "mainline" is treated as the parent. */ unsigned int mainline; - git_merge_options merge_opts; /* Options for the merging */ - git_checkout_options checkout_opts; /* Options for the checkout */ + git_merge_options merge_opts; /**< Options for the merging */ + git_checkout_options checkout_opts; /**< Options for the checkout */ } git_revert_options; #define GIT_REVERT_OPTIONS_VERSION 1 From ec510666a166958d26babbfa235846ed216b57c2 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Tue, 10 Feb 2015 15:10:32 +0000 Subject: [PATCH 50/99] Credit utf8proc for utf8 iterator --- src/util.c | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/util.c b/src/util.c index 7ee3e2ff97b..6bb7d03ee4f 100644 --- a/src/util.c +++ b/src/util.c @@ -665,6 +665,31 @@ void git__insertsort_r( git__free(swapel); } +/* + * git__utf8_iterate is taken from the utf8proc project, + * http://www.public-software-group.org/utf8proc + * + * Copyright (c) 2009 Public Software Group e. V., Berlin, Germany + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the ""Software""), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + static const int8_t utf8proc_utf8class[256] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, From 0b55ed9542aa9f9de14f58a4e0241c87d98185ee Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Tue, 10 Feb 2015 15:14:49 +0000 Subject: [PATCH 51/99] clar: use actual license and copyright --- COPYING | 36 +++++++++++++++--------------------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/COPYING b/COPYING index 1817372849d..6f6115e3bfc 100644 --- a/COPYING +++ b/COPYING @@ -388,27 +388,21 @@ Copyright (C) 1995-2010 Jean-loup Gailly and Mark Adler ---------------------------------------------------------------------- -The Clar framework is licensed under the MIT license: - -Copyright (C) 2011 by Vicent Marti - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. +The Clar framework is licensed under the ISC license: + +Copyright (c) 2011-2015 Vicent Marti + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ---------------------------------------------------------------------- From a36486ef26dac5f4f359cc54c865b91bbc31767e Mon Sep 17 00:00:00 2001 From: John Haley Date: Wed, 11 Feb 2015 10:31:54 -0700 Subject: [PATCH 52/99] Fixed error when including git2/include/sys/stream.h --- include/git2/sys/stream.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/git2/sys/stream.h b/include/git2/sys/stream.h index c249055c922..c22179fab7c 100644 --- a/include/git2/sys/stream.h +++ b/include/git2/sys/stream.h @@ -37,4 +37,6 @@ typedef struct git_stream { void (*free)(struct git_stream *); } git_stream; +GIT_END_DECL + #endif From 02eb1495a5248c8f676e15fd12e1be28d4f22480 Mon Sep 17 00:00:00 2001 From: Jameson Miller Date: Thu, 12 Feb 2015 15:32:52 -0500 Subject: [PATCH 53/99] Check rebase options version on public entry points --- src/rebase.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/rebase.c b/src/rebase.c index ceb74d39f79..2e805929ee1 100644 --- a/src/rebase.c +++ b/src/rebase.c @@ -634,7 +634,7 @@ int git_rebase_init( *out = NULL; - GITERR_CHECK_VERSION(given_opts, GIT_MERGE_OPTIONS_VERSION, "git_merge_options"); + GITERR_CHECK_VERSION(given_opts, GIT_REBASE_OPTIONS_VERSION, "git_rebase_options"); if (!onto) onto = upstream; @@ -1058,6 +1058,8 @@ int git_rebase_finish( assert(rebase); + GITERR_CHECK_VERSION(given_opts, GIT_REBASE_OPTIONS_VERSION, "git_rebase_options"); + if ((error = rebase_normalize_opts(rebase->repo, &opts, given_opts)) < 0) goto done; From 3ea78f241fb0c77822d75211752b4a1d4f299ed7 Mon Sep 17 00:00:00 2001 From: "Yury G. Kudryashov" Date: Mon, 9 Feb 2015 19:40:22 +0300 Subject: [PATCH 54/99] Add test for include.path inside included config It fails at least on my computer, though it may depend on some unpredictable factors (say, will realloc() extend the memory segment in place, or it will allocate new memory). --- tests/config/include.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/config/include.c b/tests/config/include.c index 0a931342a21..8232af489e5 100644 --- a/tests/config/include.c +++ b/tests/config/include.c @@ -102,3 +102,25 @@ void test_config_include__missing(void) git_config_free(cfg); } + +#define replicate10(s) s s s s s s s s s s +void test_config_include__depth2(void) +{ + git_config *cfg; + const char *str; + const char *content = "[include]\n" replicate10(replicate10("path=bottom\n")); + + cl_git_mkfile("top-level", "[include]\npath = middle\n[foo]\nbar = baz"); + cl_git_mkfile("middle", content); + cl_git_mkfile("bottom", "[foo]\nbar2 = baz2"); + + cl_git_pass(git_config_open_ondisk(&cfg, "top-level")); + + cl_git_pass(git_config_get_string(&str, cfg, "foo.bar")); + cl_assert_equal_s(str, "baz"); + + cl_git_pass(git_config_get_string(&str, cfg, "foo.bar2")); + cl_assert_equal_s(str, "baz2"); + + git_config_free(cfg); +} From 1713653883c3dfd199a18b51a1e2b37def183101 Mon Sep 17 00:00:00 2001 From: "Yury G. Kudryashov" Date: Thu, 5 Feb 2015 23:39:59 +0300 Subject: [PATCH 55/99] Reinit `reader` pointer after reading included config file Fixes #2869. If included file includes more files, it may reallocate cfg_file->readers, hence invalidate not only `r` pointer, but `result` pointer as well. --- src/config_file.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/config_file.c b/src/config_file.c index 4f041e7e3ad..268cced090a 100644 --- a/src/config_file.c +++ b/src/config_file.c @@ -1284,6 +1284,7 @@ static int config_parse(git_strmap *values, diskfile_backend *cfg_file, struct r if (result == 0) { result = config_parse(values, cfg_file, r, level, depth+1); r = git_array_get(cfg_file->readers, index); + reader = git_array_get(cfg_file->readers, reader_idx); } else if (result == GIT_ENOTFOUND) { giterr_clear(); From 3556a82a2ab3280ef6282de5d7f97248929a517f Mon Sep 17 00:00:00 2001 From: Leo Yang Date: Mon, 9 Feb 2015 17:35:25 -0500 Subject: [PATCH 56/99] Fix build warning on Android Always do a time_t cast on st_mtime. st_mtime on Android is not the type of time_t but has the same meaning which is the number of seconds past epoch. --- src/fileops.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fileops.c b/src/fileops.c index bd9d27c7ab6..68c2e893815 100644 --- a/src/fileops.c +++ b/src/fileops.c @@ -174,7 +174,7 @@ int git_futils_readbuffer_updated( */ if (size && *size != (size_t)st.st_size) changed = true; - if (mtime && *mtime != st.st_mtime) + if (mtime && *mtime != (time_t)st.st_mtime) changed = true; if (!size && !mtime) changed = true; From 392702ee2c88d7d8aaff25f7a84acb73606f9094 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Mon, 9 Feb 2015 23:41:13 -0500 Subject: [PATCH 57/99] allocations: test for overflow of requested size Introduce some helper macros to test integer overflow from arithmetic and set error message appropriately. --- src/array.h | 27 ++++++++++---- src/blame.c | 28 +++++++++++++-- src/blame_git.c | 8 +++-- src/buf_text.c | 12 ++++++- src/buffer.c | 62 +++++++++++++++++++++++++++++--- src/common.h | 22 ++++++++++++ src/config_file.c | 33 ++++++++++++----- src/delta-apply.c | 1 + src/delta.c | 38 ++++++++++++++++---- src/diff.c | 2 ++ src/diff_driver.c | 42 ++++++++++++++++------ src/diff_patch.c | 12 ++++++- src/diff_tform.c | 1 + src/filebuf.c | 5 +-- src/fileops.c | 6 +++- src/filter.c | 17 +++++++-- src/index.c | 18 +++++++--- src/iterator.c | 11 ++++-- src/merge.c | 2 ++ src/notes.c | 2 +- src/odb.c | 1 + src/odb_loose.c | 12 ++++++- src/odb_mempack.c | 1 + src/oid.c | 1 + src/pack-objects.c | 22 ++++++++++-- src/pack.c | 7 ++++ src/path.c | 64 +++++++++++++++++++++++++-------- src/pool.c | 6 ++-- src/refs.c | 18 +++++++--- src/remote.c | 3 +- src/revwalk.c | 6 +--- src/sortedcache.c | 4 ++- src/tag.c | 2 ++ src/transports/cred.c | 3 ++ src/transports/git.c | 4 +-- src/transports/local.c | 2 +- src/transports/smart.c | 2 +- src/transports/smart_pkt.c | 31 ++++++++++++++-- src/transports/smart_protocol.c | 4 +-- src/transports/winhttp.c | 4 +-- src/tree.c | 18 +++++++--- src/tsort.c | 5 ++- src/util.h | 15 ++++++-- src/vector.c | 7 ++-- src/win32/dir.c | 8 +++-- src/win32/utf-conv.c | 5 ++- src/zstream.c | 1 + tests/core/errors.c | 40 +++++++++++++++++++++ 48 files changed, 528 insertions(+), 117 deletions(-) diff --git a/src/array.h b/src/array.h index af9eafa43ea..ca5a35ffe5a 100644 --- a/src/array.h +++ b/src/array.h @@ -45,15 +45,28 @@ typedef git_array_t(char) git_array_generic_t; GIT_INLINE(void *) git_array_grow(void *_a, size_t item_size) { volatile git_array_generic_t *a = _a; - uint32_t new_size = (a->size < 8) ? 8 : a->asize * 3 / 2; - char *new_array = git__realloc(a->ptr, new_size * item_size); - if (!new_array) { - git_array_clear(*a); - return NULL; + uint32_t new_size; + char *new_array; + + if (a->size < 8) { + new_size = 8; } else { - a->ptr = new_array; a->asize = new_size; a->size++; - return a->ptr + (a->size - 1) * item_size; + if (GIT_ALLOC_OVERFLOW_MULTIPLY(a->size, 3 / 2)) + goto on_oom; + + new_size = a->size * 3 / 2; } + + if (GIT_ALLOC_OVERFLOW_MULTIPLY(new_size, item_size) || + (new_array = git__realloc(a->ptr, new_size * item_size)) == NULL) + goto on_oom; + + a->ptr = new_array; a->asize = new_size; a->size++; + return a->ptr + (a->size - 1) * item_size; + +on_oom: + git_array_clear(*a); + return NULL; } #define git_array_alloc(a) \ diff --git a/src/blame.c b/src/blame.c index 2cc5e552b9c..4a12cb85b2c 100644 --- a/src/blame.c +++ b/src/blame.c @@ -76,6 +76,10 @@ static git_blame_hunk* dup_hunk(git_blame_hunk *hunk) hunk->lines_in_hunk, hunk->orig_start_line_number, hunk->orig_path); + + if (!newhunk) + return NULL; + git_oid_cpy(&newhunk->orig_commit_id, &hunk->orig_commit_id); git_oid_cpy(&newhunk->final_commit_id, &hunk->final_commit_id); newhunk->boundary = hunk->boundary; @@ -221,6 +225,10 @@ static git_blame_hunk *split_hunk_in_vector( new_line_count = hunk->lines_in_hunk - rel_line; nh = new_hunk((uint16_t)(hunk->final_start_line_number+rel_line), (uint16_t)new_line_count, (uint16_t)(hunk->orig_start_line_number+rel_line), hunk->orig_path); + + if (!nh) + return NULL; + git_oid_cpy(&nh->final_commit_id, &hunk->final_commit_id); git_oid_cpy(&nh->orig_commit_id, &hunk->orig_commit_id); @@ -270,6 +278,10 @@ static git_blame_hunk* hunk_from_entry(git_blame__entry *e) { git_blame_hunk *h = new_hunk( e->lno+1, e->num_lines, e->s_lno+1, e->suspect->path); + + if (!h) + return NULL; + git_oid_cpy(&h->final_commit_id, git_commit_id(e->suspect->commit)); git_oid_cpy(&h->orig_commit_id, git_commit_id(e->suspect->commit)); git_signature_dup(&h->final_signature, git_commit_author(e->suspect->commit)); @@ -307,6 +319,8 @@ static int blame_internal(git_blame *blame) blame->final_buf_size = git_blob_rawsize(blame->final_blob); ent = git__calloc(1, sizeof(git_blame__entry)); + GITERR_CHECK_ALLOC(ent); + ent->num_lines = index_blob_lines(blame); ent->lno = blame->options.min_line - 1; ent->num_lines = ent->num_lines - blame->options.min_line + 1; @@ -322,8 +336,9 @@ static int blame_internal(git_blame *blame) cleanup: for (ent = blame->ent; ent; ) { git_blame__entry *e = ent->next; + git_blame_hunk *h = hunk_from_entry(ent); - git_vector_insert(&blame->hunks, hunk_from_entry(ent)); + git_vector_insert(&blame->hunks, h); git_blame__free_entry(ent); ent = e; @@ -392,11 +407,14 @@ static int buffer_hunk_cb( if (!blame->current_hunk) { /* Line added at the end of the file */ blame->current_hunk = new_hunk(wedge_line, 0, wedge_line, blame->path); + GITERR_CHECK_ALLOC(blame->current_hunk); + git_vector_insert(&blame->hunks, blame->current_hunk); } else if (!hunk_starts_at_or_after_line(blame->current_hunk, wedge_line)){ /* If this hunk doesn't start between existing hunks, split a hunk up so it does */ blame->current_hunk = split_hunk_in_vector(&blame->hunks, blame->current_hunk, wedge_line - blame->current_hunk->orig_start_line_number, true); + GITERR_CHECK_ALLOC(blame->current_hunk); } return 0; @@ -425,6 +443,8 @@ static int buffer_line_cb( /* Create a new buffer-blame hunk with this line */ shift_hunks_by(&blame->hunks, blame->current_diff_line, 1); blame->current_hunk = new_hunk((uint16_t)blame->current_diff_line, 1, 0, blame->path); + GITERR_CHECK_ALLOC(blame->current_hunk); + git_vector_insert_sorted(&blame->hunks, blame->current_hunk, NULL); } blame->current_diff_line++; @@ -464,10 +484,14 @@ int git_blame_buffer( assert(out && reference && buffer && buffer_len); blame = git_blame__alloc(reference->repository, reference->options, reference->path); + GITERR_CHECK_ALLOC(blame); /* Duplicate all of the hunk structures in the reference blame */ git_vector_foreach(&reference->hunks, i, hunk) { - git_vector_insert(&blame->hunks, dup_hunk(hunk)); + git_blame_hunk *h = dup_hunk(hunk); + GITERR_CHECK_ALLOC(h); + + git_vector_insert(&blame->hunks, h); } /* Diff to the reference blob */ diff --git a/src/blame_git.c b/src/blame_git.c index 72afb852ba2..05aef5d9996 100644 --- a/src/blame_git.c +++ b/src/blame_git.c @@ -35,10 +35,14 @@ static void origin_decref(git_blame__origin *o) /* Given a commit and a path in it, create a new origin structure. */ static int make_origin(git_blame__origin **out, git_commit *commit, const char *path) { - int error = 0; git_blame__origin *o; + size_t path_len = strlen(path); + int error = 0; + + GITERR_CHECK_ALLOC_ADD(sizeof(*o), path_len); + GITERR_CHECK_ALLOC_ADD(sizeof(*o) + path_len, 1); - o = git__calloc(1, sizeof(*o) + strlen(path) + 1); + o = git__calloc(1, sizeof(*o) + path_len + 1); GITERR_CHECK_ALLOC(o); o->commit = commit; o->refcnt = 1; diff --git a/src/buf_text.c b/src/buf_text.c index cb3661edb5f..ace54d72557 100644 --- a/src/buf_text.c +++ b/src/buf_text.c @@ -29,6 +29,8 @@ int git_buf_text_puts_escaped( scan += count; } + GITERR_CHECK_ALLOC_ADD(buf->size, total); + GITERR_CHECK_ALLOC_ADD(buf->size + total, 1); if (git_buf_grow(buf, buf->size + total + 1) < 0) return -1; @@ -73,8 +75,10 @@ int git_buf_text_crlf_to_lf(git_buf *tgt, const git_buf *src) return git_buf_set(tgt, src->ptr, src->size); /* reduce reallocs while in the loop */ + GITERR_CHECK_ALLOC_ADD(src->size, 1); if (git_buf_grow(tgt, src->size + 1) < 0) return -1; + out = tgt->ptr; tgt->size = 0; @@ -117,13 +121,15 @@ int git_buf_text_lf_to_crlf(git_buf *tgt, const git_buf *src) return git_buf_set(tgt, src->ptr, src->size); /* attempt to reduce reallocs while in the loop */ + GITERR_CHECK_ALLOC_ADD(src->size, src->size >> 4); + GITERR_CHECK_ALLOC_ADD(src->size + (src->size >> 4), 1); if (git_buf_grow(tgt, src->size + (src->size >> 4) + 1) < 0) return -1; tgt->size = 0; for (; next; scan = next + 1, next = memchr(scan, '\n', end - scan)) { size_t copylen = next - scan; - size_t needsize = tgt->size + copylen + 2 + 1; + size_t needsize; /* if we find mixed line endings, bail */ if (next > start && next[-1] == '\r') { @@ -131,6 +137,10 @@ int git_buf_text_lf_to_crlf(git_buf *tgt, const git_buf *src) return GIT_PASSTHROUGH; } + GITERR_CHECK_ALLOC_ADD(tgt->size, copylen); + GITERR_CHECK_ALLOC_ADD(tgt->size + copylen, 3); + needsize = tgt->size + copylen + 3; + if (tgt->asize < needsize && git_buf_grow(tgt, needsize) < 0) return -1; diff --git a/src/buffer.c b/src/buffer.c index 8013457c519..ee8bba4ecf1 100644 --- a/src/buffer.c +++ b/src/buffer.c @@ -63,6 +63,14 @@ int git_buf_try_grow( /* round allocation up to multiple of 8 */ new_size = (new_size + 7) & ~7; + if (new_size < buf->size) { + if (mark_oom) + buf->ptr = git_buf__oom; + + giterr_set_oom(); + return -1; + } + new_ptr = git__realloc(new_ptr, new_size); if (!new_ptr) { @@ -131,6 +139,7 @@ int git_buf_set(git_buf *buf, const void *data, size_t len) git_buf_clear(buf); } else { if (data != buf->ptr) { + GITERR_CHECK_ALLOC_ADD(len, 1); ENSURE_SIZE(buf, len + 1); memmove(buf->ptr, data, len); } @@ -160,6 +169,7 @@ int git_buf_sets(git_buf *buf, const char *string) int git_buf_putc(git_buf *buf, char c) { + GITERR_CHECK_ALLOC_ADD(buf->size, 2); ENSURE_SIZE(buf, buf->size + 2); buf->ptr[buf->size++] = c; buf->ptr[buf->size] = '\0'; @@ -168,6 +178,8 @@ int git_buf_putc(git_buf *buf, char c) int git_buf_putcn(git_buf *buf, char c, size_t len) { + GITERR_CHECK_ALLOC_ADD(buf->size, len); + GITERR_CHECK_ALLOC_ADD(buf->size + len, 1); ENSURE_SIZE(buf, buf->size + len + 1); memset(buf->ptr + buf->size, c, len); buf->size += len; @@ -179,6 +191,8 @@ int git_buf_put(git_buf *buf, const char *data, size_t len) { if (len) { assert(data); + GITERR_CHECK_ALLOC_ADD(buf->size, len); + GITERR_CHECK_ALLOC_ADD(buf->size + len, 1); ENSURE_SIZE(buf, buf->size + len + 1); memmove(buf->ptr + buf->size, data, len); buf->size += len; @@ -201,8 +215,12 @@ int git_buf_encode_base64(git_buf *buf, const char *data, size_t len) size_t extra = len % 3; uint8_t *write, a, b, c; const uint8_t *read = (const uint8_t *)data; + size_t blocks = (len / 3) + !!extra; - ENSURE_SIZE(buf, buf->size + 4 * ((len / 3) + !!extra) + 1); + GITERR_CHECK_ALLOC_MULTIPLY(blocks, 4); + GITERR_CHECK_ALLOC_ADD(buf->size, 4 * blocks); + GITERR_CHECK_ALLOC_ADD(buf->size + 4 * blocks, 1); + ENSURE_SIZE(buf, buf->size + 4 * blocks + 1); write = (uint8_t *)&buf->ptr[buf->size]; /* convert each run of 3 bytes into 4 output bytes */ @@ -256,6 +274,8 @@ int git_buf_decode_base64(git_buf *buf, const char *base64, size_t len) size_t orig_size = buf->size; assert(len % 4 == 0); + GITERR_CHECK_ALLOC_ADD(buf->size, len / 4 * 3); + GITERR_CHECK_ALLOC_ADD(buf->size + (len / 4 * 3), 1); ENSURE_SIZE(buf, buf->size + (len / 4 * 3) + 1); for (i = 0; i < len; i += 4) { @@ -284,7 +304,12 @@ static const char b85str[] = int git_buf_encode_base85(git_buf *buf, const char *data, size_t len) { - ENSURE_SIZE(buf, buf->size + (5 * ((len / 4) + !!(len % 4))) + 1); + size_t blocks = (len / 4) + !!(len % 4); + + GITERR_CHECK_ALLOC_MULTIPLY(blocks, 5); + GITERR_CHECK_ALLOC_ADD(buf->size, 5 * blocks); + GITERR_CHECK_ALLOC_ADD(buf->size + 5 * blocks, 1); + ENSURE_SIZE(buf, buf->size + blocks * 5 + 1); while (len) { uint32_t acc = 0; @@ -317,8 +342,14 @@ int git_buf_encode_base85(git_buf *buf, const char *data, size_t len) int git_buf_vprintf(git_buf *buf, const char *format, va_list ap) { + size_t expected_size = strlen(format); int len; - const size_t expected_size = buf->size + (strlen(format) * 2); + + GITERR_CHECK_ALLOC_MULTIPLY(expected_size, 2); + expected_size *= 2; + + GITERR_CHECK_ALLOC_ADD(expected_size, buf->size); + expected_size += buf->size; ENSURE_SIZE(buf, expected_size); @@ -345,6 +376,8 @@ int git_buf_vprintf(git_buf *buf, const char *format, va_list ap) break; } + GITERR_CHECK_ALLOC_ADD(buf->size, len); + GITERR_CHECK_ALLOC_ADD(buf->size + len, 1); ENSURE_SIZE(buf, buf->size + len + 1); } @@ -481,6 +514,9 @@ int git_buf_join_n(git_buf *buf, char separator, int nbuf, ...) /* expand buffer if needed */ if (total_size == 0) return 0; + + GITERR_CHECK_ALLOC_ADD(buf->size, total_size); + GITERR_CHECK_ALLOC_ADD(buf->size + total_size, 1); if (git_buf_grow(buf, buf->size + total_size + 1) < 0) return -1; @@ -559,6 +595,9 @@ int git_buf_join( if (str_a >= buf->ptr && str_a < buf->ptr + buf->size) offset_a = str_a - buf->ptr; + GITERR_CHECK_ALLOC_ADD(strlen_a, strlen_b); + GITERR_CHECK_ALLOC_ADD(strlen_a + strlen_b, need_sep); + GITERR_CHECK_ALLOC_ADD(strlen_a + strlen_b + need_sep, 1); if (git_buf_grow(buf, strlen_a + strlen_b + need_sep + 1) < 0) return -1; assert(buf->ptr); @@ -607,6 +646,11 @@ int git_buf_join3( sep_b = (str_b[len_b - 1] != separator); } + GITERR_CHECK_ALLOC_ADD(len_a, sep_a); + GITERR_CHECK_ALLOC_ADD(len_a + sep_a, len_b); + GITERR_CHECK_ALLOC_ADD(len_a + sep_a + len_b, sep_b); + GITERR_CHECK_ALLOC_ADD(len_a + sep_a + len_b + sep_b, len_c); + GITERR_CHECK_ALLOC_ADD(len_a + sep_a + len_b + sep_b + len_c, 1); if (git_buf_grow(buf, len_a + sep_a + len_b + sep_b + len_c + 1) < 0) return -1; @@ -660,6 +704,8 @@ int git_buf_splice( const char *data, size_t nb_to_insert) { + size_t new_size; + assert(buf && where <= git_buf_len(buf) && where + nb_to_remove <= git_buf_len(buf)); @@ -667,7 +713,13 @@ int git_buf_splice( /* Ported from git.git * https://github.com/git/git/blob/16eed7c/strbuf.c#L159-176 */ - ENSURE_SIZE(buf, buf->size + nb_to_insert - nb_to_insert + 1); + new_size = buf->size - nb_to_remove; + + GITERR_CHECK_ALLOC_ADD(new_size, nb_to_insert); + new_size += nb_to_insert; + + GITERR_CHECK_ALLOC_ADD(new_size, 1); + ENSURE_SIZE(buf, new_size + 1); memmove(buf->ptr + where + nb_to_insert, buf->ptr + where + nb_to_remove, @@ -675,7 +727,7 @@ int git_buf_splice( memcpy(buf->ptr + where, data, nb_to_insert); - buf->size = buf->size + nb_to_insert - nb_to_remove; + buf->size = new_size; buf->ptr[buf->size] = '\0'; return 0; } diff --git a/src/common.h b/src/common.h index 4b4a9977545..b53798eaf01 100644 --- a/src/common.h +++ b/src/common.h @@ -174,6 +174,28 @@ GIT_INLINE(void) git__init_structure(void *structure, size_t len, unsigned int v GITERR_CHECK_VERSION(&(VERSION), _tmpl.version, #TYPE); \ memcpy((PTR), &_tmpl, sizeof(_tmpl)); } while (0) +/** Check for integer overflow from addition or multiplication */ +#define GIT_ALLOC_OVERFLOW_ADD(one, two) \ + ((one) + (two) < (one)) + +/** Check for integer overflow from multiplication */ +#define GIT_ALLOC_OVERFLOW_MULTIPLY(one, two) \ + (one && ((one) * (two)) / (one) != (two)) + +/** Check for additive overflow, failing if it would occur. */ +#define GITERR_CHECK_ALLOC_ADD(one, two) \ + if (GIT_ALLOC_OVERFLOW_ADD(one, two)) { \ + giterr_set_oom(); \ + return -1; \ + } + +/** Check for multiplicative overflow, failing if it would occur. */ +#define GITERR_CHECK_ALLOC_MULTIPLY(nelem, elsize) \ + if (GIT_ALLOC_OVERFLOW_MULTIPLY(nelem, elsize)) { \ + giterr_set_oom(); \ + return -1; \ + } + /* NOTE: other giterr functions are in the public errors.h header file */ #include "util.h" diff --git a/src/config_file.c b/src/config_file.c index 4f041e7e3ad..36f78563bb8 100644 --- a/src/config_file.c +++ b/src/config_file.c @@ -903,9 +903,11 @@ static char *reader_readline(struct reader *reader, bool skip_whitespace) line_len = line_end - line_src; - line = git__malloc(line_len + 1); - if (line == NULL) + if (GIT_ALLOC_OVERFLOW_ADD(line_len, 1) || + (line = git__malloc(line_len + 1)) == NULL) { + giterr_set_oom(); return NULL; + } memcpy(line, line_src, line_len); @@ -958,6 +960,8 @@ static int parse_section_header_ext(struct reader *reader, const char *line, con int c, rpos; char *first_quote, *last_quote; git_buf buf = GIT_BUF_INIT; + size_t quoted_len, base_name_len = strlen(base_name); + /* * base_name is what came before the space. We should be at the * first quotation mark, except for now, line isn't being kept in @@ -966,13 +970,17 @@ static int parse_section_header_ext(struct reader *reader, const char *line, con first_quote = strchr(line, '"'); last_quote = strrchr(line, '"'); + quoted_len = last_quote - first_quote; - if (last_quote - first_quote == 0) { + if (quoted_len == 0) { set_parse_error(reader, 0, "Missing closing quotation mark in section header"); return -1; } - git_buf_grow(&buf, strlen(base_name) + last_quote - first_quote + 2); + GITERR_CHECK_ALLOC_ADD(base_name_len, quoted_len); + GITERR_CHECK_ALLOC_ADD(base_name_len + quoted_len, 2); + + git_buf_grow(&buf, base_name_len + quoted_len + 2); git_buf_printf(&buf, "%s.", base_name); rpos = 0; @@ -1029,6 +1037,7 @@ static int parse_section_header(struct reader *reader, char **section_out) int name_length, c, pos; int result; char *line; + size_t line_len; line = reader_readline(reader, true); if (line == NULL) @@ -1042,7 +1051,10 @@ static int parse_section_header(struct reader *reader, char **section_out) return -1; } - name = (char *)git__malloc((size_t)(name_end - line) + 1); + line_len = (size_t)(name_end - line); + + GITERR_CHECK_ALLOC_ADD(line_len, 1); + name = git__malloc(line_len); GITERR_CHECK_ALLOC(name); name_length = 0; @@ -1603,11 +1615,16 @@ static char *escape_value(const char *ptr) /* '\"' -> '"' etc */ static char *fixup_line(const char *ptr, int quote_count) { - char *str = git__malloc(strlen(ptr) + 1); - char *out = str, *esc; + char *str, *out, *esc; + size_t ptr_len = strlen(ptr); - if (str == NULL) + if (GIT_ALLOC_OVERFLOW_ADD(ptr_len, 1) || + (str = git__malloc(ptr_len + 1)) == NULL) { + giterr_set_oom(); return NULL; + } + + out = str; while (*ptr != '\0') { if (*ptr == '"') { diff --git a/src/delta-apply.c b/src/delta-apply.c index a39c7af5cf0..e46c9631c48 100644 --- a/src/delta-apply.c +++ b/src/delta-apply.c @@ -74,6 +74,7 @@ int git__delta_apply( return -1; } + GITERR_CHECK_ALLOC_ADD(res_sz, 1); res_dp = git__malloc(res_sz + 1); GITERR_CHECK_ALLOC(res_dp); diff --git a/src/delta.c b/src/delta.c index 8375a2c4da5..f704fdfb1cf 100644 --- a/src/delta.c +++ b/src/delta.c @@ -119,6 +119,36 @@ struct git_delta_index { struct index_entry *hash[GIT_FLEX_ARRAY]; }; +static int lookup_index_alloc( + void **out, unsigned long *out_len, size_t entries, size_t hash_count) +{ + size_t entries_len, hash_len, + index_len = sizeof(struct git_delta_index); + + GITERR_CHECK_ALLOC_MULTIPLY(entries, sizeof(struct index_entry)); + entries_len = entries * sizeof(struct index_entry); + + GITERR_CHECK_ALLOC_ADD(index_len, entries_len); + index_len += entries_len; + + GITERR_CHECK_ALLOC_MULTIPLY(hash_count, sizeof(struct index_entry *)); + hash_len = hash_count * sizeof(struct index_entry *); + + GITERR_CHECK_ALLOC_ADD(index_len, hash_len); + index_len += hash_len; + + if (!git__is_ulong(index_len)) { + giterr_set_oom(); + return -1; + } + + *out = git__malloc(index_len); + GITERR_CHECK_ALLOC(*out); + + *out_len = index_len; + return 0; +} + struct git_delta_index * git_delta_create_index(const void *buf, unsigned long bufsize) { @@ -148,13 +178,9 @@ git_delta_create_index(const void *buf, unsigned long bufsize) hsize = 1 << i; hmask = hsize - 1; - /* allocate lookup index */ - memsize = sizeof(*index) + - sizeof(*hash) * hsize + - sizeof(*entry) * entries; - mem = git__malloc(memsize); - if (!mem) + if (lookup_index_alloc(&mem, &memsize, entries, hsize) < 0) return NULL; + index = mem; mem = index->hash; hash = mem; diff --git a/src/diff.c b/src/diff.c index e23d3891f67..75e9ae9a3ba 100644 --- a/src/diff.c +++ b/src/diff.c @@ -1558,8 +1558,10 @@ int git_diff_format_email( goto on_error; } + GITERR_CHECK_ALLOC_ADD(offset, 1); summary = git__calloc(offset + 1, sizeof(char)); GITERR_CHECK_ALLOC(summary); + strncpy(summary, opts->summary, offset); } diff --git a/src/diff_driver.c b/src/diff_driver.c index c3c5f365bb4..67f1c591d5c 100644 --- a/src/diff_driver.c +++ b/src/diff_driver.c @@ -158,6 +158,29 @@ static git_diff_driver_registry *git_repository_driver_registry( return repo->diff_drivers; } +static int diff_driver_alloc( + git_diff_driver **out, size_t *namelen_out, const char *name) +{ + git_diff_driver *driver; + size_t driverlen = sizeof(git_diff_driver), + namelen = strlen(name); + + GITERR_CHECK_ALLOC_ADD(driverlen, namelen); + GITERR_CHECK_ALLOC_ADD(driverlen + namelen, 1); + + driver = git__calloc(1, driverlen + namelen + 1); + GITERR_CHECK_ALLOC(driver); + + memcpy(driver->name, name, namelen); + + *out = driver; + + if (namelen_out) + *namelen_out = namelen; + + return 0; +} + static int git_diff_driver_builtin( git_diff_driver **out, git_diff_driver_registry *reg, @@ -166,7 +189,7 @@ static int git_diff_driver_builtin( int error = 0; git_diff_driver_definition *ddef = NULL; git_diff_driver *drv = NULL; - size_t namelen, idx; + size_t idx; for (idx = 0; idx < ARRAY_SIZE(builtin_defs); ++idx) { if (!strcasecmp(driver_name, builtin_defs[idx].name)) { @@ -177,13 +200,10 @@ static int git_diff_driver_builtin( if (!ddef) goto done; - namelen = strlen(ddef->name); - - drv = git__calloc(1, sizeof(git_diff_driver) + namelen + 1); - GITERR_CHECK_ALLOC(drv); + if ((error = diff_driver_alloc(&drv, NULL, ddef->name)) < 0) + goto done; drv->type = DIFF_DRIVER_PATTERNLIST; - memcpy(drv->name, ddef->name, namelen); if (ddef->fns && (error = diff_driver_add_patterns( @@ -217,9 +237,9 @@ static int git_diff_driver_load( int error = 0; git_diff_driver_registry *reg; git_diff_driver *drv = NULL; - size_t namelen = strlen(driver_name); + size_t namelen; khiter_t pos; - git_config *cfg; + git_config *cfg = NULL; git_buf name = GIT_BUF_INIT; const git_config_entry *ce; bool found_driver = false; @@ -233,10 +253,10 @@ static int git_diff_driver_load( return 0; } - drv = git__calloc(1, sizeof(git_diff_driver) + namelen + 1); - GITERR_CHECK_ALLOC(drv); + if ((error = diff_driver_alloc(&drv, &namelen, driver_name)) < 0) + goto done; + drv->type = DIFF_DRIVER_AUTO; - memcpy(drv->name, driver_name, namelen); /* if you can't read config for repo, just use default driver */ if (git_repository_config_snapshot(&cfg, repo) < 0) { diff --git a/src/diff_patch.c b/src/diff_patch.c index a151077539d..f5eecae660d 100644 --- a/src/diff_patch.c +++ b/src/diff_patch.c @@ -388,8 +388,18 @@ static int diff_patch_with_delta_alloc( diff_patch_with_delta *pd; size_t old_len = *old_path ? strlen(*old_path) : 0; size_t new_len = *new_path ? strlen(*new_path) : 0; + size_t alloc_len = sizeof(*pd); - *out = pd = git__calloc(1, sizeof(*pd) + old_len + new_len + 2); + GITERR_CHECK_ALLOC_ADD(alloc_len, old_len); + alloc_len += old_len; + + GITERR_CHECK_ALLOC_ADD(alloc_len, new_len); + alloc_len += new_len; + + GITERR_CHECK_ALLOC_ADD(alloc_len, 2); + alloc_len += 2; + + *out = pd = git__calloc(1, alloc_len); GITERR_CHECK_ALLOC(pd); pd->patch.flags = GIT_DIFF_PATCH_ALLOCATED; diff --git a/src/diff_tform.c b/src/diff_tform.c index 9133a9b14a9..cad1356c3f1 100644 --- a/src/diff_tform.c +++ b/src/diff_tform.c @@ -831,6 +831,7 @@ int git_diff_find_similar( if ((opts.flags & GIT_DIFF_FIND_ALL) == 0) goto cleanup; + GITERR_CHECK_ALLOC_MULTIPLY(num_deltas, 2); sigcache = git__calloc(num_deltas * 2, sizeof(void *)); GITERR_CHECK_ALLOC(sigcache); diff --git a/src/filebuf.c b/src/filebuf.c index 25f6e52ef34..1a915755842 100644 --- a/src/filebuf.c +++ b/src/filebuf.c @@ -271,6 +271,7 @@ int git_filebuf_open(git_filebuf *file, const char *path, int flags, mode_t mode GITERR_CHECK_ALLOC(file->path_original); /* create the locking path by appending ".lock" to the original */ + GITERR_CHECK_ALLOC_ADD(path_len, GIT_FILELOCK_EXTLENGTH); file->path_lock = git__malloc(path_len + GIT_FILELOCK_EXTLENGTH); GITERR_CHECK_ALLOC(file->path_lock); @@ -437,8 +438,8 @@ int git_filebuf_printf(git_filebuf *file, const char *format, ...) } while ((size_t)len + 1 <= space_left); - tmp_buffer = git__malloc(len + 1); - if (!tmp_buffer) { + if (GIT_ALLOC_OVERFLOW_ADD(len, 1) || + !(tmp_buffer = git__malloc(len + 1))) { file->last_error = BUFERR_MEM; return -1; } diff --git a/src/fileops.c b/src/fileops.c index eddd5a80461..eb24013e888 100644 --- a/src/fileops.c +++ b/src/fileops.c @@ -127,6 +127,7 @@ int git_futils_readbuffer_fd(git_buf *buf, git_file fd, size_t len) git_buf_clear(buf); + GITERR_CHECK_ALLOC_ADD(len, 1); if (git_buf_grow(buf, len + 1) < 0) return -1; @@ -708,7 +709,10 @@ static int cp_link(const char *from, const char *to, size_t link_size) { int error = 0; ssize_t read_len; - char *link_data = git__malloc(link_size + 1); + char *link_data; + + GITERR_CHECK_ALLOC_ADD(link_size, 1); + link_data = git__malloc(link_size + 1); GITERR_CHECK_ALLOC(link_data); read_len = p_readlink(from, link_data, link_size); diff --git a/src/filter.c b/src/filter.c index 22eaf51a28e..d5c669f0164 100644 --- a/src/filter.c +++ b/src/filter.c @@ -228,7 +228,7 @@ int git_filter_register( const char *name, git_filter *filter, int priority) { git_filter_def *fdef; - size_t nattr = 0, nmatch = 0; + size_t nattr = 0, nmatch = 0, alloc_len; git_buf attrs = GIT_BUF_INIT; assert(name && filter); @@ -245,8 +245,16 @@ int git_filter_register( if (filter_def_scan_attrs(&attrs, &nattr, &nmatch, filter->attributes) < 0) return -1; - fdef = git__calloc( - sizeof(git_filter_def) + 2 * nattr * sizeof(char *), 1); + GITERR_CHECK_ALLOC_MULTIPLY(nattr, 2); + alloc_len = nattr * 2; + + GITERR_CHECK_ALLOC_MULTIPLY(alloc_len, sizeof(char *)); + alloc_len *= sizeof(char *); + + GITERR_CHECK_ALLOC_ADD(alloc_len, sizeof(git_filter_def)); + alloc_len += sizeof(git_filter_def); + + fdef = git__calloc(1, alloc_len); GITERR_CHECK_ALLOC(fdef); fdef->filter_name = git__strdup(name); @@ -379,6 +387,9 @@ static int filter_list_new( git_filter_list *fl = NULL; size_t pathlen = src->path ? strlen(src->path) : 0; + GITERR_CHECK_ALLOC_ADD(sizeof(git_filter_list), pathlen); + GITERR_CHECK_ALLOC_ADD(sizeof(git_filter_list) + pathlen, 1); + fl = git__calloc(1, sizeof(git_filter_list) + pathlen + 1); GITERR_CHECK_ALLOC(fl); diff --git a/src/index.c b/src/index.c index cbace3606c3..70090d12c4e 100644 --- a/src/index.c +++ b/src/index.c @@ -779,7 +779,9 @@ static int index_entry_create( return -1; } - entry = git__calloc(sizeof(struct entry_internal) + pathlen + 1, 1); + GITERR_CHECK_ALLOC_ADD(sizeof(struct entry_internal), pathlen); + GITERR_CHECK_ALLOC_ADD(sizeof(struct entry_internal) + pathlen, 1); + entry = git__calloc(1, sizeof(struct entry_internal) + pathlen + 1); GITERR_CHECK_ALLOC(entry); entry->pathlen = pathlen; @@ -826,9 +828,17 @@ static int index_entry_init( static git_index_reuc_entry *reuc_entry_alloc(const char *path) { - size_t pathlen = strlen(path); - struct reuc_entry_internal *entry = - git__calloc(sizeof(struct reuc_entry_internal) + pathlen + 1, 1); + size_t pathlen = strlen(path), + structlen = sizeof(struct reuc_entry_internal); + struct reuc_entry_internal *entry; + + if (GIT_ALLOC_OVERFLOW_ADD(structlen, pathlen) || + GIT_ALLOC_OVERFLOW_ADD(structlen + pathlen, 1)) { + giterr_set_oom(); + return NULL; + } + + entry = git__calloc(1, structlen + pathlen + 1); if (!entry) return NULL; diff --git a/src/iterator.c b/src/iterator.c index 196adddbf6c..e90cf30ff76 100644 --- a/src/iterator.c +++ b/src/iterator.c @@ -336,7 +336,7 @@ static int tree_iterator__push_frame(tree_iterator *ti) { int error = 0; tree_iterator_frame *head = ti->head, *tf = NULL; - size_t i, n_entries = 0; + size_t i, n_entries = 0, alloclen; if (head->current >= head->n_entries || !head->entries[head->current]->tree) return GIT_ITEROVER; @@ -344,8 +344,13 @@ static int tree_iterator__push_frame(tree_iterator *ti) for (i = head->current; i < head->next; ++i) n_entries += git_tree_entrycount(head->entries[i]->tree); - tf = git__calloc(sizeof(tree_iterator_frame) + - n_entries * sizeof(tree_iterator_entry *), 1); + GITERR_CHECK_ALLOC_MULTIPLY(sizeof(tree_iterator_entry *), n_entries); + alloclen = sizeof(tree_iterator_entry *) * n_entries; + + GITERR_CHECK_ALLOC_ADD(alloclen, sizeof(tree_iterator_frame)); + alloclen += sizeof(tree_iterator_frame); + + tf = git__calloc(1, alloclen); GITERR_CHECK_ALLOC(tf); tf->n_entries = n_entries; diff --git a/src/merge.c b/src/merge.c index 7c38b5692e8..25d7bd7aa39 100644 --- a/src/merge.c +++ b/src/merge.c @@ -1169,6 +1169,7 @@ int git_merge_diff_list__find_renames( goto done; if (diff_list->conflicts.length <= opts->target_limit) { + GITERR_CHECK_ALLOC_MULTIPLY(diff_list->conflicts.length, 3); cache_size = diff_list->conflicts.length * 3; cache = git__calloc(cache_size, sizeof(void *)); GITERR_CHECK_ALLOC(cache); @@ -2228,6 +2229,7 @@ static int merge_ancestor_head( assert(repo && our_head && their_heads); + GITERR_CHECK_ALLOC_ADD(their_heads_len, 1); oids = git__calloc(their_heads_len + 1, sizeof(git_oid)); GITERR_CHECK_ALLOC(oids); diff --git a/src/notes.c b/src/notes.c index f44c0bd95e9..1850e81c747 100644 --- a/src/notes.c +++ b/src/notes.c @@ -314,7 +314,7 @@ static int note_new( { git_note *note = NULL; - note = (git_note *)git__malloc(sizeof(git_note)); + note = git__malloc(sizeof(git_note)); GITERR_CHECK_ALLOC(note); git_oid_cpy(¬e->id, note_oid); diff --git a/src/odb.c b/src/odb.c index d2dff717875..80e02ef2b87 100644 --- a/src/odb.c +++ b/src/odb.c @@ -233,6 +233,7 @@ int git_odb__hashlink(git_oid *out, const char *path) char *link_data; ssize_t read_len; + GITERR_CHECK_ALLOC_ADD(size, 1); link_data = git__malloc((size_t)(size + 1)); GITERR_CHECK_ALLOC(link_data); diff --git a/src/odb_loose.c b/src/odb_loose.c index ea9bdc4a0f5..e43b261fa6e 100644 --- a/src/odb_loose.c +++ b/src/odb_loose.c @@ -64,6 +64,8 @@ static int object_file_name( git_buf *name, const loose_backend *be, const git_oid *id) { /* expand length for object root + 40 hex sha1 chars + 2 * '/' + '\0' */ + GITERR_CHECK_ALLOC_ADD(be->objects_dirlen, GIT_OID_HEXSZ); + GITERR_CHECK_ALLOC_ADD(be->objects_dirlen + GIT_OID_HEXSZ, 3); if (git_buf_grow(name, be->objects_dirlen + GIT_OID_HEXSZ + 3) < 0) return -1; @@ -268,7 +270,8 @@ static void *inflate_tail(z_stream *s, void *hb, size_t used, obj_hdr *hdr) * initial sequence of inflated data from the tail of the * head buffer, if any. */ - if ((buf = git__malloc(hdr->size + 1)) == NULL) { + if (GIT_ALLOC_OVERFLOW_ADD(hdr->size, 1) || + (buf = git__malloc(hdr->size + 1)) == NULL) { inflateEnd(s); return NULL; } @@ -321,6 +324,7 @@ static int inflate_packlike_loose_disk_obj(git_rawobj *out, git_buf *obj) /* * allocate a buffer and inflate the data into it */ + GITERR_CHECK_ALLOC_ADD(hdr.size, 1); buf = git__malloc(hdr.size + 1); GITERR_CHECK_ALLOC(buf); @@ -520,6 +524,8 @@ static int locate_object_short_oid( int error; /* prealloc memory for OBJ_DIR/xx/xx..38x..xx */ + GITERR_CHECK_ALLOC_ADD(dir_len, GIT_OID_HEXSZ); + GITERR_CHECK_ALLOC_ADD(dir_len + GIT_OID_HEXSZ, 3); if (git_buf_grow(object_location, dir_len + 3 + GIT_OID_HEXSZ) < 0) return -1; @@ -563,6 +569,8 @@ static int locate_object_short_oid( return error; /* Update the location according to the oid obtained */ + GITERR_CHECK_ALLOC_ADD(dir_len, GIT_OID_HEXSZ); + GITERR_CHECK_ALLOC_ADD(dir_len + GIT_OID_HEXSZ, 2); git_buf_truncate(object_location, dir_len); if (git_buf_grow(object_location, dir_len + GIT_OID_HEXSZ + 2) < 0) @@ -928,6 +936,8 @@ int git_odb_backend_loose( objects_dirlen = strlen(objects_dir); + GITERR_CHECK_ALLOC_ADD(sizeof(loose_backend), objects_dirlen); + GITERR_CHECK_ALLOC_ADD(sizeof(loose_backend) + objects_dirlen, 2); backend = git__calloc(1, sizeof(loose_backend) + objects_dirlen + 2); GITERR_CHECK_ALLOC(backend); diff --git a/src/odb_mempack.c b/src/odb_mempack.c index d9b3a1824dd..a71d8db4b1a 100644 --- a/src/odb_mempack.c +++ b/src/odb_mempack.c @@ -47,6 +47,7 @@ static int impl__write(git_odb_backend *_backend, const git_oid *oid, const void if (rval == 0) return 0; + GITERR_CHECK_ALLOC_ADD(sizeof(struct memobject), len); obj = git__malloc(sizeof(struct memobject) + len); GITERR_CHECK_ALLOC(obj); diff --git a/src/oid.c b/src/oid.c index 5d29200b6f3..9aac47db68d 100644 --- a/src/oid.c +++ b/src/oid.c @@ -261,6 +261,7 @@ struct git_oid_shorten { static int resize_trie(git_oid_shorten *self, size_t new_size) { + GITERR_CHECK_ALLOC_MULTIPLY(new_size, sizeof(trie_node)); self->nodes = git__realloc(self->nodes, new_size * sizeof(trie_node)); GITERR_CHECK_ALLOC(self->nodes); diff --git a/src/pack-objects.c b/src/pack-objects.c index 0040a826bc7..aea4770afd0 100644 --- a/src/pack-objects.c +++ b/src/pack-objects.c @@ -201,7 +201,11 @@ int git_packbuilder_insert(git_packbuilder *pb, const git_oid *oid, return 0; if (pb->nr_objects >= pb->nr_alloc) { + GITERR_CHECK_ALLOC_ADD(pb->nr_alloc, 1024); + GITERR_CHECK_ALLOC_MULTIPLY(pb->nr_alloc + 1024, 3 / 2); pb->nr_alloc = (pb->nr_alloc + 1024) * 3 / 2; + + GITERR_CHECK_ALLOC_MULTIPLY(pb->nr_alloc, sizeof(*po)); pb->object_list = git__realloc(pb->object_list, pb->nr_alloc * sizeof(*po)); GITERR_CHECK_ALLOC(pb->object_list); @@ -499,8 +503,13 @@ static int cb_tag_foreach(const char *name, git_oid *oid, void *data) static git_pobject **compute_write_order(git_packbuilder *pb) { unsigned int i, wo_end, last_untagged; + git_pobject **wo; - git_pobject **wo = git__malloc(sizeof(*wo) * pb->nr_objects); + if (GIT_ALLOC_OVERFLOW_MULTIPLY(pb->nr_objects, sizeof(*wo)) || + (wo = git__malloc(pb->nr_objects * sizeof(*wo))) == NULL) { + giterr_set_oom(); + return NULL; + } for (i = 0; i < pb->nr_objects; i++) { git_pobject *po = pb->object_list + i; @@ -770,10 +779,13 @@ static int try_delta(git_packbuilder *pb, struct unpacked *trg, *mem_usage += sz; } if (!src->data) { - if (git_odb_read(&obj, pb->odb, &src_object->id) < 0) + size_t obj_sz; + + if (git_odb_read(&obj, pb->odb, &src_object->id) < 0 || + !git__is_ulong(obj_sz = git_odb_object_size(obj))) return -1; - sz = (unsigned long)git_odb_object_size(obj); + sz = (unsigned long)obj_sz; src->data = git__malloc(sz); GITERR_CHECK_ALLOC(src->data); memcpy(src->data, git_odb_object_data(obj), sz); @@ -817,7 +829,9 @@ static int try_delta(git_packbuilder *pb, struct unpacked *trg, trg_object->delta_data = NULL; } if (delta_cacheable(pb, src_size, trg_size, delta_size)) { + GITERR_CHECK_ALLOC_ADD(pb->delta_cache_size, delta_size); pb->delta_cache_size += delta_size; + git_packbuilder__cache_unlock(pb); trg_object->delta_data = git__realloc(delta_buf, delta_size); @@ -1088,6 +1102,7 @@ static int ll_find_deltas(git_packbuilder *pb, git_pobject **list, return 0; } + GITERR_CHECK_ALLOC_MULTIPLY(pb->nr_threads, sizeof(*p)); p = git__malloc(pb->nr_threads * sizeof(*p)); GITERR_CHECK_ALLOC(p); @@ -1239,6 +1254,7 @@ static int prepare_pack(git_packbuilder *pb) if (pb->progress_cb) pb->progress_cb(GIT_PACKBUILDER_DELTAFICATION, 0, pb->nr_objects, pb->progress_cb_payload); + GITERR_CHECK_ALLOC_MULTIPLY(pb->nr_objects, sizeof(*delta_list)); delta_list = git__malloc(pb->nr_objects * sizeof(*delta_list)); GITERR_CHECK_ALLOC(delta_list); diff --git a/src/pack.c b/src/pack.c index 47ce854c4ef..d475b28eebd 100644 --- a/src/pack.c +++ b/src/pack.c @@ -683,8 +683,11 @@ int git_packfile_unpack( */ if (cached && stack_size == 1) { void *data = obj->data; + + GITERR_CHECK_ALLOC_ADD(obj->len, 1); obj->data = git__malloc(obj->len + 1); GITERR_CHECK_ALLOC(obj->data); + memcpy(obj->data, data, obj->len + 1); git_atomic_dec(&cached->refcount); goto cleanup; @@ -841,6 +844,7 @@ int packfile_unpack_compressed( z_stream stream; unsigned char *buffer, *in; + GITERR_CHECK_ALLOC_ADD(size, 1); buffer = git__calloc(1, size + 1); GITERR_CHECK_ALLOC(buffer); @@ -1092,6 +1096,9 @@ int git_packfile_alloc(struct git_pack_file **pack_out, const char *path) if (path_len < strlen(".idx")) return git_odb__error_notfound("invalid packfile path", NULL); + GITERR_CHECK_ALLOC_ADD(sizeof(*p), path_len); + GITERR_CHECK_ALLOC_ADD(sizeof(*p) + path_len, 2); + p = git__calloc(1, sizeof(*p) + path_len + 2); GITERR_CHECK_ALLOC(p); diff --git a/src/path.c b/src/path.c index 58d71921bb9..3dbd7187bf6 100644 --- a/src/path.c +++ b/src/path.c @@ -622,7 +622,9 @@ static bool _check_dir_contents( size_t sub_size = strlen(sub); /* leave base valid even if we could not make space for subdir */ - if (git_buf_try_grow(dir, dir_size + sub_size + 2, false, false) < 0) + if (GIT_ALLOC_OVERFLOW_ADD(dir_size, sub_size) || + GIT_ALLOC_OVERFLOW_ADD(dir_size + sub_size, 2) || + git_buf_try_grow(dir, dir_size + sub_size + 2, false, false) < 0) return false; /* save excursion */ @@ -822,6 +824,9 @@ int git_path_make_relative(git_buf *path, const char *parent) for (; (q = strchr(q, '/')) && *(q + 1); q++) depth++; + GITERR_CHECK_ALLOC_MULTIPLY(depth, 3); + GITERR_CHECK_ALLOC_ADD(depth * 3, plen); + GITERR_CHECK_ALLOC_ADD(depth * 3, 1); newlen = (depth * 3) + plen; /* save the offset as we might realllocate the pointer */ @@ -881,6 +886,7 @@ int git_path_iconv(git_path_iconv_t *ic, char **in, size_t *inlen) git_buf_clear(&ic->buf); while (1) { + GITERR_CHECK_ALLOC_ADD(wantlen, 1); if (git_buf_grow(&ic->buf, wantlen + 1) < 0) return -1; @@ -1057,6 +1063,45 @@ int git_path_direach( return error; } +static int entry_path_alloc( + char **out, + const char *path, + size_t path_len, + const char *de_path, + size_t de_len, + size_t alloc_extra) +{ + int need_slash = (path_len > 0 && path[path_len-1] != '/') ? 1 : 0; + size_t alloc_size = path_len; + char *entry_path; + + GITERR_CHECK_ALLOC_ADD(alloc_size, de_len); + alloc_size += de_len; + + GITERR_CHECK_ALLOC_ADD(alloc_size, need_slash); + alloc_size += need_slash; + + GITERR_CHECK_ALLOC_ADD(alloc_size, 1); + alloc_size++; + + GITERR_CHECK_ALLOC_ADD(alloc_size, alloc_extra); + alloc_size += alloc_extra; + + entry_path = git__calloc(1, alloc_size); + GITERR_CHECK_ALLOC(entry_path); + + if (path_len) + memcpy(entry_path, path, path_len); + + if (need_slash) + entry_path[path_len] = '/'; + + memcpy(&entry_path[path_len + need_slash], de_path, de_len); + + *out = entry_path; + return 0; +} + int git_path_dirload( const char *path, size_t prefix_len, @@ -1064,7 +1109,7 @@ int git_path_dirload( unsigned int flags, git_vector *contents) { - int error, need_slash; + int error; DIR *dir; size_t path_len; path_dirent_data de_data; @@ -1096,11 +1141,10 @@ int git_path_dirload( path += prefix_len; path_len -= prefix_len; - need_slash = (path_len > 0 && path[path_len-1] != '/') ? 1 : 0; while ((error = p_readdir_r(dir, de_buf, &de)) == 0 && de != NULL) { char *entry_path, *de_path = de->d_name; - size_t alloc_size, de_len = strlen(de_path); + size_t de_len = strlen(de_path); if (git_path_is_dot_or_dotdot(de_path)) continue; @@ -1110,17 +1154,9 @@ int git_path_dirload( break; #endif - alloc_size = path_len + need_slash + de_len + 1 + alloc_extra; - if ((entry_path = git__calloc(alloc_size, 1)) == NULL) { - error = -1; + if ((error = entry_path_alloc(&entry_path, + path, path_len, de_path, de_len, alloc_extra)) < 0) break; - } - - if (path_len) - memcpy(entry_path, path, path_len); - if (need_slash) - entry_path[path_len] = '/'; - memcpy(&entry_path[path_len + need_slash], de_path, de_len); if ((error = git_vector_insert(contents, entry_path)) < 0) { git__free(entry_path); diff --git a/src/pool.c b/src/pool.c index 7350c04c186..1599fc7a2a1 100644 --- a/src/pool.c +++ b/src/pool.c @@ -116,9 +116,11 @@ static void *pool_alloc_page(git_pool *pool, uint32_t size) pool->has_large_page_alloc = 1; } - page = git__calloc(1, alloc_size + sizeof(git_pool_page)); - if (!page) + if (GIT_ALLOC_OVERFLOW_ADD(alloc_size, sizeof(git_pool_page)) || + !(page = git__calloc(1, alloc_size + sizeof(git_pool_page)))) { + giterr_set_oom(); return NULL; + } page->size = alloc_size; page->avail = alloc_size - size; diff --git a/src/refs.c b/src/refs.c index 43c7333f2b0..33e931db575 100644 --- a/src/refs.c +++ b/src/refs.c @@ -37,10 +37,14 @@ enum { static git_reference *alloc_ref(const char *name) { git_reference *ref; - size_t namelen = strlen(name); + size_t namelen = strlen(name), reflen = sizeof(git_reference); - if ((ref = git__calloc(1, sizeof(git_reference) + namelen + 1)) == NULL) + if (GIT_ALLOC_OVERFLOW_ADD(reflen, namelen) || + GIT_ALLOC_OVERFLOW_ADD(reflen + namelen, 1) || + (ref = git__calloc(1, reflen + namelen + 1)) == NULL) { + giterr_set_oom(); return NULL; + } memcpy(ref->name, name, namelen + 1); @@ -94,10 +98,14 @@ git_reference *git_reference__set_name( git_reference *ref, const char *name) { size_t namelen = strlen(name); - git_reference *rewrite = - git__realloc(ref, sizeof(git_reference) + namelen + 1); - if (rewrite != NULL) + size_t reflen = sizeof(git_reference); + git_reference *rewrite = NULL; + + if (!GIT_ALLOC_OVERFLOW_ADD(reflen, namelen) && + !GIT_ALLOC_OVERFLOW_ADD(reflen + namelen, 1) && + (rewrite = git__realloc(ref, reflen + namelen + 1)) != NULL) memcpy(rewrite->name, name, namelen + 1); + return rewrite; } diff --git a/src/remote.c b/src/remote.c index 5ba7735aed7..d96274f1da6 100644 --- a/src/remote.c +++ b/src/remote.c @@ -383,10 +383,9 @@ int git_remote_lookup(git_remote **out, git_repository *repo, const char *name) if ((error = git_repository_config_snapshot(&config, repo)) < 0) return error; - remote = git__malloc(sizeof(git_remote)); + remote = git__calloc(1, sizeof(git_remote)); GITERR_CHECK_ALLOC(remote); - memset(remote, 0x0, sizeof(git_remote)); remote->update_fetchhead = 1; remote->name = git__strdup(name); GITERR_CHECK_ALLOC(remote->name); diff --git a/src/revwalk.c b/src/revwalk.c index e44385d48f4..2ba000c6b4f 100644 --- a/src/revwalk.c +++ b/src/revwalk.c @@ -503,13 +503,9 @@ static int prepare_walk(git_revwalk *walk) int git_revwalk_new(git_revwalk **revwalk_out, git_repository *repo) { - git_revwalk *walk; - - walk = git__malloc(sizeof(git_revwalk)); + git_revwalk *walk = git__calloc(1, sizeof(git_revwalk)); GITERR_CHECK_ALLOC(walk); - memset(walk, 0x0, sizeof(git_revwalk)); - walk->commits = git_oidmap_alloc(); GITERR_CHECK_ALLOC(walk->commits); diff --git a/src/sortedcache.c b/src/sortedcache.c index c6b22615368..021f79632c4 100644 --- a/src/sortedcache.c +++ b/src/sortedcache.c @@ -15,7 +15,9 @@ int git_sortedcache_new( pathlen = path ? strlen(path) : 0; - sc = git__calloc(sizeof(git_sortedcache) + pathlen + 1, 1); + GITERR_CHECK_ALLOC_ADD(sizeof(git_sortedcache), pathlen); + GITERR_CHECK_ALLOC_ADD(sizeof(git_sortedcache) + pathlen, 1); + sc = git__calloc(1, sizeof(git_sortedcache) + pathlen + 1); GITERR_CHECK_ALLOC(sc); if (git_pool_init(&sc->pool, 1, 0) < 0 || diff --git a/src/tag.c b/src/tag.c index dc0827b8f80..b8213140179 100644 --- a/src/tag.c +++ b/src/tag.c @@ -117,6 +117,7 @@ static int tag_parse(git_tag *tag, const char *buffer, const char *buffer_end) text_len = search - buffer; + GITERR_CHECK_ALLOC_ADD(text_len, 1); tag->tag_name = git__malloc(text_len + 1); GITERR_CHECK_ALLOC(tag->tag_name); @@ -141,6 +142,7 @@ static int tag_parse(git_tag *tag, const char *buffer, const char *buffer_end) text_len = buffer_end - ++buffer; + GITERR_CHECK_ALLOC_ADD(text_len, 1); tag->message = git__malloc(text_len + 1); GITERR_CHECK_ALLOC(tag->message); diff --git a/src/transports/cred.c b/src/transports/cred.c index 1b4d29c0a63..8e5447d1815 100644 --- a/src/transports/cred.c +++ b/src/transports/cred.c @@ -311,6 +311,9 @@ int git_cred_username_new(git_cred **cred, const char *username) assert(cred); len = strlen(username); + + GITERR_CHECK_ALLOC_ADD(sizeof(git_cred_username), len); + GITERR_CHECK_ALLOC_ADD(sizeof(git_cred_username) + len, 1); c = git__malloc(sizeof(git_cred_username) + len + 1); GITERR_CHECK_ALLOC(c); diff --git a/src/transports/git.c b/src/transports/git.c index 6f25736b13b..5ec98d86765 100644 --- a/src/transports/git.c +++ b/src/transports/git.c @@ -154,7 +154,7 @@ static int git_proto_stream_alloc( if (!stream) return -1; - s = git__calloc(sizeof(git_proto_stream), 1); + s = git__calloc(1, sizeof(git_proto_stream)); GITERR_CHECK_ALLOC(s); s->parent.subtransport = &t->parent; @@ -347,7 +347,7 @@ int git_smart_subtransport_git(git_smart_subtransport **out, git_transport *owne if (!out) return -1; - t = git__calloc(sizeof(git_subtransport), 1); + t = git__calloc(1, sizeof(git_subtransport)); GITERR_CHECK_ALLOC(t); t->owner = owner; diff --git a/src/transports/local.c b/src/transports/local.c index c01755e3454..89f2651cd43 100644 --- a/src/transports/local.c +++ b/src/transports/local.c @@ -420,7 +420,7 @@ static int local_push( const git_error *last; char *ref = spec->refspec.dst; - status = git__calloc(sizeof(push_status), 1); + status = git__calloc(1, sizeof(push_status)); if (!status) goto on_error; diff --git a/src/transports/smart.c b/src/transports/smart.c index ec0ba3784b3..69b9d22cc76 100644 --- a/src/transports/smart.c +++ b/src/transports/smart.c @@ -380,7 +380,7 @@ int git_transport_smart(git_transport **out, git_remote *owner, void *param) if (!param) return -1; - t = git__calloc(sizeof(transport_smart), 1); + t = git__calloc(1, sizeof(transport_smart)); GITERR_CHECK_ALLOC(t); t->parent.version = GIT_TRANSPORT_VERSION; diff --git a/src/transports/smart_pkt.c b/src/transports/smart_pkt.c index b5f9d6dbee5..2f83e0d7ba4 100644 --- a/src/transports/smart_pkt.c +++ b/src/transports/smart_pkt.c @@ -103,6 +103,8 @@ static int comment_pkt(git_pkt **out, const char *line, size_t len) { git_pkt_comment *pkt; + GITERR_CHECK_ALLOC_ADD(sizeof(git_pkt_comment), len); + GITERR_CHECK_ALLOC_ADD(sizeof(git_pkt_comment) + len, 1); pkt = git__malloc(sizeof(git_pkt_comment) + len + 1); GITERR_CHECK_ALLOC(pkt); @@ -122,6 +124,9 @@ static int err_pkt(git_pkt **out, const char *line, size_t len) /* Remove "ERR " from the line */ line += 4; len -= 4; + + GITERR_CHECK_ALLOC_ADD(sizeof(git_pkt_progress), len); + GITERR_CHECK_ALLOC_ADD(sizeof(git_pkt_progress) + len, 1); pkt = git__malloc(sizeof(git_pkt_err) + len + 1); GITERR_CHECK_ALLOC(pkt); @@ -141,6 +146,8 @@ static int data_pkt(git_pkt **out, const char *line, size_t len) line++; len--; + + GITERR_CHECK_ALLOC_ADD(sizeof(git_pkt_progress), len); pkt = git__malloc(sizeof(git_pkt_data) + len); GITERR_CHECK_ALLOC(pkt); @@ -159,6 +166,8 @@ static int sideband_progress_pkt(git_pkt **out, const char *line, size_t len) line++; len--; + + GITERR_CHECK_ALLOC_ADD(sizeof(git_pkt_progress), len); pkt = git__malloc(sizeof(git_pkt_progress) + len); GITERR_CHECK_ALLOC(pkt); @@ -177,6 +186,9 @@ static int sideband_error_pkt(git_pkt **out, const char *line, size_t len) line++; len--; + + GITERR_CHECK_ALLOC_ADD(sizeof(git_pkt_err), len); + GITERR_CHECK_ALLOC_ADD(sizeof(git_pkt_err) + len, 1); pkt = git__malloc(sizeof(git_pkt_err) + len + 1); GITERR_CHECK_ALLOC(pkt); @@ -220,6 +232,7 @@ static int ref_pkt(git_pkt **out, const char *line, size_t len) if (line[len - 1] == '\n') --len; + GITERR_CHECK_ALLOC_ADD(len, 1); pkt->head.name = git__malloc(len + 1); GITERR_CHECK_ALLOC(pkt->head.name); @@ -249,9 +262,13 @@ static int ok_pkt(git_pkt **out, const char *line, size_t len) pkt->type = GIT_PKT_OK; line += 3; /* skip "ok " */ - ptr = strchr(line, '\n'); + if (!(ptr = strchr(line, '\n'))) { + giterr_set(GITERR_NET, "Invalid packet line"); + return -1; + } len = ptr - line; + GITERR_CHECK_ALLOC_ADD(len, 1); pkt->ref = git__malloc(len + 1); GITERR_CHECK_ALLOC(pkt->ref); @@ -273,9 +290,13 @@ static int ng_pkt(git_pkt **out, const char *line, size_t len) pkt->type = GIT_PKT_NG; line += 3; /* skip "ng " */ - ptr = strchr(line, ' '); + if (!(ptr = strchr(line, ' '))) { + giterr_set(GITERR_NET, "Invalid packet line"); + return -1; + } len = ptr - line; + GITERR_CHECK_ALLOC_ADD(len, 1); pkt->ref = git__malloc(len + 1); GITERR_CHECK_ALLOC(pkt->ref); @@ -283,9 +304,13 @@ static int ng_pkt(git_pkt **out, const char *line, size_t len) pkt->ref[len] = '\0'; line = ptr + 1; - ptr = strchr(line, '\n'); + if (!(ptr = strchr(line, '\n'))) { + giterr_set(GITERR_NET, "Invalid packet line"); + return -1; + } len = ptr - line; + GITERR_CHECK_ALLOC_ADD(len, 1); pkt->msg = git__malloc(len + 1); GITERR_CHECK_ALLOC(pkt->msg); diff --git a/src/transports/smart_protocol.c b/src/transports/smart_protocol.c index 5f1b99892b5..f023db4df9a 100644 --- a/src/transports/smart_protocol.c +++ b/src/transports/smart_protocol.c @@ -685,7 +685,7 @@ static int add_push_report_pkt(git_push *push, git_pkt *pkt) switch (pkt->type) { case GIT_PKT_OK: - status = git__calloc(sizeof(push_status), 1); + status = git__calloc(1, sizeof(push_status)); GITERR_CHECK_ALLOC(status); status->msg = NULL; status->ref = git__strdup(((git_pkt_ok *)pkt)->ref); @@ -696,7 +696,7 @@ static int add_push_report_pkt(git_push *push, git_pkt *pkt) } break; case GIT_PKT_NG: - status = git__calloc(sizeof(push_status), 1); + status = git__calloc(1, sizeof(push_status)); GITERR_CHECK_ALLOC(status); status->ref = git__strdup(((git_pkt_ng *)pkt)->ref); status->msg = git__strdup(((git_pkt_ng *)pkt)->msg); diff --git a/src/transports/winhttp.c b/src/transports/winhttp.c index 56b7c99f9ea..278ef22c4b4 100644 --- a/src/transports/winhttp.c +++ b/src/transports/winhttp.c @@ -1178,7 +1178,7 @@ static int winhttp_stream_alloc(winhttp_subtransport *t, winhttp_stream **stream if (!stream) return -1; - s = git__calloc(sizeof(winhttp_stream), 1); + s = git__calloc(1, sizeof(winhttp_stream)); GITERR_CHECK_ALLOC(s); s->parent.subtransport = &t->parent; @@ -1329,7 +1329,7 @@ int git_smart_subtransport_http(git_smart_subtransport **out, git_transport *own if (!out) return -1; - t = git__calloc(sizeof(winhttp_subtransport), 1); + t = git__calloc(1, sizeof(winhttp_subtransport)); GITERR_CHECK_ALLOC(t); t->owner = (transport_smart *)owner; diff --git a/src/tree.c b/src/tree.c index 9693f4eca00..2c8b89291d1 100644 --- a/src/tree.c +++ b/src/tree.c @@ -84,11 +84,15 @@ int git_tree_entry_icmp(const git_tree_entry *e1, const git_tree_entry *e2) static git_tree_entry *alloc_entry(const char *filename) { git_tree_entry *entry = NULL; - size_t filename_len = strlen(filename); + size_t filename_len = strlen(filename), + tree_len = sizeof(git_tree_entry); - entry = git__malloc(sizeof(git_tree_entry) + filename_len + 1); - if (!entry) + if (GIT_ALLOC_OVERFLOW_ADD(tree_len, filename_len) || + GIT_ALLOC_OVERFLOW_ADD(tree_len + filename_len, 1) || + !(entry = git__malloc(tree_len + filename_len + 1))) { + giterr_set_oom(); return NULL; + } memset(entry, 0x0, sizeof(git_tree_entry)); memcpy(entry->filename, filename, filename_len); @@ -205,12 +209,16 @@ void git_tree_entry_free(git_tree_entry *entry) int git_tree_entry_dup(git_tree_entry **dest, const git_tree_entry *source) { - size_t total_size; + size_t total_size = sizeof(git_tree_entry); git_tree_entry *copy; assert(source); - total_size = sizeof(git_tree_entry) + source->filename_len + 1; + GITERR_CHECK_ALLOC_ADD(total_size, source->filename_len); + total_size += source->filename_len; + + GITERR_CHECK_ALLOC_ADD(total_size, 1); + total_size++; copy = git__malloc(total_size); GITERR_CHECK_ALLOC(copy); diff --git a/src/tsort.c b/src/tsort.c index 4885e435b8c..b92e520560e 100644 --- a/src/tsort.c +++ b/src/tsort.c @@ -184,7 +184,10 @@ static int check_invariant(struct tsort_run *stack, ssize_t stack_curr) static int resize(struct tsort_store *store, size_t new_size) { if (store->alloc < new_size) { - void **tempstore = git__realloc(store->storage, new_size * sizeof(void *)); + void **tempstore; + + GITERR_CHECK_ALLOC_MULTIPLY(new_size, sizeof(void *)); + tempstore = git__realloc(store->storage, new_size * sizeof(void *)); /** * Do not propagate on OOM; this will abort the sort and diff --git a/src/util.h b/src/util.h index 89816a8c952..6c94a5aa04c 100644 --- a/src/util.h +++ b/src/util.h @@ -64,7 +64,12 @@ GIT_INLINE(char *) git__strndup(const char *str, size_t n) length = p_strnlen(str, n); - ptr = (char*)git__malloc(length + 1); + if (GIT_ALLOC_OVERFLOW_ADD(length, 1)) { + giterr_set_oom(); + return NULL; + } + + ptr = git__malloc(length + 1); if (!ptr) return NULL; @@ -80,7 +85,13 @@ GIT_INLINE(char *) git__strndup(const char *str, size_t n) /* NOTE: This doesn't do null or '\0' checking. Watch those boundaries! */ GIT_INLINE(char *) git__substrdup(const char *start, size_t n) { - char *ptr = (char*)git__malloc(n+1); + char *ptr; + + if (GIT_ALLOC_OVERFLOW_ADD(n, 1) || !(ptr = git__malloc(n+1))) { + giterr_set_oom(); + return NULL; + } + memcpy(ptr, start, n); ptr[n] = '\0'; return ptr; diff --git a/src/vector.c b/src/vector.c index c769b696aaa..b636032b153 100644 --- a/src/vector.c +++ b/src/vector.c @@ -29,12 +29,12 @@ GIT_INLINE(size_t) compute_new_size(git_vector *v) GIT_INLINE(int) resize_vector(git_vector *v, size_t new_size) { - size_t new_bytes = new_size * sizeof(void *); + size_t new_bytes; void *new_contents; /* Check for overflow */ - if (new_bytes / sizeof(void *) != new_size) - GITERR_CHECK_ALLOC(NULL); + GITERR_CHECK_ALLOC_MULTIPLY(new_size, sizeof(void *)); + new_bytes = new_size * sizeof(void *); new_contents = git__realloc(v->contents, new_bytes); GITERR_CHECK_ALLOC(new_contents); @@ -51,6 +51,7 @@ int git_vector_dup(git_vector *v, const git_vector *src, git_vector_cmp cmp) assert(v && src); + GITERR_CHECK_ALLOC_MULTIPLY(src->length, sizeof(void *)); bytes = src->length * sizeof(void *); v->_alloc_size = src->length; diff --git a/src/win32/dir.c b/src/win32/dir.c index c7427ea54da..9953289f6ec 100644 --- a/src/win32/dir.c +++ b/src/win32/dir.c @@ -18,9 +18,13 @@ git__DIR *git__opendir(const char *dir) dirlen = strlen(dir); - new = git__calloc(sizeof(*new) + dirlen + 1, 1); - if (!new) + if (GIT_ALLOC_OVERFLOW_ADD(sizeof(*new), dirlen) || + GIT_ALLOC_OVERFLOW_ADD(sizeof(*new) + dirlen, 1) || + !(new = git__calloc(1, sizeof(*new) + dirlen + 1))) { + giterr_set_oom(); return NULL; + } + memcpy(new->dir, dir, dirlen); new->h = FindFirstFileW(filter_w, &new->f); diff --git a/src/win32/utf-conv.c b/src/win32/utf-conv.c index b0205b01975..62461120506 100644 --- a/src/win32/utf-conv.c +++ b/src/win32/utf-conv.c @@ -99,9 +99,8 @@ int git__utf8_to_16_alloc(wchar_t **dest, const char *src) return -1; } - *dest = git__malloc(utf16_size * sizeof(wchar_t)); - - if (!*dest) { + if (GIT_ALLOC_OVERFLOW_MULTIPLY(utf16_size, sizeof(wchar_t)) || + !(*dest = git__malloc(utf16_size * sizeof(wchar_t)))) { errno = ENOMEM; return -1; } diff --git a/src/zstream.c b/src/zstream.c index e75fb265eb4..06660e981ed 100644 --- a/src/zstream.c +++ b/src/zstream.c @@ -134,6 +134,7 @@ int git_zstream_deflatebuf(git_buf *out, const void *in, size_t in_len) while (!git_zstream_done(&zs)) { size_t step = git_zstream_suggest_output_len(&zs), written; + GITERR_CHECK_ALLOC_ADD(out->size, step); if ((error = git_buf_grow(out, out->size + step)) < 0) goto done; diff --git a/tests/core/errors.c b/tests/core/errors.c index 366d8f16a11..b6364100b7e 100644 --- a/tests/core/errors.c +++ b/tests/core/errors.c @@ -109,3 +109,43 @@ void test_core_errors__restore(void) cl_assert_equal_i(42, giterr_last()->klass); cl_assert_equal_s("Foo: bar", giterr_last()->message); } + +static int test_arraysize_multiply(size_t nelem, size_t size) +{ + GITERR_CHECK_ALLOC_MULTIPLY(nelem, size); + return 0; +} + +void test_core_errors__integer_overflow_alloc_multiply(void) +{ + cl_git_pass(test_arraysize_multiply(10, 10)); + cl_git_pass(test_arraysize_multiply(1000, 1000)); + cl_git_pass(test_arraysize_multiply(SIZE_MAX/sizeof(void *), sizeof(void *))); + cl_git_pass(test_arraysize_multiply(0, 10)); + cl_git_pass(test_arraysize_multiply(10, 0)); + + cl_git_fail(test_arraysize_multiply(SIZE_MAX-1, sizeof(void *))); + cl_git_fail(test_arraysize_multiply((SIZE_MAX/sizeof(void *))+1, sizeof(void *))); + + cl_assert_equal_i(GITERR_NOMEMORY, giterr_last()->klass); + cl_assert_equal_s("Out of memory", giterr_last()->message); +} + +static int test_arraysize_add(size_t one, size_t two) +{ + GITERR_CHECK_ALLOC_ADD(one, two); + return 0; +} + +void test_core_errors__integer_overflow_alloc_add(void) +{ + cl_git_pass(test_arraysize_add(10, 10)); + cl_git_pass(test_arraysize_add(1000, 1000)); + cl_git_pass(test_arraysize_add(SIZE_MAX-10, 10)); + + cl_git_fail(test_arraysize_multiply(SIZE_MAX-1, 2)); + cl_git_fail(test_arraysize_multiply(SIZE_MAX, SIZE_MAX)); + + cl_assert_equal_i(GITERR_NOMEMORY, giterr_last()->klass); + cl_assert_equal_s("Out of memory", giterr_last()->message); +} From 15d54fdd345dadc2854200ce5b9aad0949e3949b Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Tue, 10 Feb 2015 22:34:03 -0500 Subject: [PATCH 58/99] odb__hashlink: check st.st_size before casting --- src/odb.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/odb.c b/src/odb.c index 80e02ef2b87..fcb21c1ced8 100644 --- a/src/odb.c +++ b/src/odb.c @@ -216,28 +216,28 @@ int git_odb__hashfd_filtered( int git_odb__hashlink(git_oid *out, const char *path) { struct stat st; - git_off_t size; + size_t size; int result; if (git_path_lstat(path, &st) < 0) return -1; - size = st.st_size; - - if (!git__is_sizet(size)) { - giterr_set(GITERR_OS, "File size overflow for 32-bit systems"); + if (!git__is_sizet(st.st_size)) { + giterr_set(GITERR_FILESYSTEM, "File size overflow for 32-bit systems"); return -1; } + size = (size_t)st.st_size; + if (S_ISLNK(st.st_mode)) { char *link_data; ssize_t read_len; GITERR_CHECK_ALLOC_ADD(size, 1); - link_data = git__malloc((size_t)(size + 1)); + link_data = git__malloc(size + 1); GITERR_CHECK_ALLOC(link_data); - read_len = p_readlink(path, link_data, (size_t)size); + read_len = p_readlink(path, link_data, size); link_data[size] = '\0'; if (read_len != (ssize_t)size) { giterr_set(GITERR_OS, "Failed to read symlink data for '%s'", path); @@ -245,13 +245,13 @@ int git_odb__hashlink(git_oid *out, const char *path) return -1; } - result = git_odb_hash(out, link_data, (size_t)size, GIT_OBJ_BLOB); + result = git_odb_hash(out, link_data, size, GIT_OBJ_BLOB); git__free(link_data); } else { int fd = git_futils_open_ro(path); if (fd < 0) return -1; - result = git_odb__hashfd(out, fd, (size_t)size, GIT_OBJ_BLOB); + result = git_odb__hashfd(out, fd, size, GIT_OBJ_BLOB); p_close(fd); } From 3603cb0978b7ef21ff9cd63693ebd6d27bc2bc53 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Tue, 10 Feb 2015 23:13:49 -0500 Subject: [PATCH 59/99] git__*allocarray: safer realloc and malloc Introduce git__reallocarray that checks the product of the number of elements and element size for overflow before allocation. Also introduce git__mallocarray that behaves like calloc, but without the `c`. (It does not zero memory, for those truly worried about every cycle.) --- src/array.h | 3 +-- src/oid.c | 3 +-- src/pack-objects.c | 14 +++++--------- src/tsort.c | 3 +-- src/util.h | 22 ++++++++++++++++++++++ src/vector.c | 7 +------ src/win32/utf-conv.c | 3 +-- 7 files changed, 32 insertions(+), 23 deletions(-) diff --git a/src/array.h b/src/array.h index ca5a35ffe5a..0d07953709a 100644 --- a/src/array.h +++ b/src/array.h @@ -57,8 +57,7 @@ GIT_INLINE(void *) git_array_grow(void *_a, size_t item_size) new_size = a->size * 3 / 2; } - if (GIT_ALLOC_OVERFLOW_MULTIPLY(new_size, item_size) || - (new_array = git__realloc(a->ptr, new_size * item_size)) == NULL) + if ((new_array = git__reallocarray(a->ptr, new_size, item_size)) == NULL) goto on_oom; a->ptr = new_array; a->asize = new_size; a->size++; diff --git a/src/oid.c b/src/oid.c index 9aac47db68d..9fe2ebb6522 100644 --- a/src/oid.c +++ b/src/oid.c @@ -261,8 +261,7 @@ struct git_oid_shorten { static int resize_trie(git_oid_shorten *self, size_t new_size) { - GITERR_CHECK_ALLOC_MULTIPLY(new_size, sizeof(trie_node)); - self->nodes = git__realloc(self->nodes, new_size * sizeof(trie_node)); + self->nodes = git__reallocarray(self->nodes, new_size, sizeof(trie_node)); GITERR_CHECK_ALLOC(self->nodes); if (new_size > self->size) { diff --git a/src/pack-objects.c b/src/pack-objects.c index aea4770afd0..cd0deb29abb 100644 --- a/src/pack-objects.c +++ b/src/pack-objects.c @@ -205,9 +205,8 @@ int git_packbuilder_insert(git_packbuilder *pb, const git_oid *oid, GITERR_CHECK_ALLOC_MULTIPLY(pb->nr_alloc + 1024, 3 / 2); pb->nr_alloc = (pb->nr_alloc + 1024) * 3 / 2; - GITERR_CHECK_ALLOC_MULTIPLY(pb->nr_alloc, sizeof(*po)); - pb->object_list = git__realloc(pb->object_list, - pb->nr_alloc * sizeof(*po)); + pb->object_list = git__reallocarray(pb->object_list, + pb->nr_alloc, sizeof(*po)); GITERR_CHECK_ALLOC(pb->object_list); rehash(pb); } @@ -505,8 +504,7 @@ static git_pobject **compute_write_order(git_packbuilder *pb) unsigned int i, wo_end, last_untagged; git_pobject **wo; - if (GIT_ALLOC_OVERFLOW_MULTIPLY(pb->nr_objects, sizeof(*wo)) || - (wo = git__malloc(pb->nr_objects * sizeof(*wo))) == NULL) { + if ((wo = git__mallocarray(pb->nr_objects, sizeof(*wo))) == NULL) { giterr_set_oom(); return NULL; } @@ -1102,8 +1100,7 @@ static int ll_find_deltas(git_packbuilder *pb, git_pobject **list, return 0; } - GITERR_CHECK_ALLOC_MULTIPLY(pb->nr_threads, sizeof(*p)); - p = git__malloc(pb->nr_threads * sizeof(*p)); + p = git__mallocarray(pb->nr_threads, sizeof(*p)); GITERR_CHECK_ALLOC(p); /* Partition the work among the threads */ @@ -1254,8 +1251,7 @@ static int prepare_pack(git_packbuilder *pb) if (pb->progress_cb) pb->progress_cb(GIT_PACKBUILDER_DELTAFICATION, 0, pb->nr_objects, pb->progress_cb_payload); - GITERR_CHECK_ALLOC_MULTIPLY(pb->nr_objects, sizeof(*delta_list)); - delta_list = git__malloc(pb->nr_objects * sizeof(*delta_list)); + delta_list = git__mallocarray(pb->nr_objects, sizeof(*delta_list)); GITERR_CHECK_ALLOC(delta_list); for (i = 0; i < pb->nr_objects; ++i) { diff --git a/src/tsort.c b/src/tsort.c index b92e520560e..e5981920428 100644 --- a/src/tsort.c +++ b/src/tsort.c @@ -186,8 +186,7 @@ static int resize(struct tsort_store *store, size_t new_size) if (store->alloc < new_size) { void **tempstore; - GITERR_CHECK_ALLOC_MULTIPLY(new_size, sizeof(void *)); - tempstore = git__realloc(store->storage, new_size * sizeof(void *)); + tempstore = git__reallocarray(store->storage, new_size, sizeof(void *)); /** * Do not propagate on OOM; this will abort the sort and diff --git a/src/util.h b/src/util.h index 6c94a5aa04c..7693ef8097a 100644 --- a/src/util.h +++ b/src/util.h @@ -104,6 +104,28 @@ GIT_INLINE(void *) git__realloc(void *ptr, size_t size) return new_ptr; } +/** + * Similar to `git__realloc`, except that it is suitable for reallocing an + * array to a new number of elements of `nelem`, each of size `elsize`. + * The total size calculation is checked for overflow. + */ +GIT_INLINE(void *) git__reallocarray(void *ptr, size_t nelem, size_t elsize) +{ + void *new_ptr = NULL; + if (GIT_ALLOC_OVERFLOW_MULTIPLY(nelem, elsize) || + !(new_ptr = realloc(ptr, nelem * elsize))) + giterr_set_oom(); + return new_ptr; +} + +/** + * Similar to `git__calloc`, except that it does not zero memory. + */ +GIT_INLINE(void *) git__mallocarray(size_t nelem, size_t elsize) +{ + return git__reallocarray(NULL, nelem, elsize); +} + GIT_INLINE(void) git__free(void *ptr) { free(ptr); diff --git a/src/vector.c b/src/vector.c index b636032b153..27eafebc881 100644 --- a/src/vector.c +++ b/src/vector.c @@ -29,14 +29,9 @@ GIT_INLINE(size_t) compute_new_size(git_vector *v) GIT_INLINE(int) resize_vector(git_vector *v, size_t new_size) { - size_t new_bytes; void *new_contents; - /* Check for overflow */ - GITERR_CHECK_ALLOC_MULTIPLY(new_size, sizeof(void *)); - new_bytes = new_size * sizeof(void *); - - new_contents = git__realloc(v->contents, new_bytes); + new_contents = git__reallocarray(v->contents, new_size, sizeof(void *)); GITERR_CHECK_ALLOC(new_contents); v->_alloc_size = new_size; diff --git a/src/win32/utf-conv.c b/src/win32/utf-conv.c index 62461120506..0dad4eab06f 100644 --- a/src/win32/utf-conv.c +++ b/src/win32/utf-conv.c @@ -99,8 +99,7 @@ int git__utf8_to_16_alloc(wchar_t **dest, const char *src) return -1; } - if (GIT_ALLOC_OVERFLOW_MULTIPLY(utf16_size, sizeof(wchar_t)) || - !(*dest = git__malloc(utf16_size * sizeof(wchar_t)))) { + if (!(*dest = git__mallocarray(utf16_size, sizeof(wchar_t)))) { errno = ENOMEM; return -1; } From 4aa664ae3953d99c2ae4cd769f02818bc122eebc Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Tue, 10 Feb 2015 23:55:07 -0500 Subject: [PATCH 60/99] git_buf_grow_by: increase buf asize incrementally Introduce `git_buf_grow_by` to incrementally increase the size of a `git_buf`, performing an overflow calculation on the growth. --- src/buf_text.c | 13 ++++--------- src/buffer.c | 17 ++++++++++++++--- src/buffer.h | 12 ++++++++++++ src/transports/smart_pkt.c | 2 +- src/zstream.c | 3 +-- tests/buf/basic.c | 20 ++++++++++++++++++++ tests/buf/oom.c | 10 ++++++++++ 7 files changed, 62 insertions(+), 15 deletions(-) diff --git a/src/buf_text.c b/src/buf_text.c index ace54d72557..08b86f4ccc4 100644 --- a/src/buf_text.c +++ b/src/buf_text.c @@ -29,9 +29,8 @@ int git_buf_text_puts_escaped( scan += count; } - GITERR_CHECK_ALLOC_ADD(buf->size, total); - GITERR_CHECK_ALLOC_ADD(buf->size + total, 1); - if (git_buf_grow(buf, buf->size + total + 1) < 0) + GITERR_CHECK_ALLOC_ADD(total, 1); + if (git_buf_grow_by(buf, total + 1) < 0) return -1; for (scan = string; *scan; ) { @@ -129,7 +128,6 @@ int git_buf_text_lf_to_crlf(git_buf *tgt, const git_buf *src) for (; next; scan = next + 1, next = memchr(scan, '\n', end - scan)) { size_t copylen = next - scan; - size_t needsize; /* if we find mixed line endings, bail */ if (next > start && next[-1] == '\r') { @@ -137,11 +135,8 @@ int git_buf_text_lf_to_crlf(git_buf *tgt, const git_buf *src) return GIT_PASSTHROUGH; } - GITERR_CHECK_ALLOC_ADD(tgt->size, copylen); - GITERR_CHECK_ALLOC_ADD(tgt->size + copylen, 3); - needsize = tgt->size + copylen + 3; - - if (tgt->asize < needsize && git_buf_grow(tgt, needsize) < 0) + GITERR_CHECK_ALLOC_ADD(copylen, 3); + if (git_buf_grow_by(tgt, copylen + 3) < 0) return -1; if (next > scan) { diff --git a/src/buffer.c b/src/buffer.c index ee8bba4ecf1..8098bb1e7e2 100644 --- a/src/buffer.c +++ b/src/buffer.c @@ -101,6 +101,18 @@ int git_buf_grow(git_buf *buffer, size_t target_size) return git_buf_try_grow(buffer, target_size, true, true); } +int git_buf_grow_by(git_buf *buffer, size_t additional_size) +{ + if (GIT_ALLOC_OVERFLOW_ADD(buffer->size, additional_size)) { + buffer->ptr = git_buf__oom; + giterr_set_oom(); + return -1; + } + + return git_buf_try_grow( + buffer, buffer->size + additional_size, true, true); +} + void git_buf_free(git_buf *buf) { if (!buf) return; @@ -515,9 +527,8 @@ int git_buf_join_n(git_buf *buf, char separator, int nbuf, ...) if (total_size == 0) return 0; - GITERR_CHECK_ALLOC_ADD(buf->size, total_size); - GITERR_CHECK_ALLOC_ADD(buf->size + total_size, 1); - if (git_buf_grow(buf, buf->size + total_size + 1) < 0) + GITERR_CHECK_ALLOC_ADD(total_size, 1); + if (git_buf_grow_by(buf, total_size + 1) < 0) return -1; out = buf->ptr + buf->size; diff --git a/src/buffer.h b/src/buffer.h index 8ee4b532cf5..52342e30921 100644 --- a/src/buffer.h +++ b/src/buffer.h @@ -36,6 +36,18 @@ GIT_INLINE(bool) git_buf_is_allocated(const git_buf *buf) */ extern void git_buf_init(git_buf *buf, size_t initial_size); +/** + * Resize the buffer allocation to make more space. + * + * This will attempt to grow the buffer to accommodate the additional size. + * It is similar to `git_buf_grow`, but performs the new size calculation, + * checking for overflow. + * + * Like `git_buf_grow`, if this is a user-supplied buffer, this will allocate + * a new buffer. + */ +extern int git_buf_grow_by(git_buf *buffer, size_t additional_size); + /** * Attempt to grow the buffer to hold at least `target_size` bytes. * diff --git a/src/transports/smart_pkt.c b/src/transports/smart_pkt.c index 2f83e0d7ba4..ad81ad1b07b 100644 --- a/src/transports/smart_pkt.c +++ b/src/transports/smart_pkt.c @@ -513,7 +513,7 @@ static int buffer_want_with_caps(const git_remote_head *head, transport_smart_ca len = (unsigned int) (strlen("XXXXwant ") + GIT_OID_HEXSZ + 1 /* NUL */ + git_buf_len(&str) + 1 /* LF */); - git_buf_grow(buf, git_buf_len(buf) + len); + git_buf_grow_by(buf, len); git_oid_fmt(oid, &head->oid); git_buf_printf(buf, "%04xwant %s %s\n", len, oid, git_buf_cstr(&str)); git_buf_free(&str); diff --git a/src/zstream.c b/src/zstream.c index 06660e981ed..2130bc3ca7c 100644 --- a/src/zstream.c +++ b/src/zstream.c @@ -134,8 +134,7 @@ int git_zstream_deflatebuf(git_buf *out, const void *in, size_t in_len) while (!git_zstream_done(&zs)) { size_t step = git_zstream_suggest_output_len(&zs), written; - GITERR_CHECK_ALLOC_ADD(out->size, step); - if ((error = git_buf_grow(out, out->size + step)) < 0) + if ((error = git_buf_grow_by(out, step)) < 0) goto done; written = out->asize - out->size; diff --git a/tests/buf/basic.c b/tests/buf/basic.c index d558757a920..dcc0916d21b 100644 --- a/tests/buf/basic.c +++ b/tests/buf/basic.c @@ -15,6 +15,26 @@ void test_buf_basic__resize(void) git_buf_free(&buf1); } +void test_buf_basic__resize_incremental(void) +{ + git_buf buf1 = GIT_BUF_INIT; + + /* Presently, asking for 6 bytes will round up to 8. */ + cl_git_pass(git_buf_puts(&buf1, "Hello")); + cl_assert_equal_i(5, buf1.size); + cl_assert_equal_i(8, buf1.asize); + + /* Ensure an additional byte does not realloc. */ + cl_git_pass(git_buf_grow_by(&buf1, 1)); + cl_assert_equal_i(5, buf1.size); + cl_assert_equal_i(8, buf1.asize); + + /* But requesting many does. */ + cl_git_pass(git_buf_grow_by(&buf1, 16)); + cl_assert_equal_i(5, buf1.size); + cl_assert(buf1.asize > 8); +} + void test_buf_basic__printf(void) { git_buf buf2 = GIT_BUF_INIT; diff --git a/tests/buf/oom.c b/tests/buf/oom.c index 709439aa5ab..b9fd29cbb0a 100644 --- a/tests/buf/oom.c +++ b/tests/buf/oom.c @@ -29,3 +29,13 @@ void test_buf_oom__grow(void) git_buf_free(&buf); } + +void test_buf_oom__grow_by(void) +{ + git_buf buf = GIT_BUF_INIT; + + buf.size = SIZE_MAX-10; + + cl_assert(git_buf_grow_by(&buf, 50) == -1); + cl_assert(git_buf_oom(&buf)); +} From 2884cc42de8b20a58cec8488d014a853d47c047e Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Wed, 11 Feb 2015 09:39:38 -0500 Subject: [PATCH 61/99] overflow checking: don't make callers set oom Have the ALLOC_OVERFLOW testing macros also simply set_oom in the case where a computation would overflow, so that callers don't need to. --- src/buffer.c | 1 - src/common.h | 14 ++++---------- src/config_file.c | 2 -- src/delta.c | 2 +- src/index.c | 4 +--- src/pack-objects.c | 4 +--- src/pool.c | 4 +--- src/refs.c | 4 +--- src/tree.c | 4 +--- src/util.h | 15 ++++----------- src/win32/dir.c | 4 +--- tests/core/errors.c | 21 +++++++++++++++++++++ 12 files changed, 36 insertions(+), 43 deletions(-) diff --git a/src/buffer.c b/src/buffer.c index 8098bb1e7e2..0d031443914 100644 --- a/src/buffer.c +++ b/src/buffer.c @@ -105,7 +105,6 @@ int git_buf_grow_by(git_buf *buffer, size_t additional_size) { if (GIT_ALLOC_OVERFLOW_ADD(buffer->size, additional_size)) { buffer->ptr = git_buf__oom; - giterr_set_oom(); return -1; } diff --git a/src/common.h b/src/common.h index b53798eaf01..8b7d6936d0a 100644 --- a/src/common.h +++ b/src/common.h @@ -176,25 +176,19 @@ GIT_INLINE(void) git__init_structure(void *structure, size_t len, unsigned int v /** Check for integer overflow from addition or multiplication */ #define GIT_ALLOC_OVERFLOW_ADD(one, two) \ - ((one) + (two) < (one)) + (((one) + (two) < (one)) ? (giterr_set_oom(), 1) : 0) /** Check for integer overflow from multiplication */ #define GIT_ALLOC_OVERFLOW_MULTIPLY(one, two) \ - (one && ((one) * (two)) / (one) != (two)) + ((one && ((one) * (two)) / (one) != (two)) ? (giterr_set_oom(), 1) : 0) /** Check for additive overflow, failing if it would occur. */ #define GITERR_CHECK_ALLOC_ADD(one, two) \ - if (GIT_ALLOC_OVERFLOW_ADD(one, two)) { \ - giterr_set_oom(); \ - return -1; \ - } + if (GIT_ALLOC_OVERFLOW_ADD(one, two)) { return -1; } /** Check for multiplicative overflow, failing if it would occur. */ #define GITERR_CHECK_ALLOC_MULTIPLY(nelem, elsize) \ - if (GIT_ALLOC_OVERFLOW_MULTIPLY(nelem, elsize)) { \ - giterr_set_oom(); \ - return -1; \ - } + if (GIT_ALLOC_OVERFLOW_MULTIPLY(nelem, elsize)) { return -1; } /* NOTE: other giterr functions are in the public errors.h header file */ diff --git a/src/config_file.c b/src/config_file.c index 36f78563bb8..39e9ff84125 100644 --- a/src/config_file.c +++ b/src/config_file.c @@ -905,7 +905,6 @@ static char *reader_readline(struct reader *reader, bool skip_whitespace) if (GIT_ALLOC_OVERFLOW_ADD(line_len, 1) || (line = git__malloc(line_len + 1)) == NULL) { - giterr_set_oom(); return NULL; } @@ -1620,7 +1619,6 @@ static char *fixup_line(const char *ptr, int quote_count) if (GIT_ALLOC_OVERFLOW_ADD(ptr_len, 1) || (str = git__malloc(ptr_len + 1)) == NULL) { - giterr_set_oom(); return NULL; } diff --git a/src/delta.c b/src/delta.c index f704fdfb1cf..242f3abe3c9 100644 --- a/src/delta.c +++ b/src/delta.c @@ -138,7 +138,7 @@ static int lookup_index_alloc( index_len += hash_len; if (!git__is_ulong(index_len)) { - giterr_set_oom(); + giterr_set(GITERR_NOMEMORY, "Overly large delta"); return -1; } diff --git a/src/index.c b/src/index.c index 70090d12c4e..35f512ca4f4 100644 --- a/src/index.c +++ b/src/index.c @@ -833,10 +833,8 @@ static git_index_reuc_entry *reuc_entry_alloc(const char *path) struct reuc_entry_internal *entry; if (GIT_ALLOC_OVERFLOW_ADD(structlen, pathlen) || - GIT_ALLOC_OVERFLOW_ADD(structlen + pathlen, 1)) { - giterr_set_oom(); + GIT_ALLOC_OVERFLOW_ADD(structlen + pathlen, 1)) return NULL; - } entry = git__calloc(1, structlen + pathlen + 1); if (!entry) diff --git a/src/pack-objects.c b/src/pack-objects.c index cd0deb29abb..288077078da 100644 --- a/src/pack-objects.c +++ b/src/pack-objects.c @@ -504,10 +504,8 @@ static git_pobject **compute_write_order(git_packbuilder *pb) unsigned int i, wo_end, last_untagged; git_pobject **wo; - if ((wo = git__mallocarray(pb->nr_objects, sizeof(*wo))) == NULL) { - giterr_set_oom(); + if ((wo = git__mallocarray(pb->nr_objects, sizeof(*wo))) == NULL) return NULL; - } for (i = 0; i < pb->nr_objects; i++) { git_pobject *po = pb->object_list + i; diff --git a/src/pool.c b/src/pool.c index 1599fc7a2a1..919e13d08fb 100644 --- a/src/pool.c +++ b/src/pool.c @@ -117,10 +117,8 @@ static void *pool_alloc_page(git_pool *pool, uint32_t size) } if (GIT_ALLOC_OVERFLOW_ADD(alloc_size, sizeof(git_pool_page)) || - !(page = git__calloc(1, alloc_size + sizeof(git_pool_page)))) { - giterr_set_oom(); + !(page = git__calloc(1, alloc_size + sizeof(git_pool_page)))) return NULL; - } page->size = alloc_size; page->avail = alloc_size - size; diff --git a/src/refs.c b/src/refs.c index 33e931db575..af3190ef9ac 100644 --- a/src/refs.c +++ b/src/refs.c @@ -41,10 +41,8 @@ static git_reference *alloc_ref(const char *name) if (GIT_ALLOC_OVERFLOW_ADD(reflen, namelen) || GIT_ALLOC_OVERFLOW_ADD(reflen + namelen, 1) || - (ref = git__calloc(1, reflen + namelen + 1)) == NULL) { - giterr_set_oom(); + (ref = git__calloc(1, reflen + namelen + 1)) == NULL) return NULL; - } memcpy(ref->name, name, namelen + 1); diff --git a/src/tree.c b/src/tree.c index 2c8b89291d1..573e5644744 100644 --- a/src/tree.c +++ b/src/tree.c @@ -89,10 +89,8 @@ static git_tree_entry *alloc_entry(const char *filename) if (GIT_ALLOC_OVERFLOW_ADD(tree_len, filename_len) || GIT_ALLOC_OVERFLOW_ADD(tree_len + filename_len, 1) || - !(entry = git__malloc(tree_len + filename_len + 1))) { - giterr_set_oom(); + !(entry = git__malloc(tree_len + filename_len + 1))) return NULL; - } memset(entry, 0x0, sizeof(git_tree_entry)); memcpy(entry->filename, filename, filename_len); diff --git a/src/util.h b/src/util.h index 7693ef8097a..40e06976d09 100644 --- a/src/util.h +++ b/src/util.h @@ -64,10 +64,8 @@ GIT_INLINE(char *) git__strndup(const char *str, size_t n) length = p_strnlen(str, n); - if (GIT_ALLOC_OVERFLOW_ADD(length, 1)) { - giterr_set_oom(); + if (GIT_ALLOC_OVERFLOW_ADD(length, 1)) return NULL; - } ptr = git__malloc(length + 1); @@ -87,10 +85,8 @@ GIT_INLINE(char *) git__substrdup(const char *start, size_t n) { char *ptr; - if (GIT_ALLOC_OVERFLOW_ADD(n, 1) || !(ptr = git__malloc(n+1))) { - giterr_set_oom(); + if (GIT_ALLOC_OVERFLOW_ADD(n, 1) || !(ptr = git__malloc(n+1))) return NULL; - } memcpy(ptr, start, n); ptr[n] = '\0'; @@ -111,11 +107,8 @@ GIT_INLINE(void *) git__realloc(void *ptr, size_t size) */ GIT_INLINE(void *) git__reallocarray(void *ptr, size_t nelem, size_t elsize) { - void *new_ptr = NULL; - if (GIT_ALLOC_OVERFLOW_MULTIPLY(nelem, elsize) || - !(new_ptr = realloc(ptr, nelem * elsize))) - giterr_set_oom(); - return new_ptr; + return GIT_ALLOC_OVERFLOW_MULTIPLY(nelem, elsize) ? + NULL : realloc(ptr, nelem * elsize); } /** diff --git a/src/win32/dir.c b/src/win32/dir.c index 9953289f6ec..7f2a5a56e33 100644 --- a/src/win32/dir.c +++ b/src/win32/dir.c @@ -20,10 +20,8 @@ git__DIR *git__opendir(const char *dir) if (GIT_ALLOC_OVERFLOW_ADD(sizeof(*new), dirlen) || GIT_ALLOC_OVERFLOW_ADD(sizeof(*new) + dirlen, 1) || - !(new = git__calloc(1, sizeof(*new) + dirlen + 1))) { - giterr_set_oom(); + !(new = git__calloc(1, sizeof(*new) + dirlen + 1))) return NULL; - } memcpy(new->dir, dir, dirlen); diff --git a/tests/core/errors.c b/tests/core/errors.c index b6364100b7e..11e0bb2bbc8 100644 --- a/tests/core/errors.c +++ b/tests/core/errors.c @@ -149,3 +149,24 @@ void test_core_errors__integer_overflow_alloc_add(void) cl_assert_equal_i(GITERR_NOMEMORY, giterr_last()->klass); cl_assert_equal_s("Out of memory", giterr_last()->message); } + +void test_core_errors__integer_overflow_sets_oom(void) +{ + giterr_clear(); + cl_assert(!GIT_ALLOC_OVERFLOW_ADD(SIZE_MAX-1, 1)); + cl_assert_equal_p(NULL, giterr_last()); + + giterr_clear(); + cl_assert(!GIT_ALLOC_OVERFLOW_ADD(42, 69)); + cl_assert_equal_p(NULL, giterr_last()); + + giterr_clear(); + cl_assert(GIT_ALLOC_OVERFLOW_ADD(SIZE_MAX, SIZE_MAX)); + cl_assert_equal_i(GITERR_NOMEMORY, giterr_last()->klass); + cl_assert_equal_s("Out of memory", giterr_last()->message); + + giterr_clear(); + cl_assert(GIT_ALLOC_OVERFLOW_MULTIPLY(SIZE_MAX, SIZE_MAX)); + cl_assert_equal_i(GITERR_NOMEMORY, giterr_last()->klass); + cl_assert_equal_s("Out of memory", giterr_last()->message); +} From ec3b4d35f636c26d3c9b5703c3b7f87683800af8 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Wed, 11 Feb 2015 11:20:05 -0500 Subject: [PATCH 62/99] Use `size_t` to hold size of arrays Use `size_t` to hold the size of arrays to ease overflow checking, lest we check for overflow of a `size_t` then promptly truncate by packing the length into a smaller type. --- src/array.h | 4 ++-- src/filebuf.c | 13 +++++++------ src/pack-objects.c | 10 +++++++++- src/transports/smart_pkt.c | 17 ++++++++++++----- 4 files changed, 30 insertions(+), 14 deletions(-) diff --git a/src/array.h b/src/array.h index 0d07953709a..7c4dbdbc15f 100644 --- a/src/array.h +++ b/src/array.h @@ -23,7 +23,7 @@ * * typedef git_array_t(my_struct) my_struct_array_t; */ -#define git_array_t(type) struct { type *ptr; uint32_t size, asize; } +#define git_array_t(type) struct { type *ptr; size_t size, asize; } #define GIT_ARRAY_INIT { NULL, 0, 0 } @@ -45,7 +45,7 @@ typedef git_array_t(char) git_array_generic_t; GIT_INLINE(void *) git_array_grow(void *_a, size_t item_size) { volatile git_array_generic_t *a = _a; - uint32_t new_size; + size_t new_size; char *new_array; if (a->size < 8) { diff --git a/src/filebuf.c b/src/filebuf.c index 1a915755842..94f2bec32d3 100644 --- a/src/filebuf.c +++ b/src/filebuf.c @@ -408,8 +408,8 @@ int git_filebuf_reserve(git_filebuf *file, void **buffer, size_t len) int git_filebuf_printf(git_filebuf *file, const char *format, ...) { va_list arglist; - size_t space_left; - int len, res; + size_t space_left, len; + int written, res; char *tmp_buffer; ENSURE_BUF_OK(file); @@ -418,15 +418,16 @@ int git_filebuf_printf(git_filebuf *file, const char *format, ...) do { va_start(arglist, format); - len = p_vsnprintf((char *)file->buffer + file->buf_pos, space_left, format, arglist); + written = p_vsnprintf((char *)file->buffer + file->buf_pos, space_left, format, arglist); va_end(arglist); - if (len < 0) { + if (written < 0) { file->last_error = BUFERR_MEM; return -1; } - if ((size_t)len + 1 <= space_left) { + len = written; + if (len + 1 <= space_left) { file->buf_pos += len; return 0; } @@ -436,7 +437,7 @@ int git_filebuf_printf(git_filebuf *file, const char *format, ...) space_left = file->buf_size - file->buf_pos; - } while ((size_t)len + 1 <= space_left); + } while (len + 1 <= space_left); if (GIT_ALLOC_OVERFLOW_ADD(len, 1) || !(tmp_buffer = git__malloc(len + 1))) { diff --git a/src/pack-objects.c b/src/pack-objects.c index 288077078da..9b56234b589 100644 --- a/src/pack-objects.c +++ b/src/pack-objects.c @@ -190,6 +190,7 @@ int git_packbuilder_insert(git_packbuilder *pb, const git_oid *oid, { git_pobject *po; khiter_t pos; + size_t newsize; int ret; assert(pb && oid); @@ -203,7 +204,14 @@ int git_packbuilder_insert(git_packbuilder *pb, const git_oid *oid, if (pb->nr_objects >= pb->nr_alloc) { GITERR_CHECK_ALLOC_ADD(pb->nr_alloc, 1024); GITERR_CHECK_ALLOC_MULTIPLY(pb->nr_alloc + 1024, 3 / 2); - pb->nr_alloc = (pb->nr_alloc + 1024) * 3 / 2; + newsize = (pb->nr_alloc + 1024) * 3 / 2; + + if (!git__is_uint32(newsize)) { + giterr_set(GITERR_NOMEMORY, "Packfile too large to fit in memory."); + return -1; + } + + pb->nr_alloc = (uint32_t)newsize; pb->object_list = git__reallocarray(pb->object_list, pb->nr_alloc, sizeof(*po)); diff --git a/src/transports/smart_pkt.c b/src/transports/smart_pkt.c index ad81ad1b07b..11c6215ffc3 100644 --- a/src/transports/smart_pkt.c +++ b/src/transports/smart_pkt.c @@ -484,7 +484,7 @@ static int buffer_want_with_caps(const git_remote_head *head, transport_smart_ca { git_buf str = GIT_BUF_INIT; char oid[GIT_OID_HEXSZ +1] = {0}; - unsigned int len; + size_t len; /* Prefer multi_ack_detailed */ if (caps->multi_ack_detailed) @@ -510,12 +510,19 @@ static int buffer_want_with_caps(const git_remote_head *head, transport_smart_ca if (git_buf_oom(&str)) return -1; - len = (unsigned int) - (strlen("XXXXwant ") + GIT_OID_HEXSZ + 1 /* NUL */ + - git_buf_len(&str) + 1 /* LF */); + len = strlen("XXXXwant ") + GIT_OID_HEXSZ + 1 /* NUL */ + + git_buf_len(&str) + 1 /* LF */; + + if (len > 0xffff) { + giterr_set(GITERR_NET, + "Tried to produce packet with invalid length %d", len); + return -1; + } + git_buf_grow_by(buf, len); git_oid_fmt(oid, &head->oid); - git_buf_printf(buf, "%04xwant %s %s\n", len, oid, git_buf_cstr(&str)); + git_buf_printf(buf, + "%04xwant %s %s\n", (unsigned int)len, oid, git_buf_cstr(&str)); git_buf_free(&str); return git_buf_oom(buf); From 8d534b475829edf87c8126fc4ae30f593172f317 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Wed, 11 Feb 2015 13:01:00 -0500 Subject: [PATCH 63/99] p_read: ensure requested len is ssize_t Ensure that the given length to `p_read` is of ssize_t and ensure that callers test the return as if it were an `ssize_t`. --- src/fileops.c | 5 +++++ src/posix.c | 12 +++++++++++- src/util.h | 7 +++++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/fileops.c b/src/fileops.c index eb24013e888..420ed70a275 100644 --- a/src/fileops.c +++ b/src/fileops.c @@ -127,6 +127,11 @@ int git_futils_readbuffer_fd(git_buf *buf, git_file fd, size_t len) git_buf_clear(buf); + if (!git__is_ssizet(len)) { + giterr_set(GITERR_INVALID, "Read too large."); + return -1; + } + GITERR_CHECK_ALLOC_ADD(len, 1); if (git_buf_grow(buf, len + 1) < 0) return -1; diff --git a/src/posix.c b/src/posix.c index d5e6875b560..8d86aa8bfa9 100644 --- a/src/posix.c +++ b/src/posix.c @@ -155,6 +155,14 @@ ssize_t p_read(git_file fd, void *buf, size_t cnt) { char *b = buf; + if (!git__is_ssizet(cnt)) { +#ifdef GIT_WIN32 + SetLastError(ERROR_INVALID_PARAMETER); +#endif + errno = EINVAL; + return -1; + } + while (cnt) { ssize_t r; #ifdef GIT_WIN32 @@ -229,7 +237,9 @@ int p_mmap(git_map *out, size_t len, int prot, int flags, int fd, git_off_t offs out->data = malloc(len); GITERR_CHECK_ALLOC(out->data); - if ((p_lseek(fd, offset, SEEK_SET) < 0) || ((size_t)p_read(fd, out->data, len) != len)) { + if (!git__is_ssizet(len) || + (p_lseek(fd, offset, SEEK_SET) < 0) || + (p_read(fd, out->data, len) != (ssize_t)len)) { giterr_set(GITERR_OS, "mmap emulation failed"); return -1; } diff --git a/src/util.h b/src/util.h index 40e06976d09..dcdaf4363e7 100644 --- a/src/util.h +++ b/src/util.h @@ -153,6 +153,13 @@ GIT_INLINE(int) git__is_sizet(git_off_t p) return p == (git_off_t)r; } +/** @return true if p fits into the range of an ssize_t */ +GIT_INLINE(int) git__is_ssizet(size_t p) +{ + ssize_t r = (ssize_t)p; + return p == (size_t)r; +} + /** @return true if p fits into the range of a uint32_t */ GIT_INLINE(int) git__is_uint32(size_t p) { From 190b76a69875da49973d1a5d8fd3b96c0cd43425 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Wed, 11 Feb 2015 14:52:08 -0500 Subject: [PATCH 64/99] Introduce git__add_sizet_overflow and friends Add some helper functions to check for overflow in a type-specific manner. --- src/common.h | 6 ++-- src/integer.h | 77 ++++++++++++++++++++++++++++++++++++++++++++++ src/pack-objects.c | 4 +-- src/util.h | 28 ----------------- 4 files changed, 83 insertions(+), 32 deletions(-) create mode 100644 src/integer.h diff --git a/src/common.h b/src/common.h index 8b7d6936d0a..530e320e29a 100644 --- a/src/common.h +++ b/src/common.h @@ -58,6 +58,7 @@ #include "git2/types.h" #include "git2/errors.h" #include "thread-utils.h" +#include "integer.h" #include @@ -174,13 +175,14 @@ GIT_INLINE(void) git__init_structure(void *structure, size_t len, unsigned int v GITERR_CHECK_VERSION(&(VERSION), _tmpl.version, #TYPE); \ memcpy((PTR), &_tmpl, sizeof(_tmpl)); } while (0) + /** Check for integer overflow from addition or multiplication */ #define GIT_ALLOC_OVERFLOW_ADD(one, two) \ - (((one) + (two) < (one)) ? (giterr_set_oom(), 1) : 0) + (!git__add_sizet_overflow(NULL, (one), (two)) ? (giterr_set_oom(), 1) : 0) /** Check for integer overflow from multiplication */ #define GIT_ALLOC_OVERFLOW_MULTIPLY(one, two) \ - ((one && ((one) * (two)) / (one) != (two)) ? (giterr_set_oom(), 1) : 0) + (!git__multiply_sizet_overflow(NULL, (one), (two)) ? (giterr_set_oom(), 1) : 0) /** Check for additive overflow, failing if it would occur. */ #define GITERR_CHECK_ALLOC_ADD(one, two) \ diff --git a/src/integer.h b/src/integer.h new file mode 100644 index 00000000000..a9ed10aae9f --- /dev/null +++ b/src/integer.h @@ -0,0 +1,77 @@ +/* + * Copyright (C) the libgit2 contributors. All rights reserved. + * + * This file is part of libgit2, distributed under the GNU GPL v2 with + * a Linking Exception. For full terms see the included COPYING file. + */ +#ifndef INCLUDE_integer_h__ +#define INCLUDE_integer_h__ + +/** @return true if p fits into the range of a size_t */ +GIT_INLINE(int) git__is_sizet(git_off_t p) +{ + size_t r = (size_t)p; + return p == (git_off_t)r; +} + +/** @return true if p fits into the range of an ssize_t */ +GIT_INLINE(int) git__is_ssizet(size_t p) +{ + ssize_t r = (ssize_t)p; + return p == (size_t)r; +} + +/** @return true if p fits into the range of a uint32_t */ +GIT_INLINE(int) git__is_uint32(size_t p) +{ + uint32_t r = (uint32_t)p; + return p == (size_t)r; +} + +/** @return true if p fits into the range of an unsigned long */ +GIT_INLINE(int) git__is_ulong(git_off_t p) +{ + unsigned long r = (unsigned long)p; + return p == (git_off_t)r; +} + +/** + * Sets `one + two` into `out`, unless the arithmetic would overflow. + * @return true if the result fits in a `uint64_t`, false on overflow. + */ +GIT_INLINE(bool) git__add_uint64_overflow(uint64_t *out, uint64_t one, uint64_t two) +{ + if (UINT64_MAX - one < two) + return false; + if (out) + *out = one + two; + return true; +} + +/** + * Sets `one + two` into `out`, unless the arithmetic would overflow. + * @return true if the result fits in a `size_t`, false on overflow. + */ +GIT_INLINE(bool) git__add_sizet_overflow(size_t *out, size_t one, size_t two) +{ + if (SIZE_MAX - one < two) + return false; + if (out) + *out = one + two; + return true; +} + +/** + * Sets `one * two` into `out`, unless the arithmetic would overflow. + * @return true if the result fits in a `size_t`, false on overflow. + */ +GIT_INLINE(bool) git__multiply_sizet_overflow(size_t *out, size_t one, size_t two) +{ + if (one && SIZE_MAX / one < two) + return false; + if (out) + *out = one * two; + return true; +} + +#endif /* INCLUDE_integer_h__ */ diff --git a/src/pack-objects.c b/src/pack-objects.c index 9b56234b589..8236ef9f3da 100644 --- a/src/pack-objects.c +++ b/src/pack-objects.c @@ -833,8 +833,8 @@ static int try_delta(git_packbuilder *pb, struct unpacked *trg, trg_object->delta_data = NULL; } if (delta_cacheable(pb, src_size, trg_size, delta_size)) { - GITERR_CHECK_ALLOC_ADD(pb->delta_cache_size, delta_size); - pb->delta_cache_size += delta_size; + if (!git__add_uint64_overflow(&pb->delta_cache_size, pb->delta_cache_size, delta_size)) + return -1; git_packbuilder__cache_unlock(pb); diff --git a/src/util.h b/src/util.h index dcdaf4363e7..86139ef77f3 100644 --- a/src/util.h +++ b/src/util.h @@ -146,34 +146,6 @@ extern int git__strtol64(int64_t *n, const char *buff, const char **end_buf, int extern void git__hexdump(const char *buffer, size_t n); extern uint32_t git__hash(const void *key, int len, uint32_t seed); -/** @return true if p fits into the range of a size_t */ -GIT_INLINE(int) git__is_sizet(git_off_t p) -{ - size_t r = (size_t)p; - return p == (git_off_t)r; -} - -/** @return true if p fits into the range of an ssize_t */ -GIT_INLINE(int) git__is_ssizet(size_t p) -{ - ssize_t r = (ssize_t)p; - return p == (size_t)r; -} - -/** @return true if p fits into the range of a uint32_t */ -GIT_INLINE(int) git__is_uint32(size_t p) -{ - uint32_t r = (uint32_t)p; - return p == (size_t)r; -} - -/** @return true if p fits into the range of an unsigned long */ -GIT_INLINE(int) git__is_ulong(git_off_t p) -{ - unsigned long r = (unsigned long)p; - return p == (git_off_t)r; -} - /* 32-bit cross-platform rotl */ #ifdef _MSC_VER /* use built-in method in MSVC */ # define git__rotl(v, s) (uint32_t)_rotl(v, s) From 1ad48c8a08c4bb3b10c0344d838494de9cee087e Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Wed, 11 Feb 2015 17:36:57 -0500 Subject: [PATCH 65/99] khash: update to klib f28c067 --- src/khash.h | 57 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 33 insertions(+), 24 deletions(-) diff --git a/src/khash.h b/src/khash.h index 242204464aa..e5789c49377 100644 --- a/src/khash.h +++ b/src/khash.h @@ -46,6 +46,19 @@ int main() { */ /* + 2013-05-02 (0.2.8): + + * Use quadratic probing. When the capacity is power of 2, stepping function + i*(i+1)/2 guarantees to traverse each bucket. It is better than double + hashing on cache performance and is more robust than linear probing. + + In theory, double hashing should be more robust than quadratic probing. + However, my implementation is probably not for large hash tables, because + the second hash function is closely tied to the first hash function, + which reduce the effectiveness of double hashing. + + Reference: http://research.cs.vt.edu/AVresearch/hashing/quadratic.php + 2011-12-29 (0.2.7): * Minor code clean up; no actual effect. @@ -110,13 +123,13 @@ int main() { Generic hash table library. */ -#define AC_VERSION_KHASH_H "0.2.6" +#define AC_VERSION_KHASH_H "0.2.8" #include #include #include -/* compipler specific configuration */ +/* compiler specific configuration */ #if UINT_MAX == 0xffffffffu typedef unsigned int khint32_t; @@ -130,11 +143,13 @@ typedef unsigned long khint64_t; typedef unsigned long long khint64_t; #endif +#ifndef kh_inline #ifdef _MSC_VER #define kh_inline __inline #else #define kh_inline inline #endif +#endif /* kh_inline */ typedef khint32_t khint_t; typedef khint_t khiter_t; @@ -147,12 +162,6 @@ typedef khint_t khiter_t; #define __ac_set_isboth_false(flag, i) (flag[i>>4]&=~(3ul<<((i&0xfU)<<1))) #define __ac_set_isdel_true(flag, i) (flag[i>>4]|=1ul<<((i&0xfU)<<1)) -#ifdef KHASH_LINEAR -#define __ac_inc(k, m) 1 -#else -#define __ac_inc(k, m) (((k)>>3 ^ (k)<<3) | 1) & (m) -#endif - #define __ac_fsize(m) ((m) < 16? 1 : (m)>>4) #ifndef kroundup32 @@ -175,7 +184,7 @@ typedef khint_t khiter_t; static const double __ac_HASH_UPPER = 0.77; #define __KHASH_TYPE(name, khkey_t, khval_t) \ - typedef struct { \ + typedef struct kh_##name##_s { \ khint_t n_buckets, size, n_occupied, upper_bound; \ khint32_t *flags; \ khkey_t *keys; \ @@ -213,19 +222,19 @@ static const double __ac_HASH_UPPER = 0.77; SCOPE khint_t kh_get_##name(const kh_##name##_t *h, khkey_t key) \ { \ if (h->n_buckets) { \ - khint_t inc, k, i, last, mask; \ + khint_t k, i, last, mask, step = 0; \ mask = h->n_buckets - 1; \ k = __hash_func(key); i = k & mask; \ - inc = __ac_inc(k, mask); last = i; /* inc==1 for linear probing */ \ + last = i; \ while (!__ac_isempty(h->flags, i) && (__ac_isdel(h->flags, i) || !__hash_equal(h->keys[i], key))) { \ - i = (i + inc) & mask; \ + i = (i + (++step)) & mask; \ if (i == last) return h->n_buckets; \ } \ return __ac_iseither(h->flags, i)? h->n_buckets : i; \ } else return 0; \ } \ SCOPE int kh_resize_##name(kh_##name##_t *h, khint_t new_n_buckets) \ - { /* This function uses 0.25*n_bucktes bytes of working space instead of [sizeof(key_t+val_t)+.25]*n_buckets. */ \ + { /* This function uses 0.25*n_buckets bytes of working space instead of [sizeof(key_t+val_t)+.25]*n_buckets. */ \ khint32_t *new_flags = 0; \ khint_t j = 1; \ { \ @@ -238,11 +247,11 @@ static const double __ac_HASH_UPPER = 0.77; memset(new_flags, 0xaa, __ac_fsize(new_n_buckets) * sizeof(khint32_t)); \ if (h->n_buckets < new_n_buckets) { /* expand */ \ khkey_t *new_keys = (khkey_t*)krealloc((void *)h->keys, new_n_buckets * sizeof(khkey_t)); \ - if (!new_keys) return -1; \ + if (!new_keys) { kfree(new_flags); return -1; } \ h->keys = new_keys; \ if (kh_is_map) { \ khval_t *new_vals = (khval_t*)krealloc((void *)h->vals, new_n_buckets * sizeof(khval_t)); \ - if (!new_vals) return -1; \ + if (!new_vals) { kfree(new_flags); return -1; } \ h->vals = new_vals; \ } \ } /* otherwise shrink */ \ @@ -258,11 +267,10 @@ static const double __ac_HASH_UPPER = 0.77; if (kh_is_map) val = h->vals[j]; \ __ac_set_isdel_true(h->flags, j); \ while (1) { /* kick-out process; sort of like in Cuckoo hashing */ \ - khint_t inc, k, i; \ + khint_t k, i, step = 0; \ k = __hash_func(key); \ i = k & new_mask; \ - inc = __ac_inc(k, new_mask); \ - while (!__ac_isempty(new_flags, i)) i = (i + inc) & new_mask; \ + while (!__ac_isempty(new_flags, i)) i = (i + (++step)) & new_mask; \ __ac_set_isempty_false(new_flags, i); \ if (i < h->n_buckets && __ac_iseither(h->flags, i) == 0) { /* kick out the existing element */ \ { khkey_t tmp = h->keys[i]; h->keys[i] = key; key = tmp; } \ @@ -301,14 +309,14 @@ static const double __ac_HASH_UPPER = 0.77; } \ } /* TODO: to implement automatically shrinking; resize() already support shrinking */ \ { \ - khint_t inc, k, i, site, last, mask = h->n_buckets - 1; \ + khint_t k, i, site, last, mask = h->n_buckets - 1, step = 0; \ x = site = h->n_buckets; k = __hash_func(key); i = k & mask; \ if (__ac_isempty(h->flags, i)) x = i; /* for speed up */ \ else { \ - inc = __ac_inc(k, mask); last = i; \ + last = i; \ while (!__ac_isempty(h->flags, i) && (__ac_isdel(h->flags, i) || !__hash_equal(h->keys[i], key))) { \ if (__ac_isdel(h->flags, i)) site = i; \ - i = (i + inc) & mask; \ + i = (i + (++step)) & mask; \ if (i == last) { x = site; break; } \ } \ if (x == h->n_buckets) { \ @@ -449,7 +457,8 @@ static kh_inline khint_t __ac_Wang_hash(khint_t key) @param name Name of the hash table [symbol] @param h Pointer to the hash table [khash_t(name)*] @param k Key [type of keys] - @param r Extra return code: 0 if the key is present in the hash table; + @param r Extra return code: -1 if the operation failed; + 0 if the key is present in the hash table; 1 if the bucket is empty (never used); 2 if the element in the bucket has been deleted [int*] @return Iterator to the inserted element [khint_t] @@ -461,7 +470,7 @@ static kh_inline khint_t __ac_Wang_hash(khint_t key) @param name Name of the hash table [symbol] @param h Pointer to the hash table [khash_t(name)*] @param k Key [type of keys] - @return Iterator to the found element, or kh_end(h) is the element is absent [khint_t] + @return Iterator to the found element, or kh_end(h) if the element is absent [khint_t] */ #define kh_get(name, h, k) kh_get_##name(h, k) @@ -607,4 +616,4 @@ typedef const char *kh_cstr_t; #define KHASH_MAP_INIT_STR(name, khval_t) \ KHASH_INIT(name, kh_cstr_t, khval_t, 1, kh_str_hash_func, kh_str_hash_equal) -#endif /* __AC_KHASH_H */ +#endif /* __AC_KHASH_H */ \ No newline at end of file From 650e45f69124bd8b53ecefddeb214a82538ab2c1 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Wed, 11 Feb 2015 17:51:17 -0500 Subject: [PATCH 66/99] Update `khash.h` to pull request 42 https://github.com/attractivechaos/klib/pull/42/files introduces `kreallocarray`. Hook that up as our `git__reallocarray`. --- src/khash.h | 13 ++++++++----- src/offmap.h | 1 + src/oidmap.h | 1 + src/strmap.h | 1 + 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/khash.h b/src/khash.h index e5789c49377..818ac833bf1 100644 --- a/src/khash.h +++ b/src/khash.h @@ -177,6 +177,9 @@ typedef khint_t khiter_t; #ifndef krealloc #define krealloc(P,Z) realloc(P,Z) #endif +#ifndef kreallocarray +#define kreallocarray(P,N,Z) ((SIZE_MAX - N < Z) ? NULL : krealloc(P, (N*Z))) +#endif #ifndef kfree #define kfree(P) free(P) #endif @@ -242,15 +245,15 @@ static const double __ac_HASH_UPPER = 0.77; if (new_n_buckets < 4) new_n_buckets = 4; \ if (h->size >= (khint_t)(new_n_buckets * __ac_HASH_UPPER + 0.5)) j = 0; /* requested size is too small */ \ else { /* hash table size to be changed (shrink or expand); rehash */ \ - new_flags = (khint32_t*)kmalloc(__ac_fsize(new_n_buckets) * sizeof(khint32_t)); \ + new_flags = (khint32_t*)kreallocarray(NULL, __ac_fsize(new_n_buckets), sizeof(khint32_t)); \ if (!new_flags) return -1; \ memset(new_flags, 0xaa, __ac_fsize(new_n_buckets) * sizeof(khint32_t)); \ if (h->n_buckets < new_n_buckets) { /* expand */ \ - khkey_t *new_keys = (khkey_t*)krealloc((void *)h->keys, new_n_buckets * sizeof(khkey_t)); \ + khkey_t *new_keys = (khkey_t*)kreallocarray((void *)h->keys, new_n_buckets, sizeof(khkey_t)); \ if (!new_keys) { kfree(new_flags); return -1; } \ h->keys = new_keys; \ if (kh_is_map) { \ - khval_t *new_vals = (khval_t*)krealloc((void *)h->vals, new_n_buckets * sizeof(khval_t)); \ + khval_t *new_vals = (khval_t*)kreallocarray((void *)h->vals, new_n_buckets, sizeof(khval_t)); \ if (!new_vals) { kfree(new_flags); return -1; } \ h->vals = new_vals; \ } \ @@ -285,8 +288,8 @@ static const double __ac_HASH_UPPER = 0.77; } \ } \ if (h->n_buckets > new_n_buckets) { /* shrink the hash table */ \ - h->keys = (khkey_t*)krealloc((void *)h->keys, new_n_buckets * sizeof(khkey_t)); \ - if (kh_is_map) h->vals = (khval_t*)krealloc((void *)h->vals, new_n_buckets * sizeof(khval_t)); \ + h->keys = (khkey_t*)kreallocarray((void *)h->keys, new_n_buckets, sizeof(khkey_t)); \ + if (kh_is_map) h->vals = (khval_t*)kreallocarray((void *)h->vals, new_n_buckets, sizeof(khval_t)); \ } \ kfree(h->flags); /* free the working space */ \ h->flags = new_flags; \ diff --git a/src/offmap.h b/src/offmap.h index cd46fd687e5..9471e7b91d3 100644 --- a/src/offmap.h +++ b/src/offmap.h @@ -13,6 +13,7 @@ #define kmalloc git__malloc #define kcalloc git__calloc #define krealloc git__realloc +#define kreallocarray git__reallocarray #define kfree git__free #include "khash.h" diff --git a/src/oidmap.h b/src/oidmap.h index b871a79268e..5e3b44bfbf4 100644 --- a/src/oidmap.h +++ b/src/oidmap.h @@ -13,6 +13,7 @@ #define kmalloc git__malloc #define kcalloc git__calloc #define krealloc git__realloc +#define kreallocarray git__reallocarray #define kfree git__free #include "khash.h" diff --git a/src/strmap.h b/src/strmap.h index 8985aaf7ec2..dfbe5639f6f 100644 --- a/src/strmap.h +++ b/src/strmap.h @@ -12,6 +12,7 @@ #define kmalloc git__malloc #define kcalloc git__calloc #define krealloc git__realloc +#define kreallocarray git__reallocarray #define kfree git__free #include "khash.h" From 527ed59ad809826e43fa43024b0af262f08499db Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Fri, 13 Feb 2015 09:18:49 -0500 Subject: [PATCH 67/99] win32: limit the mode to `_wopen`/`_waccess` Win32 generally ignores Unix-like mode bits that don't make any sense on the platform (eg `0644` makes no sense to Windows). But WINE complains loudly when presented with POSIXy bits. Remove them. (Thanks @phkelley) --- src/win32/posix_w32.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/win32/posix_w32.c b/src/win32/posix_w32.c index 346f537e40e..6e005c1cddb 100644 --- a/src/win32/posix_w32.c +++ b/src/win32/posix_w32.c @@ -33,6 +33,12 @@ * inheritable on Windows, so specify the flag to get default behavior back. */ #define STANDARD_OPEN_FLAGS (_O_BINARY | _O_NOINHERIT) +/* Allowable mode bits on Win32. Using mode bits that are not supported on + * Win32 (eg S_IRWXU) is generally ignored, but Wine warns loudly about it + * so we simply remove them. + */ +#define WIN32_MODE_MASK (_S_IREAD | _S_IWRITE) + /* GetFinalPathNameByHandleW signature */ typedef DWORD(WINAPI *PFGetFinalPathNameByHandleW)(HANDLE, LPWSTR, DWORD, DWORD); @@ -343,7 +349,7 @@ int p_open(const char *path, int flags, ...) va_end(arg_list); } - return _wopen(buf, flags | STANDARD_OPEN_FLAGS, mode); + return _wopen(buf, flags | STANDARD_OPEN_FLAGS, mode & WIN32_MODE_MASK); } int p_creat(const char *path, mode_t mode) @@ -353,7 +359,9 @@ int p_creat(const char *path, mode_t mode) if (git_win32_path_from_utf8(buf, path) < 0) return -1; - return _wopen(buf, _O_WRONLY | _O_CREAT | _O_TRUNC | STANDARD_OPEN_FLAGS, mode); + return _wopen(buf, + _O_WRONLY | _O_CREAT | _O_TRUNC | STANDARD_OPEN_FLAGS, + mode & WIN32_MODE_MASK); } int p_getcwd(char *buffer_out, size_t size) @@ -607,7 +615,7 @@ int p_access(const char* path, mode_t mode) if (git_win32_path_from_utf8(buf, path) < 0) return -1; - return _waccess(buf, mode); + return _waccess(buf, mode & WIN32_MODE_MASK); } static int ensure_writable(wchar_t *fpath) From f1453c59b2afb9dab43281bfe9f1ba34cf6e0d02 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Thu, 12 Feb 2015 12:19:37 -0500 Subject: [PATCH 68/99] Make our overflow check look more like gcc/clang's Make our overflow checking look more like gcc and clang's, so that we can substitute it out with the compiler instrinsics on platforms that support it. This means dropping the ability to pass `NULL` as an out parameter. As a result, the macros also get updated to reflect this as well. --- src/array.h | 5 +- src/blame_git.c | 10 +-- src/buf_text.c | 22 +++--- src/buffer.c | 143 ++++++++++++++++++++----------------- src/common.h | 20 +++--- src/config_file.c | 24 +++---- src/delta-apply.c | 6 +- src/delta.c | 17 ++--- src/diff.c | 5 +- src/diff_driver.c | 9 +-- src/diff_patch.c | 13 ++-- src/diff_tform.c | 5 +- src/filebuf.c | 12 ++-- src/fileops.c | 18 +++-- src/filter.c | 19 ++--- src/index.c | 17 ++--- src/integer.h | 28 ++++---- src/iterator.c | 7 +- src/merge.c | 9 ++- src/odb.c | 15 ++-- src/odb_loose.c | 42 +++++------ src/odb_mempack.c | 5 +- src/pack-objects.c | 7 +- src/pack.c | 21 +++--- src/path.c | 43 +++++------ src/pool.c | 15 ++-- src/refs.c | 22 +++--- src/sortedcache.c | 8 +-- src/tag.c | 10 +-- src/transports/cred.c | 8 +-- src/transports/smart_pkt.c | 50 +++++++------ src/tree.c | 18 ++--- src/util.h | 19 +++-- src/vector.c | 3 +- src/win32/dir.c | 8 +-- tests/core/errors.c | 16 +++-- 36 files changed, 352 insertions(+), 347 deletions(-) diff --git a/src/array.h b/src/array.h index 7c4dbdbc15f..7cd9b71536e 100644 --- a/src/array.h +++ b/src/array.h @@ -51,10 +51,9 @@ GIT_INLINE(void *) git_array_grow(void *_a, size_t item_size) if (a->size < 8) { new_size = 8; } else { - if (GIT_ALLOC_OVERFLOW_MULTIPLY(a->size, 3 / 2)) + if (GIT_MULTIPLY_SIZET_OVERFLOW(&new_size, a->size, 3)) goto on_oom; - - new_size = a->size * 3 / 2; + new_size /= 2; } if ((new_array = git__reallocarray(a->ptr, new_size, item_size)) == NULL) diff --git a/src/blame_git.c b/src/blame_git.c index 05aef5d9996..e863efe2e08 100644 --- a/src/blame_git.c +++ b/src/blame_git.c @@ -36,14 +36,14 @@ static void origin_decref(git_blame__origin *o) static int make_origin(git_blame__origin **out, git_commit *commit, const char *path) { git_blame__origin *o; - size_t path_len = strlen(path); + size_t path_len = strlen(path), alloc_len; int error = 0; - GITERR_CHECK_ALLOC_ADD(sizeof(*o), path_len); - GITERR_CHECK_ALLOC_ADD(sizeof(*o) + path_len, 1); - - o = git__calloc(1, sizeof(*o) + path_len + 1); + GITERR_CHECK_ALLOC_ADD(&alloc_len, sizeof(*o), path_len); + GITERR_CHECK_ALLOC_ADD(&alloc_len, alloc_len, 1); + o = git__calloc(1, alloc_len); GITERR_CHECK_ALLOC(o); + o->commit = commit; o->refcnt = 1; strcpy(o->path, path); diff --git a/src/buf_text.c b/src/buf_text.c index 08b86f4ccc4..864e39caba7 100644 --- a/src/buf_text.c +++ b/src/buf_text.c @@ -13,7 +13,7 @@ int git_buf_text_puts_escaped( const char *esc_with) { const char *scan; - size_t total = 0, esc_len = strlen(esc_with), count; + size_t total = 0, esc_len = strlen(esc_with), count, alloclen; if (!string) return 0; @@ -29,8 +29,8 @@ int git_buf_text_puts_escaped( scan += count; } - GITERR_CHECK_ALLOC_ADD(total, 1); - if (git_buf_grow_by(buf, total + 1) < 0) + GITERR_CHECK_ALLOC_ADD(&alloclen, total, 1); + if (git_buf_grow_by(buf, alloclen) < 0) return -1; for (scan = string; *scan; ) { @@ -66,6 +66,7 @@ int git_buf_text_crlf_to_lf(git_buf *tgt, const git_buf *src) const char *scan = src->ptr; const char *scan_end = src->ptr + src->size; const char *next = memchr(scan, '\r', src->size); + size_t new_size; char *out; assert(tgt != src); @@ -74,8 +75,8 @@ int git_buf_text_crlf_to_lf(git_buf *tgt, const git_buf *src) return git_buf_set(tgt, src->ptr, src->size); /* reduce reallocs while in the loop */ - GITERR_CHECK_ALLOC_ADD(src->size, 1); - if (git_buf_grow(tgt, src->size + 1) < 0) + GITERR_CHECK_ALLOC_ADD(&new_size, src->size, 1); + if (git_buf_grow(tgt, new_size) < 0) return -1; out = tgt->ptr; @@ -113,6 +114,7 @@ int git_buf_text_lf_to_crlf(git_buf *tgt, const git_buf *src) const char *end = start + src->size; const char *scan = start; const char *next = memchr(scan, '\n', src->size); + size_t alloclen; assert(tgt != src); @@ -120,9 +122,9 @@ int git_buf_text_lf_to_crlf(git_buf *tgt, const git_buf *src) return git_buf_set(tgt, src->ptr, src->size); /* attempt to reduce reallocs while in the loop */ - GITERR_CHECK_ALLOC_ADD(src->size, src->size >> 4); - GITERR_CHECK_ALLOC_ADD(src->size + (src->size >> 4), 1); - if (git_buf_grow(tgt, src->size + (src->size >> 4) + 1) < 0) + GITERR_CHECK_ALLOC_ADD(&alloclen, src->size, src->size >> 4); + GITERR_CHECK_ALLOC_ADD(&alloclen, alloclen, 1); + if (git_buf_grow(tgt, alloclen) < 0) return -1; tgt->size = 0; @@ -135,8 +137,8 @@ int git_buf_text_lf_to_crlf(git_buf *tgt, const git_buf *src) return GIT_PASSTHROUGH; } - GITERR_CHECK_ALLOC_ADD(copylen, 3); - if (git_buf_grow_by(tgt, copylen + 3) < 0) + GITERR_CHECK_ALLOC_ADD(&alloclen, copylen, 3); + if (git_buf_grow_by(tgt, alloclen) < 0) return -1; if (next > scan) { diff --git a/src/buffer.c b/src/buffer.c index 0d031443914..3deb0329c34 100644 --- a/src/buffer.c +++ b/src/buffer.c @@ -103,13 +103,14 @@ int git_buf_grow(git_buf *buffer, size_t target_size) int git_buf_grow_by(git_buf *buffer, size_t additional_size) { - if (GIT_ALLOC_OVERFLOW_ADD(buffer->size, additional_size)) { + size_t newsize; + + if (GIT_ADD_SIZET_OVERFLOW(&newsize, buffer->size, additional_size)) { buffer->ptr = git_buf__oom; return -1; } - return git_buf_try_grow( - buffer, buffer->size + additional_size, true, true); + return git_buf_try_grow(buffer, newsize, true, true); } void git_buf_free(git_buf *buf) @@ -146,12 +147,14 @@ void git_buf_clear(git_buf *buf) int git_buf_set(git_buf *buf, const void *data, size_t len) { + size_t alloclen; + if (len == 0 || data == NULL) { git_buf_clear(buf); } else { if (data != buf->ptr) { - GITERR_CHECK_ALLOC_ADD(len, 1); - ENSURE_SIZE(buf, len + 1); + GITERR_CHECK_ALLOC_ADD(&alloclen, len, 1); + ENSURE_SIZE(buf, alloclen); memmove(buf->ptr, data, len); } @@ -180,8 +183,9 @@ int git_buf_sets(git_buf *buf, const char *string) int git_buf_putc(git_buf *buf, char c) { - GITERR_CHECK_ALLOC_ADD(buf->size, 2); - ENSURE_SIZE(buf, buf->size + 2); + size_t new_size; + GITERR_CHECK_ALLOC_ADD(&new_size, buf->size, 2); + ENSURE_SIZE(buf, new_size); buf->ptr[buf->size++] = c; buf->ptr[buf->size] = '\0'; return 0; @@ -189,9 +193,10 @@ int git_buf_putc(git_buf *buf, char c) int git_buf_putcn(git_buf *buf, char c, size_t len) { - GITERR_CHECK_ALLOC_ADD(buf->size, len); - GITERR_CHECK_ALLOC_ADD(buf->size + len, 1); - ENSURE_SIZE(buf, buf->size + len + 1); + size_t new_size; + GITERR_CHECK_ALLOC_ADD(&new_size, buf->size, len); + GITERR_CHECK_ALLOC_ADD(&new_size, new_size, 1); + ENSURE_SIZE(buf, new_size); memset(buf->ptr + buf->size, c, len); buf->size += len; buf->ptr[buf->size] = '\0'; @@ -201,10 +206,13 @@ int git_buf_putcn(git_buf *buf, char c, size_t len) int git_buf_put(git_buf *buf, const char *data, size_t len) { if (len) { + size_t new_size; + assert(data); - GITERR_CHECK_ALLOC_ADD(buf->size, len); - GITERR_CHECK_ALLOC_ADD(buf->size + len, 1); - ENSURE_SIZE(buf, buf->size + len + 1); + + GITERR_CHECK_ALLOC_ADD(&new_size, buf->size, len); + GITERR_CHECK_ALLOC_ADD(&new_size, new_size, 1); + ENSURE_SIZE(buf, new_size); memmove(buf->ptr + buf->size, data, len); buf->size += len; buf->ptr[buf->size] = '\0'; @@ -226,12 +234,13 @@ int git_buf_encode_base64(git_buf *buf, const char *data, size_t len) size_t extra = len % 3; uint8_t *write, a, b, c; const uint8_t *read = (const uint8_t *)data; - size_t blocks = (len / 3) + !!extra; + size_t blocks = (len / 3) + !!extra, alloclen; + + GITERR_CHECK_ALLOC_ADD(&blocks, blocks, 1); + GITERR_CHECK_ALLOC_MULTIPLY(&alloclen, blocks, 4); + GITERR_CHECK_ALLOC_ADD(&alloclen, alloclen, buf->size); - GITERR_CHECK_ALLOC_MULTIPLY(blocks, 4); - GITERR_CHECK_ALLOC_ADD(buf->size, 4 * blocks); - GITERR_CHECK_ALLOC_ADD(buf->size + 4 * blocks, 1); - ENSURE_SIZE(buf, buf->size + 4 * blocks + 1); + ENSURE_SIZE(buf, alloclen); write = (uint8_t *)&buf->ptr[buf->size]; /* convert each run of 3 bytes into 4 output bytes */ @@ -282,12 +291,12 @@ int git_buf_decode_base64(git_buf *buf, const char *base64, size_t len) { size_t i; int8_t a, b, c, d; - size_t orig_size = buf->size; + size_t orig_size = buf->size, new_size; assert(len % 4 == 0); - GITERR_CHECK_ALLOC_ADD(buf->size, len / 4 * 3); - GITERR_CHECK_ALLOC_ADD(buf->size + (len / 4 * 3), 1); - ENSURE_SIZE(buf, buf->size + (len / 4 * 3) + 1); + GITERR_CHECK_ALLOC_ADD(&new_size, (len / 4 * 3), buf->size); + GITERR_CHECK_ALLOC_ADD(&new_size, new_size, 1); + ENSURE_SIZE(buf, new_size); for (i = 0; i < len; i += 4) { if ((a = BASE64_DECODE_VALUE(base64[i])) < 0 || @@ -315,12 +324,13 @@ static const char b85str[] = int git_buf_encode_base85(git_buf *buf, const char *data, size_t len) { - size_t blocks = (len / 4) + !!(len % 4); + size_t blocks = (len / 4) + !!(len % 4), alloclen; - GITERR_CHECK_ALLOC_MULTIPLY(blocks, 5); - GITERR_CHECK_ALLOC_ADD(buf->size, 5 * blocks); - GITERR_CHECK_ALLOC_ADD(buf->size + 5 * blocks, 1); - ENSURE_SIZE(buf, buf->size + blocks * 5 + 1); + GITERR_CHECK_ALLOC_MULTIPLY(&alloclen, blocks, 5); + GITERR_CHECK_ALLOC_ADD(&alloclen, alloclen, buf->size); + GITERR_CHECK_ALLOC_ADD(&alloclen, alloclen, 1); + + ENSURE_SIZE(buf, alloclen); while (len) { uint32_t acc = 0; @@ -353,15 +363,11 @@ int git_buf_encode_base85(git_buf *buf, const char *data, size_t len) int git_buf_vprintf(git_buf *buf, const char *format, va_list ap) { - size_t expected_size = strlen(format); + size_t expected_size, new_size; int len; - GITERR_CHECK_ALLOC_MULTIPLY(expected_size, 2); - expected_size *= 2; - - GITERR_CHECK_ALLOC_ADD(expected_size, buf->size); - expected_size += buf->size; - + GITERR_CHECK_ALLOC_MULTIPLY(&expected_size, strlen(format), 2); + GITERR_CHECK_ALLOC_ADD(&expected_size, expected_size, buf->size); ENSURE_SIZE(buf, expected_size); while (1) { @@ -387,9 +393,9 @@ int git_buf_vprintf(git_buf *buf, const char *format, va_list ap) break; } - GITERR_CHECK_ALLOC_ADD(buf->size, len); - GITERR_CHECK_ALLOC_ADD(buf->size + len, 1); - ENSURE_SIZE(buf, buf->size + len + 1); + GITERR_CHECK_ALLOC_ADD(&new_size, buf->size, len); + GITERR_CHECK_ALLOC_ADD(&new_size, new_size, 1); + ENSURE_SIZE(buf, new_size); } return 0; @@ -516,9 +522,11 @@ int git_buf_join_n(git_buf *buf, char separator, int nbuf, ...) continue; segment_len = strlen(segment); - total_size += segment_len; + + GITERR_CHECK_ALLOC_ADD(&total_size, total_size, segment_len); + if (segment_len == 0 || segment[segment_len - 1] != separator) - ++total_size; /* space for separator */ + GITERR_CHECK_ALLOC_ADD(&total_size, total_size, 1); } va_end(ap); @@ -526,8 +534,8 @@ int git_buf_join_n(git_buf *buf, char separator, int nbuf, ...) if (total_size == 0) return 0; - GITERR_CHECK_ALLOC_ADD(total_size, 1); - if (git_buf_grow_by(buf, total_size + 1) < 0) + GITERR_CHECK_ALLOC_ADD(&total_size, total_size, 1); + if (git_buf_grow_by(buf, total_size) < 0) return -1; out = buf->ptr + buf->size; @@ -588,6 +596,7 @@ int git_buf_join( { size_t strlen_a = str_a ? strlen(str_a) : 0; size_t strlen_b = strlen(str_b); + size_t alloc_len; int need_sep = 0; ssize_t offset_a = -1; @@ -605,10 +614,10 @@ int git_buf_join( if (str_a >= buf->ptr && str_a < buf->ptr + buf->size) offset_a = str_a - buf->ptr; - GITERR_CHECK_ALLOC_ADD(strlen_a, strlen_b); - GITERR_CHECK_ALLOC_ADD(strlen_a + strlen_b, need_sep); - GITERR_CHECK_ALLOC_ADD(strlen_a + strlen_b + need_sep, 1); - if (git_buf_grow(buf, strlen_a + strlen_b + need_sep + 1) < 0) + GITERR_CHECK_ALLOC_ADD(&alloc_len, strlen_a, strlen_b); + GITERR_CHECK_ALLOC_ADD(&alloc_len, alloc_len, need_sep); + GITERR_CHECK_ALLOC_ADD(&alloc_len, alloc_len, 1); + if (git_buf_grow(buf, alloc_len) < 0) return -1; assert(buf->ptr); @@ -636,7 +645,10 @@ int git_buf_join3( const char *str_b, const char *str_c) { - size_t len_a = strlen(str_a), len_b = strlen(str_b), len_c = strlen(str_c); + size_t len_a = strlen(str_a), + len_b = strlen(str_b), + len_c = strlen(str_c), + len_total; int sep_a = 0, sep_b = 0; char *tgt; @@ -656,12 +668,12 @@ int git_buf_join3( sep_b = (str_b[len_b - 1] != separator); } - GITERR_CHECK_ALLOC_ADD(len_a, sep_a); - GITERR_CHECK_ALLOC_ADD(len_a + sep_a, len_b); - GITERR_CHECK_ALLOC_ADD(len_a + sep_a + len_b, sep_b); - GITERR_CHECK_ALLOC_ADD(len_a + sep_a + len_b + sep_b, len_c); - GITERR_CHECK_ALLOC_ADD(len_a + sep_a + len_b + sep_b + len_c, 1); - if (git_buf_grow(buf, len_a + sep_a + len_b + sep_b + len_c + 1) < 0) + GITERR_CHECK_ALLOC_ADD(&len_total, len_a, sep_a); + GITERR_CHECK_ALLOC_ADD(&len_total, len_total, len_b); + GITERR_CHECK_ALLOC_ADD(&len_total, len_total, sep_b); + GITERR_CHECK_ALLOC_ADD(&len_total, len_total, len_c); + GITERR_CHECK_ALLOC_ADD(&len_total, len_total, 1); + if (git_buf_grow(buf, len_total) < 0) return -1; tgt = buf->ptr; @@ -714,28 +726,25 @@ int git_buf_splice( const char *data, size_t nb_to_insert) { - size_t new_size; + char *splice_loc; + size_t new_size, alloc_size; - assert(buf && - where <= git_buf_len(buf) && - where + nb_to_remove <= git_buf_len(buf)); + assert(buf && where <= buf->size && nb_to_remove <= buf->size - where); + + splice_loc = buf->ptr + where; /* Ported from git.git * https://github.com/git/git/blob/16eed7c/strbuf.c#L159-176 */ - new_size = buf->size - nb_to_remove; - - GITERR_CHECK_ALLOC_ADD(new_size, nb_to_insert); - new_size += nb_to_insert; - - GITERR_CHECK_ALLOC_ADD(new_size, 1); - ENSURE_SIZE(buf, new_size + 1); + GITERR_CHECK_ALLOC_ADD(&new_size, (buf->size - nb_to_remove), nb_to_insert); + GITERR_CHECK_ALLOC_ADD(&alloc_size, new_size, 1); + ENSURE_SIZE(buf, alloc_size); - memmove(buf->ptr + where + nb_to_insert, - buf->ptr + where + nb_to_remove, - buf->size - where - nb_to_remove); + memmove(splice_loc + nb_to_insert, + splice_loc + nb_to_remove, + buf->size - where - nb_to_remove); - memcpy(buf->ptr + where, data, nb_to_insert); + memcpy(splice_loc, data, nb_to_insert); buf->size = new_size; buf->ptr[buf->size] = '\0'; diff --git a/src/common.h b/src/common.h index 530e320e29a..8d1e890642c 100644 --- a/src/common.h +++ b/src/common.h @@ -176,21 +176,21 @@ GIT_INLINE(void) git__init_structure(void *structure, size_t len, unsigned int v memcpy((PTR), &_tmpl, sizeof(_tmpl)); } while (0) -/** Check for integer overflow from addition or multiplication */ -#define GIT_ALLOC_OVERFLOW_ADD(one, two) \ - (!git__add_sizet_overflow(NULL, (one), (two)) ? (giterr_set_oom(), 1) : 0) +/** Check for additive overflow, setting an error if would occur. */ +#define GIT_ADD_SIZET_OVERFLOW(out, one, two) \ + (git__add_sizet_overflow(out, one, two) ? (giterr_set_oom(), 1) : 0) -/** Check for integer overflow from multiplication */ -#define GIT_ALLOC_OVERFLOW_MULTIPLY(one, two) \ - (!git__multiply_sizet_overflow(NULL, (one), (two)) ? (giterr_set_oom(), 1) : 0) +/** Check for additive overflow, setting an error if would occur. */ +#define GIT_MULTIPLY_SIZET_OVERFLOW(out, nelem, elsize) \ + (git__multiply_sizet_overflow(out, nelem, elsize) ? (giterr_set_oom(), 1) : 0) /** Check for additive overflow, failing if it would occur. */ -#define GITERR_CHECK_ALLOC_ADD(one, two) \ - if (GIT_ALLOC_OVERFLOW_ADD(one, two)) { return -1; } +#define GITERR_CHECK_ALLOC_ADD(out, one, two) \ + if (GIT_ADD_SIZET_OVERFLOW(out, one, two)) { return -1; } /** Check for multiplicative overflow, failing if it would occur. */ -#define GITERR_CHECK_ALLOC_MULTIPLY(nelem, elsize) \ - if (GIT_ALLOC_OVERFLOW_MULTIPLY(nelem, elsize)) { return -1; } +#define GITERR_CHECK_ALLOC_MULTIPLY(out, nelem, elsize) \ + if (GIT_MULTIPLY_SIZET_OVERFLOW(out, nelem, elsize)) { return -1; } /* NOTE: other giterr functions are in the public errors.h header file */ diff --git a/src/config_file.c b/src/config_file.c index 39e9ff84125..8ccbe64cc3b 100644 --- a/src/config_file.c +++ b/src/config_file.c @@ -885,7 +885,7 @@ static char *reader_readline(struct reader *reader, bool skip_whitespace) { char *line = NULL; char *line_src, *line_end; - size_t line_len; + size_t line_len, alloc_len; line_src = reader->read_ptr; @@ -903,8 +903,8 @@ static char *reader_readline(struct reader *reader, bool skip_whitespace) line_len = line_end - line_src; - if (GIT_ALLOC_OVERFLOW_ADD(line_len, 1) || - (line = git__malloc(line_len + 1)) == NULL) { + if (GIT_ADD_SIZET_OVERFLOW(&alloc_len, line_len, 1) || + (line = git__malloc(alloc_len)) == NULL) { return NULL; } @@ -959,7 +959,7 @@ static int parse_section_header_ext(struct reader *reader, const char *line, con int c, rpos; char *first_quote, *last_quote; git_buf buf = GIT_BUF_INIT; - size_t quoted_len, base_name_len = strlen(base_name); + size_t quoted_len, alloc_len, base_name_len = strlen(base_name); /* * base_name is what came before the space. We should be at the @@ -976,10 +976,10 @@ static int parse_section_header_ext(struct reader *reader, const char *line, con return -1; } - GITERR_CHECK_ALLOC_ADD(base_name_len, quoted_len); - GITERR_CHECK_ALLOC_ADD(base_name_len + quoted_len, 2); + GITERR_CHECK_ALLOC_ADD(&alloc_len, base_name_len, quoted_len); + GITERR_CHECK_ALLOC_ADD(&alloc_len, alloc_len, 2); - git_buf_grow(&buf, base_name_len + quoted_len + 2); + git_buf_grow(&buf, alloc_len); git_buf_printf(&buf, "%s.", base_name); rpos = 0; @@ -1050,9 +1050,7 @@ static int parse_section_header(struct reader *reader, char **section_out) return -1; } - line_len = (size_t)(name_end - line); - - GITERR_CHECK_ALLOC_ADD(line_len, 1); + GITERR_CHECK_ALLOC_ADD(&line_len, (size_t)(name_end - line), 1); name = git__malloc(line_len); GITERR_CHECK_ALLOC(name); @@ -1615,10 +1613,10 @@ static char *escape_value(const char *ptr) static char *fixup_line(const char *ptr, int quote_count) { char *str, *out, *esc; - size_t ptr_len = strlen(ptr); + size_t ptr_len = strlen(ptr), alloc_len; - if (GIT_ALLOC_OVERFLOW_ADD(ptr_len, 1) || - (str = git__malloc(ptr_len + 1)) == NULL) { + if (GIT_ADD_SIZET_OVERFLOW(&alloc_len, ptr_len, 1) || + (str = git__malloc(alloc_len)) == NULL) { return NULL; } diff --git a/src/delta-apply.c b/src/delta-apply.c index e46c9631c48..89745faa016 100644 --- a/src/delta-apply.c +++ b/src/delta-apply.c @@ -57,7 +57,7 @@ int git__delta_apply( size_t delta_len) { const unsigned char *delta_end = delta + delta_len; - size_t base_sz, res_sz; + size_t base_sz, res_sz, alloc_sz; unsigned char *res_dp; /* Check that the base size matches the data we were given; @@ -74,8 +74,8 @@ int git__delta_apply( return -1; } - GITERR_CHECK_ALLOC_ADD(res_sz, 1); - res_dp = git__malloc(res_sz + 1); + GITERR_CHECK_ALLOC_ADD(&alloc_sz, res_sz, 1); + res_dp = git__malloc(alloc_sz); GITERR_CHECK_ALLOC(res_dp); res_dp[res_sz] = '\0'; diff --git a/src/delta.c b/src/delta.c index 242f3abe3c9..d72d820d842 100644 --- a/src/delta.c +++ b/src/delta.c @@ -122,20 +122,13 @@ struct git_delta_index { static int lookup_index_alloc( void **out, unsigned long *out_len, size_t entries, size_t hash_count) { - size_t entries_len, hash_len, - index_len = sizeof(struct git_delta_index); + size_t entries_len, hash_len, index_len; - GITERR_CHECK_ALLOC_MULTIPLY(entries, sizeof(struct index_entry)); - entries_len = entries * sizeof(struct index_entry); + GITERR_CHECK_ALLOC_MULTIPLY(&entries_len, entries, sizeof(struct index_entry)); + GITERR_CHECK_ALLOC_MULTIPLY(&hash_len, hash_count, sizeof(struct index_entry *)); - GITERR_CHECK_ALLOC_ADD(index_len, entries_len); - index_len += entries_len; - - GITERR_CHECK_ALLOC_MULTIPLY(hash_count, sizeof(struct index_entry *)); - hash_len = hash_count * sizeof(struct index_entry *); - - GITERR_CHECK_ALLOC_ADD(index_len, hash_len); - index_len += hash_len; + GITERR_CHECK_ALLOC_ADD(&index_len, sizeof(struct git_delta_index), entries_len); + GITERR_CHECK_ALLOC_ADD(&index_len, index_len, hash_len); if (!git__is_ulong(index_len)) { giterr_set(GITERR_NOMEMORY, "Overly large delta"); diff --git a/src/diff.c b/src/diff.c index 75e9ae9a3ba..07eae03e7ca 100644 --- a/src/diff.c +++ b/src/diff.c @@ -1527,6 +1527,7 @@ int git_diff_format_email( char *summary = NULL, *loc = NULL; bool ignore_marker; unsigned int format_flags = 0; + size_t allocsize; int error; assert(out && diff && opts); @@ -1558,8 +1559,8 @@ int git_diff_format_email( goto on_error; } - GITERR_CHECK_ALLOC_ADD(offset, 1); - summary = git__calloc(offset + 1, sizeof(char)); + GITERR_CHECK_ALLOC_ADD(&allocsize, offset, 1); + summary = git__calloc(allocsize, sizeof(char)); GITERR_CHECK_ALLOC(summary); strncpy(summary, opts->summary, offset); diff --git a/src/diff_driver.c b/src/diff_driver.c index 67f1c591d5c..e4d9a06995c 100644 --- a/src/diff_driver.c +++ b/src/diff_driver.c @@ -163,12 +163,13 @@ static int diff_driver_alloc( { git_diff_driver *driver; size_t driverlen = sizeof(git_diff_driver), - namelen = strlen(name); + namelen = strlen(name), + alloclen; - GITERR_CHECK_ALLOC_ADD(driverlen, namelen); - GITERR_CHECK_ALLOC_ADD(driverlen + namelen, 1); + GITERR_CHECK_ALLOC_ADD(&alloclen, driverlen, namelen); + GITERR_CHECK_ALLOC_ADD(&alloclen, alloclen, 1); - driver = git__calloc(1, driverlen + namelen + 1); + driver = git__calloc(1, alloclen); GITERR_CHECK_ALLOC(driver); memcpy(driver->name, name, namelen); diff --git a/src/diff_patch.c b/src/diff_patch.c index f5eecae660d..1c4c0e8b86c 100644 --- a/src/diff_patch.c +++ b/src/diff_patch.c @@ -388,16 +388,11 @@ static int diff_patch_with_delta_alloc( diff_patch_with_delta *pd; size_t old_len = *old_path ? strlen(*old_path) : 0; size_t new_len = *new_path ? strlen(*new_path) : 0; - size_t alloc_len = sizeof(*pd); + size_t alloc_len; - GITERR_CHECK_ALLOC_ADD(alloc_len, old_len); - alloc_len += old_len; - - GITERR_CHECK_ALLOC_ADD(alloc_len, new_len); - alloc_len += new_len; - - GITERR_CHECK_ALLOC_ADD(alloc_len, 2); - alloc_len += 2; + GITERR_CHECK_ALLOC_ADD(&alloc_len, sizeof(*pd), old_len); + GITERR_CHECK_ALLOC_ADD(&alloc_len, alloc_len, new_len); + GITERR_CHECK_ALLOC_ADD(&alloc_len, alloc_len, 2); *out = pd = git__calloc(1, alloc_len); GITERR_CHECK_ALLOC(pd); diff --git a/src/diff_tform.c b/src/diff_tform.c index cad1356c3f1..8ee568cf41a 100644 --- a/src/diff_tform.c +++ b/src/diff_tform.c @@ -811,6 +811,7 @@ int git_diff_find_similar( size_t num_deltas, num_srcs = 0, num_tgts = 0; size_t tried_srcs = 0, tried_tgts = 0; size_t num_rewrites = 0, num_updates = 0, num_bumped = 0; + size_t sigcache_size; void **sigcache = NULL; /* cache of similarity metric file signatures */ diff_find_match *tgt2src = NULL; diff_find_match *src2tgt = NULL; @@ -831,8 +832,8 @@ int git_diff_find_similar( if ((opts.flags & GIT_DIFF_FIND_ALL) == 0) goto cleanup; - GITERR_CHECK_ALLOC_MULTIPLY(num_deltas, 2); - sigcache = git__calloc(num_deltas * 2, sizeof(void *)); + GITERR_CHECK_ALLOC_MULTIPLY(&sigcache_size, num_deltas, 2); + sigcache = git__calloc(sigcache_size, sizeof(void *)); GITERR_CHECK_ALLOC(sigcache); /* Label rename sources and targets diff --git a/src/filebuf.c b/src/filebuf.c index 94f2bec32d3..932b8c7d154 100644 --- a/src/filebuf.c +++ b/src/filebuf.c @@ -194,7 +194,7 @@ static int write_deflate(git_filebuf *file, void *source, size_t len) int git_filebuf_open(git_filebuf *file, const char *path, int flags, mode_t mode) { int compression, error = -1; - size_t path_len; + size_t path_len, alloc_len; /* opening an already open buffer is a programming error; * assert that this never happens instead of returning @@ -271,8 +271,8 @@ int git_filebuf_open(git_filebuf *file, const char *path, int flags, mode_t mode GITERR_CHECK_ALLOC(file->path_original); /* create the locking path by appending ".lock" to the original */ - GITERR_CHECK_ALLOC_ADD(path_len, GIT_FILELOCK_EXTLENGTH); - file->path_lock = git__malloc(path_len + GIT_FILELOCK_EXTLENGTH); + GITERR_CHECK_ALLOC_ADD(&alloc_len, path_len, GIT_FILELOCK_EXTLENGTH); + file->path_lock = git__malloc(alloc_len); GITERR_CHECK_ALLOC(file->path_lock); memcpy(file->path_lock, file->path_original, path_len); @@ -408,7 +408,7 @@ int git_filebuf_reserve(git_filebuf *file, void **buffer, size_t len) int git_filebuf_printf(git_filebuf *file, const char *format, ...) { va_list arglist; - size_t space_left, len; + size_t space_left, len, alloclen; int written, res; char *tmp_buffer; @@ -439,8 +439,8 @@ int git_filebuf_printf(git_filebuf *file, const char *format, ...) } while (len + 1 <= space_left); - if (GIT_ALLOC_OVERFLOW_ADD(len, 1) || - !(tmp_buffer = git__malloc(len + 1))) { + if (GIT_ADD_SIZET_OVERFLOW(&alloclen, len, 1) || + !(tmp_buffer = git__malloc(alloclen))) { file->last_error = BUFERR_MEM; return -1; } diff --git a/src/fileops.c b/src/fileops.c index 420ed70a275..09a8f5d4a10 100644 --- a/src/fileops.c +++ b/src/fileops.c @@ -124,6 +124,7 @@ mode_t git_futils_canonical_mode(mode_t raw_mode) int git_futils_readbuffer_fd(git_buf *buf, git_file fd, size_t len) { ssize_t read_size = 0; + size_t alloc_len; git_buf_clear(buf); @@ -132,8 +133,8 @@ int git_futils_readbuffer_fd(git_buf *buf, git_file fd, size_t len) return -1; } - GITERR_CHECK_ALLOC_ADD(len, 1); - if (git_buf_grow(buf, len + 1) < 0) + GITERR_CHECK_ALLOC_ADD(&alloc_len, len, 1); + if (git_buf_grow(buf, alloc_len) < 0) return -1; /* p_read loops internally to read len bytes */ @@ -455,7 +456,13 @@ int git_futils_mkdir_ext( } if (opts->dir_map && opts->pool) { - char *cache_path = git_pool_malloc(opts->pool, make_path.size + 1); + char *cache_path; + size_t alloc_size; + + GITERR_CHECK_ALLOC_ADD(&alloc_size, make_path.size, 1); + if (!git__is_uint32(alloc_size)) + return -1; + cache_path = git_pool_malloc(opts->pool, (uint32_t)alloc_size); GITERR_CHECK_ALLOC(cache_path); memcpy(cache_path, make_path.ptr, make_path.size + 1); @@ -715,9 +722,10 @@ static int cp_link(const char *from, const char *to, size_t link_size) int error = 0; ssize_t read_len; char *link_data; + size_t alloc_size; - GITERR_CHECK_ALLOC_ADD(link_size, 1); - link_data = git__malloc(link_size + 1); + GITERR_CHECK_ALLOC_ADD(&alloc_size, link_size, 1); + link_data = git__malloc(alloc_size); GITERR_CHECK_ALLOC(link_data); read_len = p_readlink(from, link_data, link_size); diff --git a/src/filter.c b/src/filter.c index d5c669f0164..7b54a76c064 100644 --- a/src/filter.c +++ b/src/filter.c @@ -245,14 +245,9 @@ int git_filter_register( if (filter_def_scan_attrs(&attrs, &nattr, &nmatch, filter->attributes) < 0) return -1; - GITERR_CHECK_ALLOC_MULTIPLY(nattr, 2); - alloc_len = nattr * 2; - - GITERR_CHECK_ALLOC_MULTIPLY(alloc_len, sizeof(char *)); - alloc_len *= sizeof(char *); - - GITERR_CHECK_ALLOC_ADD(alloc_len, sizeof(git_filter_def)); - alloc_len += sizeof(git_filter_def); + GITERR_CHECK_ALLOC_MULTIPLY(&alloc_len, nattr, 2); + GITERR_CHECK_ALLOC_MULTIPLY(&alloc_len, alloc_len, sizeof(char *)); + GITERR_CHECK_ALLOC_ADD(&alloc_len, alloc_len, sizeof(git_filter_def)); fdef = git__calloc(1, alloc_len); GITERR_CHECK_ALLOC(fdef); @@ -385,12 +380,12 @@ static int filter_list_new( git_filter_list **out, const git_filter_source *src) { git_filter_list *fl = NULL; - size_t pathlen = src->path ? strlen(src->path) : 0; + size_t pathlen = src->path ? strlen(src->path) : 0, alloclen; - GITERR_CHECK_ALLOC_ADD(sizeof(git_filter_list), pathlen); - GITERR_CHECK_ALLOC_ADD(sizeof(git_filter_list) + pathlen, 1); + GITERR_CHECK_ALLOC_ADD(&alloclen, sizeof(git_filter_list), pathlen); + GITERR_CHECK_ALLOC_ADD(&alloclen, alloclen, 1); - fl = git__calloc(1, sizeof(git_filter_list) + pathlen + 1); + fl = git__calloc(1, alloclen); GITERR_CHECK_ALLOC(fl); if (src->path) diff --git a/src/index.c b/src/index.c index 35f512ca4f4..75700a719b3 100644 --- a/src/index.c +++ b/src/index.c @@ -770,7 +770,7 @@ static int index_entry_create( git_repository *repo, const char *path) { - size_t pathlen = strlen(path); + size_t pathlen = strlen(path), alloclen; struct entry_internal *entry; if (!git_path_isvalid(repo, path, @@ -779,9 +779,9 @@ static int index_entry_create( return -1; } - GITERR_CHECK_ALLOC_ADD(sizeof(struct entry_internal), pathlen); - GITERR_CHECK_ALLOC_ADD(sizeof(struct entry_internal) + pathlen, 1); - entry = git__calloc(1, sizeof(struct entry_internal) + pathlen + 1); + GITERR_CHECK_ALLOC_ADD(&alloclen, sizeof(struct entry_internal), pathlen); + GITERR_CHECK_ALLOC_ADD(&alloclen, alloclen, 1); + entry = git__calloc(1, alloclen); GITERR_CHECK_ALLOC(entry); entry->pathlen = pathlen; @@ -829,14 +829,15 @@ static int index_entry_init( static git_index_reuc_entry *reuc_entry_alloc(const char *path) { size_t pathlen = strlen(path), - structlen = sizeof(struct reuc_entry_internal); + structlen = sizeof(struct reuc_entry_internal), + alloclen; struct reuc_entry_internal *entry; - if (GIT_ALLOC_OVERFLOW_ADD(structlen, pathlen) || - GIT_ALLOC_OVERFLOW_ADD(structlen + pathlen, 1)) + if (GIT_ADD_SIZET_OVERFLOW(&alloclen, structlen, pathlen) || + GIT_ADD_SIZET_OVERFLOW(&alloclen, alloclen, 1)) return NULL; - entry = git__calloc(1, structlen + pathlen + 1); + entry = git__calloc(1, alloclen); if (!entry) return NULL; diff --git a/src/integer.h b/src/integer.h index a9ed10aae9f..a4abe2bd1a9 100644 --- a/src/integer.h +++ b/src/integer.h @@ -35,6 +35,13 @@ GIT_INLINE(int) git__is_ulong(git_off_t p) return p == (git_off_t)r; } +/** @return true if p fits into the range of an int */ +GIT_INLINE(int) git__is_int(long long p) +{ + int r = (int)p; + return p == (long long)r; +} + /** * Sets `one + two` into `out`, unless the arithmetic would overflow. * @return true if the result fits in a `uint64_t`, false on overflow. @@ -42,10 +49,9 @@ GIT_INLINE(int) git__is_ulong(git_off_t p) GIT_INLINE(bool) git__add_uint64_overflow(uint64_t *out, uint64_t one, uint64_t two) { if (UINT64_MAX - one < two) - return false; - if (out) - *out = one + two; - return true; + return true; + *out = one + two; + return false; } /** @@ -55,10 +61,9 @@ GIT_INLINE(bool) git__add_uint64_overflow(uint64_t *out, uint64_t one, uint64_t GIT_INLINE(bool) git__add_sizet_overflow(size_t *out, size_t one, size_t two) { if (SIZE_MAX - one < two) - return false; - if (out) - *out = one + two; - return true; + return true; + *out = one + two; + return false; } /** @@ -68,10 +73,9 @@ GIT_INLINE(bool) git__add_sizet_overflow(size_t *out, size_t one, size_t two) GIT_INLINE(bool) git__multiply_sizet_overflow(size_t *out, size_t one, size_t two) { if (one && SIZE_MAX / one < two) - return false; - if (out) - *out = one * two; - return true; + return true; + *out = one * two; + return false; } #endif /* INCLUDE_integer_h__ */ diff --git a/src/iterator.c b/src/iterator.c index e90cf30ff76..9ddacebd127 100644 --- a/src/iterator.c +++ b/src/iterator.c @@ -344,11 +344,8 @@ static int tree_iterator__push_frame(tree_iterator *ti) for (i = head->current; i < head->next; ++i) n_entries += git_tree_entrycount(head->entries[i]->tree); - GITERR_CHECK_ALLOC_MULTIPLY(sizeof(tree_iterator_entry *), n_entries); - alloclen = sizeof(tree_iterator_entry *) * n_entries; - - GITERR_CHECK_ALLOC_ADD(alloclen, sizeof(tree_iterator_frame)); - alloclen += sizeof(tree_iterator_frame); + GITERR_CHECK_ALLOC_MULTIPLY(&alloclen, sizeof(tree_iterator_entry *), n_entries); + GITERR_CHECK_ALLOC_ADD(&alloclen, alloclen, sizeof(tree_iterator_frame)); tf = git__calloc(1, alloclen); GITERR_CHECK_ALLOC(tf); diff --git a/src/merge.c b/src/merge.c index 25d7bd7aa39..e4b60c847f9 100644 --- a/src/merge.c +++ b/src/merge.c @@ -1169,8 +1169,7 @@ int git_merge_diff_list__find_renames( goto done; if (diff_list->conflicts.length <= opts->target_limit) { - GITERR_CHECK_ALLOC_MULTIPLY(diff_list->conflicts.length, 3); - cache_size = diff_list->conflicts.length * 3; + GITERR_CHECK_ALLOC_MULTIPLY(&cache_size, diff_list->conflicts.length, 3); cache = git__calloc(cache_size, sizeof(void *)); GITERR_CHECK_ALLOC(cache); @@ -2224,13 +2223,13 @@ static int merge_ancestor_head( size_t their_heads_len) { git_oid *oids, ancestor_oid; - size_t i; + size_t i, alloc_len; int error = 0; assert(repo && our_head && their_heads); - GITERR_CHECK_ALLOC_ADD(their_heads_len, 1); - oids = git__calloc(their_heads_len + 1, sizeof(git_oid)); + GITERR_CHECK_ALLOC_ADD(&alloc_len, their_heads_len, 1); + oids = git__calloc(alloc_len, sizeof(git_oid)); GITERR_CHECK_ALLOC(oids); git_oid_cpy(&oids[0], git_commit_id(our_head->commit)); diff --git a/src/odb.c b/src/odb.c index fcb21c1ced8..b1d606b4d6e 100644 --- a/src/odb.c +++ b/src/odb.c @@ -216,30 +216,31 @@ int git_odb__hashfd_filtered( int git_odb__hashlink(git_oid *out, const char *path) { struct stat st; - size_t size; + int size; int result; if (git_path_lstat(path, &st) < 0) return -1; - if (!git__is_sizet(st.st_size)) { + if (!git__is_int(st.st_size) || (int)st.st_size < 0) { giterr_set(GITERR_FILESYSTEM, "File size overflow for 32-bit systems"); return -1; } - size = (size_t)st.st_size; + size = (int)st.st_size; if (S_ISLNK(st.st_mode)) { char *link_data; - ssize_t read_len; + int read_len; + size_t alloc_size; - GITERR_CHECK_ALLOC_ADD(size, 1); - link_data = git__malloc(size + 1); + GITERR_CHECK_ALLOC_ADD(&alloc_size, size, 1); + link_data = git__malloc(alloc_size); GITERR_CHECK_ALLOC(link_data); read_len = p_readlink(path, link_data, size); link_data[size] = '\0'; - if (read_len != (ssize_t)size) { + if (read_len != size) { giterr_set(GITERR_OS, "Failed to read symlink data for '%s'", path); git__free(link_data); return -1; diff --git a/src/odb_loose.c b/src/odb_loose.c index e43b261fa6e..bfd95588b9a 100644 --- a/src/odb_loose.c +++ b/src/odb_loose.c @@ -63,10 +63,12 @@ typedef struct { static int object_file_name( git_buf *name, const loose_backend *be, const git_oid *id) { + size_t alloclen; + /* expand length for object root + 40 hex sha1 chars + 2 * '/' + '\0' */ - GITERR_CHECK_ALLOC_ADD(be->objects_dirlen, GIT_OID_HEXSZ); - GITERR_CHECK_ALLOC_ADD(be->objects_dirlen + GIT_OID_HEXSZ, 3); - if (git_buf_grow(name, be->objects_dirlen + GIT_OID_HEXSZ + 3) < 0) + GITERR_CHECK_ALLOC_ADD(&alloclen, be->objects_dirlen, GIT_OID_HEXSZ); + GITERR_CHECK_ALLOC_ADD(&alloclen, alloclen, 3); + if (git_buf_grow(name, alloclen) < 0) return -1; git_buf_set(name, be->objects_dir, be->objects_dirlen); @@ -263,15 +265,15 @@ static int inflate_buffer(void *in, size_t inlen, void *out, size_t outlen) static void *inflate_tail(z_stream *s, void *hb, size_t used, obj_hdr *hdr) { unsigned char *buf, *head = hb; - size_t tail; + size_t tail, alloc_size; /* * allocate a buffer to hold the inflated data and copy the * initial sequence of inflated data from the tail of the * head buffer, if any. */ - if (GIT_ALLOC_OVERFLOW_ADD(hdr->size, 1) || - (buf = git__malloc(hdr->size + 1)) == NULL) { + if (GIT_ADD_SIZET_OVERFLOW(&alloc_size, hdr->size, 1) || + (buf = git__malloc(alloc_size)) == NULL) { inflateEnd(s); return NULL; } @@ -309,7 +311,7 @@ static int inflate_packlike_loose_disk_obj(git_rawobj *out, git_buf *obj) { unsigned char *in, *buf; obj_hdr hdr; - size_t len, used; + size_t len, used, alloclen; /* * read the object header, which is an (uncompressed) @@ -324,8 +326,8 @@ static int inflate_packlike_loose_disk_obj(git_rawobj *out, git_buf *obj) /* * allocate a buffer and inflate the data into it */ - GITERR_CHECK_ALLOC_ADD(hdr.size, 1); - buf = git__malloc(hdr.size + 1); + GITERR_CHECK_ALLOC_ADD(&alloclen, hdr.size, 1); + buf = git__malloc(alloclen); GITERR_CHECK_ALLOC(buf); in = ((unsigned char *)obj->ptr) + used; @@ -519,14 +521,14 @@ static int locate_object_short_oid( size_t len) { char *objects_dir = backend->objects_dir; - size_t dir_len = strlen(objects_dir); + size_t dir_len = strlen(objects_dir), alloc_len; loose_locate_object_state state; int error; /* prealloc memory for OBJ_DIR/xx/xx..38x..xx */ - GITERR_CHECK_ALLOC_ADD(dir_len, GIT_OID_HEXSZ); - GITERR_CHECK_ALLOC_ADD(dir_len + GIT_OID_HEXSZ, 3); - if (git_buf_grow(object_location, dir_len + 3 + GIT_OID_HEXSZ) < 0) + GITERR_CHECK_ALLOC_ADD(&alloc_len, dir_len, GIT_OID_HEXSZ); + GITERR_CHECK_ALLOC_ADD(&alloc_len, alloc_len, 3); + if (git_buf_grow(object_location, alloc_len) < 0) return -1; git_buf_set(object_location, objects_dir, dir_len); @@ -569,11 +571,11 @@ static int locate_object_short_oid( return error; /* Update the location according to the oid obtained */ - GITERR_CHECK_ALLOC_ADD(dir_len, GIT_OID_HEXSZ); - GITERR_CHECK_ALLOC_ADD(dir_len + GIT_OID_HEXSZ, 2); + GITERR_CHECK_ALLOC_ADD(&alloc_len, dir_len, GIT_OID_HEXSZ); + GITERR_CHECK_ALLOC_ADD(&alloc_len, alloc_len, 2); git_buf_truncate(object_location, dir_len); - if (git_buf_grow(object_location, dir_len + GIT_OID_HEXSZ + 2) < 0) + if (git_buf_grow(object_location, alloc_len) < 0) return -1; git_oid_pathfmt(object_location->ptr + dir_len, res_oid); @@ -930,15 +932,15 @@ int git_odb_backend_loose( unsigned int file_mode) { loose_backend *backend; - size_t objects_dirlen; + size_t objects_dirlen, alloclen; assert(backend_out && objects_dir); objects_dirlen = strlen(objects_dir); - GITERR_CHECK_ALLOC_ADD(sizeof(loose_backend), objects_dirlen); - GITERR_CHECK_ALLOC_ADD(sizeof(loose_backend) + objects_dirlen, 2); - backend = git__calloc(1, sizeof(loose_backend) + objects_dirlen + 2); + GITERR_CHECK_ALLOC_ADD(&alloclen, sizeof(loose_backend), objects_dirlen); + GITERR_CHECK_ALLOC_ADD(&alloclen, alloclen, 2); + backend = git__calloc(1, alloclen); GITERR_CHECK_ALLOC(backend); backend->parent.version = GIT_ODB_BACKEND_VERSION; diff --git a/src/odb_mempack.c b/src/odb_mempack.c index a71d8db4b1a..32bc8444211 100644 --- a/src/odb_mempack.c +++ b/src/odb_mempack.c @@ -38,6 +38,7 @@ static int impl__write(git_odb_backend *_backend, const git_oid *oid, const void struct memory_packer_db *db = (struct memory_packer_db *)_backend; struct memobject *obj = NULL; khiter_t pos; + size_t alloc_len; int rval; pos = kh_put(oid, db->objects, oid, &rval); @@ -47,8 +48,8 @@ static int impl__write(git_odb_backend *_backend, const git_oid *oid, const void if (rval == 0) return 0; - GITERR_CHECK_ALLOC_ADD(sizeof(struct memobject), len); - obj = git__malloc(sizeof(struct memobject) + len); + GITERR_CHECK_ALLOC_ADD(&alloc_len, sizeof(struct memobject), len); + obj = git__malloc(alloc_len); GITERR_CHECK_ALLOC(obj); memcpy(obj->data, data, len); diff --git a/src/pack-objects.c b/src/pack-objects.c index 8236ef9f3da..67d6125ff02 100644 --- a/src/pack-objects.c +++ b/src/pack-objects.c @@ -202,9 +202,8 @@ int git_packbuilder_insert(git_packbuilder *pb, const git_oid *oid, return 0; if (pb->nr_objects >= pb->nr_alloc) { - GITERR_CHECK_ALLOC_ADD(pb->nr_alloc, 1024); - GITERR_CHECK_ALLOC_MULTIPLY(pb->nr_alloc + 1024, 3 / 2); - newsize = (pb->nr_alloc + 1024) * 3 / 2; + GITERR_CHECK_ALLOC_ADD(&newsize, pb->nr_alloc, 1024); + GITERR_CHECK_ALLOC_MULTIPLY(&newsize, newsize, 3 / 2); if (!git__is_uint32(newsize)) { giterr_set(GITERR_NOMEMORY, "Packfile too large to fit in memory."); @@ -833,7 +832,7 @@ static int try_delta(git_packbuilder *pb, struct unpacked *trg, trg_object->delta_data = NULL; } if (delta_cacheable(pb, src_size, trg_size, delta_size)) { - if (!git__add_uint64_overflow(&pb->delta_cache_size, pb->delta_cache_size, delta_size)) + if (git__add_uint64_overflow(&pb->delta_cache_size, pb->delta_cache_size, delta_size)) return -1; git_packbuilder__cache_unlock(pb); diff --git a/src/pack.c b/src/pack.c index d475b28eebd..26a6036c2f3 100644 --- a/src/pack.c +++ b/src/pack.c @@ -624,7 +624,7 @@ int git_packfile_unpack( struct pack_chain_elem *elem = NULL, *stack; git_pack_cache_entry *cached = NULL; struct pack_chain_elem small_stack[SMALL_STACK_SIZE]; - size_t stack_size = 0, elem_pos; + size_t stack_size = 0, elem_pos, alloclen; git_otype base_type; /* @@ -684,8 +684,8 @@ int git_packfile_unpack( if (cached && stack_size == 1) { void *data = obj->data; - GITERR_CHECK_ALLOC_ADD(obj->len, 1); - obj->data = git__malloc(obj->len + 1); + GITERR_CHECK_ALLOC_ADD(&alloclen, obj->len, 1); + obj->data = git__malloc(alloclen); GITERR_CHECK_ALLOC(obj->data); memcpy(obj->data, data, obj->len + 1); @@ -840,17 +840,18 @@ int packfile_unpack_compressed( size_t size, git_otype type) { + size_t buf_size; int st; z_stream stream; unsigned char *buffer, *in; - GITERR_CHECK_ALLOC_ADD(size, 1); - buffer = git__calloc(1, size + 1); + GITERR_CHECK_ALLOC_ADD(&buf_size, size, 1); + buffer = git__calloc(1, buf_size); GITERR_CHECK_ALLOC(buffer); memset(&stream, 0, sizeof(stream)); stream.next_out = buffer; - stream.avail_out = (uInt)size + 1; + stream.avail_out = (uInt)buf_size; stream.zalloc = use_git_alloc; stream.zfree = use_git_free; @@ -1089,17 +1090,17 @@ int git_packfile_alloc(struct git_pack_file **pack_out, const char *path) { struct stat st; struct git_pack_file *p; - size_t path_len = path ? strlen(path) : 0; + size_t path_len = path ? strlen(path) : 0, alloc_len; *pack_out = NULL; if (path_len < strlen(".idx")) return git_odb__error_notfound("invalid packfile path", NULL); - GITERR_CHECK_ALLOC_ADD(sizeof(*p), path_len); - GITERR_CHECK_ALLOC_ADD(sizeof(*p) + path_len, 2); + GITERR_CHECK_ALLOC_ADD(&alloc_len, sizeof(*p), path_len); + GITERR_CHECK_ALLOC_ADD(&alloc_len, alloc_len, 2); - p = git__calloc(1, sizeof(*p) + path_len + 2); + p = git__calloc(1, alloc_len); GITERR_CHECK_ALLOC(p); memcpy(p->pack_name, path, path_len + 1); diff --git a/src/path.c b/src/path.c index 3dbd7187bf6..64c7b7e1293 100644 --- a/src/path.c +++ b/src/path.c @@ -620,11 +620,12 @@ static bool _check_dir_contents( bool result; size_t dir_size = git_buf_len(dir); size_t sub_size = strlen(sub); + size_t alloc_size; /* leave base valid even if we could not make space for subdir */ - if (GIT_ALLOC_OVERFLOW_ADD(dir_size, sub_size) || - GIT_ALLOC_OVERFLOW_ADD(dir_size + sub_size, 2) || - git_buf_try_grow(dir, dir_size + sub_size + 2, false, false) < 0) + if (GIT_ADD_SIZET_OVERFLOW(&alloc_size, dir_size, sub_size) || + GIT_ADD_SIZET_OVERFLOW(&alloc_size, alloc_size, 2) || + git_buf_try_grow(dir, alloc_size, false, false) < 0) return false; /* save excursion */ @@ -786,7 +787,7 @@ int git_path_cmp( int git_path_make_relative(git_buf *path, const char *parent) { const char *p, *q, *p_dirsep, *q_dirsep; - size_t plen = path->size, newlen, depth = 1, i, offset; + size_t plen = path->size, newlen, alloclen, depth = 1, i, offset; for (p_dirsep = p = path->ptr, q_dirsep = q = parent; *p && *q; p++, q++) { if (*p == '/' && *q == '/') { @@ -824,14 +825,14 @@ int git_path_make_relative(git_buf *path, const char *parent) for (; (q = strchr(q, '/')) && *(q + 1); q++) depth++; - GITERR_CHECK_ALLOC_MULTIPLY(depth, 3); - GITERR_CHECK_ALLOC_ADD(depth * 3, plen); - GITERR_CHECK_ALLOC_ADD(depth * 3, 1); - newlen = (depth * 3) + plen; + GITERR_CHECK_ALLOC_MULTIPLY(&newlen, depth, 3); + GITERR_CHECK_ALLOC_ADD(&newlen, newlen, plen); + + GITERR_CHECK_ALLOC_ADD(&alloclen, newlen, 1); /* save the offset as we might realllocate the pointer */ offset = p - path->ptr; - if (git_buf_try_grow(path, newlen + 1, 1, 0) < 0) + if (git_buf_try_grow(path, alloclen, 1, 0) < 0) return -1; p = path->ptr + offset; @@ -876,7 +877,7 @@ void git_path_iconv_clear(git_path_iconv_t *ic) int git_path_iconv(git_path_iconv_t *ic, char **in, size_t *inlen) { char *nfd = *in, *nfc; - size_t nfdlen = *inlen, nfclen, wantlen = nfdlen, rv; + size_t nfdlen = *inlen, nfclen, wantlen = nfdlen, alloclen, rv; int retry = 1; if (!ic || ic->map == (iconv_t)-1 || @@ -886,8 +887,8 @@ int git_path_iconv(git_path_iconv_t *ic, char **in, size_t *inlen) git_buf_clear(&ic->buf); while (1) { - GITERR_CHECK_ALLOC_ADD(wantlen, 1); - if (git_buf_grow(&ic->buf, wantlen + 1) < 0) + GITERR_CHECK_ALLOC_ADD(&alloclen, wantlen, 1); + if (git_buf_grow(&ic->buf, alloclen) < 0) return -1; nfc = ic->buf.ptr + ic->buf.size; @@ -1072,21 +1073,13 @@ static int entry_path_alloc( size_t alloc_extra) { int need_slash = (path_len > 0 && path[path_len-1] != '/') ? 1 : 0; - size_t alloc_size = path_len; + size_t alloc_size; char *entry_path; - GITERR_CHECK_ALLOC_ADD(alloc_size, de_len); - alloc_size += de_len; - - GITERR_CHECK_ALLOC_ADD(alloc_size, need_slash); - alloc_size += need_slash; - - GITERR_CHECK_ALLOC_ADD(alloc_size, 1); - alloc_size++; - - GITERR_CHECK_ALLOC_ADD(alloc_size, alloc_extra); - alloc_size += alloc_extra; - + GITERR_CHECK_ALLOC_ADD(&alloc_size, path_len, de_len); + GITERR_CHECK_ALLOC_ADD(&alloc_size, alloc_size, need_slash); + GITERR_CHECK_ALLOC_ADD(&alloc_size, alloc_size, 1); + GITERR_CHECK_ALLOC_ADD(&alloc_size, alloc_size, alloc_extra); entry_path = git__calloc(1, alloc_size); GITERR_CHECK_ALLOC(entry_path); diff --git a/src/pool.c b/src/pool.c index 919e13d08fb..c93d78182c5 100644 --- a/src/pool.c +++ b/src/pool.c @@ -107,21 +107,22 @@ static void pool_insert_page(git_pool *pool, git_pool_page *page) static void *pool_alloc_page(git_pool *pool, uint32_t size) { git_pool_page *page; - uint32_t alloc_size; + uint32_t new_page_size; + size_t alloc_size; if (size <= pool->page_size) - alloc_size = pool->page_size; + new_page_size = pool->page_size; else { - alloc_size = size; + new_page_size = size; pool->has_large_page_alloc = 1; } - if (GIT_ALLOC_OVERFLOW_ADD(alloc_size, sizeof(git_pool_page)) || - !(page = git__calloc(1, alloc_size + sizeof(git_pool_page)))) + if (GIT_ADD_SIZET_OVERFLOW(&alloc_size, new_page_size, sizeof(git_pool_page)) || + !(page = git__calloc(1, alloc_size))) return NULL; - page->size = alloc_size; - page->avail = alloc_size - size; + page->size = new_page_size; + page->avail = new_page_size - size; if (page->avail > 0) pool_insert_page(pool, page); diff --git a/src/refs.c b/src/refs.c index af3190ef9ac..2e88c26c3e1 100644 --- a/src/refs.c +++ b/src/refs.c @@ -36,15 +36,13 @@ enum { static git_reference *alloc_ref(const char *name) { - git_reference *ref; - size_t namelen = strlen(name), reflen = sizeof(git_reference); - - if (GIT_ALLOC_OVERFLOW_ADD(reflen, namelen) || - GIT_ALLOC_OVERFLOW_ADD(reflen + namelen, 1) || - (ref = git__calloc(1, reflen + namelen + 1)) == NULL) - return NULL; + git_reference *ref = NULL; + size_t namelen = strlen(name), reflen; - memcpy(ref->name, name, namelen + 1); + if (!GIT_ADD_SIZET_OVERFLOW(&reflen, sizeof(git_reference), namelen) && + !GIT_ADD_SIZET_OVERFLOW(&reflen, reflen, 1) && + (ref = git__calloc(1, reflen)) != NULL) + memcpy(ref->name, name, namelen + 1); return ref; } @@ -96,12 +94,12 @@ git_reference *git_reference__set_name( git_reference *ref, const char *name) { size_t namelen = strlen(name); - size_t reflen = sizeof(git_reference); + size_t reflen; git_reference *rewrite = NULL; - if (!GIT_ALLOC_OVERFLOW_ADD(reflen, namelen) && - !GIT_ALLOC_OVERFLOW_ADD(reflen + namelen, 1) && - (rewrite = git__realloc(ref, reflen + namelen + 1)) != NULL) + if (!GIT_ADD_SIZET_OVERFLOW(&reflen, sizeof(git_reference), namelen) && + !GIT_ADD_SIZET_OVERFLOW(&reflen, reflen, 1) && + (rewrite = git__realloc(ref, reflen)) != NULL) memcpy(rewrite->name, name, namelen + 1); return rewrite; diff --git a/src/sortedcache.c b/src/sortedcache.c index 021f79632c4..1195ea33930 100644 --- a/src/sortedcache.c +++ b/src/sortedcache.c @@ -11,13 +11,13 @@ int git_sortedcache_new( const char *path) { git_sortedcache *sc; - size_t pathlen; + size_t pathlen, alloclen; pathlen = path ? strlen(path) : 0; - GITERR_CHECK_ALLOC_ADD(sizeof(git_sortedcache), pathlen); - GITERR_CHECK_ALLOC_ADD(sizeof(git_sortedcache) + pathlen, 1); - sc = git__calloc(1, sizeof(git_sortedcache) + pathlen + 1); + GITERR_CHECK_ALLOC_ADD(&alloclen, sizeof(git_sortedcache), pathlen); + GITERR_CHECK_ALLOC_ADD(&alloclen, alloclen, 1); + sc = git__calloc(1, alloclen); GITERR_CHECK_ALLOC(sc); if (git_pool_init(&sc->pool, 1, 0) < 0 || diff --git a/src/tag.c b/src/tag.c index b8213140179..3b8d684edae 100644 --- a/src/tag.c +++ b/src/tag.c @@ -72,7 +72,7 @@ static int tag_parse(git_tag *tag, const char *buffer, const char *buffer_end) }; unsigned int i; - size_t text_len; + size_t text_len, alloc_len; char *search; if (git_oid__parse(&tag->target, &buffer, buffer_end, "object ") < 0) @@ -117,8 +117,8 @@ static int tag_parse(git_tag *tag, const char *buffer, const char *buffer_end) text_len = search - buffer; - GITERR_CHECK_ALLOC_ADD(text_len, 1); - tag->tag_name = git__malloc(text_len + 1); + GITERR_CHECK_ALLOC_ADD(&alloc_len, text_len, 1); + tag->tag_name = git__malloc(alloc_len); GITERR_CHECK_ALLOC(tag->tag_name); memcpy(tag->tag_name, buffer, text_len); @@ -142,8 +142,8 @@ static int tag_parse(git_tag *tag, const char *buffer, const char *buffer_end) text_len = buffer_end - ++buffer; - GITERR_CHECK_ALLOC_ADD(text_len, 1); - tag->message = git__malloc(text_len + 1); + GITERR_CHECK_ALLOC_ADD(&alloc_len, text_len, 1); + tag->message = git__malloc(alloc_len); GITERR_CHECK_ALLOC(tag->message); memcpy(tag->message, buffer, text_len); diff --git a/src/transports/cred.c b/src/transports/cred.c index 8e5447d1815..8163d311509 100644 --- a/src/transports/cred.c +++ b/src/transports/cred.c @@ -306,15 +306,15 @@ int git_cred_default_new(git_cred **cred) int git_cred_username_new(git_cred **cred, const char *username) { git_cred_username *c; - size_t len; + size_t len, allocsize; assert(cred); len = strlen(username); - GITERR_CHECK_ALLOC_ADD(sizeof(git_cred_username), len); - GITERR_CHECK_ALLOC_ADD(sizeof(git_cred_username) + len, 1); - c = git__malloc(sizeof(git_cred_username) + len + 1); + GITERR_CHECK_ALLOC_ADD(&allocsize, sizeof(git_cred_username), len); + GITERR_CHECK_ALLOC_ADD(&allocsize, allocsize, 1); + c = git__malloc(allocsize); GITERR_CHECK_ALLOC(c); c->parent.credtype = GIT_CREDTYPE_USERNAME; diff --git a/src/transports/smart_pkt.c b/src/transports/smart_pkt.c index 11c6215ffc3..d214c9fa535 100644 --- a/src/transports/smart_pkt.c +++ b/src/transports/smart_pkt.c @@ -102,10 +102,11 @@ static int pack_pkt(git_pkt **out) static int comment_pkt(git_pkt **out, const char *line, size_t len) { git_pkt_comment *pkt; + size_t alloclen; - GITERR_CHECK_ALLOC_ADD(sizeof(git_pkt_comment), len); - GITERR_CHECK_ALLOC_ADD(sizeof(git_pkt_comment) + len, 1); - pkt = git__malloc(sizeof(git_pkt_comment) + len + 1); + GITERR_CHECK_ALLOC_ADD(&alloclen, sizeof(git_pkt_comment), len); + GITERR_CHECK_ALLOC_ADD(&alloclen, alloclen, 1); + pkt = git__malloc(alloclen); GITERR_CHECK_ALLOC(pkt); pkt->type = GIT_PKT_COMMENT; @@ -120,14 +121,15 @@ static int comment_pkt(git_pkt **out, const char *line, size_t len) static int err_pkt(git_pkt **out, const char *line, size_t len) { git_pkt_err *pkt; + size_t alloclen; /* Remove "ERR " from the line */ line += 4; len -= 4; - GITERR_CHECK_ALLOC_ADD(sizeof(git_pkt_progress), len); - GITERR_CHECK_ALLOC_ADD(sizeof(git_pkt_progress) + len, 1); - pkt = git__malloc(sizeof(git_pkt_err) + len + 1); + GITERR_CHECK_ALLOC_ADD(&alloclen, sizeof(git_pkt_progress), len); + GITERR_CHECK_ALLOC_ADD(&alloclen, alloclen, 1); + pkt = git__malloc(alloclen); GITERR_CHECK_ALLOC(pkt); pkt->type = GIT_PKT_ERR; @@ -143,12 +145,13 @@ static int err_pkt(git_pkt **out, const char *line, size_t len) static int data_pkt(git_pkt **out, const char *line, size_t len) { git_pkt_data *pkt; + size_t alloclen; line++; len--; - GITERR_CHECK_ALLOC_ADD(sizeof(git_pkt_progress), len); - pkt = git__malloc(sizeof(git_pkt_data) + len); + GITERR_CHECK_ALLOC_ADD(&alloclen, sizeof(git_pkt_progress), len); + pkt = git__malloc(alloclen); GITERR_CHECK_ALLOC(pkt); pkt->type = GIT_PKT_DATA; @@ -163,12 +166,13 @@ static int data_pkt(git_pkt **out, const char *line, size_t len) static int sideband_progress_pkt(git_pkt **out, const char *line, size_t len) { git_pkt_progress *pkt; + size_t alloclen; line++; len--; - GITERR_CHECK_ALLOC_ADD(sizeof(git_pkt_progress), len); - pkt = git__malloc(sizeof(git_pkt_progress) + len); + GITERR_CHECK_ALLOC_ADD(&alloclen, sizeof(git_pkt_progress), len); + pkt = git__malloc(alloclen); GITERR_CHECK_ALLOC(pkt); pkt->type = GIT_PKT_PROGRESS; @@ -183,13 +187,14 @@ static int sideband_progress_pkt(git_pkt **out, const char *line, size_t len) static int sideband_error_pkt(git_pkt **out, const char *line, size_t len) { git_pkt_err *pkt; + size_t alloc_len; line++; len--; - GITERR_CHECK_ALLOC_ADD(sizeof(git_pkt_err), len); - GITERR_CHECK_ALLOC_ADD(sizeof(git_pkt_err) + len, 1); - pkt = git__malloc(sizeof(git_pkt_err) + len + 1); + GITERR_CHECK_ALLOC_ADD(&alloc_len, sizeof(git_pkt_err), len); + GITERR_CHECK_ALLOC_ADD(&alloc_len, alloc_len, 1); + pkt = git__malloc(alloc_len); GITERR_CHECK_ALLOC(pkt); pkt->type = GIT_PKT_ERR; @@ -209,6 +214,7 @@ static int ref_pkt(git_pkt **out, const char *line, size_t len) { int error; git_pkt_ref *pkt; + size_t alloclen; pkt = git__malloc(sizeof(git_pkt_ref)); GITERR_CHECK_ALLOC(pkt); @@ -232,8 +238,8 @@ static int ref_pkt(git_pkt **out, const char *line, size_t len) if (line[len - 1] == '\n') --len; - GITERR_CHECK_ALLOC_ADD(len, 1); - pkt->head.name = git__malloc(len + 1); + GITERR_CHECK_ALLOC_ADD(&alloclen, len, 1); + pkt->head.name = git__malloc(alloclen); GITERR_CHECK_ALLOC(pkt->head.name); memcpy(pkt->head.name, line, len); @@ -255,6 +261,7 @@ static int ok_pkt(git_pkt **out, const char *line, size_t len) { git_pkt_ok *pkt; const char *ptr; + size_t alloc_len; pkt = git__malloc(sizeof(*pkt)); GITERR_CHECK_ALLOC(pkt); @@ -268,8 +275,8 @@ static int ok_pkt(git_pkt **out, const char *line, size_t len) } len = ptr - line; - GITERR_CHECK_ALLOC_ADD(len, 1); - pkt->ref = git__malloc(len + 1); + GITERR_CHECK_ALLOC_ADD(&alloc_len, len, 1); + pkt->ref = git__malloc(alloc_len); GITERR_CHECK_ALLOC(pkt->ref); memcpy(pkt->ref, line, len); @@ -283,6 +290,7 @@ static int ng_pkt(git_pkt **out, const char *line, size_t len) { git_pkt_ng *pkt; const char *ptr; + size_t alloclen; pkt = git__malloc(sizeof(*pkt)); GITERR_CHECK_ALLOC(pkt); @@ -296,8 +304,8 @@ static int ng_pkt(git_pkt **out, const char *line, size_t len) } len = ptr - line; - GITERR_CHECK_ALLOC_ADD(len, 1); - pkt->ref = git__malloc(len + 1); + GITERR_CHECK_ALLOC_ADD(&alloclen, len, 1); + pkt->ref = git__malloc(alloclen); GITERR_CHECK_ALLOC(pkt->ref); memcpy(pkt->ref, line, len); @@ -310,8 +318,8 @@ static int ng_pkt(git_pkt **out, const char *line, size_t len) } len = ptr - line; - GITERR_CHECK_ALLOC_ADD(len, 1); - pkt->msg = git__malloc(len + 1); + GITERR_CHECK_ALLOC_ADD(&alloclen, len, 1); + pkt->msg = git__malloc(alloclen); GITERR_CHECK_ALLOC(pkt->msg); memcpy(pkt->msg, line, len); diff --git a/src/tree.c b/src/tree.c index 573e5644744..9fd4e0a0729 100644 --- a/src/tree.c +++ b/src/tree.c @@ -84,12 +84,11 @@ int git_tree_entry_icmp(const git_tree_entry *e1, const git_tree_entry *e2) static git_tree_entry *alloc_entry(const char *filename) { git_tree_entry *entry = NULL; - size_t filename_len = strlen(filename), - tree_len = sizeof(git_tree_entry); + size_t filename_len = strlen(filename), tree_len; - if (GIT_ALLOC_OVERFLOW_ADD(tree_len, filename_len) || - GIT_ALLOC_OVERFLOW_ADD(tree_len + filename_len, 1) || - !(entry = git__malloc(tree_len + filename_len + 1))) + if (GIT_ADD_SIZET_OVERFLOW(&tree_len, sizeof(git_tree_entry), filename_len) || + GIT_ADD_SIZET_OVERFLOW(&tree_len, tree_len, 1) || + !(entry = git__malloc(tree_len))) return NULL; memset(entry, 0x0, sizeof(git_tree_entry)); @@ -207,16 +206,13 @@ void git_tree_entry_free(git_tree_entry *entry) int git_tree_entry_dup(git_tree_entry **dest, const git_tree_entry *source) { - size_t total_size = sizeof(git_tree_entry); + size_t total_size; git_tree_entry *copy; assert(source); - GITERR_CHECK_ALLOC_ADD(total_size, source->filename_len); - total_size += source->filename_len; - - GITERR_CHECK_ALLOC_ADD(total_size, 1); - total_size++; + GITERR_CHECK_ALLOC_ADD(&total_size, sizeof(git_tree_entry), source->filename_len); + GITERR_CHECK_ALLOC_ADD(&total_size, total_size, 1); copy = git__malloc(total_size); GITERR_CHECK_ALLOC(copy); diff --git a/src/util.h b/src/util.h index 86139ef77f3..38dcae79bff 100644 --- a/src/util.h +++ b/src/util.h @@ -59,17 +59,13 @@ GIT_INLINE(char *) git__strdup(const char *str) GIT_INLINE(char *) git__strndup(const char *str, size_t n) { - size_t length = 0; + size_t length = 0, alloclength; char *ptr; length = p_strnlen(str, n); - if (GIT_ALLOC_OVERFLOW_ADD(length, 1)) - return NULL; - - ptr = git__malloc(length + 1); - - if (!ptr) + if (GIT_ADD_SIZET_OVERFLOW(&alloclength, length, 1) || + !(ptr = git__malloc(alloclength))) return NULL; if (length) @@ -84,8 +80,10 @@ GIT_INLINE(char *) git__strndup(const char *str, size_t n) GIT_INLINE(char *) git__substrdup(const char *start, size_t n) { char *ptr; + size_t alloclen; - if (GIT_ALLOC_OVERFLOW_ADD(n, 1) || !(ptr = git__malloc(n+1))) + if (GIT_ADD_SIZET_OVERFLOW(&alloclen, n, 1) || + !(ptr = git__malloc(alloclen))) return NULL; memcpy(ptr, start, n); @@ -107,8 +105,9 @@ GIT_INLINE(void *) git__realloc(void *ptr, size_t size) */ GIT_INLINE(void *) git__reallocarray(void *ptr, size_t nelem, size_t elsize) { - return GIT_ALLOC_OVERFLOW_MULTIPLY(nelem, elsize) ? - NULL : realloc(ptr, nelem * elsize); + size_t newsize; + return GIT_MULTIPLY_SIZET_OVERFLOW(&newsize, nelem, elsize) ? + NULL : realloc(ptr, newsize); } /** diff --git a/src/vector.c b/src/vector.c index 27eafebc881..93d09bb5b5f 100644 --- a/src/vector.c +++ b/src/vector.c @@ -46,8 +46,7 @@ int git_vector_dup(git_vector *v, const git_vector *src, git_vector_cmp cmp) assert(v && src); - GITERR_CHECK_ALLOC_MULTIPLY(src->length, sizeof(void *)); - bytes = src->length * sizeof(void *); + GITERR_CHECK_ALLOC_MULTIPLY(&bytes, src->length, sizeof(void *)); v->_alloc_size = src->length; v->_cmp = cmp ? cmp : src->_cmp; diff --git a/src/win32/dir.c b/src/win32/dir.c index 7f2a5a56e33..c15757085b1 100644 --- a/src/win32/dir.c +++ b/src/win32/dir.c @@ -11,16 +11,16 @@ git__DIR *git__opendir(const char *dir) { git_win32_path filter_w; git__DIR *new = NULL; - size_t dirlen; + size_t dirlen, alloclen; if (!dir || !git_win32__findfirstfile_filter(filter_w, dir)) return NULL; dirlen = strlen(dir); - if (GIT_ALLOC_OVERFLOW_ADD(sizeof(*new), dirlen) || - GIT_ALLOC_OVERFLOW_ADD(sizeof(*new) + dirlen, 1) || - !(new = git__calloc(1, sizeof(*new) + dirlen + 1))) + if (GIT_ADD_SIZET_OVERFLOW(&alloclen, sizeof(*new), dirlen) || + GIT_ADD_SIZET_OVERFLOW(&alloclen, alloclen, 1) || + !(new = git__calloc(1, alloclen))) return NULL; memcpy(new->dir, dir, dirlen); diff --git a/tests/core/errors.c b/tests/core/errors.c index 11e0bb2bbc8..a06ec4abc4e 100644 --- a/tests/core/errors.c +++ b/tests/core/errors.c @@ -112,7 +112,8 @@ void test_core_errors__restore(void) static int test_arraysize_multiply(size_t nelem, size_t size) { - GITERR_CHECK_ALLOC_MULTIPLY(nelem, size); + size_t out; + GITERR_CHECK_ALLOC_MULTIPLY(&out, nelem, size); return 0; } @@ -133,7 +134,8 @@ void test_core_errors__integer_overflow_alloc_multiply(void) static int test_arraysize_add(size_t one, size_t two) { - GITERR_CHECK_ALLOC_ADD(one, two); + size_t out; + GITERR_CHECK_ALLOC_ADD(&out, one, two); return 0; } @@ -152,21 +154,23 @@ void test_core_errors__integer_overflow_alloc_add(void) void test_core_errors__integer_overflow_sets_oom(void) { + size_t out; + giterr_clear(); - cl_assert(!GIT_ALLOC_OVERFLOW_ADD(SIZE_MAX-1, 1)); + cl_assert(!GIT_ADD_SIZET_OVERFLOW(&out, SIZE_MAX-1, 1)); cl_assert_equal_p(NULL, giterr_last()); giterr_clear(); - cl_assert(!GIT_ALLOC_OVERFLOW_ADD(42, 69)); + cl_assert(!GIT_ADD_SIZET_OVERFLOW(&out, 42, 69)); cl_assert_equal_p(NULL, giterr_last()); giterr_clear(); - cl_assert(GIT_ALLOC_OVERFLOW_ADD(SIZE_MAX, SIZE_MAX)); + cl_assert(GIT_ADD_SIZET_OVERFLOW(&out, SIZE_MAX, SIZE_MAX)); cl_assert_equal_i(GITERR_NOMEMORY, giterr_last()->klass); cl_assert_equal_s("Out of memory", giterr_last()->message); giterr_clear(); - cl_assert(GIT_ALLOC_OVERFLOW_MULTIPLY(SIZE_MAX, SIZE_MAX)); + cl_assert(GIT_ADD_SIZET_OVERFLOW(&out, SIZE_MAX, SIZE_MAX)); cl_assert_equal_i(GITERR_NOMEMORY, giterr_last()->klass); cl_assert_equal_s("Out of memory", giterr_last()->message); } From 8aab36a3010372edec71e8c765d4ecfd848c09b6 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Thu, 12 Feb 2015 22:14:53 +0000 Subject: [PATCH 69/99] filebuf: use an int for return check --- src/filebuf.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/filebuf.c b/src/filebuf.c index 932b8c7d154..a35b59cf0ca 100644 --- a/src/filebuf.c +++ b/src/filebuf.c @@ -446,10 +446,10 @@ int git_filebuf_printf(git_filebuf *file, const char *format, ...) } va_start(arglist, format); - len = p_vsnprintf(tmp_buffer, len + 1, format, arglist); + written = p_vsnprintf(tmp_buffer, len + 1, format, arglist); va_end(arglist); - if (len < 0) { + if (written < 0) { git__free(tmp_buffer); file->last_error = BUFERR_MEM; return -1; From 16942c6fdaddb819b71b72e53aa4aa691e3c0053 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Thu, 12 Feb 2015 17:36:48 -0500 Subject: [PATCH 70/99] integer overflow: use compiler intrinsics if supported gcc and clang support __builtin_add_overflow, use it whenever possible, falling back to our naive routines. --- src/common.h | 5 +++++ src/integer.h | 15 +++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/src/common.h b/src/common.h index 8d1e890642c..98109ae3a76 100644 --- a/src/common.h +++ b/src/common.h @@ -17,6 +17,11 @@ # define GIT_INLINE(type) static inline type #endif +/** Support for gcc/clang __has_builtin intrinsic */ +#ifndef __has_builtin +# define __has_builtin(x) 0 +#endif + #include #include #include diff --git a/src/integer.h b/src/integer.h index a4abe2bd1a9..8e86a48a52e 100644 --- a/src/integer.h +++ b/src/integer.h @@ -54,6 +54,19 @@ GIT_INLINE(bool) git__add_uint64_overflow(uint64_t *out, uint64_t one, uint64_t return false; } +/* Use clang/gcc compiler intrinsics whenever possible */ +#if (SIZE_MAX == UINT_MAX) && __has_builtin(__builtin_uadd_overflow) +# define git__add_sizet_overflow(out, one, two) \ + __builtin_uadd_overflow(one, two, out) +# define git__multiply_sizet_overflow(out, one, two) + __builtin_umul_overflow(one, two, out) +#elif (SIZE_MAX == ULONG_MAX) && __has_builtin(__builtin_uaddl_overflow) +# define git__add_sizet_overflow(out, one, two) \ + __builtin_uaddl_overflow(one, two, out) +# define git__multiply_sizet_overflow(out, one, two) \ + __builtin_umull_overflow(one, two, out) +#else + /** * Sets `one + two` into `out`, unless the arithmetic would overflow. * @return true if the result fits in a `size_t`, false on overflow. @@ -78,4 +91,6 @@ GIT_INLINE(bool) git__multiply_sizet_overflow(size_t *out, size_t one, size_t tw return false; } +#endif + #endif /* INCLUDE_integer_h__ */ From d97d9559e321ac2ef2ae81a6cfd308a841ef7581 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Thu, 12 Feb 2015 23:56:09 -0500 Subject: [PATCH 71/99] buf test: cleanup memory leak --- tests/buf/basic.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/buf/basic.c b/tests/buf/basic.c index dcc0916d21b..14ea3e7ce80 100644 --- a/tests/buf/basic.c +++ b/tests/buf/basic.c @@ -33,6 +33,8 @@ void test_buf_basic__resize_incremental(void) cl_git_pass(git_buf_grow_by(&buf1, 16)); cl_assert_equal_i(5, buf1.size); cl_assert(buf1.asize > 8); + + git_buf_free(&buf1); } void test_buf_basic__printf(void) From 0f07d54b44825399e5d13499328135771c8d0b43 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Fri, 13 Feb 2015 09:35:20 -0500 Subject: [PATCH 72/99] pack-objects: unlock the cache on integer overflow --- src/pack-objects.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/pack-objects.c b/src/pack-objects.c index 67d6125ff02..f644520ac77 100644 --- a/src/pack-objects.c +++ b/src/pack-objects.c @@ -832,13 +832,14 @@ static int try_delta(git_packbuilder *pb, struct unpacked *trg, trg_object->delta_data = NULL; } if (delta_cacheable(pb, src_size, trg_size, delta_size)) { - if (git__add_uint64_overflow(&pb->delta_cache_size, pb->delta_cache_size, delta_size)) - return -1; + bool overflow = git__add_uint64_overflow( + &pb->delta_cache_size, pb->delta_cache_size, delta_size); git_packbuilder__cache_unlock(pb); - trg_object->delta_data = git__realloc(delta_buf, delta_size); - GITERR_CHECK_ALLOC(trg_object->delta_data); + if (overflow || + !(trg_object->delta_data = git__realloc(delta_buf, delta_size))) + return -1; } else { /* create delta when writing the pack */ git_packbuilder__cache_unlock(pb); From 92e87dd74974ccad8e0dbd8dd212bfc514ba441d Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Fri, 13 Feb 2015 10:44:19 -0500 Subject: [PATCH 73/99] rebase: provide NULL `exec` on non-EXEC operations Users may want to try to pay attention to the `exec` field on all rebase operations. --- src/rebase.c | 37 ++++++++++++++++++++++++++++--------- tests/rebase/iterator.c | 1 + 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/src/rebase.c b/src/rebase.c index 2e805929ee1..229b18a38f6 100644 --- a/src/rebase.c +++ b/src/rebase.c @@ -168,10 +168,31 @@ GIT_INLINE(int) rebase_readoid( return 0; } +static git_rebase_operation *rebase_operation_alloc( + git_rebase *rebase, + git_rebase_operation_t type, + git_oid *id, + const char *exec) +{ + git_rebase_operation *operation; + + assert((type == GIT_REBASE_OPERATION_EXEC) == !id); + assert((type == GIT_REBASE_OPERATION_EXEC) == !!exec); + + if ((operation = git_array_alloc(rebase->operations)) == NULL) + return NULL; + + operation->type = type; + git_oid_cpy((git_oid *)&operation->id, id); + operation->exec = exec; + + return operation; +} + static int rebase_open_merge(git_rebase *rebase) { git_buf state_path = GIT_BUF_INIT, buf = GIT_BUF_INIT, cmt = GIT_BUF_INIT; - git_oid current_id = {{0}}; + git_oid id; git_rebase_operation *operation; size_t i, msgnum = 0, end; int error; @@ -194,7 +215,7 @@ static int rebase_open_merge(git_rebase *rebase) goto done; /* Read 'current' if it exists */ - if ((error = rebase_readoid(¤t_id, &buf, &state_path, CURRENT_FILE)) < 0 && + if ((error = rebase_readoid(&id, &buf, &state_path, CURRENT_FILE)) < 0 && error != GIT_ENOTFOUND) goto done; @@ -203,14 +224,14 @@ static int rebase_open_merge(git_rebase *rebase) GITERR_CHECK_ARRAY(rebase->operations); for (i = 0; i < end; i++) { - operation = git_array_alloc(rebase->operations); - GITERR_CHECK_ALLOC(operation); - git_buf_clear(&cmt); if ((error = git_buf_printf(&cmt, "cmt.%" PRIuZ, (i+1))) < 0 || - (error = rebase_readoid((git_oid *)&operation->id, &buf, &state_path, cmt.ptr)) < 0) + (error = rebase_readoid(&id, &buf, &state_path, cmt.ptr)) < 0) goto done; + + operation = rebase_operation_alloc(rebase, GIT_REBASE_OPERATION_PICK, &id, NULL); + GITERR_CHECK_ALLOC(operation); } /* Read 'onto_name' */ @@ -553,9 +574,7 @@ static int rebase_init_operations( if (merge) continue; - operation = git_array_alloc(rebase->operations); - operation->type = GIT_REBASE_OPERATION_PICK; - git_oid_cpy((git_oid *)&operation->id, &id); + operation = rebase_operation_alloc(rebase, GIT_REBASE_OPERATION_PICK, &id, NULL); } error = 0; diff --git a/tests/rebase/iterator.c b/tests/rebase/iterator.c index ddf4413d39f..8117a094a8b 100644 --- a/tests/rebase/iterator.c +++ b/tests/rebase/iterator.c @@ -42,6 +42,7 @@ static void test_operations(git_rebase *rebase, size_t expected_current) operation = git_rebase_operation_byindex(rebase, i); cl_assert_equal_i(GIT_REBASE_OPERATION_PICK, operation->type); cl_assert_equal_oid(&expected_oid[i], &operation->id); + cl_assert_equal_p(NULL, operation->exec); } } From 49b8293c75d287f609eebae334519872784cd893 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Fri, 13 Feb 2015 11:20:32 -0500 Subject: [PATCH 74/99] rebase: allow `NULL` branch to indicate `HEAD` Don't require the branch to rebase, if given `NULL`, simply look up `HEAD`. --- include/git2/rebase.h | 3 ++- src/rebase.c | 18 ++++++++++++++-- tests/rebase/setup.c | 49 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 3 deletions(-) diff --git a/include/git2/rebase.h b/include/git2/rebase.h index 1da3fb5aa6c..19f5cb8c966 100644 --- a/include/git2/rebase.h +++ b/include/git2/rebase.h @@ -133,7 +133,8 @@ GIT_EXTERN(int) git_rebase_init_options( * * @param out Pointer to store the rebase object * @param repo The repository to perform the rebase - * @param branch The terminal commit to rebase + * @param branch The terminal commit to rebase, or NULL to rebase the + * current branch * @param upstream The commit to begin rebasing from, or NULL to rebase all * reachable commits * @param onto The branch to rebase onto, or NULL to rebase onto the given diff --git a/src/rebase.c b/src/rebase.c index 229b18a38f6..a1c72e8a39c 100644 --- a/src/rebase.c +++ b/src/rebase.c @@ -608,11 +608,21 @@ static int rebase_init( const git_annotated_commit *onto, const git_rebase_options *opts) { + git_reference *head_ref = NULL; + git_annotated_commit *head_branch = NULL; git_buf state_path = GIT_BUF_INIT; int error; if ((error = git_buf_joinpath(&state_path, repo->path_repository, REBASE_MERGE_DIR)) < 0) - return error; + goto done; + + if (!branch) { + if ((error = git_repository_head(&head_ref, repo)) < 0 || + (error = git_annotated_commit_from_ref(&head_branch, repo, head_ref)) < 0) + goto done; + + branch = head_branch; + } rebase->repo = repo; rebase->type = GIT_REBASE_TYPE_MERGE; @@ -630,6 +640,10 @@ static int rebase_init( git_buf_free(&state_path); +done: + git_reference_free(head_ref); + git_annotated_commit_free(head_branch); + return error; } @@ -649,7 +663,7 @@ int git_rebase_init( git_checkout_options checkout_opts = GIT_CHECKOUT_OPTIONS_INIT; int error; - assert(repo && branch && (upstream || onto)); + assert(repo && (upstream || onto)); *out = NULL; diff --git a/tests/rebase/setup.c b/tests/rebase/setup.c index c81ca12459f..f1eb6a47d36 100644 --- a/tests/rebase/setup.c +++ b/tests/rebase/setup.c @@ -293,6 +293,55 @@ void test_rebase_setup__orphan_branch(void) git_rebase_free(rebase); } +/* git checkout beef && git rebase --merge master */ +void test_rebase_setup__merge_null_branch_uses_HEAD(void) +{ + git_rebase *rebase; + git_reference *upstream_ref; + git_annotated_commit *upstream_head; + git_reference *head; + git_commit *head_commit; + git_oid head_id; + git_checkout_options checkout_opts = GIT_CHECKOUT_OPTIONS_INIT; + + checkout_opts.checkout_strategy = GIT_CHECKOUT_FORCE; + + cl_assert_equal_i(GIT_REPOSITORY_STATE_NONE, git_repository_state(repo)); + + cl_git_pass(git_repository_set_head(repo, "refs/heads/beef", NULL, NULL)); + cl_git_pass(git_checkout_head(repo, &checkout_opts)); + + cl_git_pass(git_reference_lookup(&upstream_ref, repo, "refs/heads/master")); + cl_git_pass(git_annotated_commit_from_ref(&upstream_head, repo, upstream_ref)); + + cl_git_pass(git_rebase_init(&rebase, repo, NULL, upstream_head, NULL, signature, NULL)); + + cl_assert_equal_i(GIT_REPOSITORY_STATE_REBASE_MERGE, git_repository_state(repo)); + + git_oid_fromstr(&head_id, "efad0b11c47cb2f0220cbd6f5b0f93bb99064b00"); + cl_git_pass(git_repository_head(&head, repo)); + cl_git_pass(git_reference_peel((git_object **)&head_commit, head, GIT_OBJ_COMMIT)); + cl_assert_equal_oid(&head_id, git_commit_id(head_commit)); + + cl_assert_equal_file("b146bd7608eac53d9bf9e1a6963543588b555c64\n", 41, "rebase/.git/ORIG_HEAD"); + + cl_assert_equal_file("da9c51a23d02d931a486f45ad18cda05cf5d2b94\n", 41, "rebase/.git/rebase-merge/cmt.1"); + cl_assert_equal_file("8d1f13f93c4995760ac07d129246ac1ff64c0be9\n", 41, "rebase/.git/rebase-merge/cmt.2"); + cl_assert_equal_file("3069cc907e6294623e5917ef6de663928c1febfb\n", 41, "rebase/.git/rebase-merge/cmt.3"); + cl_assert_equal_file("588e5d2f04d49707fe4aab865e1deacaf7ef6787\n", 41, "rebase/.git/rebase-merge/cmt.4"); + cl_assert_equal_file("b146bd7608eac53d9bf9e1a6963543588b555c64\n", 41, "rebase/.git/rebase-merge/cmt.5"); + cl_assert_equal_file("5\n", 2, "rebase/.git/rebase-merge/end"); + cl_assert_equal_file("efad0b11c47cb2f0220cbd6f5b0f93bb99064b00\n", 41, "rebase/.git/rebase-merge/onto"); + cl_assert_equal_file("master\n", 7, "rebase/.git/rebase-merge/onto_name"); + cl_assert_equal_file("b146bd7608eac53d9bf9e1a6963543588b555c64\n", 41, "rebase/.git/rebase-merge/orig-head"); + + git_commit_free(head_commit); + git_reference_free(head); + git_annotated_commit_free(upstream_head); + git_reference_free(upstream_ref); + git_rebase_free(rebase); +} + static int rebase_is_blocked(void) { git_rebase *rebase = NULL; From ffbd51edc887021b96452394ed9737112258d8e5 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Fri, 13 Feb 2015 11:38:00 -0500 Subject: [PATCH 75/99] rebase: `CHECKOUT_SAFE` instead of `FORCE` Switch to a standard branch-switching pattern of a `SAFE` checkout, then updating `HEAD`. --- src/rebase.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/rebase.c b/src/rebase.c index a1c72e8a39c..b0a93a63b7e 100644 --- a/src/rebase.c +++ b/src/rebase.c @@ -658,8 +658,8 @@ int git_rebase_init( { git_rebase *rebase = NULL; git_rebase_options opts; - git_reference *head_ref = NULL; git_buf reflog = GIT_BUF_INIT; + git_commit *onto_commit = NULL; git_checkout_options checkout_opts = GIT_CHECKOUT_OPTIONS_INIT; int error; @@ -672,24 +672,28 @@ int git_rebase_init( if (!onto) onto = upstream; - checkout_opts.checkout_strategy = GIT_CHECKOUT_FORCE; + checkout_opts.checkout_strategy = GIT_CHECKOUT_SAFE; if ((error = rebase_normalize_opts(repo, &opts, given_opts)) < 0 || (error = git_repository__ensure_not_bare(repo, "rebase")) < 0 || (error = rebase_ensure_not_in_progress(repo)) < 0 || - (error = rebase_ensure_not_dirty(repo)) < 0) + (error = rebase_ensure_not_dirty(repo)) < 0 || + (error = git_commit_lookup( + &onto_commit, repo, git_annotated_commit_id(onto))) < 0) return error; rebase = git__calloc(1, sizeof(git_rebase)); GITERR_CHECK_ALLOC(rebase); - if ((error = rebase_init(rebase, repo, branch, upstream, onto, &opts)) < 0 || + if ((error = rebase_init( + rebase, repo, branch, upstream, onto, &opts)) < 0 || (error = rebase_setupfiles(rebase)) < 0 || (error = git_buf_printf(&reflog, "rebase: checkout %s", rebase_onto_name(onto))) < 0 || - (error = git_reference_create(&head_ref, repo, GIT_HEAD_FILE, - git_annotated_commit_id(onto), 1, signature, reflog.ptr)) < 0 || - (error = git_checkout_head(repo, &checkout_opts)) < 0) + (error = git_checkout_tree( + repo, (git_object *)onto_commit, &checkout_opts)) < 0 || + (error = git_repository_set_head_detached( + repo, git_annotated_commit_id(onto), signature, reflog.ptr)) < 0) goto done; *out = rebase; @@ -700,7 +704,7 @@ int git_rebase_init( git_rebase_free(rebase); } - git_reference_free(head_ref); + git_commit_free(onto_commit); git_buf_free(&reflog); rebase_opts_free(&opts); From 42f98a26a5da450b3ef8600b85711112dd9860f4 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Sat, 17 Jan 2015 23:31:24 -0600 Subject: [PATCH 76/99] merge test: test an actual failure, not conflict Correct the merge failed cleanup test. Merge data should not be cleaned up on conflicts, only on actual failure. And ORIG_HEAD should not be removed at all. --- tests/merge/workdir/setup.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/merge/workdir/setup.c b/tests/merge/workdir/setup.c index 099bc12113b..31ffd57024b 100644 --- a/tests/merge/workdir/setup.c +++ b/tests/merge/workdir/setup.c @@ -1018,6 +1018,7 @@ void test_merge_workdir_setup__retained_after_success(void) git_annotated_commit_free(their_heads[0]); } + void test_merge_workdir_setup__removed_after_failure(void) { git_oid our_oid; @@ -1030,16 +1031,14 @@ void test_merge_workdir_setup__removed_after_failure(void) cl_git_pass(git_reference_lookup(&octo1_ref, repo, GIT_REFS_HEADS_DIR OCTO1_BRANCH)); cl_git_pass(git_annotated_commit_from_ref(&their_heads[0], repo, octo1_ref)); - cl_git_rewritefile("merge-resolve/new-in-octo1.txt", - "Conflicting file!\n\nMerge will fail!\n"); + cl_git_write2file("merge-resolve/.git/index.lock", "foo\n", 4, O_RDWR|O_CREAT, 0666); cl_git_fail(git_merge( repo, (const git_annotated_commit **)&their_heads[0], 1, NULL, NULL)); - cl_assert(!git_path_exists("merge-resolve/" GIT_MERGE_HEAD_FILE)); - cl_assert(!git_path_exists("merge-resolve/" GIT_ORIG_HEAD_FILE)); - cl_assert(!git_path_exists("merge-resolve/" GIT_MERGE_MODE_FILE)); - cl_assert(!git_path_exists("merge-resolve/" GIT_MERGE_MSG_FILE)); + cl_assert(!git_path_exists("merge-resolve/.git/" GIT_MERGE_HEAD_FILE)); + cl_assert(!git_path_exists("merge-resolve/.git/" GIT_MERGE_MODE_FILE)); + cl_assert(!git_path_exists("merge-resolve/.git/" GIT_MERGE_MSG_FILE)); git_reference_free(octo1_ref); From 55798fd1536f055fc23a760c41d679fc60cd2ead Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Sat, 17 Jan 2015 20:49:04 -0600 Subject: [PATCH 77/99] git_indexwriter: lock then write the index Introduce `git_indexwriter`, to allow us to lock the index while performing additional operations, then complete the write (or abort, unlocking the index). --- src/index.c | 90 ++++++++++++++++++++++++++++++--------------- src/index.h | 18 +++++++++ tests/index/tests.c | 21 +++++++++++ 3 files changed, 100 insertions(+), 29 deletions(-) diff --git a/src/index.c b/src/index.c index cbace3606c3..58575fec337 100644 --- a/src/index.c +++ b/src/index.c @@ -656,39 +656,15 @@ int git_index__changed_relative_to( int git_index_write(git_index *index) { - git_filebuf file = GIT_FILEBUF_INIT; + git_indexwriter writer = GIT_INDEXWRITER_INIT; int error; - if (!index->index_file_path) - return create_index_error(-1, - "Failed to read index: The index is in-memory only"); - - if (index_sort_if_needed(index, true) < 0) - return -1; - git_vector_sort(&index->reuc); - - if ((error = git_filebuf_open( - &file, index->index_file_path, GIT_FILEBUF_HASH_CONTENTS, GIT_INDEX_FILE_MODE)) < 0) { - if (error == GIT_ELOCKED) - giterr_set(GITERR_INDEX, "The index is locked. This might be due to a concurrent or crashed process"); - - return error; - } + if ((error = git_indexwriter_init(&writer, index)) == 0) + error = git_indexwriter_commit(&writer); - if ((error = write_index(index, &file)) < 0) { - git_filebuf_cleanup(&file); - return error; - } + git_indexwriter_cleanup(&writer); - if ((error = git_filebuf_commit(&file)) < 0) - return error; - - if (git_futils_filestamp_check(&index->stamp, index->index_file_path) < 0) - /* index could not be read from disk! */; - else - index->on_disk = 1; - - return 0; + return error; } const char * git_index_path(const git_index *index) @@ -2686,3 +2662,59 @@ int git_index_snapshot_find( { return index_find_in_entries(out, entries, entry_srch, path, path_len, stage); } + +int git_indexwriter_init( + git_indexwriter *writer, + git_index *index) +{ + int error; + + writer->index = index; + + if (!index->index_file_path) + return create_index_error(-1, + "Failed to write index: The index is in-memory only"); + + if ((error = git_filebuf_open( + &writer->file, index->index_file_path, GIT_FILEBUF_HASH_CONTENTS, GIT_INDEX_FILE_MODE)) < 0) { + if (error == GIT_ELOCKED) + giterr_set(GITERR_INDEX, "The index is locked. This might be due to a concurrent or crashed process"); + + return error; + } + + return 0; +} + +int git_indexwriter_commit(git_indexwriter *writer) +{ + int error; + + if (index_sort_if_needed(writer->index, true) < 0) + return -1; + + git_vector_sort(&writer->index->reuc); + + if ((error = write_index(writer->index, &writer->file)) < 0) { + git_indexwriter_cleanup(writer); + return error; + } + + if ((error = git_filebuf_commit(&writer->file)) < 0) + return error; + + if ((error = git_futils_filestamp_check( + &writer->index->stamp, writer->index->index_file_path)) < 0) { + giterr_set(GITERR_OS, "Could not read index timestamp"); + return -1; + } + + writer->index->on_disk = 1; + + return 0; +} + +void git_indexwriter_cleanup(git_indexwriter *writer) +{ + git_filebuf_cleanup(&writer->file); +} diff --git a/src/index.h b/src/index.h index 2eb93fb1793..d151f6614be 100644 --- a/src/index.h +++ b/src/index.h @@ -94,4 +94,22 @@ extern int git_index_snapshot_find( const char *path, size_t path_len, int stage); +typedef struct { + git_index *index; + git_filebuf file; +} git_indexwriter; + +#define GIT_INDEXWRITER_INIT { NULL, GIT_FILEBUF_INIT } + +/* Lock the index for eventual writing. */ +extern int git_indexwriter_init(git_indexwriter *writer, git_index *index); + +/* Write the index and unlock it. */ +extern int git_indexwriter_commit(git_indexwriter *writer); + +/* Cleanup an index writing session, unlocking the file (if it is still + * locked and freeing any data structures. + */ +extern void git_indexwriter_cleanup(git_indexwriter *writer); + #endif diff --git a/tests/index/tests.c b/tests/index/tests.c index a63183e109c..4cf7051276b 100644 --- a/tests/index/tests.c +++ b/tests/index/tests.c @@ -677,3 +677,24 @@ void test_index_tests__reload_while_ignoring_case(void) git_index_free(index); } + +void test_index_tests__can_lock_index(void) +{ + git_index *index; + git_indexwriter one = GIT_INDEXWRITER_INIT, + two = GIT_INDEXWRITER_INIT; + + cl_git_pass(git_index_open(&index, TEST_INDEX_PATH)); + cl_git_pass(git_indexwriter_init(&one, index)); + + cl_git_fail_with(GIT_ELOCKED, git_indexwriter_init(&two, index)); + cl_git_fail_with(GIT_ELOCKED, git_index_write(index)); + + cl_git_pass(git_indexwriter_commit(&one)); + + cl_git_pass(git_index_write(index)); + + git_indexwriter_cleanup(&one); + git_indexwriter_cleanup(&two); + git_index_free(index); +} From 8639ea5f98ac53e532820e8a186f8a1d6d98f573 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Sat, 17 Jan 2015 22:47:03 -0600 Subject: [PATCH 78/99] checkout: introduce GIT_CHECKOUT_DONT_WRITE_INDEX --- include/git2/checkout.h | 8 ++++- src/checkout.c | 5 ++- tests/checkout/tree.c | 80 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 2 deletions(-) diff --git a/include/git2/checkout.h b/include/git2/checkout.h index 8314c623da0..4fe1340b9e7 100644 --- a/include/git2/checkout.h +++ b/include/git2/checkout.h @@ -135,7 +135,10 @@ typedef enum { /** Only update existing files, don't create new ones */ GIT_CHECKOUT_UPDATE_ONLY = (1u << 7), - /** Normally checkout updates index entries as it goes; this stops that */ + /** + * Normally checkout updates index entries as it goes; this stops that. + * Implies `GIT_CHECKOUT_DONT_WRITE_INDEX`. + */ GIT_CHECKOUT_DONT_UPDATE_INDEX = (1u << 8), /** Don't refresh index/config/etc before doing checkout */ @@ -166,6 +169,9 @@ typedef enum { /** Don't overwrite existing files or folders */ GIT_CHECKOUT_DONT_REMOVE_EXISTING = (1u << 22), + /** Normally checkout writes the index upon completion; this prevents that. */ + GIT_CHECKOUT_DONT_WRITE_INDEX = (1u << 23), + /** * THE FOLLOWING OPTIONS ARE NOT YET IMPLEMENTED */ diff --git a/src/checkout.c b/src/checkout.c index 3f65a9ed75a..395384030e4 100644 --- a/src/checkout.c +++ b/src/checkout.c @@ -2375,6 +2375,9 @@ static int checkout_data_init( return error; } +#define CHECKOUT_INDEX_DONT_WRITE_MASK \ + (GIT_CHECKOUT_DONT_UPDATE_INDEX | GIT_CHECKOUT_DONT_WRITE_INDEX) + int git_checkout_iterator( git_iterator *target, git_index *index, @@ -2481,7 +2484,7 @@ int git_checkout_iterator( cleanup: if (!error && data.index != NULL && - (data.strategy & GIT_CHECKOUT_DONT_UPDATE_INDEX) == 0) + (data.strategy & CHECKOUT_INDEX_DONT_WRITE_MASK) == 0) error = git_index_write(data.index); git_diff_free(data.diff); diff --git a/tests/checkout/tree.c b/tests/checkout/tree.c index 239ee55bc5b..0fabadc8d8e 100644 --- a/tests/checkout/tree.c +++ b/tests/checkout/tree.c @@ -1184,3 +1184,83 @@ void test_checkout_tree__caches_attributes_during_checkout(void) git_buf_free(&ident2); git_object_free(obj); } + +void test_checkout_tree__can_not_update_index(void) +{ + git_oid oid; + git_object *head; + unsigned int status; + git_checkout_options opts = GIT_CHECKOUT_OPTIONS_INIT; + git_index *index; + + opts.checkout_strategy |= + GIT_CHECKOUT_FORCE | GIT_CHECKOUT_DONT_UPDATE_INDEX; + + cl_git_pass(git_reference_name_to_id(&oid, g_repo, "HEAD")); + cl_git_pass(git_object_lookup(&head, g_repo, &oid, GIT_OBJ_ANY)); + + cl_git_pass(git_reset(g_repo, head, GIT_RESET_HARD, &g_opts, NULL, NULL)); + + cl_assert_equal_i(false, git_path_isdir("./testrepo/ab/")); + + cl_git_pass(git_revparse_single(&g_object, g_repo, "subtrees")); + + cl_git_pass(git_checkout_tree(g_repo, g_object, &opts)); + + cl_assert_equal_i(true, git_path_isfile("./testrepo/ab/de/2.txt")); + cl_git_pass(git_status_file(&status, g_repo, "ab/de/2.txt")); + cl_assert_equal_i(GIT_STATUS_WT_NEW, status); + + cl_git_pass(git_repository_index(&index, g_repo)); + cl_git_pass(git_index_write(index)); + + cl_git_pass(git_status_file(&status, g_repo, "ab/de/2.txt")); + cl_assert_equal_i(GIT_STATUS_WT_NEW, status); + + git_object_free(head); + git_index_free(index); +} + +void test_checkout_tree__can_update_but_not_write_index(void) +{ + git_oid oid; + git_object *head; + unsigned int status; + git_checkout_options opts = GIT_CHECKOUT_OPTIONS_INIT; + git_index *index; + git_repository *other; + + opts.checkout_strategy |= + GIT_CHECKOUT_FORCE | GIT_CHECKOUT_DONT_WRITE_INDEX; + + cl_git_pass(git_reference_name_to_id(&oid, g_repo, "HEAD")); + cl_git_pass(git_object_lookup(&head, g_repo, &oid, GIT_OBJ_ANY)); + + cl_git_pass(git_reset(g_repo, head, GIT_RESET_HARD, &g_opts, NULL, NULL)); + + cl_assert_equal_i(false, git_path_isdir("./testrepo/ab/")); + + cl_git_pass(git_revparse_single(&g_object, g_repo, "subtrees")); + + cl_git_pass(git_checkout_tree(g_repo, g_object, &opts)); + + cl_assert_equal_i(true, git_path_isfile("./testrepo/ab/de/2.txt")); + cl_git_pass(git_status_file(&status, g_repo, "ab/de/2.txt")); + cl_assert_equal_i(GIT_STATUS_INDEX_NEW, status); + + cl_git_pass(git_repository_open(&other, "testrepo")); + cl_git_pass(git_status_file(&status, other, "ab/de/2.txt")); + cl_assert_equal_i(GIT_STATUS_WT_NEW, status); + git_repository_free(other); + + cl_git_pass(git_repository_index(&index, g_repo)); + cl_git_pass(git_index_write(index)); + + cl_git_pass(git_repository_open(&other, "testrepo")); + cl_git_pass(git_status_file(&status, other, "ab/de/2.txt")); + cl_assert_equal_i(GIT_STATUS_INDEX_NEW, status); + git_repository_free(other); + + git_object_free(head); + git_index_free(index); +} From 8b0ddd5dd90e1d62f32faf14d5cb4bf88f04a095 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Sat, 17 Jan 2015 23:28:53 -0600 Subject: [PATCH 79/99] merge: lock the index at the start of the merge Always lock the index when we begin the merge, before we write any of the metdata files. This prevents a race where another client may run a commit after we have written the MERGE_HEAD but before we have updated the index, which will produce a merge commit that is treesame to one parent. The merge will finish and update the index and the resultant commit would not be a merge at all. --- src/merge.c | 21 ++++++++++++++-- tests/merge/workdir/setup.c | 49 +++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/src/merge.c b/src/merge.c index 7c38b5692e8..8201c0c1c71 100644 --- a/src/merge.c +++ b/src/merge.c @@ -2651,7 +2651,9 @@ int git_merge( git_checkout_options checkout_opts; git_annotated_commit *ancestor_head = NULL, *our_head = NULL; git_tree *ancestor_tree = NULL, *our_tree = NULL, **their_trees = NULL; - git_index *index_new = NULL; + git_index *index = NULL, *index_new = NULL; + git_indexwriter indexwriter = GIT_INDEXWRITER_INIT; + bool write_index = false; size_t i; int error = 0; @@ -2672,6 +2674,17 @@ int git_merge( ancestor_head, our_head, their_heads_len, their_heads)) < 0) goto on_error; + write_index = (checkout_opts.checkout_strategy & GIT_CHECKOUT_DONT_WRITE_INDEX) == 0; + + if (write_index) { + /* Never let checkout update the index, we'll update it ourselves. */ + checkout_opts.checkout_strategy |= GIT_CHECKOUT_DONT_WRITE_INDEX; + + if ((error = git_repository_index(&index, repo)) < 0 || + (error = git_indexwriter_init(&indexwriter, index)) < 0) + goto on_error; + } + /* Write the merge files to the repository. */ if ((error = git_merge__setup(repo, our_head, their_heads, their_heads_len)) < 0) goto on_error; @@ -2693,7 +2706,8 @@ int git_merge( if ((error = git_merge_trees(&index_new, repo, ancestor_tree, our_tree, their_trees[0], merge_opts)) < 0 || (error = git_merge__check_result(repo, index_new)) < 0 || (error = git_merge__append_conflicts_to_merge_msg(repo, index_new)) < 0 || - (error = git_checkout_index(repo, index_new, &checkout_opts)) < 0) + (error = git_checkout_index(repo, index_new, &checkout_opts)) < 0 || + (write_index && (error = git_indexwriter_commit(&indexwriter)) < 0)) goto on_error; goto done; @@ -2702,6 +2716,9 @@ int git_merge( merge_state_cleanup(repo); done: + git_indexwriter_cleanup(&indexwriter); + + git_index_free(index); git_index_free(index_new); git_tree_free(ancestor_tree); diff --git a/tests/merge/workdir/setup.c b/tests/merge/workdir/setup.c index 31ffd57024b..4aebf870135 100644 --- a/tests/merge/workdir/setup.c +++ b/tests/merge/workdir/setup.c @@ -1045,3 +1045,52 @@ void test_merge_workdir_setup__removed_after_failure(void) git_annotated_commit_free(our_head); git_annotated_commit_free(their_heads[0]); } + +void test_merge_workdir_setup__unlocked_after_success(void) +{ + git_oid our_oid; + git_reference *octo1_ref; + git_annotated_commit *our_head, *their_heads[1]; + + cl_git_pass(git_oid_fromstr(&our_oid, ORIG_HEAD)); + cl_git_pass(git_annotated_commit_lookup(&our_head, repo, &our_oid)); + + cl_git_pass(git_reference_lookup(&octo1_ref, repo, GIT_REFS_HEADS_DIR OCTO1_BRANCH)); + cl_git_pass(git_annotated_commit_from_ref(&their_heads[0], repo, octo1_ref)); + + cl_git_pass(git_merge( + repo, (const git_annotated_commit **)&their_heads[0], 1, NULL, NULL)); + + cl_assert(!git_path_exists("merge-resolve/.git/index.lock")); + + git_reference_free(octo1_ref); + + git_annotated_commit_free(our_head); + git_annotated_commit_free(their_heads[0]); +} + +void test_merge_workdir_setup__unlocked_after_conflict(void) +{ + git_oid our_oid; + git_reference *octo1_ref; + git_annotated_commit *our_head, *their_heads[1]; + + cl_git_pass(git_oid_fromstr(&our_oid, ORIG_HEAD)); + cl_git_pass(git_annotated_commit_lookup(&our_head, repo, &our_oid)); + + cl_git_pass(git_reference_lookup(&octo1_ref, repo, GIT_REFS_HEADS_DIR OCTO1_BRANCH)); + cl_git_pass(git_annotated_commit_from_ref(&their_heads[0], repo, octo1_ref)); + + cl_git_rewritefile("merge-resolve/new-in-octo1.txt", + "Conflicting file!\n\nMerge will fail!\n"); + + cl_git_fail(git_merge( + repo, (const git_annotated_commit **)&their_heads[0], 1, NULL, NULL)); + + cl_assert(!git_path_exists("merge-resolve/.git/index.lock")); + + git_reference_free(octo1_ref); + + git_annotated_commit_free(our_head); + git_annotated_commit_free(their_heads[0]); +} From be8404a7680fa1951e20abdaea704156b3345876 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Tue, 3 Feb 2015 21:51:48 -0500 Subject: [PATCH 80/99] merge-like operations: lock index while working --- src/cherrypick.c | 28 ++++++++++++++++++++++++---- src/rebase.c | 23 ++++++++++++++++++++--- src/revert.c | 27 +++++++++++++++++++++++---- 3 files changed, 67 insertions(+), 11 deletions(-) diff --git a/src/cherrypick.c b/src/cherrypick.c index e58d0ab4c7f..3754b1f77d8 100644 --- a/src/cherrypick.c +++ b/src/cherrypick.c @@ -10,6 +10,7 @@ #include "filebuf.h" #include "merge.h" #include "vector.h" +#include "index.h" #include "git2/types.h" #include "git2/merge.h" @@ -171,7 +172,9 @@ int git_cherrypick( char commit_oidstr[GIT_OID_HEXSZ + 1]; const char *commit_msg, *commit_summary; git_buf their_label = GIT_BUF_INIT; - git_index *index_new = NULL; + git_index *index = NULL, *index_new = NULL; + git_indexwriter indexwriter = GIT_INDEXWRITER_INIT; + bool write_index = false; int error = 0; assert(repo && commit); @@ -191,21 +194,38 @@ int git_cherrypick( if ((error = write_merge_msg(repo, commit_msg)) < 0 || (error = git_buf_printf(&their_label, "%.7s... %s", commit_oidstr, commit_summary)) < 0 || - (error = cherrypick_normalize_opts(repo, &opts, given_opts, git_buf_cstr(&their_label))) < 0 || - (error = write_cherrypick_head(repo, commit_oidstr)) < 0 || + (error = cherrypick_normalize_opts(repo, &opts, given_opts, git_buf_cstr(&their_label))) < 0) + goto on_error; + + write_index = (opts.checkout_opts.checkout_strategy & GIT_CHECKOUT_DONT_WRITE_INDEX) == 0; + + if (write_index) { + /* Never let checkout update the index, we'll update it ourselves. */ + opts.checkout_opts.checkout_strategy |= GIT_CHECKOUT_DONT_WRITE_INDEX; + + if ((error = git_repository_index(&index, repo)) < 0 || + (error = git_indexwriter_init(&indexwriter, index)) < 0) + goto on_error; + } + + if ((error = write_cherrypick_head(repo, commit_oidstr)) < 0 || (error = git_repository_head(&our_ref, repo)) < 0 || (error = git_reference_peel((git_object **)&our_commit, our_ref, GIT_OBJ_COMMIT)) < 0 || (error = git_cherrypick_commit(&index_new, repo, commit, our_commit, opts.mainline, &opts.merge_opts)) < 0 || (error = git_merge__check_result(repo, index_new)) < 0 || (error = git_merge__append_conflicts_to_merge_msg(repo, index_new)) < 0 || - (error = git_checkout_index(repo, index_new, &opts.checkout_opts)) < 0) + (error = git_checkout_index(repo, index_new, &opts.checkout_opts)) < 0 || + (write_index && (error = git_indexwriter_commit(&indexwriter)) < 0)) goto on_error; + goto done; on_error: cherrypick_state_cleanup(repo); done: + git_indexwriter_cleanup(&indexwriter); + git_index_free(index); git_index_free(index_new); git_commit_free(our_commit); git_reference_free(our_ref); diff --git a/src/rebase.c b/src/rebase.c index 2e805929ee1..9e6c4c1d757 100644 --- a/src/rebase.c +++ b/src/rebase.c @@ -14,6 +14,7 @@ #include "array.h" #include "config.h" #include "annotated_commit.h" +#include "index.h" #include #include @@ -725,7 +726,9 @@ static int rebase_next_merge( git_checkout_options checkout_opts = {0}; git_commit *current_commit = NULL, *parent_commit = NULL; git_tree *current_tree = NULL, *head_tree = NULL, *parent_tree = NULL; - git_index *index = NULL; + git_index *index = NULL, *index_new = NULL; + git_indexwriter indexwriter = GIT_INDEXWRITER_INIT; + bool write_index = false; git_rebase_operation *operation; char current_idstr[GIT_OID_HEXSZ]; unsigned int parent_count; @@ -755,6 +758,17 @@ static int rebase_next_merge( git_oid_fmt(current_idstr, &operation->id); + write_index = (checkout_opts.checkout_strategy & GIT_CHECKOUT_DONT_WRITE_INDEX) == 0; + + if (write_index) { + /* Never let checkout update the index, we'll update it ourselves. */ + checkout_opts.checkout_strategy |= GIT_CHECKOUT_DONT_WRITE_INDEX; + + if ((error = git_repository_index(&index, rebase->repo)) < 0 || + (error = git_indexwriter_init(&indexwriter, index)) < 0) + goto done; + } + if ((error = rebase_setupfile(rebase, MSGNUM_FILE, -1, "%d\n", rebase->current+1)) < 0 || (error = rebase_setupfile(rebase, CURRENT_FILE, -1, "%.*s\n", GIT_OID_HEXSZ, current_idstr)) < 0) goto done; @@ -762,14 +776,17 @@ static int rebase_next_merge( normalize_checkout_opts(rebase, current_commit, &checkout_opts, given_checkout_opts); if ((error = git_merge_trees(&index, rebase->repo, parent_tree, head_tree, current_tree, NULL)) < 0 || - (error = git_merge__check_result(rebase->repo, index)) < 0 || - (error = git_checkout_index(rebase->repo, index, &checkout_opts)) < 0) + (error = git_merge__check_result(rebase->repo, index_new)) < 0 || + (error = git_checkout_index(rebase->repo, index_new, &checkout_opts)) < 0 || + (write_index && (error = git_indexwriter_commit(&indexwriter)) < 0)) goto done; *out = operation; done: + git_indexwriter_cleanup(&indexwriter); git_index_free(index); + git_index_free(index_new); git_tree_free(current_tree); git_tree_free(head_tree); git_tree_free(parent_tree); diff --git a/src/revert.c b/src/revert.c index 36560a77c82..f5f1a5d6f7e 100644 --- a/src/revert.c +++ b/src/revert.c @@ -9,6 +9,7 @@ #include "repository.h" #include "filebuf.h" #include "merge.h" +#include "index.h" #include "git2/types.h" #include "git2/merge.h" @@ -174,7 +175,9 @@ int git_revert( char commit_oidstr[GIT_OID_HEXSZ + 1]; const char *commit_msg; git_buf their_label = GIT_BUF_INIT; - git_index *index_new = NULL; + git_index *index = NULL, *index_new = NULL; + git_indexwriter indexwriter = GIT_INDEXWRITER_INIT; + bool write_index = false; int error; assert(repo && commit); @@ -193,15 +196,29 @@ int git_revert( } if ((error = git_buf_printf(&their_label, "parent of %.7s... %s", commit_oidstr, commit_msg)) < 0 || - (error = revert_normalize_opts(repo, &opts, given_opts, git_buf_cstr(&their_label))) < 0 || - (error = write_revert_head(repo, commit_oidstr)) < 0 || + (error = revert_normalize_opts(repo, &opts, given_opts, git_buf_cstr(&their_label))) < 0) + goto on_error; + + write_index = (opts.checkout_opts.checkout_strategy & GIT_CHECKOUT_DONT_WRITE_INDEX) == 0; + + if (write_index) { + /* Never let checkout update the index, we'll update it ourselves. */ + opts.checkout_opts.checkout_strategy |= GIT_CHECKOUT_DONT_WRITE_INDEX; + + if ((error = git_repository_index(&index, repo)) < 0 || + (error = git_indexwriter_init(&indexwriter, index)) < 0) + goto on_error; + } + + if ((error = write_revert_head(repo, commit_oidstr)) < 0 || (error = write_merge_msg(repo, commit_oidstr, commit_msg)) < 0 || (error = git_repository_head(&our_ref, repo)) < 0 || (error = git_reference_peel((git_object **)&our_commit, our_ref, GIT_OBJ_COMMIT)) < 0 || (error = git_revert_commit(&index_new, repo, commit, our_commit, opts.mainline, &opts.merge_opts)) < 0 || (error = git_merge__check_result(repo, index_new)) < 0 || (error = git_merge__append_conflicts_to_merge_msg(repo, index_new)) < 0 || - (error = git_checkout_index(repo, index_new, &opts.checkout_opts)) < 0) + (error = git_checkout_index(repo, index_new, &opts.checkout_opts)) < 0 || + (write_index && (error = git_indexwriter_commit(&indexwriter)) < 0)) goto on_error; goto done; @@ -210,6 +227,8 @@ int git_revert( revert_state_cleanup(repo); done: + git_indexwriter_cleanup(&indexwriter); + git_index_free(index); git_index_free(index_new); git_commit_free(our_commit); git_reference_free(our_ref); From 41fae48df21c7d1140da539cb61fa3fd0598a3e7 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Tue, 3 Feb 2015 22:31:10 -0500 Subject: [PATCH 81/99] indexwriter: an indexwriter for repo operations Provide git_indexwriter_init_for_operation for the common locking pattern in merge, rebase, revert and cherry-pick. --- src/cherrypick.c | 32 +++++++++----------------------- src/index.c | 32 ++++++++++++++++++++++++++++++++ src/index.h | 11 +++++++++++ src/merge.c | 34 ++++++++++------------------------ src/rebase.c | 30 ++++++++---------------------- src/revert.c | 32 +++++++++----------------------- 6 files changed, 79 insertions(+), 92 deletions(-) diff --git a/src/cherrypick.c b/src/cherrypick.c index 3754b1f77d8..ebc9fcdd8fc 100644 --- a/src/cherrypick.c +++ b/src/cherrypick.c @@ -172,9 +172,8 @@ int git_cherrypick( char commit_oidstr[GIT_OID_HEXSZ + 1]; const char *commit_msg, *commit_summary; git_buf their_label = GIT_BUF_INIT; - git_index *index = NULL, *index_new = NULL; + git_index *index = NULL; git_indexwriter indexwriter = GIT_INDEXWRITER_INIT; - bool write_index = false; int error = 0; assert(repo && commit); @@ -194,28 +193,16 @@ int git_cherrypick( if ((error = write_merge_msg(repo, commit_msg)) < 0 || (error = git_buf_printf(&their_label, "%.7s... %s", commit_oidstr, commit_summary)) < 0 || - (error = cherrypick_normalize_opts(repo, &opts, given_opts, git_buf_cstr(&their_label))) < 0) - goto on_error; - - write_index = (opts.checkout_opts.checkout_strategy & GIT_CHECKOUT_DONT_WRITE_INDEX) == 0; - - if (write_index) { - /* Never let checkout update the index, we'll update it ourselves. */ - opts.checkout_opts.checkout_strategy |= GIT_CHECKOUT_DONT_WRITE_INDEX; - - if ((error = git_repository_index(&index, repo)) < 0 || - (error = git_indexwriter_init(&indexwriter, index)) < 0) - goto on_error; - } - - if ((error = write_cherrypick_head(repo, commit_oidstr)) < 0 || + (error = cherrypick_normalize_opts(repo, &opts, given_opts, git_buf_cstr(&their_label))) < 0 || + (error = git_indexwriter_init_for_operation(&indexwriter, repo, &opts.checkout_opts.checkout_strategy)) < 0 || + (error = write_cherrypick_head(repo, commit_oidstr)) < 0 || (error = git_repository_head(&our_ref, repo)) < 0 || (error = git_reference_peel((git_object **)&our_commit, our_ref, GIT_OBJ_COMMIT)) < 0 || - (error = git_cherrypick_commit(&index_new, repo, commit, our_commit, opts.mainline, &opts.merge_opts)) < 0 || - (error = git_merge__check_result(repo, index_new)) < 0 || - (error = git_merge__append_conflicts_to_merge_msg(repo, index_new)) < 0 || - (error = git_checkout_index(repo, index_new, &opts.checkout_opts)) < 0 || - (write_index && (error = git_indexwriter_commit(&indexwriter)) < 0)) + (error = git_cherrypick_commit(&index, repo, commit, our_commit, opts.mainline, &opts.merge_opts)) < 0 || + (error = git_merge__check_result(repo, index)) < 0 || + (error = git_merge__append_conflicts_to_merge_msg(repo, index)) < 0 || + (error = git_checkout_index(repo, index, &opts.checkout_opts)) < 0 || + (error = git_indexwriter_commit(&indexwriter)) < 0) goto on_error; goto done; @@ -226,7 +213,6 @@ int git_cherrypick( done: git_indexwriter_cleanup(&indexwriter); git_index_free(index); - git_index_free(index_new); git_commit_free(our_commit); git_reference_free(our_ref); git_buf_free(&their_label); diff --git a/src/index.c b/src/index.c index 58575fec337..6c01b40f245 100644 --- a/src/index.c +++ b/src/index.c @@ -2669,6 +2669,8 @@ int git_indexwriter_init( { int error; + GIT_REFCOUNT_INC(index); + writer->index = index; if (!index->index_file_path) @@ -2677,12 +2679,33 @@ int git_indexwriter_init( if ((error = git_filebuf_open( &writer->file, index->index_file_path, GIT_FILEBUF_HASH_CONTENTS, GIT_INDEX_FILE_MODE)) < 0) { + if (error == GIT_ELOCKED) giterr_set(GITERR_INDEX, "The index is locked. This might be due to a concurrent or crashed process"); return error; } + writer->should_write = 1; + + return 0; +} + +int git_indexwriter_init_for_operation( + git_indexwriter *writer, + git_repository *repo, + unsigned int *checkout_strategy) +{ + git_index *index; + int error; + + if ((error = git_repository_index__weakptr(&index, repo)) < 0 || + (error = git_indexwriter_init(writer, index)) < 0) + return error; + + writer->should_write = (*checkout_strategy & GIT_CHECKOUT_DONT_WRITE_INDEX) == 0; + *checkout_strategy |= GIT_CHECKOUT_DONT_WRITE_INDEX; + return 0; } @@ -2690,6 +2713,9 @@ int git_indexwriter_commit(git_indexwriter *writer) { int error; + if (!writer->should_write) + return 0; + if (index_sort_if_needed(writer->index, true) < 0) return -1; @@ -2711,10 +2737,16 @@ int git_indexwriter_commit(git_indexwriter *writer) writer->index->on_disk = 1; + git_index_free(writer->index); + writer->index = NULL; + return 0; } void git_indexwriter_cleanup(git_indexwriter *writer) { git_filebuf_cleanup(&writer->file); + + git_index_free(writer->index); + writer->index = NULL; } diff --git a/src/index.h b/src/index.h index d151f6614be..6d2904fdc00 100644 --- a/src/index.h +++ b/src/index.h @@ -97,6 +97,7 @@ extern int git_index_snapshot_find( typedef struct { git_index *index; git_filebuf file; + unsigned int should_write:1; } git_indexwriter; #define GIT_INDEXWRITER_INIT { NULL, GIT_FILEBUF_INIT } @@ -104,6 +105,16 @@ typedef struct { /* Lock the index for eventual writing. */ extern int git_indexwriter_init(git_indexwriter *writer, git_index *index); +/* Lock the index for eventual writing by a repository operation: a merge, + * revert, cherry-pick or a rebase. Note that the given checkout strategy + * will be updated for the operation's use so that checkout will not write + * the index. + */ +extern int git_indexwriter_init_for_operation( + git_indexwriter *writer, + git_repository *repo, + unsigned int *checkout_strategy); + /* Write the index and unlock it. */ extern int git_indexwriter_commit(git_indexwriter *writer); diff --git a/src/merge.c b/src/merge.c index 8201c0c1c71..a92e6aef9a6 100644 --- a/src/merge.c +++ b/src/merge.c @@ -2651,9 +2651,8 @@ int git_merge( git_checkout_options checkout_opts; git_annotated_commit *ancestor_head = NULL, *our_head = NULL; git_tree *ancestor_tree = NULL, *our_tree = NULL, **their_trees = NULL; - git_index *index = NULL, *index_new = NULL; + git_index *index = NULL; git_indexwriter indexwriter = GIT_INDEXWRITER_INIT; - bool write_index = false; size_t i; int error = 0; @@ -2667,24 +2666,12 @@ int git_merge( their_trees = git__calloc(their_heads_len, sizeof(git_tree *)); GITERR_CHECK_ALLOC(their_trees); - if ((error = merge_heads(&ancestor_head, &our_head, repo, their_heads, their_heads_len)) < 0) - goto on_error; - - if ((error = merge_normalize_checkout_opts(repo, &checkout_opts, given_checkout_opts, - ancestor_head, our_head, their_heads_len, their_heads)) < 0) + if ((error = merge_heads(&ancestor_head, &our_head, repo, their_heads, their_heads_len)) < 0 || + (error = merge_normalize_checkout_opts(repo, &checkout_opts, given_checkout_opts, + ancestor_head, our_head, their_heads_len, their_heads)) < 0 || + (error = git_indexwriter_init_for_operation(&indexwriter, repo, &checkout_opts.checkout_strategy)) < 0) goto on_error; - write_index = (checkout_opts.checkout_strategy & GIT_CHECKOUT_DONT_WRITE_INDEX) == 0; - - if (write_index) { - /* Never let checkout update the index, we'll update it ourselves. */ - checkout_opts.checkout_strategy |= GIT_CHECKOUT_DONT_WRITE_INDEX; - - if ((error = git_repository_index(&index, repo)) < 0 || - (error = git_indexwriter_init(&indexwriter, index)) < 0) - goto on_error; - } - /* Write the merge files to the repository. */ if ((error = git_merge__setup(repo, our_head, their_heads, their_heads_len)) < 0) goto on_error; @@ -2703,11 +2690,11 @@ int git_merge( /* TODO: recursive, octopus, etc... */ - if ((error = git_merge_trees(&index_new, repo, ancestor_tree, our_tree, their_trees[0], merge_opts)) < 0 || - (error = git_merge__check_result(repo, index_new)) < 0 || - (error = git_merge__append_conflicts_to_merge_msg(repo, index_new)) < 0 || - (error = git_checkout_index(repo, index_new, &checkout_opts)) < 0 || - (write_index && (error = git_indexwriter_commit(&indexwriter)) < 0)) + if ((error = git_merge_trees(&index, repo, ancestor_tree, our_tree, their_trees[0], merge_opts)) < 0 || + (error = git_merge__check_result(repo, index)) < 0 || + (error = git_merge__append_conflicts_to_merge_msg(repo, index)) < 0 || + (error = git_checkout_index(repo, index, &checkout_opts)) < 0 || + (error = git_indexwriter_commit(&indexwriter)) < 0) goto on_error; goto done; @@ -2719,7 +2706,6 @@ int git_merge( git_indexwriter_cleanup(&indexwriter); git_index_free(index); - git_index_free(index_new); git_tree_free(ancestor_tree); git_tree_free(our_tree); diff --git a/src/rebase.c b/src/rebase.c index 9e6c4c1d757..b24830bd9bb 100644 --- a/src/rebase.c +++ b/src/rebase.c @@ -726,9 +726,8 @@ static int rebase_next_merge( git_checkout_options checkout_opts = {0}; git_commit *current_commit = NULL, *parent_commit = NULL; git_tree *current_tree = NULL, *head_tree = NULL, *parent_tree = NULL; - git_index *index = NULL, *index_new = NULL; + git_index *index = NULL; git_indexwriter indexwriter = GIT_INDEXWRITER_INIT; - bool write_index = false; git_rebase_operation *operation; char current_idstr[GIT_OID_HEXSZ]; unsigned int parent_count; @@ -758,27 +757,15 @@ static int rebase_next_merge( git_oid_fmt(current_idstr, &operation->id); - write_index = (checkout_opts.checkout_strategy & GIT_CHECKOUT_DONT_WRITE_INDEX) == 0; - - if (write_index) { - /* Never let checkout update the index, we'll update it ourselves. */ - checkout_opts.checkout_strategy |= GIT_CHECKOUT_DONT_WRITE_INDEX; - - if ((error = git_repository_index(&index, rebase->repo)) < 0 || - (error = git_indexwriter_init(&indexwriter, index)) < 0) - goto done; - } - - if ((error = rebase_setupfile(rebase, MSGNUM_FILE, -1, "%d\n", rebase->current+1)) < 0 || - (error = rebase_setupfile(rebase, CURRENT_FILE, -1, "%.*s\n", GIT_OID_HEXSZ, current_idstr)) < 0) - goto done; - normalize_checkout_opts(rebase, current_commit, &checkout_opts, given_checkout_opts); - if ((error = git_merge_trees(&index, rebase->repo, parent_tree, head_tree, current_tree, NULL)) < 0 || - (error = git_merge__check_result(rebase->repo, index_new)) < 0 || - (error = git_checkout_index(rebase->repo, index_new, &checkout_opts)) < 0 || - (write_index && (error = git_indexwriter_commit(&indexwriter)) < 0)) + if ((error = git_indexwriter_init_for_operation(&indexwriter, rebase->repo, &checkout_opts.checkout_strategy)) < 0 || + (error = rebase_setupfile(rebase, MSGNUM_FILE, -1, "%d\n", rebase->current+1)) < 0 || + (error = rebase_setupfile(rebase, CURRENT_FILE, -1, "%.*s\n", GIT_OID_HEXSZ, current_idstr)) < 0 || + (error = git_merge_trees(&index, rebase->repo, parent_tree, head_tree, current_tree, NULL)) < 0 || + (error = git_merge__check_result(rebase->repo, index)) < 0 || + (error = git_checkout_index(rebase->repo, index, &checkout_opts)) < 0 || + (error = git_indexwriter_commit(&indexwriter)) < 0) goto done; *out = operation; @@ -786,7 +773,6 @@ static int rebase_next_merge( done: git_indexwriter_cleanup(&indexwriter); git_index_free(index); - git_index_free(index_new); git_tree_free(current_tree); git_tree_free(head_tree); git_tree_free(parent_tree); diff --git a/src/revert.c b/src/revert.c index f5f1a5d6f7e..f8a7f433333 100644 --- a/src/revert.c +++ b/src/revert.c @@ -175,9 +175,8 @@ int git_revert( char commit_oidstr[GIT_OID_HEXSZ + 1]; const char *commit_msg; git_buf their_label = GIT_BUF_INIT; - git_index *index = NULL, *index_new = NULL; + git_index *index = NULL; git_indexwriter indexwriter = GIT_INDEXWRITER_INIT; - bool write_index = false; int error; assert(repo && commit); @@ -196,29 +195,17 @@ int git_revert( } if ((error = git_buf_printf(&their_label, "parent of %.7s... %s", commit_oidstr, commit_msg)) < 0 || - (error = revert_normalize_opts(repo, &opts, given_opts, git_buf_cstr(&their_label))) < 0) - goto on_error; - - write_index = (opts.checkout_opts.checkout_strategy & GIT_CHECKOUT_DONT_WRITE_INDEX) == 0; - - if (write_index) { - /* Never let checkout update the index, we'll update it ourselves. */ - opts.checkout_opts.checkout_strategy |= GIT_CHECKOUT_DONT_WRITE_INDEX; - - if ((error = git_repository_index(&index, repo)) < 0 || - (error = git_indexwriter_init(&indexwriter, index)) < 0) - goto on_error; - } - - if ((error = write_revert_head(repo, commit_oidstr)) < 0 || + (error = revert_normalize_opts(repo, &opts, given_opts, git_buf_cstr(&their_label))) < 0 || + (error = git_indexwriter_init_for_operation(&indexwriter, repo, &opts.checkout_opts.checkout_strategy)) < 0 || + (error = write_revert_head(repo, commit_oidstr)) < 0 || (error = write_merge_msg(repo, commit_oidstr, commit_msg)) < 0 || (error = git_repository_head(&our_ref, repo)) < 0 || (error = git_reference_peel((git_object **)&our_commit, our_ref, GIT_OBJ_COMMIT)) < 0 || - (error = git_revert_commit(&index_new, repo, commit, our_commit, opts.mainline, &opts.merge_opts)) < 0 || - (error = git_merge__check_result(repo, index_new)) < 0 || - (error = git_merge__append_conflicts_to_merge_msg(repo, index_new)) < 0 || - (error = git_checkout_index(repo, index_new, &opts.checkout_opts)) < 0 || - (write_index && (error = git_indexwriter_commit(&indexwriter)) < 0)) + (error = git_revert_commit(&index, repo, commit, our_commit, opts.mainline, &opts.merge_opts)) < 0 || + (error = git_merge__check_result(repo, index)) < 0 || + (error = git_merge__append_conflicts_to_merge_msg(repo, index)) < 0 || + (error = git_checkout_index(repo, index, &opts.checkout_opts)) < 0 || + (error = git_indexwriter_commit(&indexwriter)) < 0) goto on_error; goto done; @@ -229,7 +216,6 @@ int git_revert( done: git_indexwriter_cleanup(&indexwriter); git_index_free(index); - git_index_free(index_new); git_commit_free(our_commit); git_reference_free(our_ref); git_buf_free(&their_label); From 8588cb0cbfadcd66061062bd0ab170a1419e006f Mon Sep 17 00:00:00 2001 From: Jason Haslam Date: Sat, 14 Feb 2015 23:43:26 -0700 Subject: [PATCH 82/99] Fix race in git_packfile_unpack. Increment refcount of newly added cache entries just like existing entries looked up from the cache. Otherwise the new entry can be evicted from the cache and destroyed while it's still in use. --- src/pack.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/pack.c b/src/pack.c index 47ce854c4ef..6891a7d0383 100644 --- a/src/pack.c +++ b/src/pack.c @@ -56,6 +56,7 @@ static git_pack_cache_entry *new_cache_object(git_rawobj *source) if (!e) return NULL; + git_atomic_inc(&e->refcount); memcpy(&e->raw, source, sizeof(git_rawobj)); return e; @@ -145,7 +146,11 @@ static void free_lowest_entry(git_pack_cache *cache) } } -static int cache_add(git_pack_cache *cache, git_rawobj *base, git_off_t offset) +static int cache_add( + git_pack_cache_entry **cached_out, + git_pack_cache *cache, + git_rawobj *base, + git_off_t offset) { git_pack_cache_entry *entry; int error, exists = 0; @@ -171,6 +176,8 @@ static int cache_add(git_pack_cache *cache, git_rawobj *base, git_off_t offset) assert(error != 0); kh_value(cache->entries, k) = entry; cache->memory_used += entry->raw.len; + + *cached_out = entry; } git_mutex_unlock(&cache->lock); /* Somebody beat us to adding it into the cache */ @@ -699,7 +706,7 @@ int git_packfile_unpack( * long as it's not already the cached one. */ if (!cached) - free_base = !!cache_add(&p->bases, obj, elem->base_key); + free_base = !!cache_add(&cached, &p->bases, obj, elem->base_key); elem = &stack[elem_pos - 1]; curpos = elem->offset; From c8e02b8776875e6372fc5eba8fc61c51f14f3392 Mon Sep 17 00:00:00 2001 From: Stefan Widgren Date: Sun, 15 Feb 2015 21:07:05 +0100 Subject: [PATCH 83/99] Remove extra semicolon outside of a function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Without this change, compiling with gcc and pedantic generates warning: ISO C does not allow extra ‘;’ outside of a function. --- src/attr.c | 2 +- src/attrcache.c | 2 +- src/checkout.c | 2 +- src/commit.c | 2 +- src/config_file.c | 2 +- src/diff_driver.c | 2 +- src/fileops.c | 2 +- src/mwindow.c | 2 +- src/odb_mempack.c | 2 +- src/offmap.h | 4 ++-- src/oidmap.h | 2 +- src/pack.h | 4 ++-- src/refdb_fs.c | 2 +- src/refs.c | 2 +- src/revwalk.h | 2 +- src/sortedcache.c | 2 +- src/strmap.h | 2 +- src/submodule.c | 2 +- src/transaction.c | 2 +- src/tree.c | 2 +- tests/core/oidmap.c | 2 +- tests/core/strmap.c | 2 +- 22 files changed, 24 insertions(+), 24 deletions(-) diff --git a/src/attr.c b/src/attr.c index 2e1ef7c2a67..44593da8132 100644 --- a/src/attr.c +++ b/src/attr.c @@ -7,7 +7,7 @@ #include "git2/oid.h" #include -GIT__USE_STRMAP; +GIT__USE_STRMAP const char *git_attr__true = "[internal]__TRUE__"; const char *git_attr__false = "[internal]__FALSE__"; diff --git a/src/attrcache.c b/src/attrcache.c index 9a9b9d09dd9..018fa48745c 100644 --- a/src/attrcache.c +++ b/src/attrcache.c @@ -5,7 +5,7 @@ #include "sysdir.h" #include "ignore.h" -GIT__USE_STRMAP; +GIT__USE_STRMAP GIT_INLINE(int) attr_cache_lock(git_attr_cache *cache) { diff --git a/src/checkout.c b/src/checkout.c index 395384030e4..880af3dff2e 100644 --- a/src/checkout.c +++ b/src/checkout.c @@ -32,7 +32,7 @@ #include "pool.h" #include "strmap.h" -GIT__USE_STRMAP; +GIT__USE_STRMAP /* See docs/checkout-internals.md for more information */ diff --git a/src/commit.c b/src/commit.c index 78c4b9de307..beb2c64c36a 100644 --- a/src/commit.c +++ b/src/commit.c @@ -400,7 +400,7 @@ GIT_COMMIT_GETTER(const char *, raw_header, commit->raw_header) GIT_COMMIT_GETTER(git_time_t, time, commit->committer->when.time) GIT_COMMIT_GETTER(int, time_offset, commit->committer->when.offset) GIT_COMMIT_GETTER(unsigned int, parentcount, (unsigned int)git_array_size(commit->parent_ids)) -GIT_COMMIT_GETTER(const git_oid *, tree_id, &commit->tree_id); +GIT_COMMIT_GETTER(const git_oid *, tree_id, &commit->tree_id) const char *git_commit_message(const git_commit *commit) { diff --git a/src/config_file.c b/src/config_file.c index b0ffd61167d..168dd548309 100644 --- a/src/config_file.c +++ b/src/config_file.c @@ -21,7 +21,7 @@ #include #include -GIT__USE_STRMAP; +GIT__USE_STRMAP typedef struct cvar_t { struct cvar_t *next; diff --git a/src/diff_driver.c b/src/diff_driver.c index e4d9a06995c..7313ab573dd 100644 --- a/src/diff_driver.c +++ b/src/diff_driver.c @@ -17,7 +17,7 @@ #include "config.h" #include "repository.h" -GIT__USE_STRMAP; +GIT__USE_STRMAP typedef enum { DIFF_DRIVER_AUTO = 0, diff --git a/src/fileops.c b/src/fileops.c index 9509b27bbb2..ec536bf2922 100644 --- a/src/fileops.c +++ b/src/fileops.c @@ -13,7 +13,7 @@ #include "win32/findfile.h" #endif -GIT__USE_STRMAP; +GIT__USE_STRMAP int git_futils_mkpath2file(const char *file_path, const mode_t mode) { diff --git a/src/mwindow.c b/src/mwindow.c index 0444cd98c4d..55c8d894b52 100644 --- a/src/mwindow.c +++ b/src/mwindow.c @@ -14,7 +14,7 @@ #include "strmap.h" #include "pack.h" -GIT__USE_STRMAP; +GIT__USE_STRMAP #define DEFAULT_WINDOW_SIZE \ (sizeof(void*) >= 8 \ diff --git a/src/odb_mempack.c b/src/odb_mempack.c index 32bc8444211..34355270f62 100644 --- a/src/odb_mempack.c +++ b/src/odb_mempack.c @@ -18,7 +18,7 @@ #include "git2/types.h" #include "git2/pack.h" -GIT__USE_OIDMAP; +GIT__USE_OIDMAP struct memobject { git_oid oid; diff --git a/src/offmap.h b/src/offmap.h index 9471e7b91d3..0d0e51272c4 100644 --- a/src/offmap.h +++ b/src/offmap.h @@ -17,11 +17,11 @@ #define kfree git__free #include "khash.h" -__KHASH_TYPE(off, git_off_t, void *); +__KHASH_TYPE(off, git_off_t, void *) typedef khash_t(off) git_offmap; #define GIT__USE_OFFMAP \ - __KHASH_IMPL(off, static kh_inline, git_off_t, void *, 1, kh_int64_hash_func, kh_int64_hash_equal); + __KHASH_IMPL(off, static kh_inline, git_off_t, void *, 1, kh_int64_hash_func, kh_int64_hash_equal) #define git_offmap_alloc() kh_init(off) #define git_offmap_free(h) kh_destroy(off, h), h = NULL diff --git a/src/oidmap.h b/src/oidmap.h index 5e3b44bfbf4..d2c451e7fe8 100644 --- a/src/oidmap.h +++ b/src/oidmap.h @@ -17,7 +17,7 @@ #define kfree git__free #include "khash.h" -__KHASH_TYPE(oid, const git_oid *, void *); +__KHASH_TYPE(oid, const git_oid *, void *) typedef khash_t(oid) git_oidmap; GIT_INLINE(khint_t) git_oidmap_hash(const git_oid *oid) diff --git a/src/pack.h b/src/pack.h index 34d37d907fc..3cebd10aca0 100644 --- a/src/pack.h +++ b/src/pack.h @@ -72,8 +72,8 @@ typedef git_array_t(struct pack_chain_elem) git_dependency_chain; #include "offmap.h" -GIT__USE_OFFMAP; -GIT__USE_OIDMAP; +GIT__USE_OFFMAP +GIT__USE_OIDMAP #define GIT_PACK_CACHE_MEMORY_LIMIT 16 * 1024 * 1024 #define GIT_PACK_CACHE_SIZE_LIMIT 1024 * 1024 /* don't bother caching anything over 1MB */ diff --git a/src/refdb_fs.c b/src/refdb_fs.c index 5cd2112fc0b..77cb1a8ce1c 100644 --- a/src/refdb_fs.c +++ b/src/refdb_fs.c @@ -26,7 +26,7 @@ #include #include -GIT__USE_STRMAP; +GIT__USE_STRMAP #define DEFAULT_NESTING_LEVEL 5 #define MAX_NESTING_LEVEL 10 diff --git a/src/refs.c b/src/refs.c index 2e88c26c3e1..e3157529b3c 100644 --- a/src/refs.c +++ b/src/refs.c @@ -24,7 +24,7 @@ #include #include -GIT__USE_STRMAP; +GIT__USE_STRMAP #define DEFAULT_NESTING_LEVEL 5 #define MAX_NESTING_LEVEL 10 diff --git a/src/revwalk.h b/src/revwalk.h index 72ddedc75da..1148a2ac97b 100644 --- a/src/revwalk.h +++ b/src/revwalk.h @@ -14,7 +14,7 @@ #include "pool.h" #include "vector.h" -GIT__USE_OIDMAP; +GIT__USE_OIDMAP struct git_revwalk { git_repository *repo; diff --git a/src/sortedcache.c b/src/sortedcache.c index 1195ea33930..1151757243b 100644 --- a/src/sortedcache.c +++ b/src/sortedcache.c @@ -1,6 +1,6 @@ #include "sortedcache.h" -GIT__USE_STRMAP; +GIT__USE_STRMAP int git_sortedcache_new( git_sortedcache **out, diff --git a/src/strmap.h b/src/strmap.h index dfbe5639f6f..5209847447e 100644 --- a/src/strmap.h +++ b/src/strmap.h @@ -16,7 +16,7 @@ #define kfree git__free #include "khash.h" -__KHASH_TYPE(str, const char *, void *); +__KHASH_TYPE(str, const char *, void *) typedef khash_t(str) git_strmap; typedef khiter_t git_strmap_iter; diff --git a/src/submodule.c b/src/submodule.c index 03714b43ba0..80f1b3789c3 100644 --- a/src/submodule.c +++ b/src/submodule.c @@ -85,7 +85,7 @@ static kh_inline int str_equal_no_trailing_slash(const char *a, const char *b) __KHASH_IMPL( str, static kh_inline, const char *, void *, 1, - str_hash_no_trailing_slash, str_equal_no_trailing_slash); + str_hash_no_trailing_slash, str_equal_no_trailing_slash) static int submodule_cache_init(git_repository *repo, int refresh); static void submodule_cache_free(git_submodule_cache *cache); diff --git a/src/transaction.c b/src/transaction.c index 1a4fffb36e4..7b45b8bf293 100644 --- a/src/transaction.c +++ b/src/transaction.c @@ -17,7 +17,7 @@ #include "git2/sys/refs.h" #include "git2/sys/refdb_backend.h" -GIT__USE_STRMAP; +GIT__USE_STRMAP typedef struct { const char *name; diff --git a/src/tree.c b/src/tree.c index 9fd4e0a0729..47ed20dbbb0 100644 --- a/src/tree.c +++ b/src/tree.c @@ -17,7 +17,7 @@ #define DEFAULT_TREE_SIZE 16 #define MAX_FILEMODE_BYTES 6 -GIT__USE_STRMAP; +GIT__USE_STRMAP static bool valid_filemode(const int filemode) { diff --git a/tests/core/oidmap.c b/tests/core/oidmap.c index ec4b5e77564..556a6ca4ac5 100644 --- a/tests/core/oidmap.c +++ b/tests/core/oidmap.c @@ -1,7 +1,7 @@ #include "clar_libgit2.h" #include "oidmap.h" -GIT__USE_OIDMAP; +GIT__USE_OIDMAP typedef struct { git_oid oid; diff --git a/tests/core/strmap.c b/tests/core/strmap.c index a120f1feb35..3b4276aea7c 100644 --- a/tests/core/strmap.c +++ b/tests/core/strmap.c @@ -1,7 +1,7 @@ #include "clar_libgit2.h" #include "strmap.h" -GIT__USE_STRMAP; +GIT__USE_STRMAP git_strmap *g_table; From fbdc9db3643e89e26d6e49f7b4e4ec1c668a4f0d Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Thu, 22 Jan 2015 16:10:06 -0600 Subject: [PATCH 84/99] filters: introduce streaming filters Add structures and preliminary functions to take a buffer, file or blob and write the contents in chunks through an arbitrary number of chained filters, finally writing into a user-provided function accept the contents. --- include/git2/filter.h | 16 +++ include/git2/sys/filter.h | 14 +++ include/git2/types.h | 3 + src/filter.c | 238 ++++++++++++++++++++++++++++++++++++-- 4 files changed, 264 insertions(+), 7 deletions(-) diff --git a/include/git2/filter.h b/include/git2/filter.h index 5b3f403943e..600356c7155 100644 --- a/include/git2/filter.h +++ b/include/git2/filter.h @@ -137,6 +137,22 @@ GIT_EXTERN(int) git_filter_list_apply_to_blob( git_filter_list *filters, git_blob *blob); +GIT_EXTERN(int) git_filter_list_stream_data( + git_filter_list *filters, + git_buf *data, + git_filter_stream *target); + +GIT_EXTERN(int) git_filter_list_stream_file( + git_filter_list *filters, + git_repository *repo, + const char *path, + git_filter_stream *target); + +GIT_EXTERN(int) git_filter_list_stream_blob( + git_filter_list *filters, + git_blob *blob, + git_filter_stream *target); + /** * Free a git_filter_list * diff --git a/include/git2/sys/filter.h b/include/git2/sys/filter.h index 60248271a71..cc06c54ad44 100644 --- a/include/git2/sys/filter.h +++ b/include/git2/sys/filter.h @@ -208,6 +208,19 @@ typedef int (*git_filter_apply_fn)( const git_buf *from, const git_filter_source *src); +struct git_filter_stream { + int (*write)(git_filter_stream *stream, const char *buffer, size_t len); + int (*close)(git_filter_stream *stream); + void (*free)(git_filter_stream *stream); +}; + +typedef int (*git_filter_stream_fn)( + git_filter_stream **out, + git_filter *self, + void **payload, + const git_filter_source *src, + git_filter_stream *next); + /** * Callback to clean up after filtering has been applied * @@ -247,6 +260,7 @@ struct git_filter { git_filter_shutdown_fn shutdown; git_filter_check_fn check; git_filter_apply_fn apply; + git_filter_stream_fn stream; git_filter_cleanup_fn cleanup; }; diff --git a/include/git2/types.h b/include/git2/types.h index 35e1573c739..3ed97d18967 100644 --- a/include/git2/types.h +++ b/include/git2/types.h @@ -410,6 +410,9 @@ typedef enum { GIT_SUBMODULE_RECURSE_ONDEMAND = 2, } git_submodule_recurse_t; +/** A stream to write filters. */ +typedef struct git_filter_stream git_filter_stream; + /** @} */ GIT_END_DECL diff --git a/src/filter.c b/src/filter.c index 7b54a76c064..1af17d80c6a 100644 --- a/src/filter.c +++ b/src/filter.c @@ -703,25 +703,249 @@ int git_filter_list_apply_to_file( return error; } +static int buf_from_blob(git_buf *out, git_blob *blob) +{ + git_off_t rawsize = git_blob_rawsize(blob); + + if (!git__is_sizet(rawsize)) { + giterr_set(GITERR_OS, "Blob is too large to filter"); + return -1; + } + + out->ptr = (char *)git_blob_rawcontent(blob); + out->asize = 0; + out->size = (size_t)rawsize; + + return 0; +} + int git_filter_list_apply_to_blob( git_buf *out, git_filter_list *filters, git_blob *blob) { git_buf in = GIT_BUF_INIT; - git_off_t rawsize = git_blob_rawsize(blob); - if (!git__is_sizet(rawsize)) { - giterr_set(GITERR_OS, "Blob is too large to filter"); + if (buf_from_blob(&in, blob) < 0) return -1; + + if (filters) + git_oid_cpy(&filters->source.oid, git_blob_id(blob)); + + return git_filter_list_apply_to_data(out, filters, &in); +} + +struct proxy_stream { + git_filter_stream base; + git_filter *filter; + const git_filter_source *source; + void **payload; + git_buf input; + git_buf output; + git_filter_stream *target; +}; + +static int proxy_stream_write( + git_filter_stream *s, const char *buffer, size_t len) +{ + struct proxy_stream *proxy_stream = (struct proxy_stream *)s; + assert(proxy_stream); + + return git_buf_put(&proxy_stream->input, buffer, len); +} + +static int proxy_stream_close(git_filter_stream *s) +{ + struct proxy_stream *proxy_stream = (struct proxy_stream *)s; + git_buf *writebuf; + int error; + + assert(proxy_stream); + + error = proxy_stream->filter->apply( + proxy_stream->filter, + proxy_stream->payload, + &proxy_stream->output, + &proxy_stream->input, + proxy_stream->source); + + if (error == GIT_PASSTHROUGH) { + writebuf = &proxy_stream->input; + } else if (error == 0) { + git_buf_sanitize(&proxy_stream->output); + writebuf = &proxy_stream->output; + } else { + return error; + } + + if ((error = proxy_stream->target->write( + proxy_stream->target, writebuf->ptr, writebuf->size)) == 0) + error = proxy_stream->target->close(proxy_stream->target); + + return error; +} + +static void proxy_stream_free(git_filter_stream *s) +{ + struct proxy_stream *proxy_stream = (struct proxy_stream *)s; + assert(proxy_stream); + + git_buf_free(&proxy_stream->input); + git_buf_free(&proxy_stream->output); + git__free(proxy_stream); +} + +static int proxy_stream_init( + git_filter_stream **out, + git_filter *filter, + void **payload, + const git_filter_source *source, + git_filter_stream *target) +{ + struct proxy_stream *proxy_stream = git__calloc(1, sizeof(struct proxy_stream)); + GITERR_CHECK_ALLOC(proxy_stream); + + proxy_stream->base.write = proxy_stream_write; + proxy_stream->base.close = proxy_stream_close; + proxy_stream->base.free = proxy_stream_free; + proxy_stream->filter = filter; + proxy_stream->payload = payload; + proxy_stream->source = source; + proxy_stream->target = target; + + *out = (git_filter_stream *)proxy_stream; + return 0; +} + +static int stream_list_init( + git_filter_stream **out, + git_vector *streams, + git_filter_list *filters, + git_filter_stream *target) +{ + git_vector filter_streams = GIT_VECTOR_INIT; + git_filter_stream *last_stream = target; + size_t i; + int error = 0; + + *out = NULL; + + if (!filters) { + *out = target; + return 0; + } + + /* Create filters last to first to get the chaining direction */ + for (i = 0; i < git_array_size(filters->filters); ++i) { + size_t filter_idx = (filters->source.mode == GIT_FILTER_TO_WORKTREE) ? + git_array_size(filters->filters) - 1 - i : i; + git_filter_entry *fe = git_array_get(filters->filters, filter_idx); + git_filter_stream *filter_stream; + git_filter_stream_fn stream_init; + + assert(fe->filter->stream || fe->filter->apply); + + /* If necessary, create a stream that proxies the one-shot apply */ + stream_init = fe->filter->stream ? + fe->filter->stream : proxy_stream_init; + + if ((error = stream_init(&filter_stream, fe->filter, &fe->payload, &filters->source, last_stream)) < 0) + return error; + + git_vector_insert(&filter_streams, filter_stream); + last_stream = filter_stream; } - in.ptr = (char *)git_blob_rawcontent(blob); - in.asize = 0; - in.size = (size_t)rawsize; + *out = last_stream; + return 0; +} + +void stream_list_free(git_vector *streams) +{ + git_filter_stream *stream; + size_t i; + + git_vector_foreach(streams, i, stream) + stream->free(stream); +} + +#define STREAM_BUFSIZE 10240 + +/* TODO: maybe not use filter_stream as a target but create one */ +int git_filter_list_stream_file( + git_filter_list *filters, + git_buf *data, + git_repository *repo, + const char *path, + git_filter_stream *target) +{ + char buf[STREAM_BUFSIZE]; + git_buf abspath = GIT_BUF_INIT, raw = GIT_BUF_INIT; + const char *base = repo ? git_repository_workdir(repo) : NULL; + git_vector filter_streams = GIT_VECTOR_INIT; + git_filter_stream *stream_start; + ssize_t readlen; + int fd, error; + + if ((error = stream_list_init( + &stream_start, &filter_streams, filters, target)) < 0 || + (error = git_path_join_unrooted(&abspath, path, base, NULL)) < 0) + goto done; + + if ((fd = git_futils_open_ro(path)) < 0) { + error = fd; + goto done; + } + + while ((readlen = p_read(fd, buf, STREAM_BUFSIZE)) > 0) { + if ((error = stream_start->write(stream_start, data->ptr, data->size)) < 0) + goto done; + } + + if (!readlen) + error = stream_start->close(stream_start); + else if (readlen < 0) + error = readlen; + + p_close(fd); + +done: + stream_list_free(&filter_streams); + git_buf_free(&abspath); + return error; +} + +int git_filter_list_stream_data( + git_filter_list *filters, + git_buf *data, + git_filter_stream *target) +{ + git_vector filter_streams = GIT_VECTOR_INIT; + git_filter_stream *stream_start; + int error = 0; + + if ((error = stream_list_init( + &stream_start, &filter_streams, filters, target)) == 0 && + (error = + stream_start->write(stream_start, data->ptr, data->size)) == 0) + error = stream_start->close(stream_start); + + stream_list_free(&filter_streams); + return error; +} + +int git_filter_list_stream_blob( + git_filter_list *filters, + git_blob *blob, + git_filter_stream *target) +{ + git_buf in = GIT_BUF_INIT; + + if (buf_from_blob(&in, blob) < 0) + return -1; if (filters) git_oid_cpy(&filters->source.oid, git_blob_id(blob)); - return git_filter_list_apply_to_data(out, filters, &in); + return git_filter_list_stream_data(filters, &in, target); } From 2d2aa1ff6e52567d4710af22fdfa11ae54a08420 Mon Sep 17 00:00:00 2001 From: Leo Yang Date: Fri, 13 Feb 2015 17:28:35 -0500 Subject: [PATCH 85/99] Fix Mac build without OpenSSL If OpenSSL is disabled on Mac the SHA1 implementation goes to the CommonCrypto from the system. In this case we should not include the generic hash impl. Otherwise there would be duplicated impls which fail the build. --- CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 69ce6fed850..81dc4c24aa4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -43,7 +43,6 @@ OPTION( VALGRIND "Configure build for valgrind" OFF ) IF(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") SET( USE_ICONV ON ) - ADD_DEFINITIONS(-DGIT_COMMON_CRYPTO) ENDIF() IF(MSVC) @@ -173,6 +172,8 @@ ENDIF() IF (WIN32 AND NOT MINGW AND NOT SHA1_TYPE STREQUAL "builtin") ADD_DEFINITIONS(-DWIN32_SHA1) FILE(GLOB SRC_SHA1 src/hash/hash_win32.c) +ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") + ADD_DEFINITIONS(-DGIT_COMMON_CRYPTO) ELSEIF (OPENSSL_FOUND AND NOT SHA1_TYPE STREQUAL "builtin") ADD_DEFINITIONS(-DOPENSSL_SHA1) IF (CMAKE_SYSTEM_NAME MATCHES "FreeBSD") From 5555696f8a48d2319a7ca0918726711e6a81a924 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Thu, 22 Jan 2015 16:11:22 -0600 Subject: [PATCH 86/99] filters: stream internally Migrate the `git_filter_list_apply_*` functions over to using the new filter streams. --- src/filter.c | 154 +++++++++++++++++++++++++++------------------------ 1 file changed, 81 insertions(+), 73 deletions(-) diff --git a/src/filter.c b/src/filter.c index 1af17d80c6a..700afbac33e 100644 --- a/src/filter.c +++ b/src/filter.c @@ -617,67 +617,70 @@ static int filter_list_out_buffer_from_raw( return 0; } -int git_filter_list_apply_to_data( - git_buf *tgt, git_filter_list *fl, git_buf *src) +struct buf_stream { + git_filter_stream base; + git_buf *target; + bool complete; +}; + +static int buf_stream_write( + git_filter_stream *s, const char *buffer, size_t len) { - int error = 0; - uint32_t i; - git_buf *dbuffer[2], local = GIT_BUF_INIT; - unsigned int si = 0; + struct buf_stream *buf_stream = (struct buf_stream *)s; + assert(buf_stream); - git_buf_sanitize(tgt); - git_buf_sanitize(src); + assert(buf_stream->complete == 0); - if (!fl) - return filter_list_out_buffer_from_raw(tgt, src->ptr, src->size); + return git_buf_put(buf_stream->target, buffer, len); +} - dbuffer[0] = src; - dbuffer[1] = tgt; +static int buf_stream_close(git_filter_stream *s) +{ + struct buf_stream *buf_stream = (struct buf_stream *)s; + assert(buf_stream); - /* if `src` buffer is reallocable, then use it, otherwise copy it */ - if (!git_buf_is_allocated(src)) { - if (git_buf_set(&local, src->ptr, src->size) < 0) - return -1; - dbuffer[0] = &local; - } + assert(buf_stream->complete == 0); + buf_stream->complete = 1; - for (i = 0; i < git_array_size(fl->filters); ++i) { - unsigned int di = 1 - si; - uint32_t fidx = (fl->source.mode == GIT_FILTER_TO_WORKTREE) ? - i : git_array_size(fl->filters) - 1 - i; - git_filter_entry *fe = git_array_get(fl->filters, fidx); - - dbuffer[di]->size = 0; - - /* Apply the filter from dbuffer[src] to the other buffer; - * if the filtering is canceled by the user mid-filter, - * we skip to the next filter without changing the source - * of the double buffering (so that the text goes through - * cleanly). - */ + return 0; +} - error = fe->filter->apply( - fe->filter, &fe->payload, dbuffer[di], dbuffer[si], &fl->source); +static void buf_stream_free(git_filter_stream *s) +{ + GIT_UNUSED(s); +} - if (error == GIT_PASSTHROUGH) { - /* PASSTHROUGH means filter decided not to process the buffer */ - error = 0; - } else if (!error) { - git_buf_sanitize(dbuffer[di]); /* force NUL termination */ - si = di; /* swap buffers */ - } else { - tgt->size = 0; - goto cleanup; - } - } +static void buf_stream_init(struct buf_stream *writer, git_buf *target) +{ + memset(writer, 0, sizeof(struct buf_stream)); - /* Ensure that the output ends up in dbuffer[1] (i.e. the dest) */ - if (si != 1) - git_buf_swap(dbuffer[0], dbuffer[1]); + writer->base.write = buf_stream_write; + writer->base.close = buf_stream_close; + writer->base.free = buf_stream_free; + writer->target = target; -cleanup: - git_buf_free(&local); /* don't leak if we allocated locally */ + git_buf_clear(target); +} +int git_filter_list_apply_to_data( + git_buf *tgt, git_filter_list *filters, git_buf *src) +{ + struct buf_stream writer; + int error; + + git_buf_sanitize(tgt); + git_buf_sanitize(src); + + if (!filters) + return filter_list_out_buffer_from_raw(tgt, src->ptr, src->size); + + buf_stream_init(&writer, tgt); + + if ((error = git_filter_list_stream_data(filters, src, + (git_filter_stream *)&writer)) < 0) + return error; + + assert(writer.complete); return error; } @@ -687,19 +690,16 @@ int git_filter_list_apply_to_file( git_repository *repo, const char *path) { + struct buf_stream writer; int error; - const char *base = repo ? git_repository_workdir(repo) : NULL; - git_buf abspath = GIT_BUF_INIT, raw = GIT_BUF_INIT; - if (!(error = git_path_join_unrooted(&abspath, path, base, NULL)) && - !(error = git_futils_readbuffer(&raw, abspath.ptr))) - { - error = git_filter_list_apply_to_data(out, filters, &raw); + buf_stream_init(&writer, out); - git_buf_free(&raw); - } + if ((error = git_filter_list_stream_file( + filters, repo, path, (git_filter_stream *)&writer)) < 0) + return error; - git_buf_free(&abspath); + assert(writer.complete); return error; } @@ -724,15 +724,17 @@ int git_filter_list_apply_to_blob( git_filter_list *filters, git_blob *blob) { - git_buf in = GIT_BUF_INIT; + struct buf_stream writer; + int error; - if (buf_from_blob(&in, blob) < 0) - return -1; + buf_stream_init(&writer, out); - if (filters) - git_oid_cpy(&filters->source.oid, git_blob_id(blob)); + if ((error = git_filter_list_stream_blob( + filters, blob, (git_filter_stream *)&writer)) < 0) + return error; - return git_filter_list_apply_to_data(out, filters, &in); + assert(writer.complete); + return error; } struct proxy_stream { @@ -823,7 +825,6 @@ static int stream_list_init( git_filter_list *filters, git_filter_stream *target) { - git_vector filter_streams = GIT_VECTOR_INIT; git_filter_stream *last_stream = target; size_t i; int error = 0; @@ -845,14 +846,19 @@ static int stream_list_init( assert(fe->filter->stream || fe->filter->apply); - /* If necessary, create a stream that proxies the one-shot apply */ + /* If necessary, create a stream that proxies the traditional + * application. + */ stream_init = fe->filter->stream ? fe->filter->stream : proxy_stream_init; - if ((error = stream_init(&filter_stream, fe->filter, &fe->payload, &filters->source, last_stream)) < 0) - return error; + error = stream_init(&filter_stream, fe->filter, + &fe->payload, &filters->source, last_stream); - git_vector_insert(&filter_streams, filter_stream); + if (error < 0) + return error; + + git_vector_insert(streams, filter_stream); last_stream = filter_stream; } @@ -867,6 +873,7 @@ void stream_list_free(git_vector *streams) git_vector_foreach(streams, i, stream) stream->free(stream); + git_vector_free(streams); } #define STREAM_BUFSIZE 10240 @@ -874,13 +881,12 @@ void stream_list_free(git_vector *streams) /* TODO: maybe not use filter_stream as a target but create one */ int git_filter_list_stream_file( git_filter_list *filters, - git_buf *data, git_repository *repo, const char *path, git_filter_stream *target) { char buf[STREAM_BUFSIZE]; - git_buf abspath = GIT_BUF_INIT, raw = GIT_BUF_INIT; + git_buf abspath = GIT_BUF_INIT; const char *base = repo ? git_repository_workdir(repo) : NULL; git_vector filter_streams = GIT_VECTOR_INIT; git_filter_stream *stream_start; @@ -898,7 +904,7 @@ int git_filter_list_stream_file( } while ((readlen = p_read(fd, buf, STREAM_BUFSIZE)) > 0) { - if ((error = stream_start->write(stream_start, data->ptr, data->size)) < 0) + if ((error = stream_start->write(stream_start, buf, readlen)) < 0) goto done; } @@ -924,6 +930,8 @@ int git_filter_list_stream_data( git_filter_stream *stream_start; int error = 0; + git_buf_sanitize(data); + if ((error = stream_list_init( &stream_start, &filter_streams, filters, target)) == 0 && (error = From e78f5c9f4282319cf8e1afc8631e6259e91653a6 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Thu, 22 Jan 2015 16:11:36 -0600 Subject: [PATCH 87/99] checkout: stream the blob into the filters Use the new streaming filter API during checkout. --- src/checkout.c | 121 +++++++++++++++++++++++++++++++++---------------- src/filter.c | 2 + 2 files changed, 84 insertions(+), 39 deletions(-) diff --git a/src/checkout.c b/src/checkout.c index 880af3dff2e..623ac92a158 100644 --- a/src/checkout.c +++ b/src/checkout.c @@ -17,6 +17,7 @@ #include "git2/diff.h" #include "git2/submodule.h" #include "git2/sys/index.h" +#include "git2/sys/filter.h" #include "refs.h" #include "repository.h" @@ -1371,39 +1372,37 @@ static int mkpath2file( return error; } -static int buffer_to_file( - checkout_data *data, - struct stat *st, - git_buf *buf, - const char *path, - mode_t file_mode) -{ - int error; - - if ((error = mkpath2file(data, path, data->opts.dir_mode)) < 0) - return error; +struct checkout_stream { + git_filter_stream base; + const char *path; + int fd; + int open; +}; - if ((error = git_futils_writebuffer( - buf, path, data->opts.file_open_flags, file_mode)) < 0) - return error; +static int checkout_stream_write( + git_filter_stream *s, const char *buffer, size_t len) +{ + struct checkout_stream *stream = (struct checkout_stream *)s; + int ret; - if (st) { - data->perfdata.stat_calls++; + if ((ret = p_write(stream->fd, buffer, len)) < 0) + giterr_set(GITERR_OS, "Could not write to '%s'", stream->path); - if ((error = p_stat(path, st)) < 0) { - giterr_set(GITERR_OS, "Error statting '%s'", path); - return error; - } - } + return ret; +} - if (GIT_PERMS_IS_EXEC(file_mode)) { - data->perfdata.chmod_calls++; +static int checkout_stream_close(git_filter_stream *s) +{ + struct checkout_stream *stream = (struct checkout_stream *)s; + assert(stream && stream->open); - if ((error = p_chmod(path, file_mode)) < 0) - giterr_set(GITERR_OS, "Failed to set permissions on '%s'", path); - } + stream->open = 0; + return 0; +} - return error; +static void checkout_stream_free(git_filter_stream *s) +{ + GIT_UNUSED(s); } static int blob_content_to_file( @@ -1411,36 +1410,80 @@ static int blob_content_to_file( struct stat *st, git_blob *blob, const char *path, - const char * hint_path, + const char *hint_path, mode_t entry_filemode) { + int flags = data->opts.file_open_flags; mode_t file_mode = data->opts.file_mode ? data->opts.file_mode : entry_filemode; - git_buf out = GIT_BUF_INIT; + struct checkout_stream writer; + mode_t mode; git_filter_list *fl = NULL; + int fd; int error = 0; if (hint_path == NULL) hint_path = path; - if (!data->opts.disable_filters) - error = git_filter_list__load_with_attr_session( + if ((error = mkpath2file(data, path, data->opts.dir_mode)) < 0) + return error; + + if (flags <= 0) + flags = O_CREAT | O_TRUNC | O_WRONLY; + if (!(mode = file_mode)) + mode = GIT_FILEMODE_BLOB; + + if ((fd = p_open(path, flags, mode)) < 0) { + giterr_set(GITERR_OS, "Could not open '%s' for writing", path); + return fd; + } + + if (!data->opts.disable_filters && + (error = git_filter_list__load_with_attr_session( &fl, data->repo, &data->attr_session, blob, hint_path, - GIT_FILTER_TO_WORKTREE, GIT_FILTER_OPT_DEFAULT); + GIT_FILTER_TO_WORKTREE, GIT_FILTER_OPT_DEFAULT))) + return error; + + /* setup the writer */ + memset(&writer, 0, sizeof(struct checkout_stream)); + writer.base.write = checkout_stream_write; + writer.base.close = checkout_stream_close; + writer.base.free = checkout_stream_free; + writer.path = path; + writer.fd = fd; + writer.open = 1; + + error = git_filter_list_stream_blob(fl, blob, (git_filter_stream *)&writer); - if (!error) - error = git_filter_list_apply_to_blob(&out, fl, blob); + assert(writer.open == 0); git_filter_list_free(fl); + p_close(fd); - if (!error) { - error = buffer_to_file(data, st, &out, path, file_mode); - st->st_mode = entry_filemode; + if (error < 0) + return error; + + if (GIT_PERMS_IS_EXEC(mode)) { + data->perfdata.chmod_calls++; + + if ((error = p_chmod(path, mode)) < 0) { + giterr_set(GITERR_OS, "Failed to set permissions on '%s'", path); + return error; + } + } - git_buf_free(&out); + if (st) { + data->perfdata.stat_calls++; + + if ((error = p_stat(path, st)) < 0) { + giterr_set(GITERR_OS, "Error statting '%s'", path); + return error; + } + + st->st_mode = entry_filemode; } - return error; + return 0; } static int blob_content_to_link( diff --git a/src/filter.c b/src/filter.c index 700afbac33e..d930b239999 100644 --- a/src/filter.c +++ b/src/filter.c @@ -522,6 +522,7 @@ int git_filter_list__load_with_attr_session( fe = git_array_alloc(fl->filters); GITERR_CHECK_ALLOC(fe); fe->filter = fdef->filter; + fe->stream = NULL; fe->payload = payload; } } @@ -590,6 +591,7 @@ int git_filter_list_push( fe = git_array_alloc(fl->filters); GITERR_CHECK_ALLOC(fe); fe->filter = filter; + fe->stream = NULL; fe->payload = payload; return 0; From 646364e780c65b1af2e28a7bdeabd22b67816454 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Tue, 17 Feb 2015 20:25:31 +0000 Subject: [PATCH 88/99] checkout: maintain temporary buffer for filters Let the filters use the checkout data's temporary buffer, instead of having to allocate new buffers each time. --- src/checkout.c | 3 +++ src/filter.c | 33 +++++++++++++++++++++------------ src/filter.h | 3 +++ 3 files changed, 27 insertions(+), 12 deletions(-) diff --git a/src/checkout.c b/src/checkout.c index 623ac92a158..35129d77142 100644 --- a/src/checkout.c +++ b/src/checkout.c @@ -1444,6 +1444,9 @@ static int blob_content_to_file( GIT_FILTER_TO_WORKTREE, GIT_FILTER_OPT_DEFAULT))) return error; + if (fl) + git_filter_list__set_temp_buf(fl, &data->tmp); + /* setup the writer */ memset(&writer, 0, sizeof(struct checkout_stream)); writer.base.write = checkout_stream_write; diff --git a/src/filter.c b/src/filter.c index d930b239999..2cae24e2090 100644 --- a/src/filter.c +++ b/src/filter.c @@ -34,6 +34,7 @@ typedef struct { struct git_filter_list { git_array_t(git_filter_entry) filters; git_filter_source source; + git_buf *temp_buf; char path[GIT_FLEX_ARRAY]; }; @@ -522,7 +523,6 @@ int git_filter_list__load_with_attr_session( fe = git_array_alloc(fl->filters); GITERR_CHECK_ALLOC(fe); fe->filter = fdef->filter; - fe->stream = NULL; fe->payload = payload; } } @@ -549,6 +549,11 @@ int git_filter_list_load( filters, repo, NULL, blob, path, mode, options); } +void git_filter_list__set_temp_buf(git_filter_list *fl, git_buf *temp_buf) +{ + fl->temp_buf = temp_buf; +} + void git_filter_list_free(git_filter_list *fl) { uint32_t i; @@ -591,7 +596,6 @@ int git_filter_list_push( fe = git_array_alloc(fl->filters); GITERR_CHECK_ALLOC(fe); fe->filter = filter; - fe->stream = NULL; fe->payload = payload; return 0; @@ -745,7 +749,8 @@ struct proxy_stream { const git_filter_source *source; void **payload; git_buf input; - git_buf output; + git_buf temp_buf; + git_buf *output; git_filter_stream *target; }; @@ -769,15 +774,15 @@ static int proxy_stream_close(git_filter_stream *s) error = proxy_stream->filter->apply( proxy_stream->filter, proxy_stream->payload, - &proxy_stream->output, + proxy_stream->output, &proxy_stream->input, proxy_stream->source); if (error == GIT_PASSTHROUGH) { writebuf = &proxy_stream->input; } else if (error == 0) { - git_buf_sanitize(&proxy_stream->output); - writebuf = &proxy_stream->output; + git_buf_sanitize(proxy_stream->output); + writebuf = proxy_stream->output; } else { return error; } @@ -795,13 +800,14 @@ static void proxy_stream_free(git_filter_stream *s) assert(proxy_stream); git_buf_free(&proxy_stream->input); - git_buf_free(&proxy_stream->output); + git_buf_free(&proxy_stream->temp_buf); git__free(proxy_stream); } static int proxy_stream_init( git_filter_stream **out, git_filter *filter, + git_buf *temp_buf, void **payload, const git_filter_source *source, git_filter_stream *target) @@ -816,6 +822,7 @@ static int proxy_stream_init( proxy_stream->payload = payload; proxy_stream->source = source; proxy_stream->target = target; + proxy_stream->output = temp_buf ? temp_buf : &proxy_stream->temp_buf; *out = (git_filter_stream *)proxy_stream; return 0; @@ -844,18 +851,20 @@ static int stream_list_init( git_array_size(filters->filters) - 1 - i : i; git_filter_entry *fe = git_array_get(filters->filters, filter_idx); git_filter_stream *filter_stream; - git_filter_stream_fn stream_init; assert(fe->filter->stream || fe->filter->apply); /* If necessary, create a stream that proxies the traditional * application. */ - stream_init = fe->filter->stream ? - fe->filter->stream : proxy_stream_init; - - error = stream_init(&filter_stream, fe->filter, + if (fe->filter->stream) + error = fe->filter->stream(&filter_stream, fe->filter, &fe->payload, &filters->source, last_stream); + else + /* Create a stream that proxies the one-shot apply */ + error = proxy_stream_init(&filter_stream, fe->filter, + filters->temp_buf, &fe->payload, &filters->source, + last_stream); if (error < 0) return error; diff --git a/src/filter.h b/src/filter.h index 390ffebad6a..53f3afdbbd9 100644 --- a/src/filter.h +++ b/src/filter.h @@ -24,6 +24,9 @@ typedef enum { GIT_CRLF_AUTO, } git_crlf_t; +extern void git_filter_list__set_temp_buf( + git_filter_list *fl, git_buf *temp_buf); + extern void git_filter_free(git_filter *filter); extern int git_filter_list__load_with_attr_session( From 8c2dfb38dbc782a6b964bfcfd43e4f46bb71526e Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Tue, 17 Feb 2015 13:27:06 -0500 Subject: [PATCH 89/99] filter: test a large file through the stream Test pushing a file on-disk into a streaming filter that compresses it into the ODB, and inflates it back into the working directory. --- tests/filter/stream.c | 221 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 221 insertions(+) create mode 100644 tests/filter/stream.c diff --git a/tests/filter/stream.c b/tests/filter/stream.c new file mode 100644 index 00000000000..ff5361a3723 --- /dev/null +++ b/tests/filter/stream.c @@ -0,0 +1,221 @@ +#include "clar_libgit2.h" +#include "posix.h" +#include "blob.h" +#include "filter.h" +#include "buf_text.h" +#include "git2/sys/filter.h" +#include "git2/sys/repository.h" + +static git_repository *g_repo = NULL; + +static git_filter *create_compress_filter(void); +static git_filter *compress_filter; + +void test_filter_stream__initialize(void) +{ + compress_filter = create_compress_filter(); + + cl_git_pass(git_filter_register("compress", compress_filter, 50)); + g_repo = cl_git_sandbox_init("empty_standard_repo"); +} + +void test_filter_stream__cleanup(void) +{ + cl_git_sandbox_cleanup(); + g_repo = NULL; + + git_filter_unregister("compress"); +} + +#define CHUNKSIZE 10240 + +struct compress_stream { + git_filter_stream base; + git_filter_stream *next; + git_filter_mode_t mode; + char current; + size_t current_chunk; +}; + +static int compress_stream_write__deflated(struct compress_stream *stream, const char *buffer, size_t len) +{ + size_t idx = 0; + + while (len > 0) { + size_t chunkremain, chunksize; + + if (stream->current_chunk == 0) + stream->current = buffer[idx]; + + chunkremain = CHUNKSIZE - stream->current_chunk; + chunksize = min(chunkremain, len); + + stream->current_chunk += chunksize; + len -= chunksize; + idx += chunksize; + + if (stream->current_chunk == CHUNKSIZE) { + cl_git_pass(stream->next->write(stream->next, &stream->current, 1)); + stream->current_chunk = 0; + } + } + + return 0; +} + +static int compress_stream_write__inflated(struct compress_stream *stream, const char *buffer, size_t len) +{ + char inflated[CHUNKSIZE]; + size_t i, j; + + for (i = 0; i < len; i++) { + for (j = 0; j < CHUNKSIZE; j++) + inflated[j] = buffer[i]; + + cl_git_pass(stream->next->write(stream->next, inflated, CHUNKSIZE)); + } + + return 0; +} + +static int compress_stream_write(git_filter_stream *s, const char *buffer, size_t len) +{ + struct compress_stream *stream = (struct compress_stream *)s; + + return (stream->mode == GIT_FILTER_TO_ODB) ? + compress_stream_write__deflated(stream, buffer, len) : + compress_stream_write__inflated(stream, buffer, len); +} + +static int compress_stream_close(git_filter_stream *s) +{ + struct compress_stream *stream = (struct compress_stream *)s; + cl_assert_equal_i(0, stream->current_chunk); + stream->next->close(stream->next); + return 0; +} + +static void compress_stream_free(git_filter_stream *stream) +{ + git__free(stream); +} + +static int compress_filter_stream_init( + git_filter_stream **out, + git_filter *self, + void **payload, + const git_filter_source *src, + git_filter_stream *next) +{ + struct compress_stream *stream = git__calloc(1, sizeof(struct compress_stream)); + cl_assert(stream); + + GIT_UNUSED(self); + GIT_UNUSED(payload); + + stream->base.write = compress_stream_write; + stream->base.close = compress_stream_close; + stream->base.free = compress_stream_free; + stream->next = next; + stream->mode = git_filter_source_mode(src); + + *out = (git_filter_stream *)stream; + return 0; +} + +static void compress_filter_free(git_filter *f) +{ + git__free(f); +} + +git_filter *create_compress_filter(void) +{ + git_filter *filter = git__calloc(1, sizeof(git_filter)); + cl_assert(filter); + + filter->version = GIT_FILTER_VERSION; + filter->attributes = "+compress"; + filter->stream = compress_filter_stream_init; + filter->shutdown = compress_filter_free; + + return filter; +} + +static void writefile(const char *filename, size_t numchunks) +{ + git_buf path = GIT_BUF_INIT; + char buf[CHUNKSIZE]; + size_t i = 0, j = 0; + int fd; + + cl_git_pass(git_buf_joinpath(&path, "empty_standard_repo", filename)); + + fd = p_open(path.ptr, O_RDWR|O_CREAT, 0666); + cl_assert(fd >= 0); + + for (i = 0; i < numchunks; i++) { + for (j = 0; j < CHUNKSIZE; j++) { + buf[j] = i % 256; + } + + cl_git_pass(p_write(fd, buf, CHUNKSIZE)); + } + p_close(fd); + + git_buf_free(&path); +} + +static void test_stream(size_t numchunks) +{ + git_index *index; + const git_index_entry *entry; + git_blob *blob; + struct stat st; + git_checkout_options checkout_opts = GIT_CHECKOUT_OPTIONS_INIT; + + checkout_opts.checkout_strategy = GIT_CHECKOUT_FORCE; + + cl_git_mkfile( + "empty_standard_repo/.gitattributes", + "* compress\n"); + + /* write a file to disk */ + writefile("streamed_file", numchunks); + + /* place it in the index */ + cl_git_pass(git_repository_index(&index, g_repo)); + cl_git_pass(git_index_add_bypath(index, "streamed_file")); + cl_git_pass(git_index_write(index)); + + /* ensure it was appropriately compressed */ + cl_assert(entry = git_index_get_bypath(index, "streamed_file", 0)); + + cl_git_pass(git_blob_lookup(&blob, g_repo, &entry->id)); + cl_assert_equal_i(numchunks, git_blob_rawsize(blob)); + + /* check the file back out */ + cl_must_pass(p_unlink("empty_standard_repo/streamed_file")); + cl_git_pass(git_checkout_index(g_repo, index, &checkout_opts)); + + /* ensure it was decompressed */ + cl_must_pass(p_stat("empty_standard_repo/streamed_file", &st)); + cl_assert_equal_sz((numchunks * CHUNKSIZE), st.st_size); + + git_index_free(index); + git_blob_free(blob); +} + +/* write a 50KB file through the "compression" stream */ +void test_filter_stream__smallfile(void) +{ + test_stream(5); +} + +/* optionally write a 500 MB file through the compression stream */ +void test_filter_stream__bigfile(void) +{ + if (!cl_getenv("GITTEST_INVASIVE_FILESYSTEM")) + cl_skip(); + + test_stream(51200); +} From b75f15aaf1587594cd398b192cdc40aa83ba8e23 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Wed, 18 Feb 2015 09:25:32 -0500 Subject: [PATCH 90/99] git_writestream: from git_filter_stream --- include/git2/filter.h | 6 ++--- include/git2/sys/filter.h | 10 ++------ include/git2/types.h | 10 ++++++-- src/checkout.c | 10 ++++---- src/filter.c | 50 +++++++++++++++++++-------------------- tests/filter/stream.c | 16 ++++++------- 6 files changed, 51 insertions(+), 51 deletions(-) diff --git a/include/git2/filter.h b/include/git2/filter.h index 600356c7155..c0aaf1508f6 100644 --- a/include/git2/filter.h +++ b/include/git2/filter.h @@ -140,18 +140,18 @@ GIT_EXTERN(int) git_filter_list_apply_to_blob( GIT_EXTERN(int) git_filter_list_stream_data( git_filter_list *filters, git_buf *data, - git_filter_stream *target); + git_writestream *target); GIT_EXTERN(int) git_filter_list_stream_file( git_filter_list *filters, git_repository *repo, const char *path, - git_filter_stream *target); + git_writestream *target); GIT_EXTERN(int) git_filter_list_stream_blob( git_filter_list *filters, git_blob *blob, - git_filter_stream *target); + git_writestream *target); /** * Free a git_filter_list diff --git a/include/git2/sys/filter.h b/include/git2/sys/filter.h index cc06c54ad44..9b560fa7581 100644 --- a/include/git2/sys/filter.h +++ b/include/git2/sys/filter.h @@ -208,18 +208,12 @@ typedef int (*git_filter_apply_fn)( const git_buf *from, const git_filter_source *src); -struct git_filter_stream { - int (*write)(git_filter_stream *stream, const char *buffer, size_t len); - int (*close)(git_filter_stream *stream); - void (*free)(git_filter_stream *stream); -}; - typedef int (*git_filter_stream_fn)( - git_filter_stream **out, + git_writestream **out, git_filter *self, void **payload, const git_filter_source *src, - git_filter_stream *next); + git_writestream *next); /** * Callback to clean up after filtering has been applied diff --git a/include/git2/types.h b/include/git2/types.h index 3ed97d18967..c90ac477651 100644 --- a/include/git2/types.h +++ b/include/git2/types.h @@ -410,8 +410,14 @@ typedef enum { GIT_SUBMODULE_RECURSE_ONDEMAND = 2, } git_submodule_recurse_t; -/** A stream to write filters. */ -typedef struct git_filter_stream git_filter_stream; +/** A type to write in a streaming fashion, for example, for filters. */ +typedef struct git_writestream git_writestream; + +struct git_writestream { + int (*write)(git_writestream *stream, const char *buffer, size_t len); + int (*close)(git_writestream *stream); + void (*free)(git_writestream *stream); +}; /** @} */ GIT_END_DECL diff --git a/src/checkout.c b/src/checkout.c index 35129d77142..1585a4fb58c 100644 --- a/src/checkout.c +++ b/src/checkout.c @@ -1373,14 +1373,14 @@ static int mkpath2file( } struct checkout_stream { - git_filter_stream base; + git_writestream base; const char *path; int fd; int open; }; static int checkout_stream_write( - git_filter_stream *s, const char *buffer, size_t len) + git_writestream *s, const char *buffer, size_t len) { struct checkout_stream *stream = (struct checkout_stream *)s; int ret; @@ -1391,7 +1391,7 @@ static int checkout_stream_write( return ret; } -static int checkout_stream_close(git_filter_stream *s) +static int checkout_stream_close(git_writestream *s) { struct checkout_stream *stream = (struct checkout_stream *)s; assert(stream && stream->open); @@ -1400,7 +1400,7 @@ static int checkout_stream_close(git_filter_stream *s) return 0; } -static void checkout_stream_free(git_filter_stream *s) +static void checkout_stream_free(git_writestream *s) { GIT_UNUSED(s); } @@ -1456,7 +1456,7 @@ static int blob_content_to_file( writer.fd = fd; writer.open = 1; - error = git_filter_list_stream_blob(fl, blob, (git_filter_stream *)&writer); + error = git_filter_list_stream_blob(fl, blob, (git_writestream *)&writer); assert(writer.open == 0); diff --git a/src/filter.c b/src/filter.c index 2cae24e2090..af5902e0653 100644 --- a/src/filter.c +++ b/src/filter.c @@ -624,13 +624,13 @@ static int filter_list_out_buffer_from_raw( } struct buf_stream { - git_filter_stream base; + git_writestream base; git_buf *target; bool complete; }; static int buf_stream_write( - git_filter_stream *s, const char *buffer, size_t len) + git_writestream *s, const char *buffer, size_t len) { struct buf_stream *buf_stream = (struct buf_stream *)s; assert(buf_stream); @@ -640,7 +640,7 @@ static int buf_stream_write( return git_buf_put(buf_stream->target, buffer, len); } -static int buf_stream_close(git_filter_stream *s) +static int buf_stream_close(git_writestream *s) { struct buf_stream *buf_stream = (struct buf_stream *)s; assert(buf_stream); @@ -651,7 +651,7 @@ static int buf_stream_close(git_filter_stream *s) return 0; } -static void buf_stream_free(git_filter_stream *s) +static void buf_stream_free(git_writestream *s) { GIT_UNUSED(s); } @@ -683,7 +683,7 @@ int git_filter_list_apply_to_data( buf_stream_init(&writer, tgt); if ((error = git_filter_list_stream_data(filters, src, - (git_filter_stream *)&writer)) < 0) + (git_writestream *)&writer)) < 0) return error; assert(writer.complete); @@ -702,7 +702,7 @@ int git_filter_list_apply_to_file( buf_stream_init(&writer, out); if ((error = git_filter_list_stream_file( - filters, repo, path, (git_filter_stream *)&writer)) < 0) + filters, repo, path, (git_writestream *)&writer)) < 0) return error; assert(writer.complete); @@ -736,7 +736,7 @@ int git_filter_list_apply_to_blob( buf_stream_init(&writer, out); if ((error = git_filter_list_stream_blob( - filters, blob, (git_filter_stream *)&writer)) < 0) + filters, blob, (git_writestream *)&writer)) < 0) return error; assert(writer.complete); @@ -744,18 +744,18 @@ int git_filter_list_apply_to_blob( } struct proxy_stream { - git_filter_stream base; + git_writestream base; git_filter *filter; const git_filter_source *source; void **payload; git_buf input; git_buf temp_buf; git_buf *output; - git_filter_stream *target; + git_writestream *target; }; static int proxy_stream_write( - git_filter_stream *s, const char *buffer, size_t len) + git_writestream *s, const char *buffer, size_t len) { struct proxy_stream *proxy_stream = (struct proxy_stream *)s; assert(proxy_stream); @@ -763,7 +763,7 @@ static int proxy_stream_write( return git_buf_put(&proxy_stream->input, buffer, len); } -static int proxy_stream_close(git_filter_stream *s) +static int proxy_stream_close(git_writestream *s) { struct proxy_stream *proxy_stream = (struct proxy_stream *)s; git_buf *writebuf; @@ -794,7 +794,7 @@ static int proxy_stream_close(git_filter_stream *s) return error; } -static void proxy_stream_free(git_filter_stream *s) +static void proxy_stream_free(git_writestream *s) { struct proxy_stream *proxy_stream = (struct proxy_stream *)s; assert(proxy_stream); @@ -805,12 +805,12 @@ static void proxy_stream_free(git_filter_stream *s) } static int proxy_stream_init( - git_filter_stream **out, + git_writestream **out, git_filter *filter, git_buf *temp_buf, void **payload, const git_filter_source *source, - git_filter_stream *target) + git_writestream *target) { struct proxy_stream *proxy_stream = git__calloc(1, sizeof(struct proxy_stream)); GITERR_CHECK_ALLOC(proxy_stream); @@ -824,17 +824,17 @@ static int proxy_stream_init( proxy_stream->target = target; proxy_stream->output = temp_buf ? temp_buf : &proxy_stream->temp_buf; - *out = (git_filter_stream *)proxy_stream; + *out = (git_writestream *)proxy_stream; return 0; } static int stream_list_init( - git_filter_stream **out, + git_writestream **out, git_vector *streams, git_filter_list *filters, - git_filter_stream *target) + git_writestream *target) { - git_filter_stream *last_stream = target; + git_writestream *last_stream = target; size_t i; int error = 0; @@ -850,7 +850,7 @@ static int stream_list_init( size_t filter_idx = (filters->source.mode == GIT_FILTER_TO_WORKTREE) ? git_array_size(filters->filters) - 1 - i : i; git_filter_entry *fe = git_array_get(filters->filters, filter_idx); - git_filter_stream *filter_stream; + git_writestream *filter_stream; assert(fe->filter->stream || fe->filter->apply); @@ -879,7 +879,7 @@ static int stream_list_init( void stream_list_free(git_vector *streams) { - git_filter_stream *stream; + git_writestream *stream; size_t i; git_vector_foreach(streams, i, stream) @@ -894,13 +894,13 @@ int git_filter_list_stream_file( git_filter_list *filters, git_repository *repo, const char *path, - git_filter_stream *target) + git_writestream *target) { char buf[STREAM_BUFSIZE]; git_buf abspath = GIT_BUF_INIT; const char *base = repo ? git_repository_workdir(repo) : NULL; git_vector filter_streams = GIT_VECTOR_INIT; - git_filter_stream *stream_start; + git_writestream *stream_start; ssize_t readlen; int fd, error; @@ -935,10 +935,10 @@ int git_filter_list_stream_file( int git_filter_list_stream_data( git_filter_list *filters, git_buf *data, - git_filter_stream *target) + git_writestream *target) { git_vector filter_streams = GIT_VECTOR_INIT; - git_filter_stream *stream_start; + git_writestream *stream_start; int error = 0; git_buf_sanitize(data); @@ -956,7 +956,7 @@ int git_filter_list_stream_data( int git_filter_list_stream_blob( git_filter_list *filters, git_blob *blob, - git_filter_stream *target) + git_writestream *target) { git_buf in = GIT_BUF_INIT; diff --git a/tests/filter/stream.c b/tests/filter/stream.c index ff5361a3723..f7456b8a475 100644 --- a/tests/filter/stream.c +++ b/tests/filter/stream.c @@ -30,8 +30,8 @@ void test_filter_stream__cleanup(void) #define CHUNKSIZE 10240 struct compress_stream { - git_filter_stream base; - git_filter_stream *next; + git_writestream base; + git_writestream *next; git_filter_mode_t mode; char current; size_t current_chunk; @@ -78,7 +78,7 @@ static int compress_stream_write__inflated(struct compress_stream *stream, const return 0; } -static int compress_stream_write(git_filter_stream *s, const char *buffer, size_t len) +static int compress_stream_write(git_writestream *s, const char *buffer, size_t len) { struct compress_stream *stream = (struct compress_stream *)s; @@ -87,7 +87,7 @@ static int compress_stream_write(git_filter_stream *s, const char *buffer, size_ compress_stream_write__inflated(stream, buffer, len); } -static int compress_stream_close(git_filter_stream *s) +static int compress_stream_close(git_writestream *s) { struct compress_stream *stream = (struct compress_stream *)s; cl_assert_equal_i(0, stream->current_chunk); @@ -95,17 +95,17 @@ static int compress_stream_close(git_filter_stream *s) return 0; } -static void compress_stream_free(git_filter_stream *stream) +static void compress_stream_free(git_writestream *stream) { git__free(stream); } static int compress_filter_stream_init( - git_filter_stream **out, + git_writestream **out, git_filter *self, void **payload, const git_filter_source *src, - git_filter_stream *next) + git_writestream *next) { struct compress_stream *stream = git__calloc(1, sizeof(struct compress_stream)); cl_assert(stream); @@ -119,7 +119,7 @@ static int compress_filter_stream_init( stream->next = next; stream->mode = git_filter_source_mode(src); - *out = (git_filter_stream *)stream; + *out = (git_writestream *)stream; return 0; } From f7c0125f47ce23d52685648e076e663bc3a8939e Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Wed, 18 Feb 2015 09:28:07 -0500 Subject: [PATCH 91/99] filter streams: base -> parent --- src/filter.c | 16 ++++++++-------- tests/filter/stream.c | 8 ++++---- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/filter.c b/src/filter.c index af5902e0653..08aa14b1bac 100644 --- a/src/filter.c +++ b/src/filter.c @@ -624,7 +624,7 @@ static int filter_list_out_buffer_from_raw( } struct buf_stream { - git_writestream base; + git_writestream parent; git_buf *target; bool complete; }; @@ -660,9 +660,9 @@ static void buf_stream_init(struct buf_stream *writer, git_buf *target) { memset(writer, 0, sizeof(struct buf_stream)); - writer->base.write = buf_stream_write; - writer->base.close = buf_stream_close; - writer->base.free = buf_stream_free; + writer->parent.write = buf_stream_write; + writer->parent.close = buf_stream_close; + writer->parent.free = buf_stream_free; writer->target = target; git_buf_clear(target); @@ -744,7 +744,7 @@ int git_filter_list_apply_to_blob( } struct proxy_stream { - git_writestream base; + git_writestream parent; git_filter *filter; const git_filter_source *source; void **payload; @@ -815,9 +815,9 @@ static int proxy_stream_init( struct proxy_stream *proxy_stream = git__calloc(1, sizeof(struct proxy_stream)); GITERR_CHECK_ALLOC(proxy_stream); - proxy_stream->base.write = proxy_stream_write; - proxy_stream->base.close = proxy_stream_close; - proxy_stream->base.free = proxy_stream_free; + proxy_stream->parent.write = proxy_stream_write; + proxy_stream->parent.close = proxy_stream_close; + proxy_stream->parent.free = proxy_stream_free; proxy_stream->filter = filter; proxy_stream->payload = payload; proxy_stream->source = source; diff --git a/tests/filter/stream.c b/tests/filter/stream.c index f7456b8a475..4dafce42ea1 100644 --- a/tests/filter/stream.c +++ b/tests/filter/stream.c @@ -30,7 +30,7 @@ void test_filter_stream__cleanup(void) #define CHUNKSIZE 10240 struct compress_stream { - git_writestream base; + git_writestream parent; git_writestream *next; git_filter_mode_t mode; char current; @@ -113,9 +113,9 @@ static int compress_filter_stream_init( GIT_UNUSED(self); GIT_UNUSED(payload); - stream->base.write = compress_stream_write; - stream->base.close = compress_stream_close; - stream->base.free = compress_stream_free; + stream->parent.write = compress_stream_write; + stream->parent.close = compress_stream_close; + stream->parent.free = compress_stream_free; stream->next = next; stream->mode = git_filter_source_mode(src); From b49edddcd0af70caee7eb4905dc26199adccabfd Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Wed, 18 Feb 2015 09:40:52 -0500 Subject: [PATCH 92/99] checkout: let the stream writer close the fd --- src/checkout.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/checkout.c b/src/checkout.c index 1585a4fb58c..e48a8557c44 100644 --- a/src/checkout.c +++ b/src/checkout.c @@ -1397,7 +1397,7 @@ static int checkout_stream_close(git_writestream *s) assert(stream && stream->open); stream->open = 0; - return 0; + return p_close(stream->fd); } static void checkout_stream_free(git_writestream *s) @@ -1461,7 +1461,6 @@ static int blob_content_to_file( assert(writer.open == 0); git_filter_list_free(fl); - p_close(fd); if (error < 0) return error; From d4cf167515d3ed7b27c1358fc2e19b9caf66e8ad Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Thu, 19 Feb 2015 10:05:33 -0500 Subject: [PATCH 93/99] buffer: introduce git_buf_attach_notowned Provide a convenience function that creates a buffer that can be provided to callers but will not be freed via `git_buf_free`, so the buffer creator maintains the allocation lifecycle of the buffer's contents. --- src/attr.c | 5 ++--- src/blob.c | 10 ++++------ src/buffer.c | 14 ++++++++++++++ src/buffer.h | 6 ++++++ src/diff_driver.c | 9 ++++----- src/filter.c | 28 +++++----------------------- 6 files changed, 35 insertions(+), 37 deletions(-) diff --git a/src/attr.c b/src/attr.c index 44593da8132..38420807a53 100644 --- a/src/attr.c +++ b/src/attr.c @@ -282,9 +282,8 @@ static int system_attr_file( * a consumer. This allows them to treat this as a regular `git_buf`, * but their call to `git_buf_free` will not attempt to free it. */ - out->ptr = attr_session->sysdir.ptr; - out->size = attr_session->sysdir.size; - out->asize = 0; + git_buf_attach_notowned( + out, attr_session->sysdir.ptr, attr_session->sysdir.size); return 0; } diff --git a/src/blob.c b/src/blob.c index 30d5b705b90..ba8769cdcf8 100644 --- a/src/blob.c +++ b/src/blob.c @@ -329,15 +329,13 @@ int git_blob_create_fromchunks( int git_blob_is_binary(const git_blob *blob) { - git_buf content; + git_buf content = GIT_BUF_INIT; assert(blob); - content.ptr = blob->odb_object->buffer; - content.size = - min(blob->odb_object->cached.size, GIT_FILTER_BYTES_TO_CHECK_NUL); - content.asize = 0; - + git_buf_attach_notowned(&content, blob->odb_object->buffer, + min(blob->odb_object->cached.size, + GIT_FILTER_BYTES_TO_CHECK_NUL)); return git_buf_text_is_binary(&content); } diff --git a/src/buffer.c b/src/buffer.c index 3deb0329c34..f633c5e02fc 100644 --- a/src/buffer.c +++ b/src/buffer.c @@ -500,6 +500,20 @@ void git_buf_attach(git_buf *buf, char *ptr, size_t asize) } } +void git_buf_attach_notowned(git_buf *buf, const char *ptr, size_t size) +{ + if (git_buf_is_allocated(buf)) + git_buf_free(buf); + + if (!size) { + git_buf_init(buf, 0); + } else { + buf->ptr = (char *)ptr; + buf->asize = 0; + buf->size = size; + } +} + int git_buf_join_n(git_buf *buf, char separator, int nbuf, ...) { va_list ap; diff --git a/src/buffer.h b/src/buffer.h index 52342e30921..093ed9b6058 100644 --- a/src/buffer.h +++ b/src/buffer.h @@ -74,6 +74,12 @@ extern void git_buf_swap(git_buf *buf_a, git_buf *buf_b); extern char *git_buf_detach(git_buf *buf); extern void git_buf_attach(git_buf *buf, char *ptr, size_t asize); +/* Populates a `git_buf` where the contents are not "owned" by the + * buffer, and calls to `git_buf_free` will not free the given buf. + */ +extern void git_buf_attach_notowned( + git_buf *buf, const char *ptr, size_t size); + /** * Test if there have been any reallocation failures with this git_buf. * diff --git a/src/diff_driver.c b/src/diff_driver.c index 7313ab573dd..049e6ef2a70 100644 --- a/src/diff_driver.c +++ b/src/diff_driver.c @@ -418,14 +418,13 @@ void git_diff_driver_update_options( int git_diff_driver_content_is_binary( git_diff_driver *driver, const char *content, size_t content_len) { - git_buf search; - - search.ptr = (char *)content; - search.size = min(content_len, GIT_FILTER_BYTES_TO_CHECK_NUL); - search.asize = 0; + git_buf search = GIT_BUF_INIT; GIT_UNUSED(driver); + git_buf_attach_notowned(&search, content, + min(content_len, GIT_FILTER_BYTES_TO_CHECK_NUL)); + /* TODO: provide encoding / binary detection callbacks that can * be UTF-8 aware, etc. For now, instead of trying to be smart, * let's just use the simple NUL-byte detection that core git uses. diff --git a/src/filter.c b/src/filter.c index 08aa14b1bac..4009a61f5a4 100644 --- a/src/filter.c +++ b/src/filter.c @@ -606,23 +606,6 @@ size_t git_filter_list_length(const git_filter_list *fl) return fl ? git_array_size(fl->filters) : 0; } -static int filter_list_out_buffer_from_raw( - git_buf *out, const void *ptr, size_t size) -{ - if (git_buf_is_allocated(out)) - git_buf_free(out); - - if (!size) { - git_buf_init(out, 0); - } else { - out->ptr = (char *)ptr; - out->asize = 0; - out->size = size; - } - - return 0; -} - struct buf_stream { git_writestream parent; git_buf *target; @@ -677,8 +660,10 @@ int git_filter_list_apply_to_data( git_buf_sanitize(tgt); git_buf_sanitize(src); - if (!filters) - return filter_list_out_buffer_from_raw(tgt, src->ptr, src->size); + if (!filters) { + git_buf_attach_notowned(tgt, src->ptr, src->size); + return 0; + } buf_stream_init(&writer, tgt); @@ -718,10 +703,7 @@ static int buf_from_blob(git_buf *out, git_blob *blob) return -1; } - out->ptr = (char *)git_blob_rawcontent(blob); - out->asize = 0; - out->size = (size_t)rawsize; - + git_buf_attach_notowned(out, git_blob_rawcontent(blob), (size_t)rawsize); return 0; } From 795eaccd667ce24c290b6211ca0c6a84f84e7c2b Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Thu, 19 Feb 2015 11:09:54 -0500 Subject: [PATCH 94/99] git_filter_opt_t -> git_filter_flag_t For consistency with the rest of the library, where an opt is an options *structure*. --- include/git2/filter.h | 10 +++++----- include/git2/sys/filter.h | 4 ++-- src/blob.c | 4 ++-- src/checkout.c | 4 ++-- src/crlf.c | 2 +- src/diff.c | 2 +- src/diff_file.c | 2 +- src/filter.c | 20 ++++++++++---------- src/filter.h | 2 +- src/repository.c | 2 +- tests/filter/crlf.c | 2 +- 11 files changed, 27 insertions(+), 27 deletions(-) diff --git a/include/git2/filter.h b/include/git2/filter.h index c0aaf1508f6..dc59e63411b 100644 --- a/include/git2/filter.h +++ b/include/git2/filter.h @@ -39,9 +39,9 @@ typedef enum { * Filter option flags. */ typedef enum { - GIT_FILTER_OPT_DEFAULT = 0u, - GIT_FILTER_OPT_ALLOW_UNSAFE = (1u << 0), -} git_filter_opt_t; + GIT_FILTER_DEFAULT = 0u, + GIT_FILTER_ALLOW_UNSAFE = (1u << 0), +} git_filter_flag_t; /** * A filter that can transform file data @@ -83,7 +83,7 @@ typedef struct git_filter_list git_filter_list; * @param blob The blob to which the filter will be applied (if known) * @param path Relative path of the file to be filtered * @param mode Filtering direction (WT->ODB or ODB->WT) - * @param options Combination of `git_filter_opt_t` flags + * @param flags Combination of `git_filter_flag_t` flags * @return 0 on success (which could still return NULL if no filters are * needed for the requested file), <0 on error */ @@ -93,7 +93,7 @@ GIT_EXTERN(int) git_filter_list_load( git_blob *blob, /* can be NULL */ const char *path, git_filter_mode_t mode, - uint32_t options); + uint32_t flags); /** * Apply filter list to a data buffer. diff --git a/include/git2/sys/filter.h b/include/git2/sys/filter.h index 9b560fa7581..5fd8d5566d8 100644 --- a/include/git2/sys/filter.h +++ b/include/git2/sys/filter.h @@ -123,9 +123,9 @@ GIT_EXTERN(const git_oid *) git_filter_source_id(const git_filter_source *src); GIT_EXTERN(git_filter_mode_t) git_filter_source_mode(const git_filter_source *src); /** - * Get the combination git_filter_opt_t options to be applied + * Get the combination git_filter_flag_t options to be applied */ -GIT_EXTERN(uint32_t) git_filter_source_options(const git_filter_source *src); +GIT_EXTERN(uint32_t) git_filter_source_flags(const git_filter_source *src); /* * struct git_filter diff --git a/src/blob.c b/src/blob.c index ba8769cdcf8..cf0329064eb 100644 --- a/src/blob.c +++ b/src/blob.c @@ -199,7 +199,7 @@ int git_blob__create_from_paths( /* Load the filters for writing this file to the ODB */ error = git_filter_list_load( &fl, repo, NULL, hint_path, - GIT_FILTER_TO_ODB, GIT_FILTER_OPT_DEFAULT); + GIT_FILTER_TO_ODB, GIT_FILTER_DEFAULT); if (error < 0) /* well, that didn't work */; @@ -357,7 +357,7 @@ int git_blob_filtered_content( if (!(error = git_filter_list_load( &fl, git_blob_owner(blob), blob, path, - GIT_FILTER_TO_WORKTREE, GIT_FILTER_OPT_DEFAULT))) { + GIT_FILTER_TO_WORKTREE, GIT_FILTER_DEFAULT))) { error = git_filter_list_apply_to_blob(out, fl, blob); diff --git a/src/checkout.c b/src/checkout.c index e48a8557c44..a4dd20999fb 100644 --- a/src/checkout.c +++ b/src/checkout.c @@ -1441,7 +1441,7 @@ static int blob_content_to_file( if (!data->opts.disable_filters && (error = git_filter_list__load_with_attr_session( &fl, data->repo, &data->attr_session, blob, hint_path, - GIT_FILTER_TO_WORKTREE, GIT_FILTER_OPT_DEFAULT))) + GIT_FILTER_TO_WORKTREE, GIT_FILTER_DEFAULT))) return error; if (fl) @@ -2054,7 +2054,7 @@ static int checkout_write_merge( if ((error = git_filter_list__load_with_attr_session( &fl, data->repo, &data->attr_session, NULL, git_buf_cstr(&path_workdir), - GIT_FILTER_TO_WORKTREE, GIT_FILTER_OPT_DEFAULT)) < 0 || + GIT_FILTER_TO_WORKTREE, GIT_FILTER_DEFAULT)) < 0 || (error = git_filter_list_apply_to_data(&out_data, fl, &in_data)) < 0) goto done; } else { diff --git a/src/crlf.c b/src/crlf.c index c0a73990fba..b5d1dbf321a 100644 --- a/src/crlf.c +++ b/src/crlf.c @@ -302,7 +302,7 @@ static int crlf_check( return error; /* downgrade FAIL to WARN if ALLOW_UNSAFE option is used */ - if ((git_filter_source_options(src) & GIT_FILTER_OPT_ALLOW_UNSAFE) && + if ((git_filter_source_flags(src) & GIT_FILTER_ALLOW_UNSAFE) && ca.safe_crlf == GIT_SAFE_CRLF_FAIL) ca.safe_crlf = GIT_SAFE_CRLF_WARN; } diff --git a/src/diff.c b/src/diff.c index 07eae03e7ca..815351b21f1 100644 --- a/src/diff.c +++ b/src/diff.c @@ -600,7 +600,7 @@ int git_diff__oid_for_entry( error = -1; } else if (!(error = git_filter_list_load( &fl, diff->repo, NULL, entry.path, - GIT_FILTER_TO_ODB, GIT_FILTER_OPT_ALLOW_UNSAFE))) + GIT_FILTER_TO_ODB, GIT_FILTER_ALLOW_UNSAFE))) { int fd = git_futils_open_ro(full_path.ptr); if (fd < 0) diff --git a/src/diff_file.c b/src/diff_file.c index 96be0942b0b..f7061ae8398 100644 --- a/src/diff_file.c +++ b/src/diff_file.c @@ -302,7 +302,7 @@ static int diff_file_content_load_workdir_file( if ((error = git_filter_list_load( &fl, fc->repo, NULL, fc->file->path, - GIT_FILTER_TO_ODB, GIT_FILTER_OPT_ALLOW_UNSAFE)) < 0) + GIT_FILTER_TO_ODB, GIT_FILTER_ALLOW_UNSAFE)) < 0) goto cleanup; /* if there are no filters, try to mmap the file */ diff --git a/src/filter.c b/src/filter.c index 4009a61f5a4..88877fead84 100644 --- a/src/filter.c +++ b/src/filter.c @@ -23,7 +23,7 @@ struct git_filter_source { git_oid oid; /* zero if unknown (which is likely) */ uint16_t filemode; /* zero if unknown */ git_filter_mode_t mode; - uint32_t options; + uint32_t flags; }; typedef struct { @@ -372,9 +372,9 @@ git_filter_mode_t git_filter_source_mode(const git_filter_source *src) return src->mode; } -uint32_t git_filter_source_options(const git_filter_source *src) +uint32_t git_filter_source_flags(const git_filter_source *src) { - return src->options; + return src->flags; } static int filter_list_new( @@ -394,7 +394,7 @@ static int filter_list_new( fl->source.repo = src->repo; fl->source.path = fl->path; fl->source.mode = src->mode; - fl->source.options = src->options; + fl->source.flags = src->flags; *out = fl; return 0; @@ -449,13 +449,13 @@ int git_filter_list_new( git_filter_list **out, git_repository *repo, git_filter_mode_t mode, - uint32_t options) + uint32_t flags) { git_filter_source src = { 0 }; src.repo = repo; src.path = NULL; src.mode = mode; - src.options = options; + src.flags = flags; return filter_list_new(out, &src); } @@ -466,7 +466,7 @@ int git_filter_list__load_with_attr_session( git_blob *blob, /* can be NULL */ const char *path, git_filter_mode_t mode, - uint32_t options) + uint32_t flags) { int error = 0; git_filter_list *fl = NULL; @@ -481,7 +481,7 @@ int git_filter_list__load_with_attr_session( src.repo = repo; src.path = path; src.mode = mode; - src.options = options; + src.flags = flags; if (blob) git_oid_cpy(&src.oid, git_blob_id(blob)); @@ -543,10 +543,10 @@ int git_filter_list_load( git_blob *blob, /* can be NULL */ const char *path, git_filter_mode_t mode, - uint32_t options) + uint32_t flags) { return git_filter_list__load_with_attr_session( - filters, repo, NULL, blob, path, mode, options); + filters, repo, NULL, blob, path, mode, flags); } void git_filter_list__set_temp_buf(git_filter_list *fl, git_buf *temp_buf) diff --git a/src/filter.h b/src/filter.h index 53f3afdbbd9..a38c6c8fab4 100644 --- a/src/filter.h +++ b/src/filter.h @@ -36,7 +36,7 @@ extern int git_filter_list__load_with_attr_session( git_blob *blob, /* can be NULL */ const char *path, git_filter_mode_t mode, - uint32_t options); + uint32_t flags); /* * Available filters diff --git a/src/repository.c b/src/repository.c index c9275078fda..23c99b0f001 100644 --- a/src/repository.c +++ b/src/repository.c @@ -1849,7 +1849,7 @@ int git_repository_hashfile( if (strlen(as_path) > 0) { error = git_filter_list_load( &fl, repo, NULL, as_path, - GIT_FILTER_TO_ODB, GIT_FILTER_OPT_DEFAULT); + GIT_FILTER_TO_ODB, GIT_FILTER_DEFAULT); if (error < 0) return error; } else { diff --git a/tests/filter/crlf.c b/tests/filter/crlf.c index a31dac9651f..406d3b6b0f7 100644 --- a/tests/filter/crlf.c +++ b/tests/filter/crlf.c @@ -123,7 +123,7 @@ void test_filter_crlf__with_safecrlf_and_unsafe_allowed(void) cl_repo_set_bool(g_repo, "core.safecrlf", true); cl_git_pass(git_filter_list_new( - &fl, g_repo, GIT_FILTER_TO_ODB, GIT_FILTER_OPT_ALLOW_UNSAFE)); + &fl, g_repo, GIT_FILTER_TO_ODB, GIT_FILTER_ALLOW_UNSAFE)); crlf = git_filter_lookup(GIT_FILTER_CRLF); cl_assert(crlf != NULL); From d05218b06ff201cd373fc764e0d87af67f7932c7 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Thu, 19 Feb 2015 11:25:26 -0500 Subject: [PATCH 95/99] filter: add `git_filter_list__load_ext` Refactor `git_filter_list__load_with_attr_reader` into `git_filter_list__load_ext`, which takes a `git_filter_options`. --- src/checkout.c | 18 ++++++++++++------ src/filter.c | 18 +++++++++++------- src/filter.h | 12 +++++++++--- 3 files changed, 32 insertions(+), 16 deletions(-) diff --git a/src/checkout.c b/src/checkout.c index a4dd20999fb..2c47147e837 100644 --- a/src/checkout.c +++ b/src/checkout.c @@ -1416,6 +1416,7 @@ static int blob_content_to_file( int flags = data->opts.file_open_flags; mode_t file_mode = data->opts.file_mode ? data->opts.file_mode : entry_filemode; + git_filter_options filter_opts = GIT_FILTER_OPTIONS_INIT; struct checkout_stream writer; mode_t mode; git_filter_list *fl = NULL; @@ -1438,10 +1439,12 @@ static int blob_content_to_file( return fd; } + filter_opts.attr_session = &data->attr_session; + if (!data->opts.disable_filters && - (error = git_filter_list__load_with_attr_session( - &fl, data->repo, &data->attr_session, blob, hint_path, - GIT_FILTER_TO_WORKTREE, GIT_FILTER_DEFAULT))) + (error = git_filter_list__load_ext( + &fl, data->repo, blob, hint_path, + GIT_FILTER_TO_WORKTREE, &filter_opts))) return error; if (fl) @@ -2003,6 +2006,7 @@ static int checkout_write_merge( git_merge_file_result result = {0}; git_filebuf output = GIT_FILEBUF_INIT; git_filter_list *fl = NULL; + git_filter_options filter_opts = GIT_FILTER_OPTIONS_INIT; int error = 0; if (data->opts.checkout_strategy & GIT_CHECKOUT_CONFLICT_STYLE_DIFF3) @@ -2052,9 +2056,11 @@ static int checkout_write_merge( in_data.ptr = (char *)result.ptr; in_data.size = result.len; - if ((error = git_filter_list__load_with_attr_session( - &fl, data->repo, &data->attr_session, NULL, git_buf_cstr(&path_workdir), - GIT_FILTER_TO_WORKTREE, GIT_FILTER_DEFAULT)) < 0 || + filter_opts.attr_session = &data->attr_session; + + if ((error = git_filter_list__load_ext( + &fl, data->repo, NULL, git_buf_cstr(&path_workdir), + GIT_FILTER_TO_WORKTREE, &filter_opts)) < 0 || (error = git_filter_list_apply_to_data(&out_data, fl, &in_data)) < 0) goto done; } else { diff --git a/src/filter.c b/src/filter.c index 88877fead84..646f1bc1153 100644 --- a/src/filter.c +++ b/src/filter.c @@ -459,14 +459,13 @@ int git_filter_list_new( return filter_list_new(out, &src); } -int git_filter_list__load_with_attr_session( +int git_filter_list__load_ext( git_filter_list **filters, git_repository *repo, - git_attr_session *attr_session, git_blob *blob, /* can be NULL */ const char *path, git_filter_mode_t mode, - uint32_t flags) + git_filter_options *filter_opts) { int error = 0; git_filter_list *fl = NULL; @@ -481,7 +480,8 @@ int git_filter_list__load_with_attr_session( src.repo = repo; src.path = path; src.mode = mode; - src.flags = flags; + src.flags = filter_opts->flags; + if (blob) git_oid_cpy(&src.oid, git_blob_id(blob)); @@ -494,7 +494,7 @@ int git_filter_list__load_with_attr_session( if (fdef->nattrs > 0) { error = filter_list_check_attributes( - &values, repo, attr_session, fdef, &src); + &values, repo, filter_opts->attr_session, fdef, &src); if (error == GIT_ENOTFOUND) { error = 0; @@ -545,8 +545,12 @@ int git_filter_list_load( git_filter_mode_t mode, uint32_t flags) { - return git_filter_list__load_with_attr_session( - filters, repo, NULL, blob, path, mode, flags); + git_filter_options filter_opts = GIT_FILTER_OPTIONS_INIT; + + filter_opts.flags = flags; + + return git_filter_list__load_ext( + filters, repo, blob, path, mode, &filter_opts); } void git_filter_list__set_temp_buf(git_filter_list *fl, git_buf *temp_buf) diff --git a/src/filter.h b/src/filter.h index a38c6c8fab4..85ef4a6f071 100644 --- a/src/filter.h +++ b/src/filter.h @@ -24,19 +24,25 @@ typedef enum { GIT_CRLF_AUTO, } git_crlf_t; +typedef struct { + git_attr_session *attr_session; + uint32_t flags; +} git_filter_options; + +#define GIT_FILTER_OPTIONS_INIT {0} + extern void git_filter_list__set_temp_buf( git_filter_list *fl, git_buf *temp_buf); extern void git_filter_free(git_filter *filter); -extern int git_filter_list__load_with_attr_session( +extern int git_filter_list__load_ext( git_filter_list **filters, git_repository *repo, - git_attr_session *attr_session, git_blob *blob, /* can be NULL */ const char *path, git_filter_mode_t mode, - uint32_t flags); + git_filter_options *filter_opts); /* * Available filters From 9c9aa1bad3b364f7dfdccd5f6abe880f7e18c499 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Thu, 19 Feb 2015 11:32:55 -0500 Subject: [PATCH 96/99] filter: take `temp_buf` in `git_filter_options` --- src/checkout.c | 5 ++--- src/filter.c | 13 ++++++------- src/filter.h | 4 +--- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/src/checkout.c b/src/checkout.c index 2c47147e837..f71be26f966 100644 --- a/src/checkout.c +++ b/src/checkout.c @@ -1440,6 +1440,7 @@ static int blob_content_to_file( } filter_opts.attr_session = &data->attr_session; + filter_opts.temp_buf = &data->tmp; if (!data->opts.disable_filters && (error = git_filter_list__load_ext( @@ -1447,9 +1448,6 @@ static int blob_content_to_file( GIT_FILTER_TO_WORKTREE, &filter_opts))) return error; - if (fl) - git_filter_list__set_temp_buf(fl, &data->tmp); - /* setup the writer */ memset(&writer, 0, sizeof(struct checkout_stream)); writer.base.write = checkout_stream_write; @@ -2057,6 +2055,7 @@ static int checkout_write_merge( in_data.size = result.len; filter_opts.attr_session = &data->attr_session; + filter_opts.temp_buf = &data->tmp; if ((error = git_filter_list__load_ext( &fl, data->repo, NULL, git_buf_cstr(&path_workdir), diff --git a/src/filter.c b/src/filter.c index 646f1bc1153..4fbf84f6a2b 100644 --- a/src/filter.c +++ b/src/filter.c @@ -517,8 +517,12 @@ int git_filter_list__load_ext( else if (error < 0) break; else { - if (!fl && (error = filter_list_new(&fl, &src)) < 0) - return error; + if (!fl) { + if ((error = filter_list_new(&fl, &src)) < 0) + return error; + + fl->temp_buf = filter_opts->temp_buf; + } fe = git_array_alloc(fl->filters); GITERR_CHECK_ALLOC(fe); @@ -553,11 +557,6 @@ int git_filter_list_load( filters, repo, blob, path, mode, &filter_opts); } -void git_filter_list__set_temp_buf(git_filter_list *fl, git_buf *temp_buf) -{ - fl->temp_buf = temp_buf; -} - void git_filter_list_free(git_filter_list *fl) { uint32_t i; diff --git a/src/filter.h b/src/filter.h index 85ef4a6f071..5062afba510 100644 --- a/src/filter.h +++ b/src/filter.h @@ -26,14 +26,12 @@ typedef enum { typedef struct { git_attr_session *attr_session; + git_buf *temp_buf; uint32_t flags; } git_filter_options; #define GIT_FILTER_OPTIONS_INIT {0} -extern void git_filter_list__set_temp_buf( - git_filter_list *fl, git_buf *temp_buf); - extern void git_filter_free(git_filter *filter); extern int git_filter_list__load_ext( From feb0e022867552c039c02fe39db7b7c3d63ae327 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Thu, 19 Feb 2015 12:14:06 -0500 Subject: [PATCH 97/99] tests: separate INVASIVE filesystem tests Introduce GITTEST_INVASIVE_FS_STRUCTURE for things that are invasive to your filesystem structure (like creating folders at your filesystem root) and GITTEST_INVASIVE_FS_SIZE for things that write lots of data. --- .travis.yml | 1 + appveyor.yml | 3 ++- tests/filter/stream.c | 2 +- tests/repo/init.c | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index fc513458b1e..68b29b1e263 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,6 +15,7 @@ compiler: env: global: - secure: "YnhS+8n6B+uoyaYfaJ3Lei7cSJqHDPiKJCKFIF2c87YDfmCvAJke8QtE7IzjYDs7UFkTCM4ox+ph2bERUrxZbSCyEkHdjIZpKuMJfYWja/jgMqTMxdyOH9y8JLFbZsSXDIXDwqBlC6vVyl1fP90M35wuWcNTs6tctfVWVofEFbs=" + - GITTEST_INVASIVE_FS_SIZE=1 matrix: - OPTIONS="-DTHREADSAFE=ON -DCMAKE_BUILD_TYPE=Release" - OPTIONS="-DTHREADSAFE=OFF -DBUILD_EXAMPLES=ON" diff --git a/appveyor.yml b/appveyor.yml index 8ac6728c3ab..d155485fdb1 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -3,7 +3,8 @@ branches: only: - master environment: - GITTEST_INVASIVE_FILESYSTEM: 1 + GITTEST_INVASIVE_FS_STRUCTURE: 1 + GITTEST_INVASIVE_FS_SIZE: 1 matrix: - GENERATOR: "Visual Studio 11" diff --git a/tests/filter/stream.c b/tests/filter/stream.c index 4dafce42ea1..603f1949476 100644 --- a/tests/filter/stream.c +++ b/tests/filter/stream.c @@ -214,7 +214,7 @@ void test_filter_stream__smallfile(void) /* optionally write a 500 MB file through the compression stream */ void test_filter_stream__bigfile(void) { - if (!cl_getenv("GITTEST_INVASIVE_FILESYSTEM")) + if (!cl_getenv("GITTEST_INVASIVE_FS_SIZE")) cl_skip(); test_stream(51200); diff --git a/tests/repo/init.c b/tests/repo/init.c index 91747c9f5d6..07615681752 100644 --- a/tests/repo/init.c +++ b/tests/repo/init.c @@ -722,7 +722,7 @@ void test_repo_init__at_filesystem_root(void) git_buf root = GIT_BUF_INIT; int root_len; - if (!cl_getenv("GITTEST_INVASIVE_FILESYSTEM")) + if (!cl_getenv("GITTEST_INVASIVE_FS_STRUCTURE")) cl_skip(); root_len = git_path_root(sandbox); From c92987d1575b93eac3b6fa4d1b4bc166137305ac Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 20 Feb 2015 10:21:32 -0500 Subject: [PATCH 98/99] Fix MAX 32 bit build problem described in libgit2/libgit2#2917 --- src/integer.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/integer.h b/src/integer.h index 8e86a48a52e..b08094c2f14 100644 --- a/src/integer.h +++ b/src/integer.h @@ -58,7 +58,7 @@ GIT_INLINE(bool) git__add_uint64_overflow(uint64_t *out, uint64_t one, uint64_t #if (SIZE_MAX == UINT_MAX) && __has_builtin(__builtin_uadd_overflow) # define git__add_sizet_overflow(out, one, two) \ __builtin_uadd_overflow(one, two, out) -# define git__multiply_sizet_overflow(out, one, two) +# define git__multiply_sizet_overflow(out, one, two) \ __builtin_umul_overflow(one, two, out) #elif (SIZE_MAX == ULONG_MAX) && __has_builtin(__builtin_uaddl_overflow) # define git__add_sizet_overflow(out, one, two) \ From ecf1c3c3589229e5d26ead78607d6abbb881c6d8 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Mon, 23 Feb 2015 11:18:52 -0500 Subject: [PATCH 99/99] README: provide some more explanation about TLS --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b9a42d76d7e..882924231e1 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,10 @@ before calling any other libgit2 functions. You can call this function many time git_libgit2_shutdown(); -will free the resources. +will free the resources. Note that if you have worker threads, you should +call `git_libgit2_shutdown` *after* those threads have exited. If you +require assistance coordinating this, simply have the worker threads call +`git_libgit2_init` at startup and `git_libgit2_shutdown` at shutdown. Threading =========