10000 Streaming filters by ethomson · Pull Request #2911 · libgit2/libgit2 · GitHub
[go: up one dir, main page]

Skip to content

Streaming filters #2911

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 13 commits into from
Feb 19, 2015
Prev Previous commit
Next Next commit
checkout: maintain temporary buffer for filters
Let the filters use the checkout data's temporary buffer, instead
of having to allocate new buffers each time.
  • Loading branch information
ethomson authored and Edward Thomson committed Feb 17, 2015
commit 646364e780c65b1af2e28a7bdeabd22b67816454
3 changes: 3 additions & 0 deletions src/checkout.c
Original file line number Diff line number Diff line change
Expand Up @@ -1444,6 +1444,9 @@ static int blob_content_to_file(
GIT_FILTER_TO_WORKTREE, GIT_FILTER_OPT_DEFAULT)))
return error;

if (fl)
git_filter_list__set_temp_buf(fl, &data->tmp);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This if looks odd. Can we not have the filter list here? We seem to assume later on that we do have it.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We can not - the existing filters system let you pass a NULL filter_list to simply skip the filtering. (https://github.com/libgit2/libgit2/blob/master/src/filter.c#L631) This maintains that.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fair enough, I hadn't noticed that we always use functions around fl and do check if it's NULL each time.


/* setup the writer */
memset(&writer, 0, sizeof(struct checkout_stream));
writer.base.write = checkout_stream_write;
Expand Down
33 changes: 21 additions & 12 deletions src/filter.c
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ typedef struct {
struct git_filter_list {
git_array_t(git_filter_entry) filters;
git_filter_source source;
git_buf *temp_buf;
char path[GIT_FLEX_ARRAY];
};

Expand Down Expand Up @@ -522,7 +523,6 @@ int git_filter_list__load_with_attr_session(
fe = git_array_alloc(fl->filters);
GITERR_CHECK_ALLOC(fe);
fe->filter = fdef->filter;
fe->stream = NULL;
fe->payload = payload;
}
}
Expand All @@ -549,6 +549,11 @@ int git_filter_list_load(
filters, repo, NULL, blob, path, mode, options);
}

void git_filter_list__set_temp_buf(git_filter_list *fl, git_buf *temp_buf)
{
fl->temp_buf = temp_buf;
}

void git_filter_list_free(git_filter_list *fl)
{
uint32_t i;
Expand Down Expand Up @@ -591,7 +596,6 @@ int git_filter_list_push(
fe = git_array_alloc(fl->filters);
GITERR_CHECK_ALLOC(fe);
fe->filter = filter;
fe->stream = NULL;
fe->payload = payload;

return 0;
Expand Down Expand Up @@ -745,7 +749,8 @@ struct proxy_stream {
const git_filter_source *source;
void **payload;
git_buf input;
git_buf output;
git_buf temp_buf;
git_buf *output;
git_filter_stream *target;
};

Expand All @@ -769,15 +774,15 @@ static int proxy_stream_close(git_filter_stream *s)
error = proxy_stream->filter->apply(
proxy_stream->filter,
proxy_stream->payload,
&proxy_stream->output,
proxy_stream->output,
&proxy_stream->input,
proxy_stream->source);

if (error == GIT_PASSTHROUGH) {
writebuf = &proxy_stream->input;
} else if (error == 0) {
git_buf_sanitize(&proxy_stream->output);
writebuf = &proxy_stream->output;
git_buf_sanitize(proxy_stream->output);
writebuf = proxy_stream->output;
} else {
return error;
}
Expand All @@ -795,13 +800,14 @@ static void proxy_stream_free(git_filter_stream *s)
assert(proxy_stream);

git_buf_free(&proxy_stream->input);
git_buf_free(&proxy_stream->output);
git_buf_free(&proxy_stream->temp_buf);
git__free(proxy_stream);
}

static int proxy_stream_init(
git_filter_stream **out,
git_filter *filter,
git_buf *temp_buf,
void **payload,
const git_filter_source *source,
git_filter_stream *target)
Expand All @@ -816,6 +822,7 @@ static int proxy_stream_init(
proxy_stream->payload = payload;
proxy_stream->source = source;
proxy_stream->target = target;
proxy_stream->output = temp_buf ? temp_buf : &proxy_stream->temp_buf;

*out = (git_filter_stream *)proxy_stream;
return 0;
Expand Down Expand Up @@ -844,18 +851,20 @@ static int stream_list_init(
git_array_size(filters->filters) - 1 - i : i;
git_filter_entry *fe = git_array_get(filters->filters, filter_idx);
git_filter_stream *filter_stream;
git_filter_stream_fn stream_init;

assert(fe->filter->stream || fe->filter->apply);

/* If necessary, create a stream that proxies the traditional
* application.
*/
stream_init = fe->filter->stream ?
fe->filter->stream : proxy_stream_init;

error = stream_init(&filter_stream, fe->filter,
if (fe->filter->stream)
error = fe->filter->stream(&filter_stream, fe->filter,
&fe->payload, &filters->source, last_stream);
else
/* Create a stream that proxies the one-shot apply */
error = proxy_stream_init(&filter_stream, fe->filter,
filters->temp_buf, &fe->payload, &filters->source,
last_stream);

if (error < 0)
return error;
Expand Down
3 changes: 3 additions & 0 deletions src/filter.h
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,9 @@ typedef enum {
GIT_CRLF_AUTO,
} git_crlf_t;

extern void git_filter_list__set_temp_buf(
git_filter_list *fl, git_buf *temp_buf);

extern void git_filter_free(git_filter *filter);

extern int git_filter_list__load_with_attr_session(
Expand Down
0