8000 Streaming filters by ethomson · Pull Request #2911 · libgit2/libgit2 · GitHub
[go: up one dir, main page]
Skip to content

Streaming filters #2911

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 13 commits into from
Feb 19, 2015
Merged

Streaming filters #2911

Changes from 1 commit
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
fbdc9db
filters: introduce streaming filters
Jan 22, 2015
5555696
filters: stream internally
Jan 22, 2015
e78f5c9
checkout: stream the blob into the filters
Jan 22, 2015
646364e
checkout: maintain temporary buffer for filters
ethomson Feb 17, 2015
8c2dfb3
filter: test a large file through the stream
Feb 17, 2015
b75f15a
git_writestream: from git_filter_stream
Feb 18, 2015
f7c0125
filter streams: base -> parent
Feb 18, 2015
b49eddd
checkout: let the stream writer close the fd
Feb 18, 2015
d4cf167
buffer: introduce git_buf_attach_notowned
Feb 19, 2015
795eacc
git_filter_opt_t -> git_filter_flag_t
Feb 19, 2015
d05218b
filter: add `git_filter_list__load_ext`
Feb 19, 2015
9c9aa1b
filter: take `temp_buf` in `git_filter_options`
Feb 19, 2015
feb0e02
tests: separate INVASIVE filesystem tests
Feb 19, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
filters: stream internally 
Migrate the `git_filter_list_apply_*` functions over to using the
new filter streams.
  • Loading branch information
Edward Thomson committed Feb 17, 2015
commit 5555696f8a48d2319a7ca0918726711e6a81a924
154 changes: 81 additions & 73 deletions src/filter.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -617,67 +617,70 @@ static int filter_list_out_buffer_from_raw(
return 0;
}

int git_filter_list_apply_to_data(
git_buf *tgt, git_filter_list *fl, git_buf *src)
struct buf_stream {
git_filter_stream base;
git_buf *target;
bool complete;
};

static int buf_stream_write(
git_filter_stream *s, const char *buffer, size_t len)
{
int error = 0;
uint32_t i;
git_buf *dbuffer[2], local = GIT_BUF_INIT;
unsigned int si = 0;
struct buf_stream *buf_stream = (struct buf_stream *)s;
assert(buf_stream);

git_buf_sanitize(tgt);
git_buf_sanitize(src);
assert(buf_stream->complete == 0);

if (!fl)
return filter_list_out_buffer_from_raw(tgt, src->ptr, src->size);
return git_buf_put(buf_stream->target, buffer, len);
}

dbuffer[0] = src;
dbuffer[1] = tgt;
static int buf_stream_close(git_filter_stream *s)
{
struct buf_stream *buf_stream = (struct buf_stream *)s;
assert(buf_stream);

/* if `src` buffer is reallocable, then use it, otherwise copy it */
if (!git_buf_is_allocated(src)) {
if (git_buf_set(&local, src->ptr, src->size) < 0)
return -1;
dbuffer[0] = &local;
}
assert(buf_stream->complete == 0);
buf_stream->complete = 1;

for (i = 0; i < git_array_size(fl->filters); ++i) {
unsigned int di = 1 - si;
uint32_t fidx = (fl->source.mode == GIT_FILTER_TO_WORKTREE) ?
i : git_array_size(fl->filters) - 1 - i;
git_filter_entry *fe = git_array_get(fl->filters, fidx);

dbuffer[di]->size = 0;

/* Apply the filter from dbuffer[src] to the other buffer;
* if the filtering is canceled by the user mid-filter,
* we skip to the next filter without changing the source
* of the double buffering (so that the text goes through
* cleanly).
*/
return 0;
}

error = fe->filter->apply(
fe->filter, &fe->payload, dbuffer[di], dbuffer[si], &fl->source);
static void buf_stream_free(git_filter_stream *s)
{
GIT_UNUSED(s);
}

if (error == GIT_PASSTHROUGH) {
/* PASSTHROUGH means filter decided not to process the buffer */
error = 0;
} else if (!error) {
git_buf_sanitize(dbuffer[di]); /* force NUL termination */
si = di; /* swap buffers */
} else {
tgt->size = 0;
goto cleanup;
}
}
static void buf_stream_init(struct buf_stream *writer, git_buf *target)
{
memset(writer, 0, sizeof(struct buf_stream));

/* Ensure that the output ends up in dbuffer[1] (i.e. the dest) */
if (si != 1)
git_buf_swap(dbuffer[0], dbuffer[1]);
writer->base.write = buf_stream_write;
writer->base.close = buf_stream_close;
writer->base.free = buf_stream_free;
writer->target = target;

cleanup:
git_buf_free(&local); /* don't leak if we allocated locally */
git_buf_clear(target);
}

int git_filter_list_apply_to_data(
git_buf *tgt, git_filter_list *filters, git_buf *src)
{
struct buf_stream writer;
int error;

git_buf_sanitize(tgt);
git_buf_sanitize(src);

if (!filters)
return filter_list_out_buffer_from_raw(tgt, src->ptr, src->size);

buf_stream_init(&writer, tgt);

if ((error = git_filter_list_stream_data(filters, src,
(git_filter_stream *)&writer)) < 0)
return error;

assert(writer.complete);
return error;
}

Expand All @@ -687,19 +690,16 @@ int git_filter_list_apply_to_file(
git_repository *repo,
const char *path)
{
struct buf_stream writer;
int error;
const char *base = repo ? git_repository_workdir(repo) : NULL;
git_buf abspath = GIT_BUF_INIT, raw = GIT_BUF_INIT;

if (!(error = git_path_join_unrooted(&abspath, path, base, NULL)) &&
!(error = git_futils_readbuffer(&raw, abspath.ptr)))
{
error = git_filter_list_apply_to_data(out, filters, &raw);
buf_stream_init(&writer, out);

git_buf_free(&raw);
}
if ((error = git_filter_list_stream_file(
filters, repo, path, (git_filter_stream *)&writer)) < 0)
return error;

git_buf_free(&abspath);
assert(writer.complete);
return error;
}

Expand All @@ -724,15 +724,17 @@ int git_filter_list_apply_to_blob(
git_filter_list *filters,
git_blob *blob)
{
git_buf in = GIT_BUF_INIT;
struct buf_stream writer;
int error;

if (buf_from_blob(&in, blob) < 0)
return -1;
buf_stream_init(&writer, out);

if (filters)
git_oid_cpy(&filters->source.oid, git_blob_id(blob));
if ((error = git_filter_list_stream_blob(
filters, blob, (git_filter_stream *)&writer)) < 0)
return error;

return git_filter_list_apply_to_data(out, filters, &in);
assert(writer.complete);
return error;
}

struct proxy_stream {
Expand Down Expand Up @@ -823,7 +825,6 @@ static int stream_list_init(
git_filter_list *filters,
git_filter_stream *target)
{
git_vector filter_streams = GIT_VECTOR_INIT;
git_filter_stream *last_stream = target;
size_t i;
int error = 0;
Expand All @@ -845,14 +846,19 @@ static int stream_list_init(

assert(fe->filter->stream || fe->filter->apply);

/* If necessary, create a stream that proxies the one-shot apply */
/* If necessary, create a stream that proxies the traditional
* application.
*/
stream_init = fe->filter->stream ?
fe->filter->stream : proxy_stream_init;

if ((error = stream_init(&filter_stream, fe->filter, &fe->payload, &filters->source, last_stream)) < 0)
return error;
error = stream_init(&filter_stream, fe->filter,
&fe->payload, &filters->source, last_stream);

git_vector_insert(&filter_streams, filter_stream);
if (error < 0)
return error;

git_vector_insert(streams, filter_stream);
last_stream = filter_stream;
}

Expand All @@ -867,20 +873,20 @@ void stream_list_free(git_vector *streams)

git_vector_foreach(streams, i, stream)
stream->free(stream);
git_vector_free(streams);
}

#define STREAM_BUFSIZE 10240

/* TODO: maybe not use filter_stream as a target but create one */
int git_filter_list_stream_file(
git_filter_list *filters,
git_buf *data,
git_repository *repo,
const char *path,
git_filter_stream *target)
{
char buf[STREAM_BUFSIZE];
git_buf abspath = GIT_BUF_INIT, raw = GIT_BUF_INIT;
git_buf abspath = GIT_BUF_INIT;
const char *base = repo ? git_repository_workdir(repo) : NULL;
git_vector filter_streams = GIT_VECTOR_INIT;
git_filter_stream *stream_start;
Expand All @@ -898,7 +904,7 @@ int git_filter_list_stream_file(
}

while ((readlen = p_read(fd, buf, STREAM_BUFSIZE)) > 0) {
if ((error = stream_start->write(stream_start, data->ptr, data->size)) < 0)
if ((error = stream_start->write(stream_start, buf, readlen)) < 0)
goto done;
}

Expand All @@ -924,6 +930,8 @@ int git_filter_list_stream_data(
git_filter_stream *stream_start;
int error = 0;

git_buf_sanitize(data);

if ((error = stream_list_init(
&stream_start, &filter_streams, filters, target)) == 0 &&
(error =
Expand Down
0