10000 output: use io.Reader implementation more internally by bobheadxi · Pull Request #35 · sourcegraph/run · GitHub
[go: up one dir, main page]
Skip to content

output: use io.Reader implementation more internally #35

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
bobheadxi merged 1 commit into from
May 30, 2022
Merged

output: use io.Reader implementation more internally #35

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 21 additions & 10 deletions jq.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"context"
"encoding/json"
"fmt"
"io"

"github.com/itchyny/gojq"
)
Expand All @@ -22,18 +23,28 @@ func buildJQ(query string) (*gojq.Code, error) {
return jqCode, nil
}

// execJQ executes the compiled jq query against content.
func execJQ(ctx context.Context, jqCode *gojq.Code, content []byte) ([]byte, error) {
// execJQBytes can be used to execute a compiled jq query against small content bytes,
// e.g. lines. Errors are annotated with the provided content for ease of debugging.
func execJQBytes(ctx context.Context, jqCoode *gojq.Code, content []byte) ([]byte, error) {
if len(content) == 0 {
return nil, nil
}
result, err := execJQ(ctx, jqCoode, bytes.NewReader(content))
if err != nil {
// Embed the consumed content
return nil, fmt.Errorf("%w: %s", err, string(content))
}
return result, nil
}

// execJQ executes the compiled jq query against content from reader.
func execJQ(ctx context.Context, jqCode *gojq.Code, reader io.Reader) ([]byte, error) {
var input interface{}
if err := json.NewDecoder(bytes.NewReader(content)).Decode(&input); err != nil {
return nil, fmt.Errorf("json: %w: %s", err, string(content))
if err := json.NewDecoder(reader).Decode(&input); err != nil {
return nil, fmt.Errorf("json: %w", err)
}

var newLine bytes.Buffer
var result bytes.Buffer
iter := jqCode.RunWithContext(ctx, input)
for {
v, ok := iter.Next()
Expand All @@ -42,14 +53,14 @@ func execJQ(ctx context.Context, jqCode *gojq.Code, content []byte) ([]byte, err
}

if err, ok := v.(error); ok {
return nil, fmt.Errorf("jq: %w: %s", err, string(content))
return nil, fmt.Errorf("jq: %w", err)
}

result, err := gojq.Marshal(v)
encoded, err := gojq.Marshal(v)
if err != nil {
return nil, fmt.Errorf("jq: %w: %s", err, string(content))
return nil, fmt.Errorf("jq: %w", err)
}
newLine.Write(result)
result.Write(encoded)
}
return newLine.Bytes(), nil
return result.Bytes(), nil
}
2 changes: 1 addition & 1 deletion map.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ func MapJQ(query string) (LineMap, error) {
}

return func(ctx context.Context, line []byte, dst io.Writer) (int, error) {
b, err := execJQ(ctx, jqCode, line)
b, err := execJQBytes(ctx, jqCode, line)
if err != nil {
return 0, err
}
Expand Down
17 changes: 7 additions & 10 deletions output.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -180,22 +180,19 @@ func (o *commandOutput) JQ(query string) ([]byte, error) {
return nil, err
}

var buffer bytes.Buffer
if err := o.Stream(&buffer); err != nil {
return nil, err
}

b, err := execJQ(o.ctx, jqCode, buffer.Bytes())
result, err := execJQ(o.ctx, jqCode, o)
if err != nil {
return nil, err
}
return b, nil
return result, nil
}

func (o *commandOutput) String() (string, error) {
var sb strings.Builder
err := o.Stream(&sb)
return strings.TrimSuffix(sb.String(), "\n"), err
b, err := io.ReadAll(o)
if err != nil {
return "", err
}
return strings.TrimSuffix(string(b), "\n"), nil
Copy link
Contributor
@jhchabran jhchabran May 30, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Isn't strings.TrimSpace what we're looking for here?

Copy link
Member Author
@bobheadxi bobheadxi May 30, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We are specifically trimming only the tailing new line that we added as we write output back line-by-line - we don't want to trim indentation, for example

}

func (o *commandOutput) Read(p []byte) (int, error) {
Expand Down
0