8000 refactor: add json operators to SQLGlot compiler by chelsea-lin · Pull Request #1887 · googleapis/python-bigquery-dataframes · GitHub
[go: up one dir, main page]
Skip to content

refactor: add json operators to SQLGlot compiler #1887

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
chelsea-lin merged 1 commit into from
Jul 8, 2025
Merged

refactor: add json operators to SQLGlot compiler #1887

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions bigframes/core/compile/sqlglot/expressions/binary_compiler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,3 +42,8 @@ def _(op, left: TypedExpr, right: TypedExpr) -> sge.Expression:
@BINARY_OP_REGISTRATION.register(ops.ge_op)
def _(op, left: TypedExpr, right: TypedExpr) -> sge.Expression:
return sge.GTE(this=left.expr, expression=right.expr)


@BINARY_OP_REGISTRATION.register(ops.JSONSet)
def _(op, left: TypedExpr, right: TypedExpr) -> sge.Expression:
return sge.func("JSON_SET", left.expr, sge.convert(op.json_path), right.expr)
46 changes: 46 additions & 0 deletions bigframes/core/compile/sqlglot/expressions/unary_compiler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,3 +70,49 @@ def _(op: ops.ArraySliceOp, expr: TypedExpr) -> sge.Expression:
)

return sge.array(selected_elements)


# JSON Ops
@UNARY_OP_REGISTRATION.register(ops.JSONExtract)
def _(op: ops.JSONExtract, expr: TypedExpr) -> sge.Expression:
return sge.func("JSON_EXTRACT", expr.expr, sge.convert(op.json_path))


@UNARY_OP_REGISTRATION.register(ops.JSONExtractArray)
def _(op: ops.JSONExtractArray, expr: TypedExpr) -> sge.Expression:
return sge.func("JSON_EXTRACT_ARRAY", expr.expr, sge.convert(op.json_path))


@UNARY_OP_REGISTRATION.register(ops.JSONExtractStringArray)
def _(op: ops.JSONExtractStringArray, expr: TypedExpr) -> sge.Expression:
return sge.func("JSON_EXTRACT_STRING_ARRAY", expr.expr, sge.convert(op.json_path))


@UNARY_OP_REGISTRATION.register(ops.JSONQuery)
def _(op: ops.JSONQuery, expr: TypedExpr) -> sge.Expression:
return sge.func("JSON_QUERY", expr.expr, sge.convert(op.json_path))


@UNARY_OP_REGISTRATION.register(ops.JSONQueryArray)
def _(op: ops.JSONQueryArray, expr: TypedExpr) -> sge.Expression:
return sge.func("JSON_QUERY_ARRAY", expr.expr, sge.convert(op.json_path))


@UNARY_OP_REGISTRATION.register(ops.JSONValue)
def _(op: ops.JSONValue, expr: TypedExpr) -> sge.Expression:
return sge.func("JSON_VALUE", expr.expr, sge.convert(op.json_path))


@UNARY_OP_REGISTRATION.register(ops.JSONValueArray)
def _(op: ops.JSONValueArray, expr: TypedExpr) -> sge.Expression:
return sge.func("JSON_VALUE_ARRAY", expr.expr, sge.convert(op.json_path))


@UNARY_OP_REGISTRATION.register(ops.ParseJSON)
def _(op: ops.ParseJSON, expr: TypedExpr) -> sge.Expression:
return sge.func("PARSE_JSON", expr.expr)


@UNARY_OP_REGISTRATION.register(ops.ToJSONString)
def _(op: ops.ToJSONString, expr: TypedExpr) -> sge.Expression:
return sge.func("TO_JSON_STRING", expr.expr)
20 changes: 20 additions & 0 deletions ...nit/core/compile/sqlglot/expressions/snapshots/test_binary_compiler/test_json_set/out.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
WITH `bfcte_0` AS (
SELECT
`rowindex` AS `bfcol_0`,
`json_col` AS `bfcol_1`
FROM `bigframes-dev`.`sqlglot_test`.`json_types`
), `bfcte_1` AS (
SELECT
*,
JSON_SET(`bfcol_1`, '$.a', 100) AS `bfcol_4`
FROM `bfcte_0`
), `bfcte_2` AS (
SELECT
*,
JSON_SET(`bfcol_4`, '$.b', 'hi') AS `bfcol_7`
FROM `bfcte_1`
)
SELECT
`bfcol_0` AS `rowindex`,
`bfcol_7` AS `json_col`
FROM `bfcte_2`
15 changes: 15 additions & 0 deletions .../core/compile/sqlglot/expressions/snapshots/test_unary_compiler/test_json_extract/out.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
WITH `bfcte_0` AS (
SELECT
`rowindex` AS `bfcol_0`,
`json_col` AS `bfcol_1`
FROM `bigframes-dev`.`sqlglot_test`.`json_types`
), `bfcte_1` AS (
SELECT
*,
JSON_EXTRACT(`bfcol_1`, '$') AS `bfcol_4`
FROM `bfcte_0`
)
SELECT
`bfcol_0` AS `rowindex`,
`bfcol_4` AS `json_col`
FROM `bfcte_1`
15 changes: 15 additions & 0 deletions ...it/core/compile/sqlglot/expressions/snapshots/test_unary_compiler/test_parse_json/out.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
WITH `bfcte_0` AS (
SELECT
`rowindex` AS `bfcol_0`,
`string_col` AS `bfcol_1`
FROM `bigframes-dev`.`sqlglot_test`.`scalar_types`
), `bfcte_1` AS (
SELECT
*,
JSON_VALUE(`bfcol_1`, '$') AS `bfcol_4`
FROM `bfcte_0`
)
SELECT
`bfcol_0` AS `rowindex`,
`bfcol_4` AS `string_col`
FROM `bfcte_1`
6 changes: 6 additions & 0 deletions tests/unit/core/compile/sqlglot/expressions/test_binary_compiler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@

import pytest

import bigframes.bigquery as bbq
import bigframes.pandas as bpd

pytest.importorskip("pytest_snapshot")
Expand Down Expand Up @@ -41,3 +42,8 @@ def test_add_string(scalar_types_df: bpd.DataFrame, snapshot):
bf_df["string_col"] = bf_df["string_col"] + "a"

snapshot.assert_match(bf_df.sql, "out.sql")


def test_json_set(json_types_df: bpd.DataFrame, snapshot):
result = bbq.json_set(json_types_df["json_col"], [("$.a", 100), ("$.b", "hi")])
snapshot.assert_match(result.to_frame().sql, "out.sql")
47 changes: 45 additions & 2 deletions tests/unit/core/compile/sqlglot/expressions/test_unary_compiler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,14 +14,14 @@

import pytest

from bigframes import bigquery
import bigframes.bigquery as bbq
import bigframes.pandas as bpd

pytest.importorskip("pytest_snapshot")


def test_array_to_string(repeated_types_df: bpd.DataFrame, snapshot):
result = bigquery.array_to_string(repeated_types_df["string_list_col"], ".")
result = bbq.array_to_string(repeated_types_df["string_list_col"], ".")

snapshot.assert_match(result.to_frame().sql, "out.sql")

Expand All @@ -42,3 +42,46 @@ def test_array_slice_with_start_and_stop(repeated_types_df: bpd.DataFrame, snaps
result = repeated_types_df["string_list_col"].list[1:5]

snapshot.assert_match(result.to_frame().sql, "out.sql")


# JSON Ops
def test_json_extract(json_types_df: bpd.DataFrame, snapshot):
result = bbq.json_extract(json_types_df["json_col"], "$")
expected_sql = "JSON_EXTRACT(`bfcol_1`, '$') AS `bfcol_4`"
assert expected_sql in result.to_frame().sql
snapshot.assert_match(result.to_frame().sql, "out.sql")


def test_json_extract_array(json_types_df: bpd.DataFrame):
result = bbq.json_extract_array(json_types_df["json_col"], "$")
expected_sql = "JSON_EXTRACT_ARRAY(`bfcol_1`, '$') AS `bfcol_4`"
assert expected_sql in result.to_frame().sql
Comment on lines +57 to +58
Copy link
Contributor
@sycai sycai Jul 7, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Non-rhetorical question: why we only check substrings instead of using snapshot to match the full output?

Copy link
Contributor Author
@chelsea-lin chelsea-lin Jul 8, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am trying to reduce the number of out.sql for less maintain cost, specially for these hundreds scalar operators, but open for discussions.



def test_json_extract_string_array(json_types_df: bpd.DataFrame):
result = bbq.json_extract_string_array(json_types_df["json_col"], "$")
expected_sql = "JSON_EXTRACT_STRING_ARRAY(`bfcol_1`, '$') AS `bfcol_4`"
assert expected_sql in result.to_frame().sql


def test_json_query(json_types_df: bpd.DataFrame):
result = bbq.json_query(json_types_df["json_col"], "$")
expected_sql = "JSON_QUERY(`bfcol_1`, '$') AS `bfcol_4`"
assert expected_sql in result.to_frame().sql


def test_json_query_array(json_types_df: bpd.DataFrame):
result = bbq.json_query_array(json_types_df["json_col"], "$")
expected_sql = "JSON_QUERY_ARRAY(`bfcol_1`, '$') AS `bfcol_4`"
assert expected_sql in result.to_frame().sql


def test_json_value(json_types_df: bpd.DataFrame):
result = bbq.json_value(json_types_df["json_col"], "$")
expected_sql = "JSON_VALUE(`bfcol_1`, '$') AS `bfcol_4`"
assert expected_sql in result.to_frame().sql


def test_parse_json(scalar_types_df: bpd.DataFrame, snapshot):
result = bbq.json_value(scalar_types_df["string_col"], "$")
snapshot.assert_match(result.to_frame().sql, "out.sql")
0