8000 fix: cloudevent conversion conformance tests (#139) · jrmfg/functions-framework-python@4900780 · GitHub
[go: up one dir, main page]
Skip to content

Commit 4900780

Browse files
matthewrobertsonmatthewrobertson
authored
fix: cloudevent conversion conformance tests (GoogleCloudPlatform#139)
This commit updates the legacy event to cloud event conversion logic to: - include the `messageId` and `publishTime` fields in the data payload of Pub/Sub events - include the location in the source field of firebasedatabase events fixes GoogleCloudPlatform#139
1 parent c7f66fd commit 4900780

File tree

7 files changed

+168
-25
lines changed

7 files changed

+168
-25
lines changed

‎.github/workflows/conformance.yml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,23 +24,23 @@ jobs:
2424
go-version: '1.15'
2525

2626
- name: Run HTTP conformance tests
27-
uses: GoogleCloudPlatform/functions-framework-conformance/action@v0.3.10
27+
uses: GoogleCloudPlatform/functions-framework-conformance/action@v0.3.12
2828
with:
2929
functionType: 'http'
3030
useBuildpacks: false
3131
validateMapping: false
3232
cmd: "'functions-framework --source tests/conformance/main.py --target write_http --signature-type http'"
3333

3434
- name: Run event conformance tests
35-
uses: GoogleCloudPlatform/functions-framework-conformance/action@v0.3.10
35+
uses: GoogleCloudPlatform/functions-framework-conformance/action@v0.3.12
3636
with:
3737
functionType: 'legacyevent'
3838
useBuildpacks: false
3939
validateMapping: true
4040
cmd: "'functions-framework --source tests/conformance/main.py --target write_legacy_event --signature-type event'"
4141

4242
- name: Run cloudevent conformance tests
43-
uses: GoogleCloudPlatform/functions-framework-conformance/action@v0.3.10
43+
uses: GoogleCloudPlatform/functions-framework-conformance/action@v0.3.12
4444
with:
4545
functionType: 'cloudevent'
4646
useBuildpacks: false

‎.gitignore

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,6 @@ build/
66
dist/
77
.coverage
88
.vscode/
9+
function_output.json
10+
serverlog_stderr.txt
11+
serverlog_stdout.txt

‎src/functions_framework/event_conversion.py

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,7 @@
9494
# for the subject.
9595
_CE_SERVICE_TO_RESOURCE_RE = {
9696
_FIREBASE_CE_SERVICE: re.compile(r"^(projects/[^/]+)/(events/[^/]+)$"),
97-
_FIREBASE_DB_CE_SERVICE: re.compile(r"^(projects/[^/]/instances/[^/]+)/(refs/.+)$"),
97+
_FIREBASE_DB_CE_SERVICE: re.compile(r"^projects/_/(instances/[^/]+)/(refs/.+)$"),
9898
_FIRESTORE_CE_SERVICE: re.compile(
9999
r"^(projects/[^/]+/databases/\(default\))/(documents/.+)$"
100100
),
@@ -131,9 +131,14 @@ def background_event_to_cloudevent(request) -> CloudEvent:
131131
new_type = _BACKGROUND_TO_CE_TYPE[context.event_type]
132132

133133
service, resource, subject = _split_resource(context)
134+
source = f"//{service}/{resource}"
134135

135136
# Handle Pub/Sub events.
136137
if service == _PUBSUB_CE_SERVICE:
138+
if "messageId" not in data:
139+
data["messageId"] = context.event_id
140+
if "publishTime" not in data:
141+
data["publishTime"] = context.timestamp
137142
data = {"message": data}
138143

139144
# Handle Firebase Auth events.
@@ -147,13 +152,30 @@ def background_event_to_cloudevent(request) -> CloudEvent:
147152
uid = data["uid"]
148153
subject = f"users/{uid}"
149154

155+
# Handle Firebase DB events.
156+
if service == _FIREBASE_DB_CE_SERVICE:
157+
# The CE source of firebasedatabase cloudevents includes location information
158+
# that is inferred from the 'domain' field of legacy events.
159+
if "domain" not in event_data:
160+
raise EventConversionException(
161+
"Invalid FirebaseDB event payload: missing 'domain'"
162+
)
163+
164+
domain = event_data["domain"]
165+
location = "us-central1"
166+
if domain != "firebaseio.com":
167+
location = domain.split(".")[0]
168+
169+
resource = f"projects/_/locations/{location}/{resource}"
170+
source = f"//{service}/{resource}"
171+
150172
metadata = {
151173
"id": context.event_id,
152174
"time": context.timestamp,
153175
"specversion": _CLOUDEVENT_SPEC_VERSION,
154176
"datacontenttype": "application/json",
155177
"type": new_type,
156-
"source": f"//{service}/{resource}",
178+
"source": source,
157179
}
158180

159181
if subject:
@@ -201,6 +223,10 @@ def cloudevent_to_background_event(request) -> Tuple[Any, Context]:
201223
resource = {"service": service, "name": name, "type": _PUBSUB_MESSAGE_TYPE}
202224
if "message" in data:
203225
data = data["message"]
226+
if "messageId" in data:
227+
del data["messageId"]
228+
if "publishTime" in data:
229+
del data["publishTime"]
204230
elif service == _FIREBASE_AUTH_CE_SERVICE:
205231
resource = name
206232
if "metadata" in data:
@@ -214,6 +240,9 @@ def cloudevent_to_background_event(request) -> Tuple[Any, Context]:
214240
"service": service,
215241
"type": data["kind"],
216242
}
243+
elif service == _FIREBASE_DB_CE_SERVICE:
244+
name = re.sub("/locations/[^/]+", "", name)
245+
resource = f"{name}/{event['subject']}"
217246
else:
218247
resource = f"{name}/{event['subject']}"
219248

‎tests/test_convert.py

Lines changed: 95 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,8 @@
7575
"data": {
7676
"message": {
7777
"data": "10",
78+
"publishTime": "2020-05-18T12:13:19Z",
79+
"messageId": "1215011316659232",
7880
},
7981
},
8082
}
@@ -122,7 +124,10 @@ def marshalled_pubsub_request():
122124
def raw_pubsub_cloudevent_output(marshalled_pubsub_request):
123125
event = PUBSUB_CLOUD_EVENT.copy()
124126
# the data payload is more complex for the raw pubsub request
125-
event["data"] = {"message": marshalled_pubsub_request["data"]}
127+
data = marshalled_pubsub_request["data"]
128+
data["messageId"] = event["id"]
129+
data["publishTime"] = event["time"]
130+
event["data"] = {"message": data}
126131
return from_json(json.dumps(event))
127132

128133

@@ -138,6 +143,18 @@ def firebase_auth_cloudevent_output():
138143
return from_json(f.read())
139144

140145

146+
@pytest.fixture
147+
def firebase_db_background_input():
148+
with open(TEST_DATA_DIR / "firebase-db-legacy-input.json", "r") as f:
149+
return json.load(f)
150+
151+
152+
@pytest.fixture
153+
def firebase_db_cloudevent_output():
154+
with open(TEST_DATA_DIR / "firebase-db-cloudevent-output.json", "r") as f:
155+
return from_json(f.read())
156+
157+
141158
@pytest.fixture
142159
def create_ce_headers():
143160
return lambda event_type, source: {
@@ -207,6 +224,41 @@ def test_firebase_auth_event_to_cloudevent_no_uid(
207224
assert cloudevent == firebase_auth_cloudevent_output
208225

209226

227+
def test_firebase_db_event_to_cloudevent_default_location(
228+
firebase_db_background_input, firebase_db_cloudevent_output
229+
):
230+
req = flask.Request.from_values(json=firebase_db_background_input)
231+
cloudevent = event_conversion.background_event_to_cloudevent(req)
232+
assert cloudevent == firebase_db_cloudevent_output
233+
234+
235+
def test_firebase_db_event_to_cloudevent_location_subdomain(
236+
firebase_db_background_input, firebase_db_cloudevent_output
237+
):
238+
firebase_db_background_input["domain"] = "europe-west1.firebasedatabase.app"
239+
firebase_db_cloudevent_output["source"] = firebase_db_cloudevent_output[
240+
"source"
241+
].replace("us-central1", "europe-west1")
242+
243+
req = flask.Request.from_values(json=firebase_db_background_input)
244+
cloudevent = event_conversion.background_event_to_cloudevent(req)
245+
assert cloudevent == firebase_db_cloudevent_output
246+
247+
248+
def test_firebase_db_event_to_cloudevent_missing_domain(
249+
firebase_db_background_input, firebase_db_cloudevent_output
250+
):
251+
del firebase_db_background_input["domain"]
252+
req = flask.Request.from_values(json=firebase_db_background_input)
253+
254+
with pytest.raises(EventConversionException) as exc_info:
255+
event_conversion.background_event_to_cloudevent(req)
256+
257+
assert (
258+
"Invalid FirebaseDB event payload: missing 'domain'" in exc_info.value.args[0]
259+
)
260+
261+
210262
@pytest.mark.parametrize(
211263
"background_resource",
212264
[
@@ -299,34 +351,39 @@ def test_marshal_background_event_data_with_topic_path(
299351
assert payload == marshalled_pubsub_request
300352

301353

354+
@pytest.mark.parametrize(
355+
"request_fixture, overrides",
356+
[
357+
(
358+
"raw_pubsub_request",
359+
{
360+
"request_path": "x/projects/sample-project/topics/gcf-test?pubsub_trigger=true",
361+
},
362+
),
363+
("raw_pubsub_request", {"source": "//pubsub.googleapis.com/"}),
364+
("marshalled_pubsub_request", {}),
365+
],
366+
)
302367
def test_pubsub_emulator_request_to_cloudevent(
303-
raw_pubsub_request, raw_pubsub_cloudevent_output
368+
raw_pubsub_cloudevent_output, request_fixture, overrides, request
304369
):
370+
request_path = overrides.get("request_path", "/")
371+
payload = request.getfixturevalue(request_fixture)
305372
req = flask.Request.from_values(
306-
json=raw_pubsub_request,
307-
path="x/projects/sample-project/topics/gcf-test?pubsub_trigger=true",
373+
path=request_path,
374+
json=payload,
308375
)
309376
cloudevent = event_conversion.background_event_to_cloudevent(req)
310377

311378
# Remove timestamps as they are generated on the fly.
312379
del raw_pubsub_cloudevent_output["time"]
380+
del raw_pubsub_cloudevent_output.data["message"]["publishTime"]
313381
del cloudevent["time"]
382+
del cloudevent.data["message"]["publishTime"]
314383

315-
assert cloudevent == raw_pubsub_cloudevent_output
316-
317-
318-
def test_pubsub_emulator_request_to_cloudevent_without_topic_path(
319-
raw_pubsub_request, raw_pubsub_cloudevent_output
320-
):
321-
req = flask.Request.from_values(json=raw_pubsub_request, path="/")
322-
cloudevent = event_conversion.background_event_to_cloudevent(req)
323-
324-
# Remove timestamps as they are generated on the fly.
325-
del raw_pubsub_cloudevent_output["time"]
326-
del cloudevent["time"]
327-
328-
# Default to the service name, when the topic is not configured subscription's pushEndpoint.
329-
raw_pubsub_cloudevent_output["source"] = "//pubsub.googleapis.com/"
384+
if "source" in overrides:
385+
# Default to the service name, when the topic is not configured subscription's pushEndpoint.
386+
raw_pubsub_cloudevent_output["source"] = overrides["source"]
330387

331388
assert cloudevent == raw_pubsub_cloudevent_output
332389

@@ -378,6 +435,18 @@ def test_pubsub_emulator_request_with_invalid_message(
378435
"providers/firebase.auth/eventTypes/user.create",
379436
"projects/my-project-id",
380437
),
438+
(
439+
"google.firebase.database.document.v1.written",
440+
"//firebasedatabase.googleapis.com/projects/_/locations/us-central1/instances/my-project-id",
441+
"providers/google.firebase.database/eventTypes/ref.write",
442+
"projects/_/instances/my-project-id/my/subject",
443+
),
444+
(
445+
"google.cloud.firestore.document.v1.written",
446+
"//firestore.googleapis.com/projects/project-id/databases/(default)",
447+
"providers/cloud.firestore/eventTypes/document.write",
448+
"projects/project-id/databases/(default)/my/subject",
449+
),
381450
],
382451
)
383452
def test_cloudevent_to_legacy_event(
@@ -406,7 +475,13 @@ def test_cloudevent_to_legacy_event_with_pubsub_message_payload(
406475
"google.cloud.pubsub.topic.v1.messagePublished",
407476
"//pubsub.googleapis.com/projects/sample-project/topics/gcf-test",
408477
)
409-
data = {"message": {"data": "fizzbuzz"}}
478+
data = {
479+
"message": {
480+
"data": "fizzbuzz",
481+
"messageId": "aaaaaa-1111-bbbb-2222-cccccccccccc",
482+
"publishTime": "2020-09-29T11:32:00.000Z",
483+
}
484+
}
410485
req = flask.Request.from_values(headers=headers, json=data)
411486

412487
(res_data, res_context) = event_conversion.cloudevent_to_background_event(req)

‎tests/test_data/firebase-db-cloudevent-output.json

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
{
2+
"specversion": "1.0",
3+
"type": "google.firebase.database.document.v1.written",
4+
"source": "//firebasedatabase.googleapis.com/projects/_/locations/us-central1/instances/my-project-id",
5+
"subject": "refs/gcf-test/xyz",
6+
"id": "aaaaaa-1111-bbbb-2222-cccccccccccc",
7+
"time": "2020-09-29T11:32:00.000Z",
8+
"datacontenttype": "application/json",
9+
"data": {
10+
"data": null,
11+
"delta": {
12+
"grandchild": "other"
13+
}
14+
}
15+
}

‎tests/test_data/firebase-db-legacy-input.json

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
{
2+
"eventType": "providers/google.firebase.database/eventTypes/ref.write",
3+
"params": {
4+
"child": "xyz"
5+
},
6+
"auth": {
7+
"admin": true
8+
},
9+
"domain": "firebaseio.com",
10+
"data": {
11+
"data": null,
12+
"delta": {
13+
"grandchild": "other"
14+
}
15+
},
16+
"resource": "projects/_/instances/my-project-id/refs/gcf-test/xyz",
17+
"timestamp": "2020-09-29T11:32:00.000Z",
18+
"eventId": "aaaaaa-1111-bbbb-2222-cccccccccccc"
19+
}

‎tests/test_functions/cloudevents/converted_background_event.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@ def function(cloudevent):
3535
"attr1": "attr1-value",
3636
},
3737
"data": "dGVzdCBtZXNzYWdlIDM=",
38+
"messageId": "aaaaaa-1111-bbbb-2222-cccccccccccc",
39+
"publishTime": "2020-09-29T11:32:00.000Z",
3840
},
3941
}
4042

0 commit comments

Comments
 (0)
0