fix: cloudevent conversion conformance tests (GoogleCloudPlatform#139)

This commit updates the legacy event to cloud event conversion logic to:

- include the `messageId` and `publishTime` fields in the data payload of Pub/Sub events
- include the location in the source field of firebasedatabase events

fixes GoogleCloudPlatform#139

Author: matthewrobertson

.github/workflows/conformance.yml

@@ -24,23 +24,23 @@ jobs:
         go-version: '1.15'
 
     - name: Run HTTP conformance tests
-      uses: GoogleCloudPlatform/functions-framework-conformance/action@v0.3.10
+      uses: GoogleCloudPlatform/functions-framework-conformance/action@v0.3.12
       with:
         functionType: 'http'
         useBuildpacks: false
         validateMapping: false
         cmd: "'functions-framework --source tests/conformance/main.py --target write_http --signature-type http'"
 
     - name: Run event conformance tests
-      uses: GoogleCloudPlatform/functions-framework-conformance/action@v0.3.10
+      uses: GoogleCloudPlatform/functions-framework-conformance/action@v0.3.12
       with:
         functionType: 'legacyevent'
         useBuildpacks: false
         validateMapping: true
         cmd: "'functions-framework --source tests/conformance/main.py --target write_legacy_event --signature-type event'"
 
     - name: Run cloudevent conformance tests
-      uses: GoogleCloudPlatform/functions-framework-conformance/action@v0.3.10
+      uses: GoogleCloudPlatform/functions-framework-conformance/action@v0.3.12
       with:
         functionType: 'cloudevent'
         useBuildpacks: false

.gitignore

@@ -6,3 +6,6 @@ build/
 dist/
 .coverage
 .vscode/
+function_output.json
+serverlog_stderr.txt
+serverlog_stdout.txt

src/functions_framework/event_conversion.py

@@ -94,7 +94,7 @@
 # for the subject.
 _CE_SERVICE_TO_RESOURCE_RE = {
     _FIREBASE_CE_SERVICE: re.compile(r"^(projects/[^/]+)/(events/[^/]+)$"),
-    _FIREBASE_DB_CE_SERVICE: re.compile(r"^(projects/[^/]/instances/[^/]+)/(refs/.+)$"),
+    _FIREBASE_DB_CE_SERVICE: re.compile(r"^projects/_/(instances/[^/]+)/(refs/.+)$"),
     _FIRESTORE_CE_SERVICE: re.compile(
         r"^(projects/[^/]+/databases/\(default\))/(documents/.+)$"
     ),
@@ -131,9 +131,14 @@ def background_event_to_cloudevent(request) -> CloudEvent:
     new_type = _BACKGROUND_TO_CE_TYPE[context.event_type]
 
     service, resource, subject = _split_resource(context)
+    source = f"//{service}/{resource}"
 
     # Handle Pub/Sub events.
     if service == _PUBSUB_CE_SERVICE:
+        if "messageId" not in data:
+            data["messageId"] = context.event_id
+        if "publishTime" not in data:
+            data["publishTime"] = context.timestamp
         data = {"message": data}
 
     # Handle Firebase Auth events.
@@ -147,13 +152,30 @@ def background_event_to_cloudevent(request) -> CloudEvent:
             uid = data["uid"]
             subject = f"users/{uid}"
 
+    # Handle Firebase DB events.
+    if service == _FIREBASE_DB_CE_SERVICE:
+        # The CE source of firebasedatabase cloudevents includes location information
+        # that is inferred from the 'domain' field of legacy events.
+        if "domain" not in event_data:
+            raise EventConversionException(
+                "Invalid FirebaseDB event payload: missing 'domain'"
+            )
+
+        domain = event_data["domain"]
+        location = "us-central1"
+        if domain != "firebaseio.com":
+            location = domain.split(".")[0]
+
+        resource = f"projects/_/locations/{location}/{resource}"
+        source = f"//{service}/{resource}"
+
     metadata = {
         "id": context.event_id,
         "time": context.timestamp,
         "specversion": _CLOUDEVENT_SPEC_VERSION,
         "datacontenttype": "application/json",
         "type": new_type,
-        "source": f"//{service}/{resource}",
+        "source": source,
     }
 
     if subject:
@@ -201,6 +223,10 @@ def cloudevent_to_background_event(request) -> Tuple[Any, Context]:
             resource = {"service": service, "name": name, "type": _PUBSUB_MESSAGE_TYPE}
             if "message" in data:
                 data = data["message"]
+            if "messageId" in data:
+                del data["messageId"]
+            if "publishTime" in data:
+                del data["publishTime"]
         elif service == _FIREBASE_AUTH_CE_SERVICE:
             resource = name
             if "metadata" in data:
@@ -214,6 +240,9 @@ def cloudevent_to_background_event(request) -> Tuple[Any, Context]:
                 "service": service,
                 "type": data["kind"],
             }
+        elif service == _FIREBASE_DB_CE_SERVICE:
+            name = re.sub("/locations/[^/]+", "", name)
+            resource = f"{name}/{event['subject']}"
         else:
             resource = f"{name}/{event['subject']}"
 

tests/test_convert.py

@@ -75,6 +75,8 @@
     "data": {
         "message": {
             "data": "10",
+            "publishTime": "2020-05-18T12:13:19Z",
+            "messageId": "1215011316659232",
         },
     },
 }
@@ -122,7 +124,10 @@ def marshalled_pubsub_request():
 def raw_pubsub_cloudevent_output(marshalled_pubsub_request):
     event = PUBSUB_CLOUD_EVENT.copy()
     # the data payload is more complex for the raw pubsub request
-    event["data"] = {"message": marshalled_pubsub_request["data"]}
+    data = marshalled_pubsub_request["data"]
+    data["messageId"] = event["id"]
+    data["publishTime"] = event["time"]
+    event["data"] = {"message": data}
     return from_json(json.dumps(event))
 
 
@@ -138,6 +143,18 @@ def firebase_auth_cloudevent_output():
         return from_json(f.read())
 
 
+@pytest.fixture
+def firebase_db_background_input():
+    with open(TEST_DATA_DIR / "firebase-db-legacy-input.json", "r") as f:
+        return json.load(f)
+
+
+@pytest.fixture
+def firebase_db_cloudevent_output():
+    with open(TEST_DATA_DIR / "firebase-db-cloudevent-output.json", "r") as f:
+        return from_json(f.read())
+
+
 @pytest.fixture
 def create_ce_headers():
     return lambda event_type, source: {
@@ -207,6 +224,41 @@ def test_firebase_auth_event_to_cloudevent_no_uid(
     assert cloudevent == firebase_auth_cloudevent_output
 
 
+def test_firebase_db_event_to_cloudevent_default_location(
+    firebase_db_background_input, firebase_db_cloudevent_output
+):
+    req = flask.Request.from_values(json=firebase_db_background_input)
+    cloudevent = event_conversion.background_event_to_cloudevent(req)
+    assert cloudevent == firebase_db_cloudevent_output
+
+
+def test_firebase_db_event_to_cloudevent_location_subdomain(
+    firebase_db_background_input, firebase_db_cloudevent_output
+):
+    firebase_db_background_input["domain"] = "europe-west1.firebasedatabase.app"
+    firebase_db_cloudevent_output["source"] = firebase_db_cloudevent_output[
+        "source"
+    ].replace("us-central1", "europe-west1")
+
+    req = flask.Request.from_values(json=firebase_db_background_input)
+    cloudevent = event_conversion.background_event_to_cloudevent(req)
+    assert cloudevent == firebase_db_cloudevent_output
+
+
+def test_firebase_db_event_to_cloudevent_missing_domain(
+    firebase_db_background_input, firebase_db_cloudevent_output
+):
+    del firebase_db_background_input["domain"]
+    req = flask.Request.from_values(json=firebase_db_background_input)
+
+    with pytest.raises(EventConversionException) as exc_info:
+        event_conversion.background_event_to_cloudevent(req)
+
+    assert (
+        "Invalid FirebaseDB event payload: missing 'domain'" in exc_info.value.args[0]
+    )
+
+
 @pytest.mark.parametrize(
     "background_resource",
     [
@@ -299,34 +351,39 @@ def test_marshal_background_event_data_with_topic_path(
     assert payload == marshalled_pubsub_request
 
 
+@pytest.mark.parametrize(
+    "request_fixture, overrides",
+    [
+        (
+            "raw_pubsub_request",
+            {
+                "request_path": "x/projects/sample-project/topics/gcf-test?pubsub_trigger=true",
+            },
+        ),
+        ("raw_pubsub_request", {"source": "//pubsub.googleapis.com/"}),
+        ("marshalled_pubsub_request", {}),
+    ],
+)
 def test_pubsub_emulator_request_to_cloudevent(
-    raw_pubsub_request, raw_pubsub_cloudevent_output
+    raw_pubsub_cloudevent_output, request_fixture, overrides, request
 ):
+    request_path = overrides.get("request_path", "/")
+    payload = request.getfixturevalue(request_fixture)
     req = flask.Request.from_values(
-        json=raw_pubsub_request,
-        path="x/projects/sample-project/topics/gcf-test?pubsub_trigger=true",
+        path=request_path,
+        json=payload,
     )
     cloudevent = event_conversion.background_event_to_cloudevent(req)
 
     # Remove timestamps as they are generated on the fly.
     del raw_pubsub_cloudevent_output["time"]
+    del raw_pubsub_cloudevent_output.data["message"]["publishTime"]
     del cloudevent["time"]
+    del cloudevent.data["message"]["publishTime"]
 
-    assert cloudevent == raw_pubsub_cloudevent_output
-
-
-def test_pubsub_emulator_request_to_cloudevent_without_topic_path(
-    raw_pubsub_request, raw_pubsub_cloudevent_output
-):
-    req = flask.Request.from_values(json=raw_pubsub_request, path="/")
-    cloudevent = event_conversion.background_event_to_cloudevent(req)
-
-    # Remove timestamps as they are generated on the fly.
-    del raw_pubsub_cloudevent_output["time"]
-    del cloudevent["time"]
-
-    # Default to the service name, when the topic is not configured subscription's pushEndpoint.
-    raw_pubsub_cloudevent_output["source"] = "//pubsub.googleapis.com/"
+    if "source" in overrides:
+        # Default to the service name, when the topic is not configured subscription's pushEndpoint.
+        raw_pubsub_cloudevent_output["source"] = overrides["source"]
 
     assert cloudevent == raw_pubsub_cloudevent_output
 
@@ -378,6 +435,18 @@ def test_pubsub_emulator_request_with_invalid_message(
             "providers/firebase.auth/eventTypes/user.create",
             "projects/my-project-id",
         ),
+        (
+            "google.firebase.database.document.v1.written",
+            "//firebasedatabase.googleapis.com/projects/_/locations/us-central1/instances/my-project-id",
+            "providers/google.firebase.database/eventTypes/ref.write",
+            "projects/_/instances/my-project-id/my/subject",
+        ),
+        (
+            "google.cloud.firestore.document.v1.written",
+            "//firestore.googleapis.com/projects/project-id/databases/(default)",
+            "providers/cloud.firestore/eventTypes/document.write",
+            "projects/project-id/databases/(default)/my/subject",
+        ),
     ],
 )
 def test_cloudevent_to_legacy_event(
@@ -406,7 +475,13 @@ def test_cloudevent_to_legacy_event_with_pubsub_message_payload(
         "google.cloud.pubsub.topic.v1.messagePublished",
         "//pubsub.googleapis.com/projects/sample-project/topics/gcf-test",
     )
-    data = {"message": {"data": "fizzbuzz"}}
+    data = {
+        "message": {
+            "data": "fizzbuzz",
+            "messageId": "aaaaaa-1111-bbbb-2222-cccccccccccc",
+            "publishTime": "2020-09-29T11:32:00.000Z",
+        }
+    }
     req = flask.Request.from_values(headers=headers, json=data)
 
     (res_data, res_context) = event_conversion.cloudevent_to_background_event(req)

tests/test_data/firebase-db-cloudevent-output.json

@@ -0,0 +1,15 @@
+{
+    "specversion": "1.0",
+    "type": "google.firebase.database.document.v1.written",
+    "source": "//firebasedatabase.googleapis.com/projects/_/locations/us-central1/instances/my-project-id",
+    "subject": "refs/gcf-test/xyz",
+    "id": "aaaaaa-1111-bbbb-2222-cccccccccccc",
+    "time": "2020-09-29T11:32:00.000Z",
+    "datacontenttype": "application/json",
+    "data": {
+      "data": null,
+      "delta": {
+        "grandchild": "other"
+      }
+    }
+  }

tests/test_data/firebase-db-legacy-input.json

@@ -0,0 +1,19 @@
+{
+    "eventType": "providers/google.firebase.database/eventTypes/ref.write",
+    "params": {
+      "child": "xyz"
+    },
+    "auth": {
+      "admin": true
+    },
+    "domain": "firebaseio.com",
+    "data": {
+      "data": null,
+      "delta": {
+        "grandchild": "other"
+      }
+    },
+    "resource": "projects/_/instances/my-project-id/refs/gcf-test/xyz",
+    "timestamp": "2020-09-29T11:32:00.000Z",
+    "eventId": "aaaaaa-1111-bbbb-2222-cccccccccccc"
+  }

tests/test_functions/cloudevents/converted_background_event.py

@@ -35,6 +35,8 @@ def function(cloudevent):
                 "attr1": "attr1-value",
             },
             "data": "dGVzdCBtZXNzYWdlIDM=",
+            "messageId": "aaaaaa-1111-bbbb-2222-cccccccccccc",
+            "publishTime": "2020-09-29T11:32:00.000Z",
         },
     }