|
| 1 | +import json |
| 2 | +import logging |
| 3 | +import re |
| 4 | + |
| 5 | +from localstack import config |
| 6 | +from localstack.aws.api.lambda_ import FilterCriteria |
| 7 | +from localstack.utils.event_matcher import matches_event |
| 8 | +from localstack.utils.strings import first_char_to_lower |
| 9 | + |
| 10 | +LOG = logging.getLogger(__name__) |
| 11 | + |
| 12 | + |
| 13 | +class InvalidEventPatternException(Exception): |
| 14 | + reason: str |
| 15 | + |
| 16 | + def __init__(self, reason=None, message=None) -> None: |
| 17 | + self.reason = reason |
| 18 | + self.message = message or f"Event pattern is not valid. Reason: {reason}" |
| 19 | + |
| 20 | + |
| 21 | +def filter_stream_records(records, filters: list[FilterCriteria]): |
| 22 | + filtered_records = [] |
| 23 | + for record in records: |
| 24 | + for filter in filters: |
| 25 | + for rule in filter["Filters"]: |
| 26 | + if config.EVENT_RULE_ENGINE == "java": |
| 27 | + event_str = json.dumps(record) |
| 28 | + event_pattern_str = rule["Pattern"] |
| 29 | + match_result = matches_event(event_pattern_str, event_str) |
| 30 | + else: |
| 31 | + filter_pattern: dict[str, any] = json.loads(rule["Pattern"]) |
| 32 | + match_result = does_match_event(filter_pattern, record) |
| 33 | + if match_result: |
| 34 | + filtered_records.append(record) |
| 35 | + break |
| 36 | + return filtered_records |
| 37 | + |
| 38 | + |
| 39 | +def does_match_event(event_pattern: dict[str, any], event: dict[str, any]) -> bool: |
| 40 | + """Decides whether an event pattern matches an event or not. |
| 41 | + Returns True if the `event_pattern` matches the given `event` and False otherwise. |
| 42 | +
|
| 43 | + Implements "Amazon EventBridge event patterns": |
| 44 | + https://docs.aws.amazon.com/eventbridge/latest/userguide/eb-event-patterns.html |
| 45 | + Used in different places: |
| 46 | + * EventBridge: https://docs.aws.amazon.com/eventbridge/latest/userguide/eb-event-patterns.html |
| 47 | + * Lambda ESM: https://docs.aws.amazon.com/lambda/latest/dg/invocation-eventfiltering.html |
| 48 | + * EventBridge Pipes: https://docs.aws.amazon.com/eventbridge/latest/userguide/eb-pipes-event-filtering.html |
| 49 | + * SNS: https://docs.aws.amazon.com/sns/latest/dg/sns-subscription-filter-policies.html |
| 50 | +
|
| 51 | + Open source AWS rule engine: https://github.com/aws/event-ruler |
| 52 | + """ |
| 53 | + # TODO: test this conditional: https://coveralls.io/builds/66584026/source?filename=localstack%2Fservices%2Flambda_%2Fevent_source_listeners%2Futils.py#L25 |
| 54 | + if not event_pattern: |
| 55 | + return True |
| 56 | + does_match_results = [] |
| 57 | + for key, value in event_pattern.items(): |
| 58 | + # check if rule exists in event |
| 59 | + event_value = event.get(key) if isinstance(event, dict) else None |
| 60 | + does_pattern_match = False |
| 61 | + if event_value is not None: |
| 62 | + # check if filter rule value is a list (leaf of rule tree) or a dict (recursively call function) |
| 63 | + if isinstance(value, list): |
| 64 | + if len(value) > 0: |
| 65 | + if isinstance(value[0], (str, int)): |
| 66 | + does_pattern_match = event_value in value |
| 67 | + if isinstance(value[0], dict): |
| 68 | + does_pattern_match = verify_dict_filter(event_value, value[0]) |
| 69 | + else: |
| 70 | + LOG.warning("Empty lambda filter: %s", key) |
| 71 | + elif isinstance(value, dict): |
| 72 | + does_pattern_match = does_match_event(value, event_value) |
| 73 | + else: |
| 74 | + # special case 'exists' |
| 75 | + def _filter_rule_value_list(val): |
| 76 | + if isinstance(val[0], dict): |
| 77 | + return not val[0].get("exists", True) |
| 78 | + elif val[0] is None: |
| 79 | + # support null filter |
| 80 | + return True |
| 81 | + |
| 82 | + def _filter_rule_value_dict(val): |
| 83 | + for k, v in val.items(): |
| 84 | + return ( |
| 85 | + _filter_rule_value_list(val[k]) |
| 86 | + if isinstance(val[k], list) |
| 87 | + else _filter_rule_value_dict(val[k]) |
| 88 | + ) |
| 89 | + return True |
| 90 | + |
| 91 | + if isinstance(value, list) and len(value) > 0: |
| 92 | + does_pattern_match = _filter_rule_value_list(value) |
| 93 | + elif isinstance(value, dict): |
| 94 | + # special case 'exists' for S type, e.g. {"S": [{"exists": false}]} |
| 95 | + # https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Streams.Lambda.Tutorial2.html |
| 96 | + does_pattern_match = _filter_rule_value_dict(value) |
| 97 | + |
| 98 | + does_match_results.append(does_pattern_match) |
| 99 | + return all(does_match_results) |
| 100 | + |
| 101 | + |
| 102 | +def verify_dict_filter(record_value: any, dict_filter: dict[str, any]) -> bool: |
| 103 | + # https://docs.aws.amazon.com/lambda/latest/dg/invocation-eventfiltering.html#filtering-syntax |
| 104 | + does_match_filter = False |
| 105 | + for key, filter_value in dict_filter.items(): |
| 106 | + if key == "anything-but": |
| 107 | + does_match_filter = record_value not in filter_value |
| 108 | + elif key == "numeric": |
| 109 | + does_match_filter = handle_numeric_conditions(record_value, filter_value) |
| 110 | + elif key == "exists": |
| 111 | + does_match_filter = bool( |
| 112 | + filter_value |
| 113 | + ) # exists means that the key exists in the event record |
| 114 | + elif key == "prefix": |
| 115 | + if not isinstance(record_value, str): |
| 116 | + LOG.warning("Record Value %s does not seem to be a valid string.", record_value) |
| 117 | + does_match_filter = isinstance(record_value, str) and record_value.startswith( |
| 118 | + str(filter_value) |
| 119 | + ) |
| 120 | + if does_match_filter: |
| 121 | + return True |
| 122 | + |
| 123 | + return does_match_filter |
| 124 | + |
| 125 | + |
| 126 | +def handle_numeric_conditions( |
| 127 | + first_operand: int | float, conditions: list[str | int | float] |
| 128 | +) -> bool: |
| 129 | + """Implements numeric matching for a given list of conditions. |
| 130 | + Example: { "numeric": [ ">", 0, "<=", 5 ] } |
| 131 | +
|
| 132 | + Numeric matching works with values that are JSON numbers. |
| 133 | + It is limited to values between -5.0e9 and +5.0e9 inclusive, with 15 digits of precision, |
| 134 | + or six digits to the right of the decimal point. |
| 135 | + https://docs.aws.amazon.com/eventbridge/latest/userguide/eb-event-patterns-content-based-filtering.html#filtering-numeric-matchinghttps://docs.aws.amazon.com/eventbridge/latest/userguide/eb-event-patterns-content-based-filtering.html#filtering-numeric-matching |
| 136 | + """ |
| 137 | + # Invalid example for uneven list: { "numeric": [ ">", 0, "<" ] } |
| 138 | + if len(conditions) % 2 > 0: |
| 139 | + raise InvalidEventPatternException("Bad numeric range operator") |
| 140 | + |
| 141 | + if not isinstance(first_operand, (int, float)): |
| 142 | + raise InvalidEventPatternException( |
| 143 | + f"The value {first_operand} for the numeric comparison {conditions} is not a valid number" |
| 144 | + ) |
| 145 | + |
| 146 | + for i in range(0, len(conditions), 2): |
| 147 | + operator = conditions[i] |
| 148 | + second_operand_str = conditions[i + 1] |
| 149 | + try: |
| 150 | + second_operand = float(second_operand_str) |
| 151 | + except ValueError: |
| 152 | + raise InvalidEventPatternException( |
| 153 | + f"Could not convert filter value {second_operand_str} to a valid number" |
| 154 | + ) from ValueError |
| 155 | + |
| 156 | + if operator == ">" and not (first_operand > second_operand): |
| 157 | + return False |
| 158 | + if operator == ">=" and not (first_operand >= second_operand): |
| 159 | + return False |
| 160 | + if operator == "=" and not (first_operand == second_operand): |
| 161 | + return False |
| 162 | + if operator == "<" and not (first_operand < second_operand): |
| 163 | + return False |
| 164 | + if operator == "<=" and not (first_operand <= second_operand): |
| 165 | + return False |
| 166 | + return True |
| 167 | + |
| 168 | + |
| 169 | +def contains_list(filter: dict) -> bool: |
| 170 | + if isinstance(filter, dict): |
| 171 | + for key, value in filter.items(): |
| 172 | + if isinstance(value, list) and len(value) > 0: |
| 173 | + return True |
| 174 | + return contains_list(value) |
| 175 | + return False |
| 176 | + |
| 177 | + |
| 178 | +def validate_filters(filter: FilterCriteria) -> bool: |
| 179 | + # filter needs to be json serializeable |
| 180 | + for rule in filter["Filters"]: |
| 181 | + try: |
| 182 | + if not (filter_pattern := json.loads(rule["Pattern"])): |
| 183 | + return False |
| 184 | + return contains_list(filter_pattern) |
| 185 | + except json.JSONDecodeError: |
| 186 | + return False |
| 187 | + # needs to contain on what to filter (some list with citerias) |
| 188 | + # https://docs.aws.amazon.com/lambda/latest/dg/invocation-eventfiltering.html#filtering-syntax |
| 189 | + |
| 190 | + return True |
| 191 | + |
| 192 | + |
| 193 | +def message_attributes_to_lower(message_attrs): |
| 194 | + """Convert message attribute details (first characters) to lower case (e.g., stringValue, dataType).""" |
| 195 | + message_attrs = message_attrs or {} |
| 196 | + for _, attr in message_attrs.items(): |
| 197 | + if not isinstance(attr, dict): |
| 198 | + continue |
| 199 | + for key, value in dict(attr).items(): |
| 200 | + attr[first_char_to_lower(key)] = attr.pop(key) |
| 201 | + return message_attrs |
| 202 | + |
| 203 | + |
| 204 | +def event_source_arn_matches(mapped: str, searched: str) -> bool: |
| 205 | + if not mapped: |
| 206 | + return False |
| 207 | + if not searched or mapped == searched: |
| 208 | + return True |
| 209 | + # Some types of ARNs can end with a path separated by slashes, for |
| 210 | + # example the ARN of a DynamoDB stream is tableARN/stream/ID. It's |
| 211 | + # a little counterintuitive that a more specific mapped ARN can |
| 212 | + # match a less specific ARN on the event, but some integration tests |
| 213 | + # rely on it for things like subscribing to a stream and matching an |
| 214 | + # event labeled with the table ARN. |
| 215 | + if re.match(r"^%s$" % searched, mapped): |
| 216 | + return True |
| 217 | + if mapped.startswith(searched): |
| 218 | + suffix = mapped[len(searched) :] |
| 219 | + return suffix[0] == "/" |
| 220 | + return False |
| 221 | + |
| 222 | + |
| 223 | +def has_data_filter_criteria(filters: list[FilterCriteria]) -> bool: |
| 224 | + for filter in filters: |
| 225 | + for rule in filter.get("Filters", []): |
| 226 | + parsed_pattern = json.loads(rule["Pattern"]) |
| 227 | + if "data" in parsed_pattern: |
| 228 | + return True |
| 229 | + return False |
| 230 | + |
| 231 | + |
| 232 | +def has_data_filter_criteria_parsed(parsed_filters: list[dict]) -> bool: |
| 233 | + for filter in parsed_filters: |
| 234 | + if "data" in filter: |
| 235 | + return True |
| 236 | + return False |
0 commit comments