XSUP-78726: Generic API Event Collector - fix pagination crash, integer ID cursor, and unresolved placeholders

Packs/GenericAPIEventCollector/Integrations/GenericAPIEventCollector/GenericAPIEventCollector.py

@@ -253,11 +253,13 @@ def fetch_events(
             if id_type.lower() not in ALL_ID_TYPES:
                 return_error(f"ID type {id_type} but must be one of {', '.join(ALL_ID_TYPES)}")
                 return {}, []
+            id_type_lower = id_type.lower()
             demisto.debug(f"ID type:{id_type}")
         else:
             return_error("ID type was not specified")
             return {}, []
 
+    previous_page_value: Any = None
     while pagination_needed:
         raw_events = client.search_events(
             endpoint=endpoint, http_method=http_method, request_data=request_data, ok_codes=ok_codes
@@ -266,15 +268,32 @@ def fetch_events(
         all_events_list.extend(events_list)
         demisto.debug(f"{len(all_events_list)} events fetched")
         pagination_needed, next_page_value = is_pagination_needed(raw_events, pagination_logic)
+        # Stop on an empty page or a cursor that did not advance, otherwise APIs that always return
+        # the pagination flag (e.g. a sequence cursor) would loop forever.
+        if pagination_needed and (
+            not events_list or next_page_value in (None, previous_page_value) or len(all_events_list) >= MAX_INCIDENTS_TO_FETCH
+        ):
+            demisto.debug(f"Stopping pagination: page size {len(events_list)}, next page value {next_page_value}")
+            pagination_needed = False
         if pagination_needed:
-            request_json = {pagination_logic.pagination_field_name: next_page_value}
-            request_data = RequestData(request_data.request_data, request_json, request_data.query_params)
+            previous_page_value = next_page_value
+            # pagination_field_name is a dot-separated path in the response; the request uses its last key.
+            page_param = {pagination_logic.pagination_field_name[-1]: next_page_value}
+            if http_method.upper() == "GET":
+                request_data = RequestData(
+                    request_data.request_data, request_data.request_json, {**(request_data.query_params or {}), **page_param}
+                )
+            else:
+                request_data = RequestData(
+                    request_data.request_data, {**(request_data.request_json or {}), **page_param}, request_data.query_params
+                )
 
     # endregion
 
     # region Collect all events based on their last fetch time.
     latest_created_datetime: datetime = last_fetched_datetime
     last_fetched_id: Any | None = last_run.get(PlaceHolders.LAST_FETCHED_ID.value)
+    previous_fetched_id: Any | None = last_fetched_id
     returned_event_list: list[dict[str, Any]] = []
     for event in all_events_list:
         try:
@@ -283,9 +302,16 @@ def fetch_events(
             demisto.error(f"Error parsing timestamp for event: {event} exception: {e}")
             continue
         event["_time"] = incident_created_dt.isoformat()
+        current_id: Any = dict_safe_get(event, id_keys) if id_keys else None
+        current_id_int = arg_to_number(current_id) if id_type_lower == IdTypes.INTEGER.value else None
+        previous_id_int = arg_to_number(previous_fetched_id) if id_type_lower == IdTypes.INTEGER.value else None
+
+        # An integer ID above the last fetched ID is new even if its timestamp is older, e.g. feeds
+        # ordered by an update sequence where the timestamp is the original content date.
+        is_new_by_id = current_id_int is not None and previous_id_int is not None and current_id_int > previous_id_int
 
         # to prevent duplicates, we are only adding events with creation_time > last fetched incident
-        if incident_created_dt > last_fetched_datetime:
+        if incident_created_dt > last_fetched_datetime or is_new_by_id:
             demisto.debug(f"Adding event with creation time: {incident_created_dt}")
             returned_event_list.append(event)
             latest_created_datetime = max(latest_created_datetime, incident_created_dt)
@@ -294,10 +320,9 @@ def fetch_events(
 
         # Handle the last event id.
         if id_keys:
-            current_id: Any = dict_safe_get(event, id_keys)
             demisto.debug(f"Current event id: {current_id}")
-            if last_fetched_id is not None and id_type_lower == IdTypes.INTEGER.value:
-                last_fetched_id = str(max(int(last_fetched_id), int(current_id)))  # noqa
+            if current_id_int is not None and last_fetched_id is not None:
+                last_fetched_id = str(max(int(last_fetched_id), current_id_int))
             else:
                 # We assume the last event contains the last id.
                 last_fetched_id = current_id

Packs/GenericAPIEventCollector/Integrations/GenericAPIEventCollector/GenericAPIEventCollector_test.py

@@ -944,6 +944,124 @@ def search_events(self, **kwargs):
     assert "NOT-A-TIMESTAMP" in mock_error.call_args[0][0]
 
 
+class RecordingClient:
+    """Returns the given pages in order and records the request of every call."""
+
+    def __init__(self, pages):
+        self.pages = list(pages)
+        self.requests: list[RequestData] = []
+
+    def search_events(self, **kwargs):
+        self.requests.append(kwargs["request_data"])
+        return self.pages.pop(0) if self.pages else {"items": []}
+
+
+def run_fetch(client, params, last_run, http_method="GET"):
+    return fetch_events(
+        client=client,
+        params=params,
+        last_run=last_run,
+        first_fetch_datetime=datetime(2026, 10, 11),
+        endpoint="/e",
+        http_method=http_method,
+        ok_codes=[200],
+        events_keys=["items"],
+        timestamp_field_config=TimestampFieldConfig(["datetime"], "%Y-%m-%dT%H:%M:%S"),
+    )
+
+
+def test_fetch_events_pagination_sends_cursor_as_query_param():
+    """
+    Given: pagination configured with a cursor field that is a list (as built by argToList).
+    When: fetch_events paginates a GET endpoint.
+    Then: no "unhashable type: 'list'" error, the cursor is sent as a query parameter on the next
+          request, and pagination stops on an empty page (regression test, XSUP-78726).
+    """
+    client = RecordingClient(
+        [
+            {"items": [{"datetime": "2026-10-12T00:00:00", "seqUpdate": 5}], "seqUpdate": 5},
+            {"items": [], "seqUpdate": 5},
+        ]
+    )
+    params = {"pagination_needed": "true", "pagination_field_name": "seqUpdate", "pagination_flag": "seqUpdate"}
+
+    _, events = run_fetch(client, params, {"@first_fetch_datetime": "2026-10-11T00:00:00"})
+
+    assert len(events) == 1
+    assert len(client.requests) == 2
+    assert client.requests[1].query_params == {"seqUpdate": 5}
+
+
+def test_fetch_events_pagination_stops_when_cursor_does_not_advance():
+    """
+    Given: an API that always returns the pagination flag with the same cursor.
+    When: fetch_events paginates.
+    Then: it stops instead of looping forever.
+    """
+    page = {"items": [{"datetime": "2026-10-12T00:00:00"}], "next": "abc"}
+    client = RecordingClient([page, page, page])
+    params = {"pagination_needed": "true", "pagination_field_name": "next", "pagination_flag": "next"}
+
+    run_fetch(client, params, {"@first_fetch_datetime": "2026-10-11T00:00:00"}, http_method="POST")
+
+    assert len(client.requests) == 2
+    assert client.requests[1].request_json == {"next": "abc"}
+
+
+def test_fetch_events_integer_id_cursor_accepts_events_with_old_timestamps():
+    """
+    Given: a feed ordered by an integer update sequence whose timestamp is the original content date
+           (Group-IB darkweb items), and a last run holding the previous sequence.
+    When: fetch_events runs with ID type "integer".
+    Then: events with a higher sequence are ingested even though their timestamp is old, events at
+          or below the stored sequence are dropped, and the cursor becomes the maximum sequence
+          even when the API returns items out of order (regression test, XSUP-78726).
+    """
+    client = RecordingClient(
+        [
+            {
+                "items": [
+                    {"datetime": "2014-06-04T01:28:00", "seqUpdate": 300},
+                    {"datetime": "2025-03-09T17:15:00", "seqUpdate": 100},
+                    {"datetime": "2023-12-23T15:28:00", "seqUpdate": 200},
+                ]
+            }
+        ]
+    )
+    params = {"id_keys": "seqUpdate", "id_type": "integer"}
+    last_run = {
+        "@first_fetch_datetime": "2026-10-11T00:00:00",
+        "@last_fetched_datetime": "2026-10-11T00:00:00",
+        "@last_fetched_id": "100",
+    }
+
+    next_run, events = run_fetch(client, params, last_run)
+
+    assert [event["seqUpdate"] for event in events] == [300, 200]
+    assert next_run["@last_fetched_id"] == "300"
+    assert next_run["@last_fetched_datetime"] == "2026-10-11T00:00:00"
+
+
+def test_fetch_events_string_id_keeps_time_filter():
+    """
+    Given: ID type "string".
+    When: fetch_events runs with events older than the last fetched time.
+    Then: the time filter still applies (string IDs are not comparable), and the last event's ID is kept.
+    """
+    client = RecordingClient([{"items": [{"datetime": "2020-01-01T00:00:00", "id": "b"}]}])
+    params = {"id_keys": "id", "id_type": "string"}
+    last_run = {
+        "@first_fetch_datetime": "2026-10-11T00:00:00",
+        "@last_fetched_datetime": "2026-10-11T00:00:00",
+        "@last_fetched_id": "a",
+    }
+
+    next_run, events = run_fetch(client, params, last_run)
+
+    assert events == []
+    assert next_run["@last_fetched_id"] == "b"
+
+
 def test_derive_authorize_url_swaps_last_path_segment():
     """
     Given: a Microsoft Entra style token endpoint.

Packs/GenericAPIEventCollector/ReleaseNotes/1_1_1.md

@@ -0,0 +1,9 @@
+
+#### Integrations
+
+##### Generic API Event Collector (Beta)
+
+- Fixed an issue where fetching events failed with the *unhashable type: 'list'* error when *Is pagination needed* was enabled. The next page value is now sent as a query parameter for GET requests and in the JSON body for POST requests.
+- Fixed an issue where pagination could loop indefinitely when the API kept returning the same next page value.
+- Fixed an issue where the *integer* ID type was ignored, so the last fetched ID was the ID of the last event returned instead of the highest ID.
+- Improved implementation so that when the ID type is *integer*, events with an ID higher than the last fetched ID are collected even if their timestamp is older than the last fetch time.

Packs/GenericAPIEventCollector/pack_metadata.json

@@ -3,7 +3,7 @@
     "description": "This pack provides a generic API event collector integration that can be used to collect events from various sources.",
     "support": "xsoar",
     "serverMinVersion": "6.0.0",
-    "currentVersion": "1.1.0",
+    "currentVersion": "1.1.1",
     "author": "Cortex XSOAR",
     "url": "https://www.paloaltonetworks.com/cortex",
     "email": "",