From 926bdf67b523791777a41d09b10483a526e4d6af Mon Sep 17 00:00:00 2001 From: Ajinkya Date: Fri, 10 Jan 2025 13:50:45 +0530 Subject: [PATCH] added filters support in threat detection APIs --- .../SuspectSampleDataAction.java | 76 +++++++++++-------- .../components/SusDataTable.jsx | 6 -- .../service/MaliciousEventService.java | 28 ++++++- .../backend/service/ThreatActorService.java | 23 ++++++ .../backend/service/ThreatApiService.java | 21 +++++ .../dashboard_service/v1/service.proto | 27 +++++++ 6 files changed, 142 insertions(+), 39 deletions(-) diff --git a/apps/dashboard/src/main/java/com/akto/action/threat_detection/SuspectSampleDataAction.java b/apps/dashboard/src/main/java/com/akto/action/threat_detection/SuspectSampleDataAction.java index c4aa7600a5..35d8678b05 100644 --- a/apps/dashboard/src/main/java/com/akto/action/threat_detection/SuspectSampleDataAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/threat_detection/SuspectSampleDataAction.java @@ -42,20 +42,39 @@ public SuspectSampleDataAction() { } public String fetchSampleData() { - HttpPost post = - new HttpPost( - String.format("%s/api/dashboard/list_malicious_requests", this.getBackendUrl())); + HttpPost post = new HttpPost( + String.format("%s/api/dashboard/list_malicious_requests", this.getBackendUrl())); post.addHeader("Authorization", "Bearer " + this.getApiToken()); post.addHeader("Content-Type", "application/json"); - Map body = - new HashMap() { - { - put("skip", skip); - put("limit", LIMIT); - put("sort", sort); - } - }; + Map filter = new HashMap<>(); + if (this.ips != null && !this.ips.isEmpty()) { + filter.put("ips", this.ips); + } + + if (this.urls != null && !this.urls.isEmpty()) { + filter.put("urls", this.urls); + } + + Map time_range = new HashMap<>(); + if (this.startTimestamp > 0) { + time_range.put("start", this.startTimestamp); + } + + if (this.endTimestamp > 0) { + time_range.put("end", this.endTimestamp); + } + + filter.put("detected_at_time_range", time_range); + + Map body = new HashMap() { + { + put("skip", skip); + put("limit", LIMIT); + put("sort", sort); + put("filter", filter); + } + }; String msg = objectMapper.valueToTree(body).toString(); StringEntity requestEntity = new StringEntity(msg, ContentType.APPLICATION_JSON); @@ -65,24 +84,22 @@ public String fetchSampleData() { String responseBody = EntityUtils.toString(resp.getEntity()); ProtoMessageUtils.toProtoMessage( - ListMaliciousRequestsResponse.class, responseBody) + ListMaliciousRequestsResponse.class, responseBody) .ifPresent( m -> { - this.maliciousEvents = - m.getMaliciousEventsList().stream() - .map( - smr -> - new DashboardMaliciousEvent( - smr.getId(), - smr.getActor(), - smr.getFilterId(), - smr.getEndpoint(), - URLMethods.Method.fromString(smr.getMethod()), - smr.getApiCollectionId(), - smr.getIp(), - smr.getCountry(), - smr.getDetectedAt())) - .collect(Collectors.toList()); + this.maliciousEvents = m.getMaliciousEventsList().stream() + .map( + smr -> new DashboardMaliciousEvent( + smr.getId(), + smr.getActor(), + smr.getFilterId(), + smr.getEndpoint(), + URLMethods.Method.fromString(smr.getMethod()), + smr.getApiCollectionId(), + smr.getIp(), + smr.getCountry(), + smr.getDetectedAt())) + .collect(Collectors.toList()); this.total = m.getTotal(); }); } catch (Exception e) { @@ -94,8 +111,7 @@ public String fetchSampleData() { } public String fetchFilters() { - HttpGet get = - new HttpGet(String.format("%s/api/dashboard/fetch_filters", this.getBackendUrl())); + HttpGet get = new HttpGet(String.format("%s/api/dashboard/fetch_filters", this.getBackendUrl())); get.addHeader("Authorization", "Bearer " + this.getApiToken()); get.addHeader("Content-Type", "application/json"); @@ -103,7 +119,7 @@ public String fetchFilters() { String responseBody = EntityUtils.toString(resp.getEntity()); ProtoMessageUtils.toProtoMessage( - FetchAlertFiltersResponse.class, responseBody) + FetchAlertFiltersResponse.class, responseBody) .ifPresent( msg -> { this.ips = msg.getActorsList(); diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/threat_detection/components/SusDataTable.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/threat_detection/components/SusDataTable.jsx index 24c8d8387a..8b1755786f 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/threat_detection/components/SusDataTable.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/threat_detection/components/SusDataTable.jsx @@ -149,12 +149,6 @@ function SusDataTable({ currDateRange, rowClicked }) { }); filters = [ - { - key: "apiCollectionId", - label: "Collection", - title: "Collection", - choices: apiCollectionFilterChoices, - }, { key: "sourceIps", label: "Source IP", diff --git a/apps/threat-detection-backend/src/main/java/com/akto/threat/backend/service/MaliciousEventService.java b/apps/threat-detection-backend/src/main/java/com/akto/threat/backend/service/MaliciousEventService.java index 9868373d1a..7bd58e6888 100644 --- a/apps/threat-detection-backend/src/main/java/com/akto/threat/backend/service/MaliciousEventService.java +++ b/apps/threat-detection-backend/src/main/java/com/akto/threat/backend/service/MaliciousEventService.java @@ -10,6 +10,7 @@ import com.akto.proto.generated.threat_detection.service.dashboard_service.v1.FetchAlertFiltersResponse; import com.akto.proto.generated.threat_detection.service.dashboard_service.v1.ListMaliciousRequestsRequest; import com.akto.proto.generated.threat_detection.service.dashboard_service.v1.ListMaliciousRequestsResponse; +import com.akto.proto.generated.threat_detection.service.dashboard_service.v1.TimeRangeFilter; import com.akto.proto.generated.threat_detection.service.malicious_alert_service.v1.RecordMaliciousEventRequest; import com.akto.threat.backend.client.IPLookupClient; import com.akto.threat.backend.constants.KafkaTopic; @@ -17,7 +18,6 @@ import com.akto.threat.backend.db.AggregateSampleMaliciousEventModel; import com.akto.threat.backend.db.MaliciousEventModel; import com.akto.threat.backend.utils.KafkaUtils; -import com.mongodb.BasicDBObject; import com.mongodb.client.DistinctIterable; import com.mongodb.client.MongoClient; import com.mongodb.client.MongoCollection; @@ -28,6 +28,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import org.bson.Document; import org.bson.conversions.Bson; public class MaliciousEventService { @@ -135,6 +136,7 @@ public ListMaliciousRequestsResponse listMaliciousRequests( int limit = request.getLimit(); int skip = request.hasSkip() ? request.getSkip() : 0; Map sort = request.getSortMap(); + ListMaliciousRequestsRequest.Filter filter = request.getFilter(); MongoCollection coll = this.mongoClient @@ -142,11 +144,31 @@ public ListMaliciousRequestsResponse listMaliciousRequests( .getCollection( MongoDBCollection.ThreatDetection.MALICIOUS_EVENTS, MaliciousEventModel.class); - BasicDBObject query = new BasicDBObject(); + Document query = new Document(); + if (!filter.getActorsList().isEmpty()) { + query.append("actor", new Document("$in", filter.getActorsList())); + } + + if (!filter.getUrlsList().isEmpty()) { + query.append("latestApiEndpoint", new Document("$in", filter.getUrlsList())); + } + + if (!filter.getIpsList().isEmpty()) { + query.append("latestApiIp", new Document("$in", filter.getIpsList())); + } + + if (filter.hasDetectedAtTimeRange()) { + TimeRangeFilter timeRange = filter.getDetectedAtTimeRange(); + long start = timeRange.hasStart() ? timeRange.getStart() : 0; + long end = timeRange.hasEnd() ? timeRange.getEnd() : Long.MAX_VALUE; + + query.append("detectedAt", new Document("$gte", start).append("$lte", end)); + } + long total = coll.countDocuments(query); try (MongoCursor cursor = coll.find(query) - .sort(new BasicDBObject("detectedAt", sort.getOrDefault("detectedAt", -1))) + .sort(new Document("detectedAt", sort.getOrDefault("detectedAt", -1))) .skip(skip) .limit(limit) .cursor()) { diff --git a/apps/threat-detection-backend/src/main/java/com/akto/threat/backend/service/ThreatActorService.java b/apps/threat-detection-backend/src/main/java/com/akto/threat/backend/service/ThreatActorService.java index c1d8c47874..31495f3ec0 100644 --- a/apps/threat-detection-backend/src/main/java/com/akto/threat/backend/service/ThreatActorService.java +++ b/apps/threat-detection-backend/src/main/java/com/akto/threat/backend/service/ThreatActorService.java @@ -31,7 +31,30 @@ public ListThreatActorResponse listThreatActors( .getDatabase(accountId) .getCollection(MongoDBCollection.ThreatDetection.MALICIOUS_EVENTS, Document.class); + ListThreatActorsRequest.Filter filter = request.getFilter(); + List base = new ArrayList<>(); + + Document match = new Document(); + + if (!filter.getActorsList().isEmpty()) { + match.append("actor", new Document("$in", filter.getActorsList())); + } + + if (!filter.getLatestIpsList().isEmpty()) { + match.append("latestApiIp", new Document("$in", filter.getLatestIpsList())); + } + + if (filter.hasDetectedAtTimeRange()) { + long start = filter.getDetectedAtTimeRange().getStart(); + long end = filter.getDetectedAtTimeRange().getEnd(); + match.append("detectedAt", new Document("$gte", start).append("$lte", end)); + } + + if (!match.isEmpty()) { + base.add(new Document("$match", match)); + } + base.add(new Document("$sort", new Document("detectedAt", -1))); base.add( new Document( diff --git a/apps/threat-detection-backend/src/main/java/com/akto/threat/backend/service/ThreatApiService.java b/apps/threat-detection-backend/src/main/java/com/akto/threat/backend/service/ThreatApiService.java index 6fc455553f..b2dbc258a9 100644 --- a/apps/threat-detection-backend/src/main/java/com/akto/threat/backend/service/ThreatApiService.java +++ b/apps/threat-detection-backend/src/main/java/com/akto/threat/backend/service/ThreatApiService.java @@ -31,6 +31,27 @@ public ListThreatApiResponse listThreatApis(String accountId, ListThreatApiReque .getCollection(MongoDBCollection.ThreatDetection.MALICIOUS_EVENTS, Document.class); List base = new ArrayList<>(); + ListThreatApiRequest.Filter filter = request.getFilter(); + + Document match = new Document(); + if (!filter.getMethodsList().isEmpty()) { + match.append("latestApiMethod", new Document("$in", filter.getMethodsList())); + } + + if (!filter.getUrlsList().isEmpty()) { + match.append("latestApiEndpoint", new Document("$in", filter.getUrlsList())); + } + + if (filter.hasDetectedAtTimeRange()) { + long start = filter.getDetectedAtTimeRange().getStart(); + long end = filter.getDetectedAtTimeRange().getEnd(); + match.append("detectedAt", new Document("$gte", start).append("$lte", end)); + } + + if (!match.isEmpty()) { + base.add(new Document("$match", match)); + } + base.add(new Document("$sort", new Document("detectedAt", -1))); // sort base.add( new Document( diff --git a/protobuf/threat_detection/service/dashboard_service/v1/service.proto b/protobuf/threat_detection/service/dashboard_service/v1/service.proto index 90bc1d057c..a3f31b1c20 100644 --- a/protobuf/threat_detection/service/dashboard_service/v1/service.proto +++ b/protobuf/threat_detection/service/dashboard_service/v1/service.proto @@ -28,11 +28,24 @@ message ListMaliciousRequestsResponse { uint64 total = 2; } +message TimeRangeFilter { + optional uint64 start = 1; + optional uint64 end = 2; +} + message ListMaliciousRequestsRequest { + message Filter { + repeated string actors = 1; + repeated string urls = 2; + repeated string ips = 3; + optional TimeRangeFilter detected_at_time_range = 4; + } + // The number of alerts to return optional uint32 skip = 1; uint32 limit = 2; map sort = 3; + Filter filter = 4; } message FetchAlertFiltersRequest {} @@ -43,9 +56,16 @@ message FetchAlertFiltersResponse { } message ListThreatActorsRequest { + message Filter { + repeated string actors = 1; + repeated string latest_ips = 2; + optional TimeRangeFilter detected_at_time_range = 3; + } + optional uint32 skip = 1; uint32 limit = 2; map sort = 3; + Filter filter = 4; } message ListThreatActorResponse { @@ -62,9 +82,16 @@ message ListThreatActorResponse { } message ListThreatApiRequest { + message Filter { + repeated string urls = 1; + repeated string methods = 2; + optional TimeRangeFilter detected_at_time_range = 3; + } + optional uint32 skip = 1; uint32 limit = 2; map sort = 3; + Filter filter = 4; } message ListThreatApiResponse {