Skip to content

Commit

Permalink
Merge branch '2.11' into backport/backport-730-to-2.11
Browse files Browse the repository at this point in the history 
Signed-off-by: Chase <[email protected]>
  • Loading branch information
@engechas
engechas authored Apr 10, 2024
2 parents 7243f99 + 7c6b79d commit a943c2d
Show file tree
Hide file tree
Showing 44 changed files with 3,887 additions and 533 deletions.
7 changes: 2 additions & 5 deletions release-notes/opensearch-security-analytics.release-notes-2.11.1.0.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,10 @@ Compatible with OpenSearch 2.11.1
### Maintenance
* [AUTO] Incremented version to 2.11.1. ([#668](https://github.com/opensearch-project/security-analytics/pull/668))

### Enhancements
* add rollover & archival mechanism for correlation history indices ([#670](https://github.com/opensearch-project/security-analytics/pull/670))

### Bug Fixes
* fix for doc level query constructor change ([#651](https://github.com/opensearch-project/security-analytics/pull/651))
* add rollover & archival mechanism for correlation history indices ([#670](https://github.com/opensearch-project/security-analytics/pull/670))
* return rule fields which do not have aliases ([#652](https://github.com/opensearch-project/security-analytics/pull/652))
* fix detector writeTo() method missing fields ([#695](https://github.com/opensearch-project/security-analytics/pull/695))

### Documentation
* Added 2.11.1 release notes.([#727](https://github.com/opensearch-project/security-analytics/pull/727))
* Added 2.11.1 release notes.([#727](https://github.com/opensearch-project/security-analytics/pull/727))
88 changes: 44 additions & 44 deletions ...erated/org/opensearch/securityanalytics/rules/condition/aggregation/AggregationLexer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,50 +129,50 @@ public AggregationLexer(CharStream input) {
"\u0001\u000f\u0001\u000f\u0000\u0000\u0010\u0001\u0001\u0003\u0002\u0005"+
"\u0003\u0007\u0004\t\u0005\u000b\u0006\r\u0007\u000f\b\u0011\t\u0013\n"+
"\u0015\u000b\u0017\f\u0019\r\u001b\u000e\u001d\u000f\u001f\u0010\u0001"+
"\u0000\u0004\u0001\u000009\u0004\u0000**AZ__az\u0004\u000009AZ__az\u0003"+
"\u0000\t\n\f\r n\u0000\u0001\u0001\u0000\u0000\u0000\u0000\u0003\u0001"+
"\u0000\u0000\u0000\u0000\u0005\u0001\u0000\u0000\u0000\u0000\u0007\u0001"+
"\u0000\u0000\u0000\u0000\t\u0001\u0000\u0000\u0000\u0000\u000b\u0001\u0000"+
"\u0000\u0000\u0000\r\u0001\u0000\u0000\u0000\u0000\u000f\u0001\u0000\u0000"+
"\u0000\u0000\u0011\u0001\u0000\u0000\u0000\u0000\u0013\u0001\u0000\u0000"+
"\u0000\u0000\u0015\u0001\u0000\u0000\u0000\u0000\u0017\u0001\u0000\u0000"+
"\u0000\u0000\u0019\u0001\u0000\u0000\u0000\u0000\u001b\u0001\u0000\u0000"+
"\u0000\u0000\u001d\u0001\u0000\u0000\u0000\u0000\u001f\u0001\u0000\u0000"+
"\u0000\u0001!\u0001\u0000\u0000\u0000\u0003#\u0001\u0000\u0000\u0000\u0005"+
"&\u0001\u0000\u0000\u0000\u0007(\u0001\u0000\u0000\u0000\t+\u0001\u0000"+
"\u0000\u0000\u000b.\u0001\u0000\u0000\u0000\r4\u0001\u0000\u0000\u0000"+
"\u000f8\u0001\u0000\u0000\u0000\u0011<\u0001\u0000\u0000\u0000\u0013@"+
"\u0001\u0000\u0000\u0000\u0015D\u0001\u0000\u0000\u0000\u0017G\u0001\u0000"+
"\u0000\u0000\u0019I\u0001\u0000\u0000\u0000\u001bL\u0001\u0000\u0000\u0000"+
"\u001d[\u0001\u0000\u0000\u0000\u001fc\u0001\u0000\u0000\u0000!\"\u0005"+
">\u0000\u0000\"\u0002\u0001\u0000\u0000\u0000#$\u0005>\u0000\u0000$%\u0005"+
"=\u0000\u0000%\u0004\u0001\u0000\u0000\u0000&\'\u0005<\u0000\u0000\'\u0006"+
"\u0001\u0000\u0000\u0000()\u0005<\u0000\u0000)*\u0005=\u0000\u0000*\b"+
"\u0001\u0000\u0000\u0000+,\u0005=\u0000\u0000,-\u0005=\u0000\u0000-\n"+
"\u0001\u0000\u0000\u0000./\u0005c\u0000\u0000/0\u0005o\u0000\u000001\u0005"+
"u\u0000\u000012\u0005n\u0000\u000023\u0005t\u0000\u00003\f\u0001\u0000"+
"\u0000\u000045\u0005s\u0000\u000056\u0005u\u0000\u000067\u0005m\u0000"+
"\u00007\u000e\u0001\u0000\u0000\u000089\u0005m\u0000\u00009:\u0005i\u0000"+
"\u0000:;\u0005n\u0000\u0000;\u0010\u0001\u0000\u0000\u0000<=\u0005m\u0000"+
"\u0000=>\u0005a\u0000\u0000>?\u0005x\u0000\u0000?\u0012\u0001\u0000\u0000"+
"\u0000@A\u0005a\u0000\u0000AB\u0005v\u0000\u0000BC\u0005g\u0000\u0000"+
"C\u0014\u0001\u0000\u0000\u0000DE\u0005b\u0000\u0000EF\u0005y\u0000\u0000"+
"F\u0016\u0001\u0000\u0000\u0000GH\u0005(\u0000\u0000H\u0018\u0001\u0000"+
"\u0000\u0000IJ\u0005)\u0000\u0000J\u001a\u0001\u0000\u0000\u0000KM\u0005"+
"-\u0000\u0000LK\u0001\u0000\u0000\u0000LM\u0001\u0000\u0000\u0000MO\u0001"+
"\u0000\u0000\u0000NP\u0007\u0000\u0000\u0000ON\u0001\u0000\u0000\u0000"+
"PQ\u0001\u0000\u0000\u0000QO\u0001\u0000\u0000\u0000QR\u0001\u0000\u0000"+
"\u0000RY\u0001\u0000\u0000\u0000SU\u0005.\u0000\u0000TV\u0007\u0000\u0000"+
"\u0000UT\u0001\u0000\u0000\u0000VW\u0001\u0000\u0000\u0000WU\u0001\u0000"+
"\u0000\u0000WX\u0001\u0000\u0000\u0000XZ\u0001\u0000\u0000\u0000YS\u0001"+
"\u0000\u0000\u0000YZ\u0001\u0000\u0000\u0000Z\u001c\u0001\u0000\u0000"+
"\u0000[_\u0007\u0001\u0000\u0000\\^\u0007\u0002\u0000\u0000]\\\u0001\u0000"+
"\u0000\u0000^a\u0001\u0000\u0000\u0000_]\u0001\u0000\u0000\u0000_`\u0001"+
"\u0000\u0000\u0000`\u001e\u0001\u0000\u0000\u0000a_\u0001\u0000\u0000"+
"\u0000bd\u0007\u0003\u0000\u0000cb\u0001\u0000\u0000\u0000de\u0001\u0000"+
"\u0000\u0000ec\u0001\u0000\u0000\u0000ef\u0001\u0000\u0000\u0000fg\u0001"+
"\u0000\u0000\u0000gh\u0006\u000f\u0000\u0000h \u0001\u0000\u0000\u0000"+
"\u0007\u0000LQWY_e\u0001\u0006\u0000\u0000";
"\u0000\u0004\u0001\u000009\u0005\u0000**..AZ__az\u0005\u0000..09AZ__a"+
"z\u0003\u0000\t\n\f\r n\u0000\u0001\u0001\u0000\u0000\u0000\u0000\u0003"+
"\u0001\u0000\u0000\u0000\u0000\u0005\u0001\u0000\u0000\u0000\u0000\u0007"+
"\u0001\u0000\u0000\u0000\u0000\t\u0001\u0000\u0000\u0000\u0000\u000b\u0001"+
"\u0000\u0000\u0000\u0000\r\u0001\u0000\u0000\u0000\u0000\u000f\u0001\u0000"+
"\u0000\u0000\u0000\u0011\u0001\u0000\u0000\u0000\u0000\u0013\u0001\u0000"+
"\u0000\u0000\u0000\u0015\u0001\u0000\u0000\u0000\u0000\u0017\u0001\u0000"+
"\u0000\u0000\u0000\u0019\u0001\u0000\u0000\u0000\u0000\u001b\u0001\u0000"+
"\u0000\u0000\u0000\u001d\u0001\u0000\u0000\u0000\u0000\u001f\u0001\u0000"+
"\u0000\u0000\u0001!\u0001\u0000\u0000\u0000\u0003#\u0001\u0000\u0000\u0000"+
"\u0005&\u0001\u0000\u0000\u0000\u0007(\u0001\u0000\u0000\u0000\t+\u0001"+
"\u0000\u0000\u0000\u000b.\u0001\u0000\u0000\u0000\r4\u0001\u0000\u0000"+
"\u0000\u000f8\u0001\u0000\u0000\u0000\u0011<\u0001\u0000\u0000\u0000\u0013"+
"@\u0001\u0000\u0000\u0000\u0015D\u0001\u0000\u0000\u0000\u0017G\u0001"+
"\u0000\u0000\u0000\u0019I\u0001\u0000\u0000\u0000\u001bL\u0001\u0000\u0000"+
"\u0000\u001d[\u0001\u0000\u0000\u0000\u001fc\u0001\u0000\u0000\u0000!"+
"\"\u0005>\u0000\u0000\"\u0002\u0001\u0000\u0000\u0000#$\u0005>\u0000\u0000"+
"$%\u0005=\u0000\u0000%\u0004\u0001\u0000\u0000\u0000&\'\u0005<\u0000\u0000"+
"\'\u0006\u0001\u0000\u0000\u0000()\u0005<\u0000\u0000)*\u0005=\u0000\u0000"+
"*\b\u0001\u0000\u0000\u0000+,\u0005=\u0000\u0000,-\u0005=\u0000\u0000"+
"-\n\u0001\u0000\u0000\u0000./\u0005c\u0000\u0000/0\u0005o\u0000\u0000"+
"01\u0005u\u0000\u000012\u0005n\u0000\u000023\u0005t\u0000\u00003\f\u0001"+
"\u0000\u0000\u000045\u0005s\u0000\u000056\u0005u\u0000\u000067\u0005m"+
"\u0000\u00007\u000e\u0001\u0000\u0000\u000089\u0005m\u0000\u00009:\u0005"+
"i\u0000\u0000:;\u0005n\u0000\u0000;\u0010\u0001\u0000\u0000\u0000<=\u0005"+
"m\u0000\u0000=>\u0005a\u0000\u0000>?\u0005x\u0000\u0000?\u0012\u0001\u0000"+
"\u0000\u0000@A\u0005a\u0000\u0000AB\u0005v\u0000\u0000BC\u0005g\u0000"+
"\u0000C\u0014\u0001\u0000\u0000\u0000DE\u0005b\u0000\u0000EF\u0005y\u0000"+
"\u0000F\u0016\u0001\u0000\u0000\u0000GH\u0005(\u0000\u0000H\u0018\u0001"+
"\u0000\u0000\u0000IJ\u0005)\u0000\u0000J\u001a\u0001\u0000\u0000\u0000"+
"KM\u0005-\u0000\u0000LK\u0001\u0000\u0000\u0000LM\u0001\u0000\u0000\u0000"+
"MO\u0001\u0000\u0000\u0000NP\u0007\u0000\u0000\u0000ON\u0001\u0000\u0000"+
"\u0000PQ\u0001\u0000\u0000\u0000QO\u0001\u0000\u0000\u0000QR\u0001\u0000"+
"\u0000\u0000RY\u0001\u0000\u0000\u0000SU\u0005.\u0000\u0000TV\u0007\u0000"+
"\u0000\u0000UT\u0001\u0000\u0000\u0000VW\u0001\u0000\u0000\u0000WU\u0001"+
"\u0000\u0000\u0000WX\u0001\u0000\u0000\u0000XZ\u0001\u0000\u0000\u0000"+
"YS\u0001\u0000\u0000\u0000YZ\u0001\u0000\u0000\u0000Z\u001c\u0001\u0000"+
"\u0000\u0000[_\u0007\u0001\u0000\u0000\\^\u0007\u0002\u0000\u0000]\\\u0001"+
"\u0000\u0000\u0000^a\u0001\u0000\u0000\u0000_]\u0001\u0000\u0000\u0000"+
"_`\u0001\u0000\u0000\u0000`\u001e\u0001\u0000\u0000\u0000a_\u0001\u0000"+
"\u0000\u0000bd\u0007\u0003\u0000\u0000cb\u0001\u0000\u0000\u0000de\u0001"+
"\u0000\u0000\u0000ec\u0001\u0000\u0000\u0000ef\u0001\u0000\u0000\u0000"+
"fg\u0001\u0000\u0000\u0000gh\u0006\u000f\u0000\u0000h \u0001\u0000\u0000"+
"\u0000\u0007\u0000LQWY_e\u0001\u0006\u0000\u0000";
public static final ATN _ATN =
new ATNDeserializer().deserialize(_serializedATN.toCharArray());
static {
Expand Down
2 changes: 1 addition & 1 deletion src/main/grammars/Aggregation.g4
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ RPAREN : ')' ;

DECIMAL : '-'?[0-9]+('.'[0-9]+)? ;

IDENTIFIER : [a-zA-Z*_][a-zA-Z_0-9]* ;
IDENTIFIER : [a-zA-Z*_.][a-zA-Z_0-9.]* ;
WS : [ \r\t\u000C\n]+ -> skip ;

comparison_expr : comparison_operand comp_operator comparison_operand # ComparisonExpressionWithOperator
Expand Down
54 changes: 50 additions & 4 deletions src/main/java/org/opensearch/securityanalytics/action/GetFindingsRequest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
package org.opensearch.securityanalytics.action;

import java.io.IOException;
import java.time.Instant;
import java.util.List;
import java.util.Locale;
import org.opensearch.action.ActionRequest;
import org.opensearch.action.ActionRequestValidationException;
Expand All @@ -18,9 +20,14 @@

public class GetFindingsRequest extends ActionRequest {

private List<String> findingIds;
private Instant startTime;
private Instant endTime;
private String logType;
private String detectorId;
private Table table;
private String severity;
private String detectionType;

public static final String DETECTOR_ID = "detector_id";

Expand All @@ -32,22 +39,36 @@ public GetFindingsRequest(StreamInput sin) throws IOException {
this(
sin.readOptionalString(),
sin.readOptionalString(),
Table.readFrom(sin)
Table.readFrom(sin),
sin.readOptionalString(),
sin.readOptionalString(),
sin.readOptionalStringList(),
sin.readOptionalInstant(),
sin.readOptionalInstant()
);
}

public GetFindingsRequest(String detectorId, String logType, Table table) {
public GetFindingsRequest(String detectorId, String logType, Table table, String severity, String detectionType, List<String> findingIds, Instant startTime, Instant endTime) {
this.detectorId = detectorId;
this.logType = logType;
this.table = table;
this.severity = severity;
this.detectionType = detectionType;
this.findingIds = findingIds;
this.startTime = startTime;
this.endTime = endTime;
}

@Override
public ActionRequestValidationException validate() {
ActionRequestValidationException validationException = null;
if ((detectorId == null || detectorId.length() == 0) && logType == null) {
if (detectorId != null && detectorId.length() == 0) {
validationException = addValidationError(String.format(Locale.getDefault(),
"detector_id is missing"),
validationException);
} else if(startTime != null && endTime != null && startTime.isAfter(endTime)) {
validationException = addValidationError(String.format(Locale.getDefault(),
"At least one of detector type or detector id needs to be passed", DETECTOR_ID),
"startTime should be less than endTime"),
validationException);
}
return validationException;
Expand All @@ -58,17 +79,42 @@ public void writeTo(StreamOutput out) throws IOException {
out.writeOptionalString(detectorId);
out.writeOptionalString(logType);
table.writeTo(out);
out.writeOptionalString(severity);
out.writeOptionalString(detectionType);
out.writeOptionalStringCollection(findingIds);
out.writeOptionalInstant(startTime);
out.writeOptionalInstant(endTime);
}

public String getDetectorId() {
return detectorId;
}

public String getSeverity() {
return severity;
}

public String getDetectionType() {
return detectionType;
}

public String getLogType() {
return logType;
}

public Table getTable() {
return table;
}

public List<String> getFindingIds() {
return findingIds;
}

public Instant getStartTime() {
return startTime;
}

public Instant getEndTime() {
return endTime;
}
}
13 changes: 12 additions & 1 deletion src/main/java/org/opensearch/securityanalytics/correlation/VectorEmbeddingsEngine.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.opensearch.OpenSearchStatusException;
import org.opensearch.ResourceNotFoundException;
import org.opensearch.cluster.routing.Preference;
import org.opensearch.core.action.ActionListener;
import org.opensearch.action.bulk.BulkRequest;
Expand Down Expand Up @@ -84,6 +85,11 @@ public void onResponse(SearchResponse response) {
correlateFindingAction.onFailures(new OpenSearchStatusException("Search request timed out", RestStatus.REQUEST_TIMEOUT));
}

if (response.getHits().getHits().length == 0) {
correlateFindingAction.onFailures(
new ResourceNotFoundException("Failed to find hits in metadata index for finding id {}", finding.getId()));
}

Map<String, Object> hitSource = response.getHits().getHits()[0].getSourceAsMap();
long counter = Long.parseLong(hitSource.get("counter").toString());

Expand Down Expand Up @@ -125,7 +131,7 @@ public void onResponse(MultiSearchResponse items) {
continue;
}

long totalHits = response.getResponse().getHits().getTotalHits().value;
long totalHits = response.getResponse().getHits().getHits().length;
totalNeighbors += totalHits;

for (int idx = 0; idx < totalHits; ++idx) {
Expand Down Expand Up @@ -229,6 +235,11 @@ public void onFailure(Exception e) {
}

public void insertOrphanFindings(String detectorType, Finding finding, float timestampFeature, Map<String, CustomLogType> logTypes) {
if (logTypes.get(detectorType) == null) {
log.error("LogTypes Index is missing the detector type {}", detectorType);
correlateFindingAction.onFailures(new OpenSearchStatusException("LogTypes Index is missing the detector type", RestStatus.INTERNAL_SERVER_ERROR));
}

Map<String, Object> tags = logTypes.get(detectorType).getTags();
String correlationId = tags.get("correlation_id").toString();

Expand Down
Loading

0 comments on commit a943c2d

Please sign in to comment.