Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

TIF Integ Testing (WIP) #674

Closed
wants to merge 3 commits into from
Closed

TIF Integ Testing (WIP) #674

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
2555ee2
merge conflicts
jowg-amazon Oct 18, 2023
12c53a5
fixed java wildcards and changed update key name
jowg-amazon Oct 17, 2023
f57ca0b
integ test failing
jowg-amazon Oct 18, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,6 @@ opensearchplugin {
name 'opensearch-security-analytics'
description 'OpenSearch Security Analytics plugin'
classname 'org.opensearch.securityanalytics.SecurityAnalyticsPlugin'
// extendedPlugins = ['opensearch-job-scheduler']
}

javaRestTest {
Expand Down
7 changes: 5 additions & 2 deletions src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,12 @@
*/
package org.opensearch.securityanalytics;

import java.util.*;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.function.Supplier;
import java.util.Optional;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.opensearch.core.action.ActionListener;
Expand Down Expand Up @@ -68,7 +72,6 @@
import org.opensearch.securityanalytics.util.DetectorIndices;
import org.opensearch.securityanalytics.util.RuleIndices;
import org.opensearch.securityanalytics.util.RuleTopicIndices;
import org.opensearch.threadpool.ExecutorBuilder;
import org.opensearch.threadpool.ThreadPool;
import org.opensearch.watcher.ResourceWatcherService;

Expand Down
3 changes: 1 addition & 2 deletions src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,8 +122,7 @@ public class SecurityAnalyticsSettings {
// threat intel settings
public static final Setting<TimeValue> TIF_UPDATE_INTERVAL = Setting.timeSetting(
"plugins.security_analytics.threatintel.tifjob.update_interval",
TimeValue.timeValueHours(24),
TimeValue.timeValueHours(1),
TimeValue.timeValueSeconds(1),
Setting.Property.NodeScope,
Setting.Property.Dynamic
);
Expand Down
13 changes: 13 additions & 0 deletions src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedDataService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,8 @@
import java.util.Arrays;
import java.util.Optional;
import java.util.concurrent.CountDownLatch;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

import static org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter.THREAT_INTEL_DATA_INDEX_NAME_PREFIX;
Expand Down Expand Up @@ -197,6 +199,10 @@ public void parseAndSaveThreatIntelFeedDataCSV(
String iocType = tifMetadata.getIocType(); //todo make generic in upcoming versions
Integer colNum = tifMetadata.getIocCol();
String iocValue = record.values()[colNum].split(" ")[0];
if (iocType.equals("ip") && !isValidIp(iocValue)) {
log.info("Invalid IP address, skipping this ioc record.");
continue;
}
String feedId = tifMetadata.getFeedId();
Instant timestamp = Instant.now();
ThreatIntelFeedData threatIntelFeedData = new ThreatIntelFeedData(iocType, iocValue, feedId, timestamp);
Expand All @@ -218,6 +224,13 @@ public void parseAndSaveThreatIntelFeedDataCSV(
freezeIndex(indexName);
}

public static boolean isValidIp(String ip) {
String ipPattern = "^\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}$";
Pattern pattern = Pattern.compile(ipPattern);
Matcher matcher = pattern.matcher(ip);
return matcher.matches();
}

public void saveTifds(BulkRequest bulkRequest, TimeValue timeout) {

try {
Expand Down
3 changes: 2 additions & 1 deletion src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobExtension.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,13 @@

package org.opensearch.securityanalytics.threatIntel.jobscheduler;

import org.opensearch.jobscheduler.spi.JobSchedulerExtension;
import org.opensearch.jobscheduler.spi.ScheduledJobParser;
import org.opensearch.jobscheduler.spi.ScheduledJobRunner;

import java.util.Map;

public class TIFJobExtension implements org.opensearch.jobscheduler.spi.JobSchedulerExtension {
public class TIFJobExtension implements JobSchedulerExtension {
/**
* Job index name for a TIF job
*/
Expand Down
89 changes: 72 additions & 17 deletions src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,14 @@
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.common.io.stream.Writeable;
import org.opensearch.core.xcontent.ConstructingObjectParser;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.*;
import org.opensearch.jobscheduler.spi.ScheduledJobParameter;
import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.jobscheduler.spi.schedule.ScheduleParser;
import org.opensearch.securityanalytics.threatIntel.action.PutTIFJobRequest;
import org.opensearch.securityanalytics.threatIntel.common.TIFJobState;
import org.opensearch.securityanalytics.threatIntel.common.TIFLockService;
import org.opensearch.securityanalytics.threatIntel.common.TIFMetadata;

import java.io.IOException;
import java.time.Instant;
Expand All @@ -29,34 +31,32 @@

import static org.opensearch.common.time.DateUtils.toInstant;

import org.opensearch.securityanalytics.threatIntel.action.PutTIFJobRequest;
import org.opensearch.securityanalytics.threatIntel.common.TIFJobState;
import org.opensearch.securityanalytics.threatIntel.common.TIFLockService;
import org.opensearch.securityanalytics.threatIntel.common.TIFMetadata;

public class TIFJobParameter implements Writeable, ScheduledJobParameter {
/**
* Prefix of indices having threatIntel data
*/
public static final String THREAT_INTEL_DATA_INDEX_NAME_PREFIX = ".opensearch-sap-threatintel";




/**
* Default fields for job scheduling
*/
private static final ParseField NAME_FIELD = new ParseField("name");
private static final ParseField ENABLED_FIELD = new ParseField("update_enabled");
private static final ParseField LAST_UPDATE_TIME_FIELD = new ParseField("last_update_time");
private static final ParseField LAST_UPDATE_TIME_FIELD_READABLE = new ParseField("last_update_time_field");
public static final ParseField NAME_FIELD = new ParseField("name");
public static final ParseField ENABLED_FIELD = new ParseField("update_enabled");
public static final ParseField LAST_UPDATE_TIME_FIELD = new ParseField("last_update_time");
public static final ParseField LAST_UPDATE_TIME_FIELD_READABLE = new ParseField("last_update_time_field");
public static final ParseField SCHEDULE_FIELD = new ParseField("schedule");
private static final ParseField ENABLED_TIME_FIELD = new ParseField("enabled_time");
private static final ParseField ENABLED_TIME_FIELD_READABLE = new ParseField("enabled_time_field");
public static final ParseField ENABLED_TIME_FIELD = new ParseField("enabled_time");
public static final ParseField ENABLED_TIME_FIELD_READABLE = new ParseField("enabled_time_field");

/**
* Additional fields for tif job
*/
private static final ParseField STATE_FIELD = new ParseField("state");
private static final ParseField INDICES_FIELD = new ParseField("indices");
private static final ParseField UPDATE_STATS_FIELD = new ParseField("update_stats");
public static final ParseField STATE_FIELD = new ParseField("state");
public static final ParseField INDICES_FIELD = new ParseField("indices");
public static final ParseField UPDATE_STATS_FIELD = new ParseField("update_stats");


/**
Expand Down Expand Up @@ -113,6 +113,61 @@ public class TIFJobParameter implements Writeable, ScheduledJobParameter {
*/
private UpdateStats updateStats;

public static TIFJobParameter parse(XContentParser xcp, String id, Long version) throws IOException {
String name = null;
Instant lastUpdateTime = null;
Boolean isEnabled = null;
TIFJobState state = null;

xcp.nextToken();
XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp);
while (xcp.nextToken() != XContentParser.Token.END_OBJECT) {
String fieldName = xcp.currentName();
xcp.nextToken();

switch (fieldName) {
case "name":
name = xcp.text();
break;
case "last_update_time":
lastUpdateTime = Instant.ofEpochMilli(xcp.longValue());
break;
case "update_enabled":
isEnabled = xcp.booleanValue();
break;
case "state":
state = toState(xcp.text());
break;
default:
xcp.skipChildren();
}
}
return new TIFJobParameter(name, lastUpdateTime, isEnabled, state);
}

public static TIFJobState toState(String stateName){
if (stateName.equals("CREATING")){
return TIFJobState.CREATING;
}
if (stateName.equals("AVAILABLE")){
return TIFJobState.AVAILABLE;
}
if (stateName.equals("CREATE_FAILED")){
return TIFJobState.CREATE_FAILED;
}
if (stateName.equals("DELETING")){
return TIFJobState.DELETING;
}
return null;
}

public TIFJobParameter(final String name, final Instant lastUpdateTime, final Boolean isEnabled, TIFJobState state) {
this.name = name;
this.lastUpdateTime = lastUpdateTime;
this.isEnabled = isEnabled;
this.state = state;
}

/**
* tif job parser
*/
Expand Down
Loading
Loading