Merge pull request #164 from logzio/asc-alerts

Asc alerts and additional fields support, Changelog

CHANGELOG.md

@@ -0,0 +1,41 @@
+# Change log
+
+## v0.0.7 - Aug. 11, 2021
+
+**Added:**
+- ASC (Azure Security Center) alerts api
+- User can now add to the yaml configuration additional fields that will be sent with the data.
+- Changelog
+
+## v0.0.6 - Aug. 3, 2021
+
+**Added:**
+- Risky sign ins api
+- The ability to add different apis more generically
+
+## v0.0.5 - Jul. 28, 2021
+
+**Updated:**
+- Project dependencies
+
+## v0.0.4- Mar. 10, 2020
+
+**Updated:**
+- com.fasterxml.jackson.core:jackson-databind:2.9.10.3
+
+## v0.0.3 - Feb. 10, 2020
+
+**Updated:**
+- com.fasterxml.jackson.core:jackson-databind:2.9.10.2
+
+## v0.0.2 - Jan. 5, 2020
+
+**Updated:**
+- com.fasterxml.jackson.core:jackson-databind:2.9.10.1
+- io.logz.sender:logzio-sender:1.1.1
+
+## v0.0.1 - Oct. 20, 2019
+
+**Added:**
+- First release
+

Dockerfile

@@ -1,12 +1,12 @@
 
-FROM openjdk:8-jre-alpine
+FROM openjdk:11-jre-alpine
 
 MAINTAINER Tamir Michaeli <tamir.michaeli@logz.io>
 
 RUN apk --no-cache add curl
 
-RUN wget https://github.com/logzio/microsoft-graph/releases/download/v0.0.6/logzio-msgraph-0.0.6-app.jar
+RUN wget https://github.com/logzio/microsoft-graph/releases/download/v0.0.7/logzio-msgraph-0.0.7-app.jar
 
-RUN cp logzio-msgraph-0.0.6-app.jar /logzio-msgraph.jar
+RUN cp logzio-msgraph-0.0.7-app.jar /logzio-msgraph.jar
 
 CMD java -jar logzio-msgraph.jar config.yaml

README.md

@@ -1,12 +1,19 @@
 # Microsoft Graph API Integration
+**Version 0.0.7**  [Change log](CHANGELOG.md)
 
 You can ship logs available from the Microsoft Graph APIs with Logzio-MSGraph.
 Logzio-MSGraph is a self-hosted application.
 
-Logzio-MSGraph supports these APIs:
+**Logzio-MSGraph supports these APIs:**
 
-* Azure Active Directory audit logs
-* Azure Active Directory sign-in logs
+**Azure Active Directory:**
+
+- Audit logs
+
+- Sign-in logs (Including risky sign-in)
+
+**ASC (Azure Security Center):**
+- Alerts
 
 There are many other APIs available through Microsoft Graph.
 If you don't see your API in the list,
@@ -46,11 +53,15 @@ and click **Add a permission**.
 
 Select **Microsoft Graph > Application permissions**.
 
-Select these items:
+Select the following items, for each desired API:
 
+Sign Ins (Including risky sign ins) and Directory Audits:
 * **AuditLog.Read.All**
 * **Directory.Read.All**
 
+ASC(Azure Security Center) Alerts:
+* **SecurityEvents.Read.All**
+
 Click **Add permissions**.
 
 Click **Grant admin consent for Default Directory**, and then click **Yes** to confirm.
@@ -79,8 +90,13 @@ targetApis:
   ADApis:
     - <<supportedApi1>>
     - <<supportedApi2>>
+  ASCApis:
+    - <<supportedApi1>>
 
 logLevel: INFO
+
+additionalFields:
+  <<KEY>>: "<<VALUE>>"
 ```
 
 **Parameters**
@@ -95,8 +111,11 @@ logLevel: INFO
 | azureADClient.clientId | **Required**. Application client ID. <br> You can find this in the _Overview_ section of the app you registered in step 1. |
 | azureADClient.clientSecret | **Required**. The Application Client Secret you created in step 2. |
 | azureADClient.pullIntervalSeconds | **Default**: `300` <br>  Time interval, in seconds, to pull the logs with the Graph API. |
-| targetApis.ADApis | **Default**: `300` <br>  List of AD apis to run. Required at least 1 api to run. Current supported apis: directoryAudits, signIns, riskySignIns. All apis are case sensitive and should be configured as mentioned here.|
+| targetApis | **Required**. <br> Specifies types of api lists to run, each API provider has its own list. Must contain at least 1 list with 1 api in the list. Current supported providers: ADApis, ASCApis. |
+| targetApis.ADApis | **Optional**. <br>  List of AD apis to run. Current supported apis: directoryAudits, signIns (this includes risky sign ins), riskySignIns. All apis are case sensitive and should be configured as mentioned here. |
+| targetApis.ASCApis | **Optional**. <br>  List of ASC apis to run. Current supported apis: alerts. All apis are case sensitive and should be configured as mentioned here. |
 | logLevel | **Default**: `INFO` <br> Log level for Logizo-MSGraph to omit. Can be one of: `OFF`, `ERROR`, `WARN`, `INFO`, `DEBUG`, `TRACE`, `ALL`. |
+| additionalFields | **Optional**. <br> List of additional fields to be added to the sent data. Pairs of key and value.
 
 #### <span id="if-fromdisk-true">If fromDisk=true</span>
 

pom.xml

@@ -5,7 +5,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>io.logz</groupId>
     <artifactId>logzio-msgraph</artifactId>
-    <version>0.0.6</version>
+    <version>0.0.7</version>
     <build>
         <plugins>
             <plugin>

src/main/java/api/MSGraphRequestExecutor.java

@@ -69,7 +69,7 @@ public JSONArray getAllPages(String api, String timeField,String optionalFilter)
     }
 
     public JSONArray getAllPages(String url) throws IOException, JSONException, AuthenticationException {
-        logger.debug("API URL: " + url);
+        logger.debug("Thread: "+Thread.currentThread().getName()+", API URL: " + url);
         Response response = executeRequest(url);
         String responseBody = response.body().string();
         JSONObject resultJson = new JSONObject(responseBody);

src/main/java/api/Office365Apis.java

@@ -16,6 +16,7 @@ public class Office365Apis {
     private static final String AD_DIRECTORY_AUDITS = "auditLogs/directoryaudits";
     private static final String ACTIVITY_DATE_TIME_FIELD = "activityDateTime";
     private static final String GRAPH_API_URL = "https://graph.microsoft.com/v1.0/";
+    private static final String ASC_ALERTS="security/alerts";
     private static final String AD_RISKY_SIGN_INS_FILTER_SUFFIX=" and (riskState eq 'atRisk' or riskState eq 'confirmedCompromised' or riskState eq 'unknownFutureValue')";
     private final MSGraphRequestExecutor requestExecutor;
 
@@ -56,4 +57,8 @@ public RequestDataResult getDirectoryAudits() {
     public RequestDataResult getRiskySignIns(){
         return office365request(GRAPH_API_URL+AD_SINGINS,CREATED_DATE_TIME_FIELD,AD_RISKY_SIGN_INS_FILTER_SUFFIX);
     }
+
+    public RequestDataResult getASCAlerts(){
+        return  office365request(GRAPH_API_URL+ASC_ALERTS,CREATED_DATE_TIME_FIELD);
+    }
 }

src/main/java/main/FetchSendManager.java

@@ -6,18 +6,22 @@
 import io.logz.sender.exceptions.LogzioParameterErrorException;
 import objects.JsonArrayRequest;
 import objects.LogzioJavaSenderParams;
+import objects.MSGraphConfiguration;
 import objects.RequestDataResult;
 import org.apache.log4j.Logger;
 import org.awaitility.Awaitility;
 import org.awaitility.core.ConditionTimeoutException;
 import org.json.JSONException;
+import org.json.JSONObject;
 import utils.exceptions.ConfigurationException;
 import utils.HangupInterceptor;
 import utils.Shutdownable;
 import utils.StatusReporterFactory;
 
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
@@ -32,13 +36,13 @@ public class FetchSendManager implements Shutdownable {
     private static final int RETRY_TIMEOUT_DURATION_SEC = 60;
     private static final int TERMINATION_TIMEOUT_SEC = 20;
     private static final int FIBONACCI_OFFSET = 4;
-
     private final ScheduledExecutorService taskScheduler;
     private final ArrayList<JsonArrayRequest> dataRequests;
     private final LogzioJavaSenderParams logzioSenderParams;
     private final LogzioSender sender;
     private final int interval;
     private ScheduledExecutorService senderExecutors;
+    private Map<String,String> additionalFields;
 
 
     public FetchSendManager(ArrayList<JsonArrayRequest> dataRequests, LogzioJavaSenderParams senderParams, int interval) {
@@ -47,6 +51,24 @@ public FetchSendManager(ArrayList<JsonArrayRequest> dataRequests, LogzioJavaSend
         this.dataRequests = dataRequests;
         this.sender = getLogzioSender();
         this.interval = interval;
+        this.additionalFields= new HashMap<>();
+    }
+
+    public FetchSendManager(ArrayList<JsonArrayRequest> requests, MSGraphConfiguration config) {
+        this(requests,config.getSenderParams(),config.getAzureADClient().getPullIntervalSeconds());
+        additionalFields =initAdditionalFieldsMap(config);
+
+    }
+
+    private Map<String, String> initAdditionalFieldsMap(MSGraphConfiguration config) {
+        Map<String,String> additionalFields=new HashMap<>();
+        additionalFields.put("tenantId",config.getAzureADClient().getTenantId());
+        additionalFields.put("clientId",config.getAzureADClient().getClientId());
+        if(config.getAdditionalFields()!=null){
+            config.getAdditionalFields().forEach(additionalFields::put);
+        }
+
+        return additionalFields;
     }
 
     public void start() {
@@ -85,8 +107,12 @@ public void pullAndSendData(JsonArrayRequest request) throws ConfigurationExcept
     }
 
     private void convertAndSendResults(RequestDataResult dataResult) {
+        JSONObject data;
         for (int i = 0; i < dataResult.getData().length(); i++) {
             try {
+                data=dataResult.getData().getJSONObject(i);
+                JSONObject finalData = data;
+                additionalFields.forEach(finalData::put);
                 byte[] jsonAsBytes = StandardCharsets.UTF_8.encode(dataResult.getData().getJSONObject(i).toString()).array();
                 synchronized (this) {
                     sender.send(jsonAsBytes);

src/main/java/main/MSClient.java

@@ -5,6 +5,7 @@
 import objects.JsonArrayRequest;
 import objects.MSGraphConfiguration;
 import objects.RequestDataResult;
+import objects.TargetApi;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.log4j.Level;
 import org.apache.log4j.Logger;
@@ -48,45 +49,50 @@ public void start() throws ConfigurationException {
             logger.error(e.getMessage(), e);
             return;
         }
+
         Office365Apis officeApis = new Office365Apis(executor);
-        ArrayList<JsonArrayRequest>  requests = new ArrayList(getApiTargets(officeApis));
-        FetchSendManager manager = new FetchSendManager(requests, configuration.getSenderParams(), configuration.getAzureADClient().getPullIntervalSeconds());
+        ArrayList<JsonArrayRequest> requests = new ArrayList(getApiTargets(officeApis));
+        FetchSendManager manager = new FetchSendManager(requests,configuration);
         manager.start();
     }
 
     /**
-     *
      * @param office365Apis
      * @return Method reference list matching yaml api configuration
      */
     private List<JsonArrayRequest> getApiTargets(Office365Apis office365Apis) throws ConfigurationException {
         List<String> adApis = configuration.getTargetApi().getADApis();
+        List<String> ascApis = configuration.getTargetApi().getAscApis();
         List<String> apiMethods = new java.util.LinkedList<>();
         if (adApis != null) {
             apiMethods.addAll(adApis.stream().map(api -> "get" + StringUtils.capitalize(api)).collect(Collectors.toList()));
         }
 
+        if (ascApis != null) {
+            apiMethods.addAll(ascApis.stream().map(api -> "getASC" + StringUtils.capitalize(api)).collect(Collectors.toList()));
+        }
+
         List<JsonArrayRequest> apis = new ArrayList<>();
         for (String api : apiMethods) {
             apis.add(() -> (RequestDataResult) getApiMethodRef(office365Apis, api));
             logger.info("Initialized api: " + api);
         }
 
-        if(apis.size()!=apiMethods.size()){
-            throw new ConfigurationException("Invalid configuration of apis in configuration yaml, review the configured apis: "+adApis);
+        if (apis.size() != apiMethods.size()) {
+            throw new ConfigurationException("Invalid configuration of apis in configuration yaml, review the configured apis: " + adApis);
         }
 
         return apis;
     }
 
-    private Callable<RequestDataResult> getApiMethodRef(Office365Apis office365Apis, String api) throws ConfigurationException {
-        RequestDataResult res = null;
+    private RequestDataResult getApiMethodRef(Office365Apis office365Apis, String api) throws ConfigurationException {
+        RequestDataResult res;
         try {
             res = (RequestDataResult) office365Apis.getClass().getMethod(api).invoke(office365Apis);
         } catch (ReflectiveOperationException illegalAccessException) {
             throw new ConfigurationException("Invalid configuration of apis in configuration yaml");
         }
-        return (Callable<RequestDataResult>) res;
+        return res;
     }
 
     public MSGraphConfiguration getConfiguration() {
@@ -105,6 +111,16 @@ private MSGraphConfiguration loadMSGraphConfig(String yamlFile) throws FileNotFo
         checkNotNull(config.getAzureADClient().getClientId(), "Parameter azureADClient.clientId is mandatory");
         checkNotNull(config.getAzureADClient().getClientSecret(), "Parameter azureADClient.clientSecret is mandatory");
         checkNotNull(config.getTargetApi(), "Parameter targetApi is mandatory");
+        checkNotNull(checkApiCount(config), "At least one api must be specified in configuration file");
         return config;
     }
+
+    private Boolean checkApiCount(MSGraphConfiguration config) {
+        List<String> ascApis = config.getTargetApi().getAscApis();
+        List<String> adApis = config.getTargetApi().getADApis();
+        int apiCount = 0;
+        apiCount+=ascApis!=null?ascApis.size():0;
+        apiCount+=adApis!=null?adApis.size():0;
+        return apiCount > 0 ? Boolean.TRUE : null;
+    }
 }

src/main/java/objects/MSGraphConfiguration.java

@@ -1,11 +1,13 @@
 package objects;
 
+import java.util.HashMap;
+
 public class MSGraphConfiguration {
     private AzureADClient azureADClient;
     private LogzioJavaSenderParams senderParams;
     private String logLevel = "INFO";
     private TargetApi targetApi;
-
+    private HashMap<String,String> additionalFields;
 
     public MSGraphConfiguration() {
     }
@@ -41,4 +43,12 @@ public TargetApi getTargetApi() {
     public void setTargetApi(TargetApi targetApi) {
         this.targetApi = targetApi;
     }
+
+    public HashMap<String, String> getAdditionalFields() {
+        return additionalFields;
+    }
+
+    public void setAdditionalFields(HashMap<String, String> additionalFields) {
+        this.additionalFields = additionalFields;
+    }
 }

src/main/java/objects/TargetApi.java

@@ -5,6 +5,7 @@
 public class TargetApi {
 
     private List<String> ADApis;
+    private List<String> ASCApis;
 
     public TargetApi() {
     }
@@ -16,4 +17,12 @@ public List<String> getADApis() {
     public void setADApis(List<String> ADApis) {
         this.ADApis = ADApis;
     }
+
+    public List<String> getAscApis() {
+        return ASCApis;
+    }
+
+    public void setASCApis(List<String> ASCApis) {
+        this.ASCApis = ASCApis;
+    }
 }

src/test/resources/testFullConfig.yaml

@@ -17,4 +17,6 @@ targetApi:
   ADApis:
     - signIns
     - directoryAudits
-    - riskySignIns
+    - riskySignIns
+  ASCApis:
+    - alerts