From 292c70b89998268e8a9aa4c6f38d1c408dd4567c Mon Sep 17 00:00:00 2001 From: Mohamed Date: Tue, 16 Oct 2018 15:16:55 -0500 Subject: [PATCH 01/12] Readme Updates --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 143a11d..a021a26 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ Installation requires the following steps: 1. Set up the required Active Directory security permissions for the application to authorize it to read threat intelligence data and activity reports for your organization. 1. Create an Access Key that will allow the application to connect to the Alert Logic Cloud Defender and Cloud Insight backend. 1. Download and deploy a custom ARM template to Microsoft Azure to create functions for collecting and managing O365 log data -1. Verify the installation was successful using the Alert Logic UI under Configuration -> Deployments -> All Deployments -> Log Sources -> Filter by `Push (Office 365, CloudWatch)` collection method. +1. Verify that installation was successful using the Alertlogic UI under Configuration -> Deployments -> All Deployments -> Log Sources -> Filter by "Office 365" collection method. ## Register a New O365 Web Application in O365 @@ -41,7 +41,7 @@ In order to install O365 Log collector: 1. On the `Settings` panel of the application, select `Keys`. 1. Enter key `Description` and set `Duration` to `Never expires` then click `Save`. **Note:** please save the key value, it is needed later during template deployment. -1. Get the `Service Principal ID` associated with the application by navigating back to the `Registered App` blade, click on the link under `Managed application in local directory` then click `Properties`. The `Service Principal ID` +1. Get the `Service Principal Id` associated with the application by navigating back to the `Registered App` blade, click on the link under `Managed application in local directory` then click `Properties`. The `Service Principal Id` is labeled as `Object ID` on the properties page. **Caution:** This is not the same `Object ID` found under the `Registered app` view or under the `Settings`. @@ -51,13 +51,13 @@ From the Bash command line in [Azure Cloud Shell](https://docs.microsoft.com/en- ``` export AL_USERNAME='' -auth=$(curl -SX POST -u $AL_USERNAME https://api.global-services.global.alertlogic.com/aims/v1/authenticate); export AL_ACCOUNT_ID=$(echo $auth | jq -r '.authentication.account.id'); export AL_USER_ID=$(echo $auth | jq -r '.authentication.user.id'); export AL_TOKEN=$(echo $auth | jq -r '.authentication.token'); if [ -z $AL_TOKEN ]; then echo "Authentication failure"; else roles=$(curl -SX GET -H "x-aims-auth-token: $AL_TOKEN" https://api.global-services.global.alertlogic.com/aims/v1/$AL_ACCOUNT_ID/users/$AL_USER_ID/roles | jq -r '.roles[].name'); if [ "$roles" != "Administrator" ]; then echo "The $AL_USERNAME doesn’t have Administrator role. Assigned role is '$roles'"; else curl -SX POST -H "x-aims-auth-token: $AL_TOKEN" https://api.global-services.global.alertlogic.com/aims/v1/$AL_ACCOUNT_ID/users/$AL_USER_ID/access_keys | jq .; fi; fi; unset AL_USERNAME; +auth=$(curl -SX POST -d '{"mfa_code": "" }' -u $AL_USERNAME https://api.global-services.global.alertlogic.com/aims/v1/authenticate); export AL_ACCOUNT_ID=$(echo $auth | jq -r '.authentication.account.id'); export AL_USER_ID=$(echo $auth | jq -r '.authentication.user.id'); export AL_TOKEN=$(echo $auth | jq -r '.authentication.token'); if [ -z $AL_TOKEN ]; then echo "Authentication failure"; else roles=$(curl -SX GET -H "x-aims-auth-token: $AL_TOKEN" https://api.global-services.global.alertlogic.com/aims/v1/$AL_ACCOUNT_ID/users/$AL_USER_ID/roles | jq -r '.roles[].name'); if [ "$roles" != "Administrator" ]; then echo "The $AL_USERNAME doesn’t have Administrator role. Assigned role is '$roles'"; else curl -SX POST -H "x-aims-auth-token: $AL_TOKEN" https://api.global-services.global.alertlogic.com/aims/v1/$AL_ACCOUNT_ID/users/$AL_USER_ID/access_keys | jq .; fi; fi; unset AL_USERNAME; ``` For accounts where MFA is enabled: ``` export AL_USERNAME='' -auth=$(curl -SX POST -d '{"mfa_code": "" }' -u $AL_USERNAME https://api.global-services.global.alertlogic.com/aims/v1/authenticate); export AL_ACCOUNT_ID=$(echo $auth | jq -r '.authentication.account.id'); export AL_USER_ID=$(echo $auth | jq -r '.authentication.user.id'); export AL_TOKEN=$(echo $auth | jq -r '.authentication.token'); if [ -z $AL_TOKEN ]; then echo "Authentication failure"; else roles=$(curl -SX GET -H "x-aims-auth-token: $AL_TOKEN" https://api.global-services.global.alertlogic.com/aims/v1/$AL_ACCOUNT_ID/users/$AL_USER_ID/roles | jq -r '.roles[].name'); if [ "$roles" != "Administrator" ]; then echo "The $AL_USERNAME doesn’t have Administrator role. Assigned role is '$roles'"; else curl -SX POST -H "x-aims-auth-token: $AL_TOKEN" https://api.global-services.global.alertlogic.com/aims/v1/$AL_ACCOUNT_ID/users/$AL_USER_ID/access_keys | jq .; fi; fi; unset AL_USERNAME; +auth=$(curl -sX POST -d '{"mfa_code": "" }' -u $AL_USERNAME https://api.global-services.global.alertlogic.com/aims/v1/authenticate); export AL_ACCOUNT_ID=$(echo $auth | jq -r '.authentication.account.id'); export AL_USER_ID=$(echo $auth | jq -r '.authentication.user.id'); export AL_TOKEN=$(echo $auth | jq -r '.authentication.token'); if [ -z $AL_TOKEN ]; then echo "Authentication failure"; else roles=$(curl -s -X GET -H "x-aims-auth-token: $AL_TOKEN" https://api.global-services.global.alertlogic.com/aims/v1/$AL_ACCOUNT_ID/users/$AL_USER_ID/roles | jq -r '.roles[].name'); if [ "$roles" != "Administrator" ]; then echo "The $AL_USERNAME doesn’t have Administrator role. Assigned role is '$roles'"; else curl -s -X POST -H "x-aims-auth-token: $AL_TOKEN" https://api.global-services.global.alertlogic.com/aims/v1/$AL_ACCOUNT_ID/users/$AL_USER_ID/access_keys | jq .; fi; fi; unset AL_USERNAME; ``` An example of a successful response is: From b7076ea2456af7ff3e6ff606e5532d2864c01570 Mon Sep 17 00:00:00 2001 From: Mohamed Date: Wed, 17 Oct 2018 11:27:54 -0500 Subject: [PATCH 02/12] additional minor updates per feedback --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index a021a26..143a11d 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ Installation requires the following steps: 1. Set up the required Active Directory security permissions for the application to authorize it to read threat intelligence data and activity reports for your organization. 1. Create an Access Key that will allow the application to connect to the Alert Logic Cloud Defender and Cloud Insight backend. 1. Download and deploy a custom ARM template to Microsoft Azure to create functions for collecting and managing O365 log data -1. Verify that installation was successful using the Alertlogic UI under Configuration -> Deployments -> All Deployments -> Log Sources -> Filter by "Office 365" collection method. +1. Verify the installation was successful using the Alert Logic UI under Configuration -> Deployments -> All Deployments -> Log Sources -> Filter by `Push (Office 365, CloudWatch)` collection method. ## Register a New O365 Web Application in O365 @@ -41,7 +41,7 @@ In order to install O365 Log collector: 1. On the `Settings` panel of the application, select `Keys`. 1. Enter key `Description` and set `Duration` to `Never expires` then click `Save`. **Note:** please save the key value, it is needed later during template deployment. -1. Get the `Service Principal Id` associated with the application by navigating back to the `Registered App` blade, click on the link under `Managed application in local directory` then click `Properties`. The `Service Principal Id` +1. Get the `Service Principal ID` associated with the application by navigating back to the `Registered App` blade, click on the link under `Managed application in local directory` then click `Properties`. The `Service Principal ID` is labeled as `Object ID` on the properties page. **Caution:** This is not the same `Object ID` found under the `Registered app` view or under the `Settings`. @@ -51,13 +51,13 @@ From the Bash command line in [Azure Cloud Shell](https://docs.microsoft.com/en- ``` export AL_USERNAME='' -auth=$(curl -SX POST -d '{"mfa_code": "" }' -u $AL_USERNAME https://api.global-services.global.alertlogic.com/aims/v1/authenticate); export AL_ACCOUNT_ID=$(echo $auth | jq -r '.authentication.account.id'); export AL_USER_ID=$(echo $auth | jq -r '.authentication.user.id'); export AL_TOKEN=$(echo $auth | jq -r '.authentication.token'); if [ -z $AL_TOKEN ]; then echo "Authentication failure"; else roles=$(curl -SX GET -H "x-aims-auth-token: $AL_TOKEN" https://api.global-services.global.alertlogic.com/aims/v1/$AL_ACCOUNT_ID/users/$AL_USER_ID/roles | jq -r '.roles[].name'); if [ "$roles" != "Administrator" ]; then echo "The $AL_USERNAME doesn’t have Administrator role. Assigned role is '$roles'"; else curl -SX POST -H "x-aims-auth-token: $AL_TOKEN" https://api.global-services.global.alertlogic.com/aims/v1/$AL_ACCOUNT_ID/users/$AL_USER_ID/access_keys | jq .; fi; fi; unset AL_USERNAME; +auth=$(curl -SX POST -u $AL_USERNAME https://api.global-services.global.alertlogic.com/aims/v1/authenticate); export AL_ACCOUNT_ID=$(echo $auth | jq -r '.authentication.account.id'); export AL_USER_ID=$(echo $auth | jq -r '.authentication.user.id'); export AL_TOKEN=$(echo $auth | jq -r '.authentication.token'); if [ -z $AL_TOKEN ]; then echo "Authentication failure"; else roles=$(curl -SX GET -H "x-aims-auth-token: $AL_TOKEN" https://api.global-services.global.alertlogic.com/aims/v1/$AL_ACCOUNT_ID/users/$AL_USER_ID/roles | jq -r '.roles[].name'); if [ "$roles" != "Administrator" ]; then echo "The $AL_USERNAME doesn’t have Administrator role. Assigned role is '$roles'"; else curl -SX POST -H "x-aims-auth-token: $AL_TOKEN" https://api.global-services.global.alertlogic.com/aims/v1/$AL_ACCOUNT_ID/users/$AL_USER_ID/access_keys | jq .; fi; fi; unset AL_USERNAME; ``` For accounts where MFA is enabled: ``` export AL_USERNAME='' -auth=$(curl -sX POST -d '{"mfa_code": "" }' -u $AL_USERNAME https://api.global-services.global.alertlogic.com/aims/v1/authenticate); export AL_ACCOUNT_ID=$(echo $auth | jq -r '.authentication.account.id'); export AL_USER_ID=$(echo $auth | jq -r '.authentication.user.id'); export AL_TOKEN=$(echo $auth | jq -r '.authentication.token'); if [ -z $AL_TOKEN ]; then echo "Authentication failure"; else roles=$(curl -s -X GET -H "x-aims-auth-token: $AL_TOKEN" https://api.global-services.global.alertlogic.com/aims/v1/$AL_ACCOUNT_ID/users/$AL_USER_ID/roles | jq -r '.roles[].name'); if [ "$roles" != "Administrator" ]; then echo "The $AL_USERNAME doesn’t have Administrator role. Assigned role is '$roles'"; else curl -s -X POST -H "x-aims-auth-token: $AL_TOKEN" https://api.global-services.global.alertlogic.com/aims/v1/$AL_ACCOUNT_ID/users/$AL_USER_ID/access_keys | jq .; fi; fi; unset AL_USERNAME; +auth=$(curl -SX POST -d '{"mfa_code": "" }' -u $AL_USERNAME https://api.global-services.global.alertlogic.com/aims/v1/authenticate); export AL_ACCOUNT_ID=$(echo $auth | jq -r '.authentication.account.id'); export AL_USER_ID=$(echo $auth | jq -r '.authentication.user.id'); export AL_TOKEN=$(echo $auth | jq -r '.authentication.token'); if [ -z $AL_TOKEN ]; then echo "Authentication failure"; else roles=$(curl -SX GET -H "x-aims-auth-token: $AL_TOKEN" https://api.global-services.global.alertlogic.com/aims/v1/$AL_ACCOUNT_ID/users/$AL_USER_ID/roles | jq -r '.roles[].name'); if [ "$roles" != "Administrator" ]; then echo "The $AL_USERNAME doesn’t have Administrator role. Assigned role is '$roles'"; else curl -SX POST -H "x-aims-auth-token: $AL_TOKEN" https://api.global-services.global.alertlogic.com/aims/v1/$AL_ACCOUNT_ID/users/$AL_USER_ID/access_keys | jq .; fi; fi; unset AL_USERNAME; ``` An example of a successful response is: From 8d7ba345ccae453c061ac31d9262badca7b6e583 Mon Sep 17 00:00:00 2001 From: rhosmo Date: Wed, 28 Nov 2018 08:34:52 -0600 Subject: [PATCH 03/12] Update README.md (#47) * Update README.md Info Dev edits applied to Integration branch. * Removed comments; Removed DLP.All list. --- README.md | 184 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 102 insertions(+), 82 deletions(-) diff --git a/README.md b/README.md index 143a11d..4a7a4d9 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# azure-collector +# Alert Logic Microsoft Office 365 Log Collector [![Build Status](https://secure.travis-ci.org/alertlogic/azure-collector.png?branch=master)](http://travis-ci.org/alertlogic/azure-collector) @@ -6,55 +6,61 @@ AlertLogic Office 365 Log Collector # Overview -This repository contains Azure Web application Node.js source code and an ARM template for setting up a data collector in Azure which will collect and forward Office 365 log data to the Alert Logic Cloud Defender Log Manager (LM) feature. +This repository contains the Microsoft Azure web application Node.js source code and an Azure Resource Manager (ARM) template to set up a data collector in Azure, which collects and forwards Microsoft Office 365 log data to Alert Logic Log Management. # Installation -Installation requires the following steps: +To perform the set up required to grant Alert Logic permission access to collect Office 365 logs, you must have access to the following: + +* A Microsoft Office 365 subscription with administrative privileges +* A Microsoft Azure account with administrative privileges +* An Alert Logic user account with administrative privileges -1. Register a new O365 web application in O365 portal for collecting O365 logs. -1. Set up the required Active Directory security permissions for the application to authorize it to read threat intelligence data and activity reports for your organization. -1. Create an Access Key that will allow the application to connect to the Alert Logic Cloud Defender and Cloud Insight backend. -1. Download and deploy a custom ARM template to Microsoft Azure to create functions for collecting and managing O365 log data -1. Verify the installation was successful using the Alert Logic UI under Configuration -> Deployments -> All Deployments -> Log Sources -> Filter by `Push (Office 365, CloudWatch)` collection method. ## Register a New O365 Web Application in O365 -In order to install O365 Log collector: +In the Office 365 portal, you must register a new Office 365 web application to collect Office 365 logs. + +**To register an Office 365 web application to collect logs:** -1. Log into [O365 portal](https://portal.office.com) as AD tenant administrator. +1. Log into the [Office 365 portal](https://portal.office.com) as an Active Directory tenant administrator. 1. Navigate to `Admin Centers` and `Azure AD`. -1. On the left side panel click `Azure Active Directory` then select `App Registrations`. -1. Click `+ New application registration`, fill in configuration parameters and click `Create`: - * `Name` - for example `alo365collector`. +1. On the left side panel click `Azure Active Directory`, and then select `App Registrations`. +1. Click `+ New application registration`, and then provide the following configuration parameters: + * `Name`- Provide a name for the new application (For example `alo365collector`). * Select `Web app/ API` as `Application type`. - * In `Sign-on URL` enter some URL, for example `http://alo365collector.com`. **Note**, it is not used anywhere within your subscription. - -1. After application is created select `All apps`, click on the application name that was created and make a note of the `Application ID`, for example, `a261478c-84fb-42f9-84c2-de050a4babe3` + * `Sign-on URL` - Type a URL for the application (for example `http://alo365collector.com`). + **Note** This information is not used anywhere within your subscription. +1. Click `Create`. +1. From the `All applications` tab on the `App registration (Preview)` blade, select `All apps`, and then click the application name you created. +1. Note the `Application ID`, for example, `a261478c-84fb-42f9-84c2-de050a4babe3` ## Set Up the Required Active Directory Security Permissions -1. On the `Settings` panel under the newly created Application, select `Required permissions` and click `+ Add` -1. Hit `Select an API` and chose `Office 365 Management APIs`, click `Select` -1. In `Application permissions` select `Read service health information for your organization`, `Read activity data for your organization`, `Read threat intelligence data for your organization` and `Read activity reports for your organization`. Click `Select` and then `Done`. -1. Click the `Grant Permissions` button and confirm by clicking `Yes`. **Note:** only AD tenant admin can grant permissions to an Azure AD application. -1. On the `Settings` panel of the application, select `Keys`. -1. Enter key `Description` and set `Duration` to `Never expires` then click `Save`. -**Note:** please save the key value, it is needed later during template deployment. -1. Get the `Service Principal ID` associated with the application by navigating back to the `Registered App` blade, click on the link under `Managed application in local directory` then click `Properties`. The `Service Principal ID` -is labeled as `Object ID` on the properties page. -**Caution:** This is not the same `Object ID` found under the `Registered app` view or under the `Settings`. +1. On the `Settings` panel, under the newly created Application, select `Required permissions`, and click then click `+ Add`. +1. Click `Select an API` -> `Office 365 Management APIs`, and then click `Select`. +1. In `Application permissions`, click `Read service health information for your organization` -> `Read activity data for your organization` -> `Read threat intelligence data for your organization` -> `Read activity reports for your organization`. +1. Click `Select`, and then click `Done`. +1. Click `Grant Permissions`, and then click `Yes`. +**Note:** Only the Active Directory tenant administrator can grant permissions to an Azure Active Directory application. +1. On the `Settings` panel for the application, select `Keys`. +1. Type a key `Description`, and then set `Duration` to `Never expires`. +1. Click `Save`. +**Note:** Save the key value, which you need during ARM template deployment. +1. From the `Registered App` blade, click the link under `Managed application in local directory`, and then click `Properties`. +1. Get the `Service Principal ID` associated with the application. (The `Service Principal ID`is labeled as `Object ID` on the properties page.) +**Caution:** This ID is not the same `Object ID` found under the `Registered app` view or under the `Settings`. ## Create an Alert Logic Access Key -From the Bash command line in [Azure Cloud Shell](https://docs.microsoft.com/en-us/azure/cloud-shell/quickstart) run the following commands, where `` is your Alert Logic user and `` is your Alert Logic password: +**From the Bash command line in [Azure Cloud Shell](https://docs.microsoft.com/en-us/azure/cloud-shell/quickstart) run the following commands, where `` is your Alert Logic user name and `` is your Alert Logic password:** ``` export AL_USERNAME='' auth=$(curl -SX POST -u $AL_USERNAME https://api.global-services.global.alertlogic.com/aims/v1/authenticate); export AL_ACCOUNT_ID=$(echo $auth | jq -r '.authentication.account.id'); export AL_USER_ID=$(echo $auth | jq -r '.authentication.user.id'); export AL_TOKEN=$(echo $auth | jq -r '.authentication.token'); if [ -z $AL_TOKEN ]; then echo "Authentication failure"; else roles=$(curl -SX GET -H "x-aims-auth-token: $AL_TOKEN" https://api.global-services.global.alertlogic.com/aims/v1/$AL_ACCOUNT_ID/users/$AL_USER_ID/roles | jq -r '.roles[].name'); if [ "$roles" != "Administrator" ]; then echo "The $AL_USERNAME doesn’t have Administrator role. Assigned role is '$roles'"; else curl -SX POST -H "x-aims-auth-token: $AL_TOKEN" https://api.global-services.global.alertlogic.com/aims/v1/$AL_ACCOUNT_ID/users/$AL_USER_ID/access_keys | jq .; fi; fi; unset AL_USERNAME; ``` -For accounts where MFA is enabled: +**For accounts with multi-factor authentication (MFA) enabled:** ``` export AL_USERNAME='' auth=$(curl -SX POST -d '{"mfa_code": "" }' -u $AL_USERNAME https://api.global-services.global.alertlogic.com/aims/v1/authenticate); export AL_ACCOUNT_ID=$(echo $auth | jq -r '.authentication.account.id'); export AL_USER_ID=$(echo $auth | jq -r '.authentication.user.id'); export AL_TOKEN=$(echo $auth | jq -r '.authentication.token'); if [ -z $AL_TOKEN ]; then echo "Authentication failure"; else roles=$(curl -SX GET -H "x-aims-auth-token: $AL_TOKEN" https://api.global-services.global.alertlogic.com/aims/v1/$AL_ACCOUNT_ID/users/$AL_USER_ID/roles | jq -r '.roles[].name'); if [ "$roles" != "Administrator" ]; then echo "The $AL_USERNAME doesn’t have Administrator role. Assigned role is '$roles'"; else curl -SX POST -H "x-aims-auth-token: $AL_TOKEN" https://api.global-services.global.alertlogic.com/aims/v1/$AL_ACCOUNT_ID/users/$AL_USER_ID/access_keys | jq .; fi; fi; unset AL_USERNAME; @@ -69,90 +75,104 @@ An example of a successful response is: } ``` -**Note:** if the output is blank please double-check the Alert Logic user permission, you should have administrator access. More details about AIMS APIs can be found [here](https://console.product.dev.alertlogic.com/api/aims/). +**Note:** If the output is blank, verify the Alert Logic user account permissions. You must have administrator access. For more information about AIMS APIs, see [Access and Identity Management Service](https://console.product.dev.alertlogic.com/api/aims/). -Make a note of the `access_key_id` and `secret_key` values for use in the deployment steps below. +Note the `access_key_id` and the `secret_key` values for use in the deployment steps below. -**Note:** Only five access keys can be created per user. If you get a "limit exceeded" response you will need to delete some keys in order to create new ones. Use the following command to list access keys: +**Note:** An account can create only five access keys. If you receive a "limit exceeded" response, you must delete some keys to create more. Use the following command to list access keys: ``` curl -s -X GET -H "x-aims-auth-token: $AL_TOKEN" https://api.global-services.global.alertlogic.com/aims/v1/$AL_ACCOUNT_ID/users/$AL_USER_ID/access_keys | jq ``` -Then use the selected access_key_id in the following curl command to delete it: +Then use the selected access_key_id in the following curl command to delete the key: ``` curl -X DELETE -H "x-aims-auth-token: $AL_TOKEN" https://api.global-services.global.alertlogic.com/aims/v1/$AL_ACCOUNT_ID/users/$AL_USER_ID/access_keys/ ``` -## Function deployment +## Download and Deploy the ARM Template + +You can use either the Microsoft Azure portal or a command line to deploy the template. To perform either procedure, you must log into the [Azure portal](https://portal.azure.com). -Log into [Azure portal](https://portal.azure.com). **Note**, In order to perform steps below you should have an active Azure subscription, to find out visit [Azure subscriptions blade](https://portal.azure.com/#blade/Microsoft_Azure_Billing/SubscriptionsBlade). +**Note:** The steps in this section require an active Azure subscription. To verify your Azure subscrption, visit [Azure subscriptions blade](https://portal.azure.com/#blade/Microsoft_Azure_Billing/SubscriptionsBlade). -If multiple Active Directory tenants are used within your organization please login into the same tenant where application registration was created during [Register a New O365 Web Application in O365](#register-a-new-o365-web-application-in-o365). How to find Office365 tenant id see [here](https://support.office.com/en-gb/article/find-your-office-365-tenant-id-6891b561-a52d-4ade-9f39-b492285e2c9b). +If your organization uses multiple Active Directory tenants, log into the same tenant used to [Register a New Office 365 Web Application](#register-a-new-office-365-web-application). To find your Office 365 tenant ID, see [Find your Office 365 tenant ID](https://support.office.com/en-gb/article/find-your-office-365-tenant-id-6891b561-a52d-4ade-9f39-b492285e2c9b). -### Deploy via the Custom ARM Template in an Azure Subscription +### Deploy with the Custom ARM Template in an Azure Subscription + +Click the button below to start deployment. [![Deploy to Azure](https://azuredeploy.net/deploybutton.png)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2Falertlogic%2Fazure-collector%2Fmaster%2Ftemplate.json) -Fill in required template parameters and click the `Purchase` button to start a deployment: - - `Name` - This is the name of the log source that will show in the Alert Logic UI - - `Storage Name` - Any Storage Account name (that does not currently exist) - - `Alert Logic Access Key ID` - `access_key_id` returned from AIMs [above](#create_an_alert_logic_access_key) - - `Alert Logic Secret Key` - `secret_key` returned from AIMs [above](#create_an_alert_logic_access_key) - - `Alert Logic API endpoint` - leave it as `api.global-services.global.alertlogic.com` - - `Alert Logic Data Residency` - leave it as `default` - - `Office365 Content Streams` - The list of streams you would like to collect. Valid values are: - - ["Audit.AzureActiveDirectory","Audit.Exchange","Audit.SharePoint","Audit.General", "DLP.All"] - - `Service Principal ID` - The `Object ID` of the application that created the subscription. You can obtain it from _Azure_ -> _AD_ -> _App registrations_ -> _Your app name_ -> Link under _Managed application in local directory_ -> _Properties_ -> _Object ID_ - - `App Client ID` - The GUID of your application that created the subscription. You can obtain it from _Azure_ -> _AD_ -> _App registrations_ -> _Your app name_ +1. Provide the following required template parameters and click the `Purchase` button to start a deployment: + - `Name` - Type the name of the log source to appear in the Alert Logic console. + - `Storage Name` - Any storage account name (that does not currently exist). + - `Alert Logic Access Key ID` - The `access_key_id` you created above + - `Alert Logic Secret Key` - The `secret_key` you created above. + - `Alert Logic API endpoint` - Leave the default value (api.global-services.global.alertlogic.com). + - `Alert Logic Data Residency` - Leave the value as `default`. + - `Office365 Content Streams` - The log types you want to collect. Valid values are: + ["Audit.AzureActiveDirectory","Audit.Exchange","Audit.SharePoint","Audit.General"] + - `Service Principal ID` - The `Object ID` of the application that created the subscription. + **Noe** You can obtain this value from _Azure_ -> _AD_ -> _App registrations_ -> _Your app name_ -> Link under _Managed application in local directory_ -> _Properties_ -> _Object ID_. + - `App Client ID` - The GUID of your application that created the subscription. + **Note** You can obtain this value from _Azure_ -> _AD_ -> _App registrations_ -> _Your app name_ - `App Client Secret` - The secret key of your application from _App Registrations_ +1. Click `Purchase`. + +### Deploy through the Azure CLI -### Deploy via Azure CLI +If you want to deploy the template through the Azure command line (CLI), you can use either [Azure Cloud Shell](https://docs.microsoft.com/en-gb/azure/cloud-shell/quickstart#start-cloud-shell) or a local installation of [Azure CLI](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli?view=azure-cli-latest). -You can use either [Azure Cloud Shell](https://docs.microsoft.com/en-gb/azure/cloud-shell/quickstart#start-cloud-shell) or local installation of [Azure CLI](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli?view=azure-cli-latest). +**To deploy through the Azure CLI:** -1. Create a new resource group in, for example, the "Central US" location by executing following command: +1. In the command line, type the following to create a new resource group. +**Note** The example below creates a new resource group in the "Central US" location.) ``` az group create --name --location "Central US" ``` -1. Once created go to `Resource groups` blade and select the resource group. -1. Select `Access Control (IAM)` and add `Website Contributor` role to AD application identity created above. -1. Deploy a template by using following command, during its execution enter required parameters when asked. +1. In the Azure portal, access the `Resource groups` blade, and then select the resource group you created. +1. Select `Access Control (IAM)`, and add `Website Contributor` role to the Active Directory application identity you created above. +1. In the command line, type the following command to deploy a template, and enter the required parameters when prompted. ``` az group deployment create \ --resource-group \ --template-uri "https://raw.githubusercontent.com/alertlogic/azure-collector/master/template.json" ``` -Wait until it is deployed successfully. - ## Verify the Installation +**To verify successful installation of the template:** -1. Go to `Function Apps` and choose the Alert Logic O365 collector function. The recent log entry under `Functions-> Master-> Monitor` should read an OK status (Example: `O365 source checkin OK`) and should not contain any error messages. -1. Log into Alert Logic UI and navigate into Configuration -> Deployments -> All Deployments -> Log Sources -> Filter by `Push (Office 365, CloudWatch)` collection method. Check new O365 log source (with a name provided during `az group deployment create` above) has been created and source status is `ok`. +1. In the Azure portal, access `Function Apps`, and then choose the Alert Logic Office 365 collector function. +1. Click `Functions` -> `Master` -> `Monitor` and verify the recent log entry has the status of "OK" and contains no error messages. +**Example:** `O365 source checkin OK`. +1. In the Alert Logic console, navigate to `Configuration` -> `Deployments` -> `All Deployments` -> `Log Sources`, and then filter the list by `Push (Office 365, CloudWatch)` collection method. +1. Verify a new Office 365 log source with the name provided during `az group deployment create` above appears with the source status as `OK`. -# How It Works +# How the Office 365 Collector Works -**Note:** the following Azure functions use Application/O365 tenant id (`APP_TENANT_ID` web application setting) as a `PublisherIdentifier` during O365 management API requests. More info about `PublisherIdentifier` can be found [here](https://msdn.microsoft.com/en-us/office-365/troubleshooting-the-office-365-management-activity-api#requesting-content-blobs-and-throttling). +**Note:** The following Azure functions use the Application/O365 tenant ID (`APP_TENANT_ID` web application setting) as a `PublisherIdentifier` during Office 365 management API requests. For more information about `PublisherIdentifier`, see [Requesting content blobs and throttling](https://msdn.microsoft.com/en-us/office-365/troubleshooting-the-office-365-management-activity-api#requesting-content-blobs-and-throttling). ## Master Function -The `Master` function is a timer trigger function which is responsible for: -- registering the Azure web app in Alertlogic backend; -- reporting health-checks to the backed; -- performing log source configuration updates, which happen via Alertlogic UI. +The `Master` function is a timer trigger function responsible for: +- Registering the Azure web app in Alertlogic backend +- Reporting health-checks to the backed +- Performing log source configuration updates, which happen via Alertlogic UI -**Note:** When releasing a new version of the collector please remember to increment the version number in -npm package.json file. To display the current version locally, issue `npm run local-version` +**Note:** When you release a new version of the collector, remember to increment the version number in +npm package.json file. To display the current version locally, issue `npm run local-version` ## Updater Function -The `Updater` is a timer triggered function runs deployment sync operation every 12 hours in order to keep entire Web application up to date. +The `Updater` function is a timer triggered function that runs a deployment sync operation every 12 hours to keep web application up to date. ## O365WebHook Function -The `O365WebHook` function exposes an HTTP API endpoint `https:///o365/webhook` which is registered as an [Office 365 webhook](https://msdn.microsoft.com/en-us/office-365/office-365-management-activity-api-reference#start-a-subscription) and processes O365 activity notifications. Below is a notification example, +The `O365WebHook` function exposes an HTTP API endpoint `https:///o365/webhook` that is registered as an [Office 365 webhook](https://msdn.microsoft.com/en-us/office-365/office-365-management-activity-api-reference#start-a-subscription) and processes Office 365 activity notifications + +**Notification Example** ``` [ @@ -177,32 +197,32 @@ The `O365WebHook` function exposes an HTTP API endpoint `https:///o365 ] ``` -A notification contains a link to the actual data which is retrieved by the `O365WebHook`, wrapped into a protobuf [structure](proto/common_proto.piqi.proto) and is sent into Alertlogic Ingest service. +A notification contains a link to the actual data, which is retrieved by the `O365WebHook`, wrapped into a protobuf [structure](proto/common_proto.piqi.proto), and then sent to Alert Logic Ingest service. -**Note:** it can take up to 24 hours before audit content is available. Please follow [this link](https://support.office.com/en-us/article/Search-the-audit-log-in-the-Office-365-Security-Compliance-Center-0d4d0f35-390b-4518-800e-0c7ec95e946c?ui=en-US&rs=en-US&ad=US#PickTab=BYB) to find the time it takes for the different services in Office 365. +**Note:** Audit content may not be available for up to 24 hours. Please follow [this link](https://support.office.com/en-us/article/Search-the-audit-log-in-the-Office-365-Security-Compliance-Center-0d4d0f35-390b-4518-800e-0c7ec95e946c?ui=en-US&rs=en-US&ad=US#PickTab=BYB) to find the time it takes for the different services in Office 365. # Local Development -1. Clone repo `git clone git@github.com:alertlogic/azure-collector.git` +1. Clone the repo `git clone git@github.com:alertlogic/azure-collector.git`. 1. `cd azure-collector` -1. Run `./local_dev/setup.sh` -1. Edit `./local_dev/dev_config.js` -1. Run Master function locally: `npm run local-master` -1. Run Updater function locally: `npm run local-updater` -1. Run O365WebHook function locally: `npm run local-o365webhook` -1. Run `npm test` in order to perform code analysis and unit tests. +1. Run `./local_dev/setup.sh`. +1. Edit `./local_dev/dev_config.js`. +1. Run the Master function locally: `npm run local-master`. +1. Run the Updater function locally: `npm run local-updater`. +1. Run the O365WebHook function locally: `npm run local-o365webhook`. +1. Run `npm test` to perform code analysis and unit tests. Please use the following [code style](https://github.com/airbnb/javascript) as much as possible. ## Setting environment in dev_config.js -- `process.env.APP_TENANT_ID` - The GUID of the tenant i.e. 'alazurealertlogic.onmicrosoft.com' -- `process.evn.APP_RESOURCE_GROUP` - The name of the resource group where your application is deployed. +- `process.env.APP_TENANT_ID` - The GUID of the tenant (such as `alazurealertlogic.onmicrosoft.com`) +- `process.evn.APP_RESOURCE_GROUP` - The name of the resource group where you deployed your application. - `process.env.CUSTOMCONNSTR_APP_CLIENT_ID` - The GUID of your application that created the subscription. -You can obtain it from _Azure_ -> _AD_ -> _App registrations_ -> _Your app name_ -- `process.env.CUSTOMCONNSTR_APP_CLIENT_SECRET` - A secret key of your application from _App Registrations_. -- `process.env.CUSTOMCONNSTR_APP_CI_ACCESS_KEY_ID` - access key returned from AIMs [above](#create_an_alert_logic_access_key). -- `process.env.CUSTOMCONNSTR_APP_CI_SECRET_KEY`- secret key returned from AIMs [above](#create_an_alert_logic_access_key). +**Note** You can obtain this value from _Azure_ -> _AD_ -> _App registrations_ -> _Your app name_ +- `process.env.CUSTOMCONNSTR_APP_CLIENT_SECRET` - The secret key of your application from _App Registrations_. +- `process.env.CUSTOMCONNSTR_APP_CI_ACCESS_KEY_ID` - The access key returned from AIMs [above](#create_an_alert_logic_access_key). +- `process.env.CUSTOMCONNSTR_APP_CI_SECRET_KEY`- The secret key returned from AIMs [above](#create_an_alert_logic_access_key). # Known Issues/ Open Questions From 9a7002364e194530beb06f98b68d50a5adcb06d7 Mon Sep 17 00:00:00 2001 From: JDragovichAlertLogic <47750771+JDragovichAlertLogic@users.noreply.github.com> Date: Fri, 22 Mar 2019 14:08:36 +0000 Subject: [PATCH 04/12] Refactor al azure collector to use al-azure-collector-js (#50) * Readme updates from integration (#49) * Readme Updates * additional minor updates per feedback * Update README.md (#47) * Update README.md Info Dev edits applied to Integration branch. * Removed comments; Removed DLP.All list. * Refactor collector to new Azure framework --- .gitignore | 1 + LICENSE | 8 + Master/appsettings.js | 82 ------- Master/appstats.js | 138 ------------ Master/endpoints.js | 81 ------- Master/healthchecks.js | 65 ++++++ Master/index.js | 72 +++---- Master/o365collector.js | 121 ----------- O365WebHook/formatO365Log.js | 43 ++++ O365WebHook/ingest.js | 46 ---- O365WebHook/ingest_proto.js | 181 ---------------- O365WebHook/o365content.js | 166 ++++----------- Updater/index.js | 124 +---------- package.json | 8 +- template.json | 7 +- test/formatO365Log_test.js | 25 +++ test/master_appsettings_test.js | 116 ---------- test/master_appstats_test.js | 121 ----------- test/master_endpoints_test.js | 87 -------- test/master_o365collector_test.js | 342 ++++-------------------------- test/mock.js | 7 + test/o365webhook_content_test.js | 44 +--- 22 files changed, 289 insertions(+), 1596 deletions(-) create mode 100644 LICENSE delete mode 100644 Master/appsettings.js delete mode 100644 Master/appstats.js delete mode 100644 Master/endpoints.js create mode 100644 Master/healthchecks.js delete mode 100644 Master/o365collector.js create mode 100644 O365WebHook/formatO365Log.js delete mode 100644 O365WebHook/ingest.js delete mode 100644 O365WebHook/ingest_proto.js create mode 100644 test/formatO365Log_test.js delete mode 100644 test/master_appsettings_test.js delete mode 100644 test/master_appstats_test.js delete mode 100644 test/master_endpoints_test.js diff --git a/.gitignore b/.gitignore index 4e08845..723ca95 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ .idea/ node_modules/ +*.swp diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..6eac43d --- /dev/null +++ b/LICENSE @@ -0,0 +1,8 @@ +Copyright 2019 ALERT LOGIC + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + diff --git a/Master/appsettings.js b/Master/appsettings.js deleted file mode 100644 index 0a9033a..0000000 --- a/Master/appsettings.js +++ /dev/null @@ -1,82 +0,0 @@ -/* ----------------------------------------------------------------------------- - * @copyright (C) 2017, Alert Logic, Inc - * @doc - * - * The module for updating Azure application settings. - * - * @end - * ----------------------------------------------------------------------------- - */ - -const async = require('async'); - -const m_alUtil = require('../lib/al_util'); - -var azureRest = require('ms-rest-azure'); -var azureArmClient = require('azure-arm-resource').ResourceManagementClient; -var azureArmWebsite = require('azure-arm-website'); - -var fileTokenCache = require('azure/lib/util/fileTokenCache'); -var tokenCache = new fileTokenCache(m_alUtil.getADCacheFilename( - { - creds : { - client_id : process.env.CUSTOMCONNSTR_APP_CLIENT_ID, - tenant_id : process.env.APP_TENANT_ID - }, - resource : 'https://management.azure.com' - })); - -var g_appAdCreds = new azureRest.ApplicationTokenCredentials( - process.env.CUSTOMCONNSTR_APP_CLIENT_ID, - process.env.APP_TENANT_ID, - process.env.CUSTOMCONNSTR_APP_CLIENT_SECRET, - { 'tokenCache': tokenCache }); - -var g_websiteClient = new azureArmWebsite(g_appAdCreds, process.env.APP_SUBSCRIPTION_ID); - - -var _updateAppsettings = function(newSettings, callback) { - async.waterfall([ - function(asyncCallback) { - return _getAppsettings(asyncCallback); - }, - function(appSettings, asyncCallback) { - let updatedProps = Object.assign({}, appSettings.properties, newSettings); - let updatedEnv = Object.assign({}, process.env, newSettings); - process.env = updatedEnv; - appSettings.properties = updatedProps; - return _setAppsettings(appSettings, asyncCallback); - }], - callback - ); -}; - -var _getAppsettings = function(callback) { - return g_websiteClient.webApps.listApplicationSettings( - process.env.APP_RESOURCE_GROUP, process.env.WEBSITE_SITE_NAME, null, - function(err, result, request, response) { - if (err) { - return callback(err); - } else { - return callback(null, result); - } - }); -}; - -var _setAppsettings = function(settings, callback) { - return g_websiteClient.webApps.updateApplicationSettings( - process.env.APP_RESOURCE_GROUP, process.env.WEBSITE_SITE_NAME, settings, null, - function(err, result, request, response) { - if (err) { - return callback(err); - } else { - return callback(null); - } - }); -}; - -module.exports = { - getAppsettings : _getAppsettings, - setAppsettings : _setAppsettings, - updateAppsettings : _updateAppsettings -}; diff --git a/Master/appstats.js b/Master/appstats.js deleted file mode 100644 index af777d9..0000000 --- a/Master/appstats.js +++ /dev/null @@ -1,138 +0,0 @@ -/* ----------------------------------------------------------------------------- - * @copyright (C) 2017, Alert Logic, Inc - * @doc - * - * The module for getting storage account settings. - * - * @end - * ----------------------------------------------------------------------------- - */ - -const async = require('async'); -const util = require('util'); -const moment = require('moment'); -const parse = require('parse-key-value'); -const azureStorage = require('azure-storage'); - - -const m_alUtil = require('../lib/al_util'); - -const STATS_PERIOD_MINUTES = 15; -const APP_FUNCTIONS = ['Master', 'O365WebHook', 'Updater']; - -var TableQuery = azureStorage.TableQuery; -var TableUtilities = azureStorage.TableUtilities; -var g_tableService = null; - -var _initTableService = function() { - const storageParams = parse(process.env.AzureWebJobsStorage); - g_tableService = azureStorage.createTableService( - storageParams.AccountName, - storageParams.AccountKey, - storageParams.AccountName + '.table.core.windows.net'); - return g_tableService; -}; - -var getTableService = function() { - return (g_tableService) ? g_tableService : _initTableService(); -}; - -var getLogTableName = function() { - return 'AzureWebJobsHostLogs' + moment.utc().format('YYYYMM'); -}; - -var getInvocationsQuery = function(functionName, timestamp) { - var functionFilter = TableQuery.stringFilter( - 'FunctionName', - TableUtilities.QueryComparisons.EQUAL, - functionName); - var dateFilter = TableQuery.dateFilter( - 'StartTime', - TableUtilities.QueryComparisons.GREATER_THAN_OR_EQUAL, - new Date(moment(timestamp).utc().subtract(STATS_PERIOD_MINUTES, 'minutes'))); - var whereFilter = TableQuery.combineFilters( - dateFilter, - TableUtilities.TableOperators.AND, - functionFilter); - - return new TableQuery().where(whereFilter); -}; - -var _getInvocationStats = function(entities, accStats) { - accStats.invocations += entities.length; - - return entities.reduce(function(acc, current) { - if (current.ErrorDetails) { - acc.errors++; - } - return acc; - }, - accStats); -}; - -var _getFunctionStats = function(functionName, timestamp, callback) { - var accStats = { - invocations : 0, - errors : 0 - }; - return _getFunctionStatsAcc(functionName, timestamp, null, accStats, callback); -}; - -var _getFunctionStatsAcc = function(functionName, timestamp, contToken, accStats, callback) { - var tableService = getTableService(); - var obj = {}; - - tableService.queryEntities( - getLogTableName(), - getInvocationsQuery(functionName, timestamp), - contToken, - function(error, result) { - if (error) { - obj[functionName] = { - error : `Error getting stats ${error}` - }; - return callback(null, obj); - } else { - if (result.continuationToken) { - return _getFunctionStatsAcc( - functionName, - timestamp, - result.continuationToken, - _getInvocationStats(result.entries, accStats), - callback); - } else { - obj[functionName] = _getInvocationStats(result.entries, accStats); - return callback(null, obj); - } - } - }); -}; - -// Returns application stats for the last 15 mins starting from provided 'timestamp'. -// Stats include: function invocations total and invocation error count. -// Returns: -// [{"Master": -// {"invocations":2,"errors":0} -// }, -// {"O365WebHook": -// {"invocations":10,"errors":1} -// }, -// {"Updater": -// {"invocations":0,"errors":0} -// }] - -var _getAppStats = function(timestamp, callback) { - async.map(APP_FUNCTIONS, - function(fname, mapCallback){ - _getFunctionStats(fname, timestamp, mapCallback); - }, - function (mapErr, mapsResult) { - return (mapErr) ? callback(mapErr) : callback(null, mapsResult); - }); -}; - -module.exports = { - getFunctionStats : _getFunctionStats, - getAppStats : _getAppStats, - getTableService : getTableService -}; diff --git a/Master/endpoints.js b/Master/endpoints.js deleted file mode 100644 index 2a0208e..0000000 --- a/Master/endpoints.js +++ /dev/null @@ -1,81 +0,0 @@ -/* ----------------------------------------------------------------------------- - * @copyright (C) 2017, Alert Logic, Inc - * @doc - * - * The module for communicating with Alertlogic Endpoints service. - * - * @end - * ----------------------------------------------------------------------------- - */ - -const async = require('async'); - -const m_alServiceC = require('../lib/al_servicec'); -const m_appSettings = require('./appsettings'); - - -exports.checkUpdate = function (context, AlertlogicMasterTimer, callback) { - if (process.env.APP_INGEST_ENDPOINT && process.env.APP_AZCOLLECT_ENDPOINT) { - context.log.verbose('Reuse Ingest endpoint', process.env.APP_INGEST_ENDPOINT); - context.log.verbose('Reuse Azcollect endpoint', process.env.APP_AZCOLLECT_ENDPOINT); - return callback(null); - } else { - // Endpoint settings do not exist. Update them. - let alApiEndpoint = process.env.CUSTOMCONNSTR_APP_AL_API_ENDPOINT; - let alResidency = process.env.CUSTOMCONNSTR_APP_AL_RESIDENCY; - let aimsCreds = { - access_key_id : process.env.CUSTOMCONNSTR_APP_CI_ACCESS_KEY_ID, - secret_key : process.env.CUSTOMCONNSTR_APP_CI_SECRET_KEY - }; - let endpointsSvc = new Endpoints(alApiEndpoint, aimsCreds); - async.map(['ingest', 'azcollect'], - function(service, mapCallback){ - endpointsSvc.getEndpoint(service, alResidency) - .then(resp => { - return mapCallback(null, resp); - }) - .catch(function(exception) { - return mapCallback(`Endpoints update failure ${exception}`); - }); - }, - function (mapErr, mapsRsult) { - if (mapErr) { - return callback(mapErr); - } else { - let endpoints = { - APP_INGEST_ENDPOINT : mapsRsult[0].ingest, - APP_AZCOLLECT_ENDPOINT : mapsRsult[1].azcollect - }; - m_appSettings.updateAppsettings(endpoints, function(settingsError) { - if (settingsError) { - return callback(settingsError); - } else { - return callback(null); - } - }); - } - }); - } -}; - -/** - * @class - * HTTPS client for Alertlogic Endpoints service. - * - * @constructor - * @param {string} apiEndpoint - Alertlogic API hostname. - * @param {Object} aisCreds - Alertlogic API credentials. - * @param {string} [aisCreds.access_key_id] - Aertlogic API access key id. - * @param {string} [aisCreds.secret_key] - Alertlogic API secret key. - * - */ -class Endpoints extends m_alServiceC.AlServiceC { - constructor(apiEndpoint, aimsCreds) { - super(apiEndpoint, 'endpoints', 'v1', aimsCreds, process.env.TMP); - } - getEndpoint(serivceName, residency) { - return this.get(`/residency/${residency}/services/${serivceName}/endpoint`, {}); - } -} - -exports.Endpoints = Endpoints; diff --git a/Master/healthchecks.js b/Master/healthchecks.js new file mode 100644 index 0000000..273c37f --- /dev/null +++ b/Master/healthchecks.js @@ -0,0 +1,65 @@ +/* ---------------------------------------------------------------------------- + * @copyright (C) 2019, Alert Logic, Inc + * @doc + * + * Various O365 collector health checks. + * The last error code is O365000002 + * + * @end + * ---------------------------------------------------------------------------- + */ + +const async = require('async'); +const m_o365mgmnt = require('../lib/o365_mgmnt'); + +const checkStreams = function(master, callback) { + async.waterfall([ + function(asyncCallback){ + m_o365mgmnt.subscriptionsList(asyncCallback); + }, + function(subscriptions, httpRequest, response, asyncCallback){ + _checkEnableAuditStreams(master, subscriptions, asyncCallback); + } + ], + callback); +}; + +const _checkEnableAuditStreams = function(master, listedStreams, callback) { + try { + let o365AuditStreams = JSON.parse(process.env.O365_CONTENT_STREAMS); + // TODO: take webhook path from O365Webhook/function.json + let webhookURL = 'https://' + process.env.WEBSITE_HOSTNAME + + '/api/o365/webhook'; + async.map(o365AuditStreams, + function(stream, asyncCallback) { + let currentStream = listedStreams.find( + obj => obj.contentType === stream); + if (currentStream && currentStream.status === 'enabled' && + currentStream.webhook && + currentStream.webhook.status === 'enabled' && + currentStream.webhook.address === webhookURL) { + return asyncCallback(null, stream); + } else { + let webhook = { webhook : { + address : webhookURL, + expiration : "" + }}; + return m_o365mgmnt.subscriptionsStart(stream, JSON.stringify(webhook), + function(err, result, httpRequest, response) { + if (err) { + return asyncCallback(master.errorStatusFmt('O365000001', `Unable to start subscription ${err}`)); + } else { + return asyncCallback(null, stream); + } + }); + } + }, + callback); + } catch (ex) { + return callback(master.errorStatusFmt('O365000002', `Exception thrown during health check ${ex}`)); + } +}; + +module.exports = { + checkStreams +}; diff --git a/Master/index.js b/Master/index.js index 939daf6..47c011a 100644 --- a/Master/index.js +++ b/Master/index.js @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- - * @copyright (C) 2017, Alert Logic, Inc + * @copyright (C) 2019, Alert Logic, Inc * @doc * * The purpose of this function is to check updates of collector configuration, @@ -10,51 +10,34 @@ */ const async = require('async'); +const pkg = require('../package.json'); +const { AlAzureMaster } = require('al-azure-collector-js'); +const { checkStreams } = require('./healthchecks.js'); -const m_endpoints = require('./endpoints'); -const m_azcollect = require('./azcollect'); -const m_o365collector = require('./o365collector'); +//get the old o365 collector parameters if they exist +const g_aimsCreds = {}; +if(process.env.CUSTOMCONNSTR_APP_CI_ACCESS_KEY_ID) g_aimsCreds.aimsKeyId = process.env.CUSTOMCONNSTR_APP_CI_ACCESS_KEY_ID; +if(process.env.CUSTOMCONNSTR_APP_CI_SECRET_KEY) g_aimsCreds.aimsKeySecret = process.env.CUSTOMCONNSTR_APP_CI_SECRET_KEY; -const g_aimsCreds = { - access_key_id : process.env.CUSTOMCONNSTR_APP_CI_ACCESS_KEY_ID, - secret_key : process.env.CUSTOMCONNSTR_APP_CI_SECRET_KEY -}; +const APP_FUNCTIONS = ['Master', 'Updater', 'O365WebHook']; module.exports = function (context, AlertlogicMasterTimer) { + const healthFuns = [ + checkStreams + ]; + const master = new AlAzureMaster(context, 'o365', pkg.version, healthFuns, null, g_aimsCreds, {}, APP_FUNCTIONS); async.waterfall([ function(asyncCallback) { - return m_endpoints.checkUpdate(context, AlertlogicMasterTimer, - function(endpointsError) { - if (endpointsError) { - return asyncCallback(endpointsError); - } - context.log.info('Alertlogic endpoints updated.'); - return asyncCallback(null); - }); - }, - function(asyncCallback) { - let azcollectSvc = new m_azcollect.Azcollect( - process.env.APP_AZCOLLECT_ENDPOINT, g_aimsCreds); - return m_o365collector.checkRegister(context, - AlertlogicMasterTimer, azcollectSvc, - function(azcollectError, collectorId) { - if (azcollectError) { - return asyncCallback(azcollectError); - } - context.log.info('O365 source registered', collectorId); - return asyncCallback(null, azcollectSvc); - }); + return master.register(_o365RegisterBody(), asyncCallback); }, - function(azcollectSvc, asyncCallback) { - return m_o365collector.checkin(context, - AlertlogicMasterTimer.last, azcollectSvc, - function(azcollectError, checkinResp) { - if (azcollectError) { - return asyncCallback(`Checkin failed ${azcollectError}`); - } - context.log.info('O365 source checkin OK', checkinResp); - return asyncCallback(null); - }); + function(hostId, sourceId, asyncCallback) { + return master.checkin(AlertlogicMasterTimer.last, (checkinErr, checkinRes) => { + if (checkinErr) { + return asyncCallback(`Checkin failed ${checkinErr}`); + } + context.log.info(`O365 source checkin OK`, checkinRes); + return asyncCallback(null, {}); + }); } ], function(error, results) { @@ -64,3 +47,14 @@ module.exports = function (context, AlertlogicMasterTimer) { context.done(error); }); }; + +function _o365RegisterBody() { + let o365AuditStreams = JSON.parse(process.env.O365_CONTENT_STREAMS); + return { + config : { + type : 'o365', + office_tenant_id : process.env.APP_TENANT_ID, + content_streams: o365AuditStreams + } + }; +} diff --git a/Master/o365collector.js b/Master/o365collector.js deleted file mode 100644 index f2206b2..0000000 --- a/Master/o365collector.js +++ /dev/null @@ -1,121 +0,0 @@ -/* ----------------------------------------------------------------------------- - * @copyright (C) 2017, Alert Logic, Inc - * @doc - * - * The module for checking/enabling to O365 Management API subscriptions and - * reprting status to Alertlogic backend. - * - * @end - * ----------------------------------------------------------------------------- - */ - -const async = require('async'); - -const m_alServiceC = require('../lib/al_servicec'); -const m_appSettings = require('./appsettings'); -const m_appStats = require('./appstats'); -const m_o365mgmnt = require('../lib/o365_mgmnt'); - - -exports.checkRegister = function (context, AlertlogicMasterTimer, azcollectSvc, callback) { - if (process.env.O365_COLLECTOR_ID && process.env.O365_HOST_ID) { - context.log.verbose('Reuse collector id', process.env.O365_COLLECTOR_ID); - return callback(null, process.env.O365_COLLECTOR_ID); - } else { - // Collector is not registered. - azcollectSvc.register_o365().then(resp => { - let newSettings = { - O365_COLLECTOR_ID: resp.source.id, - O365_HOST_ID: resp.source.host.id - }; - m_appSettings.updateAppsettings(newSettings, - function(settingsError) { - if (settingsError) { - return callback(settingsError); - } else { - return callback(null, resp.source.id); - } - }); - }).catch(function(exception) { - return callback(`Registration failure ${exception}`); - }); - } -}; - -exports.checkin = function (context, AlertlogicMasterTimer, azcollectSvc, callback) { - async.waterfall([ - function(asyncCallback) { - m_o365mgmnt.subscriptionsList(asyncCallback); - }, - function(subscriptions, httpRequest, response, asyncCallback) { - _checkEnableAuditStreams(context, subscriptions, asyncCallback); - }], - function(error, checkResults) { - m_appStats.getAppStats(AlertlogicMasterTimer, function(statsErr, appStats) { - var stats = null; - if (statsErr) { - stats = [{ - error : `Error getting application stats: ${statsErr}` - }]; - } else { - stats = appStats; - } - if (error) { - azcollectSvc.checkin('o365', - process.env.O365_COLLECTOR_ID, 'error', `${error}`, stats) - .then(resp => { - return callback(null, resp); - }) - .catch(function(exception) { - return callback(`Unable to checkin ${exception}`); - }); - } else { - azcollectSvc.checkin('o365', - process.env.O365_COLLECTOR_ID, 'ok', `${checkResults}`, stats) - .then(resp => { - return callback(null, resp); - }) - .catch(function(exception) { - return callback(`Unable to checkin ${exception}`); - }); - } - }); - }); -}; - -var _checkEnableAuditStreams = function(context, listedStreams, callback) { - try { - let o365AuditStreams = JSON.parse(process.env.O365_CONTENT_STREAMS); - // TODO: take webhook path from O365Webhook/function.json - let webhookURL = 'https://' + process.env.WEBSITE_HOSTNAME + - '/api/o365/webhook'; - async.map(o365AuditStreams, - function(stream, asyncCallback) { - let currentStream = listedStreams.find( - obj => obj.contentType === stream); - if (currentStream && currentStream.status === 'enabled' && - currentStream.webhook && - currentStream.webhook.status === 'enabled' && - currentStream.webhook.address === webhookURL) { - context.log.verbose('Stream already enabled', stream); - return asyncCallback(null, stream); - } else { - let webhook = { webhook : { - address : webhookURL, - expiration : "" - }}; - return m_o365mgmnt.subscriptionsStart(stream, JSON.stringify(webhook), - function(err, result, httpRequest, response) { - if (err) { - return asyncCallback(err); - } else { - return asyncCallback(null, stream); - } - }); - } - }, - callback); - } catch (ex) { - return callback(ex); - } -}; diff --git a/O365WebHook/formatO365Log.js b/O365WebHook/formatO365Log.js new file mode 100644 index 0000000..fdaed09 --- /dev/null +++ b/O365WebHook/formatO365Log.js @@ -0,0 +1,43 @@ +/* ---------------------------------------------------------------------------- + * @copyright (C) 2019, Alert Logic, Inc + * @doc + * + * Log formatting function for O365 logs + * + * @end + * ---------------------------------------------------------------------------- + */ + +const Parse = require('al-collector-js').Parse; + +module.exports = function(item) { + //Paths from https://docs.microsoft.com/en-us/office/office-365-management-api/office-365-management-activity-api-schema#common-schema + const creationTimePaths = [ + {path: ['CreationTime']} + ]; + const messageTypeIdPaths = [ + {path: ['RecordType']} + ]; + + let message; + try { + message = JSON.stringify(item); + } + catch(err) { + throw new Error(`Unable to stringify content. ${err}`); + } + + let creationTime = Parse.getMsgTs(item, creationTimePaths); + let messageTypeId = Parse.getMsgTypeId(item, messageTypeIdPaths); + + return { + messageTs: creationTime.sec, + priority: 11, + progName: 'o365webhook', + pid: undefined, + message: message, + messageType: 'json/azure.o365', + messageTypeId: `${messageTypeId}`, + messageTsUs: creationTime.usec ? creationTime.usec : undefined + }; +}; diff --git a/O365WebHook/ingest.js b/O365WebHook/ingest.js deleted file mode 100644 index f312842..0000000 --- a/O365WebHook/ingest.js +++ /dev/null @@ -1,46 +0,0 @@ -/* ----------------------------------------------------------------------------- - * @copyright (C) 2017, Alert Logic, Inc - * @doc - * - * The module for communicating with Alertlogic Ingest service. - * - * @end - * ----------------------------------------------------------------------------- - */ - -const m_ingestProto = require('./ingest_proto'); -const m_alServiceC = require('../lib/al_servicec'); - -/** - * @class - * HTTPS client for Alertlogic Ingest service. - * - * @constructor - * @param {string} apiEndpoint - Alertlogic API hostname. - * @param {Object} aisCreds - Alertlogic API credentials. - * @param {string} [aisCreds.access_key_id] - Aertlogic API access key id. - * @param {string} [aisCreds.secret_key] - Alertlogic API secret key. - * - */ -class Ingest extends m_alServiceC.AlServiceC { - constructor(apiEndpoint, aimsCreds) { - super(apiEndpoint, 'ingest', 'v1', - aimsCreds, process.env.TMP); - } - - sendO365Data(data) { - let payload = { - json : false, - headers : { - 'Content-Type': 'alertlogic.com/pass-through', - 'x-invoked-by' : 'azure_function', - 'Content-Encoding' : 'deflate', - 'Content-Length' : Buffer.byteLength(data) - }, - body : data - }; - return this.post(`/data/aicspmsgs`, payload); - } -} - -exports.Ingest = Ingest; diff --git a/O365WebHook/ingest_proto.js b/O365WebHook/ingest_proto.js deleted file mode 100644 index 7b0bd23..0000000 --- a/O365WebHook/ingest_proto.js +++ /dev/null @@ -1,181 +0,0 @@ -/* ----------------------------------------------------------------------------- - * @copyright (C) 2017, Alert Logic, Inc - * @doc - * - * Use protobuf to create an ingest payload. - * - * @end - * ----------------------------------------------------------------------------- - */ - - -const protobuf = require('protobufjs'); -const async = require('async'); -const Long = require('long'); -const path = require('path'); -const crypto = require('crypto'); - -// FIXME - protobuf load -// We have to load PROTO_DEF every invocation. Maybe the solution can to to use -// another library such as bpf which compiles proto to js. -module.exports.load = function(context, callback) { - protobuf.load(getCommonProtoPath(), function(err, root) { - if (err) - context.log.error('Unable to load proto files.', err); - - callback(err, root); - }); -}; - - -module.exports.setMessage = function(context, root, content, callback) { - async.reduce(content, [], function(memo, item, callback) { - parseMessage(context, root, memo, item, callback); - }, - function(err, result) { - if (err) - context.log.error('Unable to build messages.'); - - callback(err, result); - } - ); -}; - - -module.exports.setHostMetadata = function(context, root, content, callback) { - var hostmetaType = root.lookupType('host_metadata.metadata'); - var hostmetaData = getHostmeta(context, root); - var meta = { - hostUuid : process.env.O365_HOST_ID, - data : hostmetaData, - dataChecksum : new Buffer('') - }; - var sha = crypto.createHash('sha1'); - var hashPayload = hostmetaType.encode(meta).finish(); - hashValue = sha.update(hashPayload).digest(); - - var metadataPayload = { - hostUuid : process.env.O365_HOST_ID, - dataChecksum : hashValue, - timestamp : Math.floor(Date.now() / 1000), - data : hostmetaData - }; - - build(hostmetaType, metadataPayload, function(err, buf) { - if (err) - context.log.error('Unable to build host_metadata.'); - - return callback(err, buf); - }); -}; - - -module.exports.setBatch = function(context, root, metadata, messages, callback) { - var batchType = root.lookupType('common_proto.collected_batch'); - - var batchPayload = { - sourceId: process.env.O365_COLLECTOR_ID, - metadata: metadata, - message: messages - }; - - build(batchType, batchPayload, function(err, buf) { - if (err) - context.log.error('Unable to build collected_batch.'); - - return callback(err, buf); - }); -}; - - -module.exports.setBatchList = function(context, root, batches, callback) { - var batchListType = root.lookupType('common_proto.collected_batch_list'); - - var batchListPayload = { - elem: [batches] - }; - - build(batchListType, batchListPayload, function(err, buf) { - if (err) - context.log.error('Unable to build collected_batch_list.'); - - return callback(err, buf); - }); -}; - -module.exports.encode = function(context, root, batchList, callback) { - var batchListType = root.lookupType('common_proto.collected_batch_list'); - var buf = batchListType.encode(batchList).finish(); - return callback(null, buf); -}; - - -// Private functions - -function build(type, payload, callback) { - var verify = type.verify(payload); - if (verify) - return callback(verify); - - var payloadCreated = type.create(payload); - - return callback(null, payloadCreated); -} - - -function buildSync(type, payload) { - var verify = type.verify(payload); - if (verify) - throw("Error: Protobuf build failed. " + verify); - - return type.create(payload); -} - - -function parseMessage(context, root, memo, content, callback) { - var messageType = root.lookupType('common_proto.collected_message'); - - var messagePayload = { - messageTs: content.message_ts, - priority: 11, - progName: 'o365webhook', - pid: undefined, - message: content.message, - messageType: 'json/azure.o365', - messageTypeId: content.record_type, - messageTsUs: undefined - }; - - build(messageType, messagePayload, function(err, buf) { - if (err) - context.log.error('Unable to build collected_message.'); - - memo.push(buf); - return callback(err, memo); - }); -} - -function getHostmeta(context, root) { - var dictType = root.lookupType('alc_dict.dict'); - var elemType = root.lookupType('alc_dict.elem'); - var valueType = root.lookupType('alc_dict.value'); - - var hostTypeElem = { - key: 'host_type', - value: {str: 'azure_fun'} - }; - var localHostnameElem = { - key: 'local_hostname', - value: {str: process.env.WEBSITE_HOSTNAME} - }; - var dict = { - elem: [localHostnameElem, hostTypeElem] - }; - - return buildSync(dictType, dict); -} - - -function getCommonProtoPath() { - return path.join(__dirname, '../', 'proto', 'common_proto.piqi.proto'); -} diff --git a/O365WebHook/o365content.js b/O365WebHook/o365content.js index bb1ec23..d081666 100644 --- a/O365WebHook/o365content.js +++ b/O365WebHook/o365content.js @@ -11,24 +11,24 @@ */ const async = require('async'); -const zlib = require('zlib'); +const pkg = require('../package.json'); const m_o365mgmnt = require('../lib/o365_mgmnt'); -const m_ingestProto = require('./ingest_proto'); -const m_ingest = require('./ingest'); - -const g_ingestc = new m_ingest.Ingest( - process.env.APP_INGEST_ENDPOINT, - { - access_key_id : process.env.CUSTOMCONNSTR_APP_CI_ACCESS_KEY_ID, - secret_key: process.env.CUSTOMCONNSTR_APP_CI_SECRET_KEY - } -); +const AlAzureCollector = require('al-azure-collector-js').AlAzureCollector; +const formatO365Log = require('./formatO365Log'); // One O365 content message is about 1KB. -var MAX_BATCH_MESSAGES = 1500; +let MAX_BATCH_MESSAGES = 1500; module.exports.processNotifications = function(context, notifications, callback) { + //get the old o365 collector parameters if they exist + const collectorKeys = {}; + if(process.env.O365_COLLECTOR_ID) collectorKeys.sourceId = process.env.O365_COLLECTOR_ID; + if(process.env.O365_HOST_ID) collectorKeys.hostId = process.env.O365_HOST_ID; + if(process.env.CUSTOMCONNSTR_APP_CI_ACCESS_KEY_ID) collectorKeys.aimsKeyId = process.env.CUSTOMCONNSTR_APP_CI_ACCESS_KEY_ID; + if(process.env.CUSTOMCONNSTR_APP_CI_SECRET_KEY) collectorKeys.aimsKeySecret = process.env.CUSTOMCONNSTR_APP_CI_SECRET_KEY; + + const collector = new AlAzureCollector(context, 'o365', pkg.version, collectorKeys); async.map(notifications, function(notification, asyncCallback) { return m_o365mgmnt.getContent(notification.contentUri, asyncCallback); }, function(fetchErr, mapResult) { @@ -37,25 +37,41 @@ module.exports.processNotifications = function(context, notifications, callback) } else { const flattenResult = [].concat.apply([], mapResult); context.log.verbose('Messages fetched:', flattenResult.length); - return processContent(context, flattenResult, callback); + return processContent(context, flattenResult, collector, callback); } }); }; -function processContent(context, content, callback) { +function processContent(context, content, collector, callback) { const slices = getSliceIndexes(content.length); - return async.map(slices, function(slice, asyncCallback){ - const contentSlice = content.slice(slice.start, slice.end); - parseContent(context, contentSlice, - function(err, parsedContent) { - if (err) { - return asyncCallback(err); - } - else { - return sendToIngest(context, parsedContent, asyncCallback); + const acc = {processed: 0, skip:0}; + return async.mapLimit(slices, process.env.concurrentLogProcesses || 5, + function(slice, asyncCallback){ + const contentSlice = content.slice(slice.start, slice.end); + collector.processLog(contentSlice, formatO365Log, null, + function(err, parsedContent) { + if (err) { + acc.skip += contentSlice.length; + context.log.error(`error from log processing ${JSON.stringify(err)}`); + } + else { + acc.processed += contentSlice.length; + } + return asyncCallback(null, acc); + }); + }, + function(err){ + if(err){ + context.log.error('Records skipped:', content.length); + return callback(err); + } else { + context.log.info('Processed:', acc.processed); + if(acc.skip) { + context.log.info('Records skipped:', acc.skip); } - }); - }, callback); + } + return callback(null, acc); + }); } function getSliceIndexes(contentLength) { @@ -72,104 +88,4 @@ function getSliceIndexes(contentLength) { return sliceArray; } -// Parse each message into: -// { -// hostname: -// message_ts: -// message: -// } -function parseContent(context, parsedContent, callback) { - async.reduce(parsedContent, [], function(memo, item, callback) { - var message; - try { - message = JSON.stringify(item); - } - catch(err) { - return callback(`Unable to stringify content. ${err}`); - } - - var creationTime; - if (item.CreationTime == undefined) { - context.log.warn('Unable to parse CreationTime from content.'); - creationTime = Math.floor(Date.now() / 1000); - } - else { - creationTime = Math.floor(Date.parse(item.CreationTime) / 1000); - } - var newItem = { - message_ts: creationTime, - record_type: (item.RecordType) ? - item.RecordType.toString() : - item.RecordType, - message: message - }; - - memo.push(newItem); - return callback(null, memo); - }, - function(err, result) { - if (err) { - return callback(`Content parsing failure. ${err}`); - } else { - return callback(null, result); - } - } - ); -} - -function sendToIngest(context, content, callback) { - async.waterfall([ - function(asyncCallback) { - m_ingestProto.load(context, function(err, root) { - asyncCallback(err, root); - }); - }, - function(root, asyncCallback) { - m_ingestProto.setMessage(context, root, content, function(err, msg) { - asyncCallback(err, root, msg); - }); - }, - function(root, msg, asyncCallback) { - m_ingestProto.setHostMetadata(context, root, content, function(err, meta) { - asyncCallback(err, root, meta, msg); - }); - }, - function(root, meta, msg, asyncCallback) { - m_ingestProto.setBatch(context, root, meta, msg, function(err, batch) { - asyncCallback(err, root, batch); - }); - }, - function(root, batchBuf, asyncCallback) { - m_ingestProto.setBatchList(context, root, batchBuf, - function(err, batchList) { - asyncCallback(err, root, batchList); - }); - }, - function(root, batchList, asyncCallback) { - m_ingestProto.encode(context, root, batchList, asyncCallback); - }], - function(err, result) { - if (err) { - return callback(err); - } - - zlib.deflate(result, function(err, compressed) { - if (err) { - return callback(`Unable to compress. ${err}`); - } else { - if (compressed.byteLength > 700000) - context.log.warn(`Compressed log batch length`, - `(${compressed.byteLength}) exceeds maximum allowed value.`); - return g_ingestc.sendO365Data(compressed) - .then(resp => { - context.log.verbose('Bytes sent to Ingest: ', compressed.byteLength); - return callback(null, resp); - }) - .catch(function(exception){ - return callback(`Unable to send to Ingest. ${exception}`); - }); - } - }); - }); -} diff --git a/Updater/index.js b/Updater/index.js index c31c556..d05cdaf 100644 --- a/Updater/index.js +++ b/Updater/index.js @@ -1,5 +1,5 @@ /* ----------------------------------------------------------------------------- - * @copyright (C) 2017, Alert Logic, Inc + * @copyright (C) 2019, Alert Logic, Inc * @doc * * The purpose of this function is to sync web app every 12 hours with @@ -10,125 +10,17 @@ * ----------------------------------------------------------------------------- */ -const https = require('https'); -const util = require('util'); - +const AlAzureUpdater = require('al-azure-collector-js').AlAzureUpdater; module.exports = function (context, AlertlogicUpdaterTimer) { - var creds = { - client_id : process.env.CUSTOMCONNSTR_APP_CLIENT_ID, - tenant_id : process.env.APP_TENANT_ID, - client_secret : process.env.CUSTOMCONNSTR_APP_CLIENT_SECRET - }; - - requestNewToken(context, creds, function(tokenError, adToken) { - if (tokenError) { - context.log.error('Error getting AD token: ', - tokenError.statusCode, tokenError.statusMessage); - context.done(tokenError); + const updater = new AlAzureUpdater(); + updater.syncWebApp(function(syncError){ + if(syncError){ + context.log.error('Application sync failed: ', syncError); } else { - siteSync(context, adToken, function(syncError) { - if (syncError) { - context.log.error('Site sync failed: ', syncError); - context.done(syncError); - } else { - context.log.info('Site sync OK'); - context.done(); - } - }); + context.log.info('Application sync OK'); } + context.done(syncError); }); }; -/* - * Internal functions - */ - -function siteSync(context, adToken, callback) { - var subscriptionId = process.env.APP_SUBSCRIPTION_ID; - var resourceGroupName = process.env.APP_RESOURCE_GROUP; - var webAppName = process.env.WEBSITE_SITE_NAME; - var options = { - hostname: 'management.azure.com', - path: - '/subscriptions/' + encodeURIComponent(subscriptionId) + - '/resourceGroups/' + encodeURIComponent(resourceGroupName) + - '/providers/Microsoft.Web/sites/' + encodeURIComponent(webAppName) + - '/sync?api-version=2016-08-01', - method: 'POST', - headers: { - 'Accept': 'application/json', - 'Content-Type' : 'application/x-www-form-urlencoded', - 'Content-Length' : 0, - 'Authorization' : 'Bearer ' + encodeURIComponent(adToken) - } - }; - var syncResp = ''; - var req = https.request(options, function(response) { - response.on('data', function (chunk) { - syncResp += chunk; - }); - - response.on('end', function () { - if (response.statusCode == 200) { - return callback(null, response); - } else { - return callback(response); - } - }); - }); - - req.on('error', function(reqError) { - return callback(reqError); - }); - req.end(); -} - -/* - * We don't want to have any unnecessary dependencies like token_cache in - * Updater function in order to minimize potential failure. - */ -function requestNewToken(context, creds, callback) { - var postData = - 'grant_type=client_credentials&' + - 'client_id=' + encodeURIComponent(creds.client_id) +'&'+ - 'resource=' + encodeURIComponent('https://management.azure.com/') +'&'+ - 'client_secret=' + encodeURIComponent(creds.client_secret); - var options = { - hostname: 'login.windows.net', - path: '/' + encodeURI(creds.tenant_id) + - '/oauth2/token', - method: 'POST', - headers: { - 'Accept': '*/*', - 'Content-Type': 'application/x-www-form-urlencoded', - 'Content-Length' : Buffer.byteLength(postData) - } - }; - var loginResp = ''; - var req = https.request(options, function(response) { - response.on('data', function (chunk) { - loginResp += chunk; - }); - - response.on('end', function () { - if (response.statusCode == 200) { - var respJson; - try { - respJson = JSON.parse(loginResp); - return callback(null, respJson.access_token); - } catch (exception) { - return callback(exception); - } - } else { - return callback(response); - } - }); - }); - - req.on('error', function(reqError) { - callback(reqError); - }); - req.write(postData); - req.end(); -} diff --git a/package.json b/package.json index c730b69..d650802 100644 --- a/package.json +++ b/package.json @@ -1,13 +1,11 @@ { "name": "azure_collector", - "version": "1.0.8", + "version": "1.1.0", "dependencies": { - "async": "*", + "async": "^2.6.2", "azure": "^2.0.0-preview", - "moment": "^2.20.1", - "parse-key-value": "^1.0.0", "path": "^0.12.7", - "protobufjs": "*", + "al-azure-collector-js": "git://github.com/alertlogic/al-azure-collector-js#master", "request-promise-native": "^1.0.4" }, "scripts": { diff --git a/template.json b/template.json index 2b2ed0f..e99a8da 100644 --- a/template.json +++ b/template.json @@ -18,7 +18,8 @@ "type": "String", "defaultValue": "api.global-services.global.alertlogic.com", "allowedValues" : [ - "api.global-services.global.alertlogic.com" + "api.global-services.global.alertlogic.com", + "api.global-integration.product.dev.alertlogic.com" ] }, "Alert Logic Data Residency": { @@ -125,12 +126,12 @@ "connectionString": "[parameters('App Client Secret')]" }, { - "name": "APP_CI_ACCESS_KEY_ID", + "name": "APP_AL_ACCESS_KEY_ID", "type": "Custom", "connectionString": "[parameters('Alert Logic Access Key ID')]" }, { - "name": "APP_CI_SECRET_KEY", + "name": "APP_AL_SECRET_KEY", "type": "Custom", "connectionString": "[parameters('Alert Logic Secret Key')]" }, diff --git a/test/formatO365Log_test.js b/test/formatO365Log_test.js new file mode 100644 index 0000000..369b7c9 --- /dev/null +++ b/test/formatO365Log_test.js @@ -0,0 +1,25 @@ +const assert = require('assert'); +const formatO365Log = require('../O365WebHook/formatO365Log'); +const mock = require('./mock'); + +describe('formatO365 units', function(){ + it('Formats a O365 log correctly', function(done){ + const logRecord = mock.o365Content[0]; + const formattedRecord = formatO365Log(logRecord); + + const expectedRecord = { + messageTs: new Date(logRecord.CreationTime).getTime() / 1000, + priority: 11, + progName: 'o365webhook', + pid: undefined, + message: JSON.stringify(logRecord), + messageType: 'json/azure.o365', + messageTypeId: `${logRecord.RecordType}`, + messageTsUs: undefined + }; + + + assert.deepEqual(formattedRecord, expectedRecord); + done(); + }); +}); diff --git a/test/master_appsettings_test.js b/test/master_appsettings_test.js deleted file mode 100644 index cc80bf4..0000000 --- a/test/master_appsettings_test.js +++ /dev/null @@ -1,116 +0,0 @@ -/* ----------------------------------------------------------------------------- - * @copyright (C) 2017, Alert Logic, Inc - * @doc - * - * Unit tests for Master function - * - * @end - * ----------------------------------------------------------------------------- - */ - -var testMock = require('./mock'); - -var assert = require('assert'); -var rewire = require('rewire'); -var sinon = require('sinon'); - -var m_appsettings = rewire('../Master/appsettings'); - -describe('Master Function appsettings.js Units', function() { - var private_appAdCreds; - var private_websiteClient; - var msListApplicationSettingsStub; - var msUpdateApplicationSettingsStub; - - before(function() { - private_appAdCreds = m_appsettings.__get__('g_appAdCreds'); - private_websiteClient = m_appsettings.__get__('g_websiteClient'); - - retrieveTokenFromCacheStub = sinon.stub(private_appAdCreds, '_retrieveTokenFromCache').callsFake( - function fakeFn(callback) { - var mockToken = { - 'tokenType' : 'Bearer', - 'expiresIn' : 3599, - 'expiresOn': '2017-09-26T11:34:40.703Z', - 'resource' : 'https://management.azure.com', - 'accessToken' : 'some-token', - 'isMRRT' : true, - '_clientId' : process.env.CUSTOMCONNSTR_APP_CLIENT_ID, - '_authority' :' https://login.microsoftonline.com/' + process.env.APP_TENANT_ID - }; - return callback(null, mockToken); - }); - msListApplicationSettingsStub = sinon.stub(private_websiteClient.webApps, 'listApplicationSettings').callsFake( - function fakeFn(rgName, name, options, callback) { - var mockSettings = { - properties : { - MOCK_SETTING : 'mock-value' - } - }; - return callback(null, mockSettings, null, null); - }); - msUpdateApplicationSettingsStub = sinon.stub(private_websiteClient.webApps, 'updateApplicationSettings').callsFake( - function fakeFn(rgName, name, settings, options, callback) { - return callback(null, null, null, null); - }); - }); - after(function() { - msListApplicationSettingsStub.restore(); - msUpdateApplicationSettingsStub.restore(); - retrieveTokenFromCacheStub.restore(); - }); - beforeEach(function() { - msListApplicationSettingsStub.resetHistory(); - msUpdateApplicationSettingsStub.resetHistory(); - }); - - describe('Azure web application setting tests', function() { - it('checks getAppsettings()', function(done) { - m_appsettings.getAppsettings(function(err, settings){ - if (err) - return done(err); - - sinon.assert.callCount(msListApplicationSettingsStub, 1); - done(); - }); - }); - - it('checks setAppsettings()', function(done) { - var testSettings = { - test : 'test' - }; - m_appsettings.setAppsettings(testSettings, function(err, settings) { - if (err) - return done(err); - - sinon.assert.callCount(msUpdateApplicationSettingsStub, 1); - done(); - }); - }); - - it('checks updateAppsettings()', function(done) { - var testSettings = { - test : 'test' - }; - m_appsettings.updateAppsettings(testSettings, function(err, settings) { - if (err) - return done(err); - - var expectedSettings = { - properties : { - MOCK_SETTING : 'mock-value', - test : 'test' - } - }; - sinon.assert.callCount(msListApplicationSettingsStub, 1); - sinon.assert.callCount(msUpdateApplicationSettingsStub, 1); - sinon.assert.calledWith(msUpdateApplicationSettingsStub, - process.env.APP_RESOURCE_GROUP, - process.env.WEBSITE_SITE_NAME, - expectedSettings, - null); - done(); - }); - }); - }); -}); diff --git a/test/master_appstats_test.js b/test/master_appstats_test.js deleted file mode 100644 index fbe6ce1..0000000 --- a/test/master_appstats_test.js +++ /dev/null @@ -1,121 +0,0 @@ -/* ----------------------------------------------------------------------------- - * @copyright (C) 2017, Alert Logic, Inc - * @doc - * - * Unit tests for Master function - * - * @end - * ----------------------------------------------------------------------------- - */ - -var testMock = require('./mock'); - -var assert = require('assert'); -var rewire = require('rewire'); -var sinon = require('sinon'); - -var m_appstats = rewire('../Master/appstats'); -var azureStorage = require('azure-storage'); - -describe('Master Function appstats.js Units', function() { - - before(function() { - }); - after(function() { - }); - beforeEach(function() { - m_appstats.__set__({g_tableService : null}); - }); - afterEach(function() { - }); - - describe('Azure web application statistics tests', function() { - it('checks getAppStats() empty', function(done) { - var msTableServiceStub = sinon.stub(azureStorage, 'createTableService').callsFake( - function fakeFn(account, key, host) { - var mockObj = { - queryEntities : function(table, query, token, callback){ - return callback(null, {entries : []}); - } - }; - return mockObj; - } - ); - var expectedStats = [ - { Master: { invocations: 0, errors: 0 } }, - { O365WebHook: { invocations: 0, errors: 0 } }, - { Updater: { invocations: 0, errors: 0 } } - ]; - - m_appstats.getAppStats('2017-12-22T14:31:39', function(err, stats) { - msTableServiceStub.restore(); - assert.deepEqual(expectedStats, stats); - done(); - }); - }); - - it('checks getAppStats() success', function(done) { - var msTableServiceStub = sinon.stub(azureStorage, 'createTableService').callsFake( - function fakeFn(account, key, host) { - var mockObj = { - queryEntities : function(table, query, token, callback) { - if (query == 'Master') - return callback(null, testMock.masterAuditLogs); - if (query == 'O365WebHook') - return callback(null, testMock.o365webhookAuditLogs); - if (query == 'Updater') - return callback(null, testMock.updaterAuditLogs); - return callback(null, {entries : []}); - } - }; - return mockObj; - } - ); - rewireGetInvocationsQuery = m_appstats.__set__( - { - getInvocationsQuery: function(functionName, ts) { - return functionName; - } - } - ); - var expectedStats = [ - { Master: { invocations: 3, errors: 2 } }, - { O365WebHook: { invocations: 3, errors: 1 } }, - { Updater: { invocations: 2, errors: 1 } } - ]; - - m_appstats.getAppStats('2017-12-22T14:31:39', function(err, stats) { - msTableServiceStub.restore(); - assert.deepEqual(expectedStats, stats); - done(); - }); - }); - - it('checks getAppStats() errors', function(done) { - var msTableServiceStub = sinon.stub(azureStorage, 'createTableService').callsFake( - function fakeFn(account, key, host) { - var mockObj = { - queryEntities : function(table, query, token, callback){ - return callback('Error: getaddrinfo ENOTFOUND test.table.core.windows.net test.table.core.windows.net:443'); - } - }; - return mockObj; - } - ); - var expectedErrorStats = [ - {"Master":{"error":"Error getting stats Error: getaddrinfo ENOTFOUND test.table.core.windows.net test.table.core.windows.net:443"}}, - {"O365WebHook":{"error":"Error getting stats Error: getaddrinfo ENOTFOUND test.table.core.windows.net test.table.core.windows.net:443"}}, - {"Updater":{"error":"Error getting stats Error: getaddrinfo ENOTFOUND test.table.core.windows.net test.table.core.windows.net:443"}} - ]; - - m_appstats.getAppStats('2017-12-22T14:31:40', function(err, stats) { - azureStorage.createTableService.restore(); - msTableServiceStub.restore(); - assert.deepEqual(expectedErrorStats, stats); - - done(); - }); - }); - - }); -}); diff --git a/test/master_endpoints_test.js b/test/master_endpoints_test.js deleted file mode 100644 index de0894c..0000000 --- a/test/master_endpoints_test.js +++ /dev/null @@ -1,87 +0,0 @@ -/* ----------------------------------------------------------------------------- - * @copyright (C) 2017, Alert Logic, Inc - * @doc - * - * Unit tests for Master function - * - * @end - * ----------------------------------------------------------------------------- - */ - -var assert = require('assert'); -var sinon = require('sinon'); - -var testMock = require('./mock'); - -var m_endpoints = require('../Master/endpoints'); -var m_appsettings = require('../Master/appsettings'); - -describe('Master Function endpoints.js Units', function() { - var updateSettingsStub = null; - var endpointsStub = null; - - before(function() { - var residency = process.env.CUSTOMCONNSTR_APP_AL_RESIDENCY; - - updateSettingsStub = sinon.stub(m_appsettings, 'updateAppsettings').callsFake( - function fakeFn(settings, callback) { - return callback(null, settings); - }); - endpointsStub = sinon.stub(m_endpoints.Endpoints.prototype, 'getEndpoint'); - endpointsStub.withArgs('ingest', residency).resolves({ - ingest : 'new-ingest-endpoint' - }); - endpointsStub.withArgs('azcollect', residency).resolves({ - azcollect : 'new-azcollect-endpoint' - }); - }); - after(function() { - if (updateSettingsStub) updateSettingsStub.restore(); - if (endpointsStub) endpointsStub.restore(); - }); - beforeEach(function() { - if (updateSettingsStub) updateSettingsStub.resetHistory(); - if (endpointsStub) endpointsStub.resetHistory(); - }); - - describe('AL endpoints retrieval tests', function() { - it('checks endpoints values are reused if already fetched', function(done) { - process.env.APP_INGEST_ENDPOINT = 'existing-ingest-endpoint'; - process.env.APP_AZCOLLECT_ENDPOINT = 'existing-azcollect-endpoint'; - m_endpoints.checkUpdate(testMock.context, testMock.timer, - function(err){ - if (err) { - return done(err); - } else { - sinon.assert.callCount(endpointsStub, 0); - sinon.assert.callCount(updateSettingsStub, 0); - return done(); - } - }); - }); - - it('checks endpoints values are saved as app settings', function(done) { - process.env.APP_INGEST_ENDPOINT = null; - process.env.APP_AZCOLLECT_ENDPOINT = null; - var residency = process.env.CUSTOMCONNSTR_APP_AL_RESIDENCY; - - m_endpoints.checkUpdate(testMock.context, testMock.timer, - function(err){ - if (err) { - return done(err); - } else { - var expectedSettings = { - APP_INGEST_ENDPOINT : 'new-ingest-endpoint', - APP_AZCOLLECT_ENDPOINT : 'new-azcollect-endpoint' - }; - sinon.assert.callCount(endpointsStub, 2); - sinon.assert.calledWith(endpointsStub, 'ingest', residency); - sinon.assert.calledWith(endpointsStub, 'azcollect', residency); - sinon.assert.callCount(updateSettingsStub, 1); - sinon.assert.calledWith(updateSettingsStub, expectedSettings); - return done(); - } - }); - }); - }); -}); diff --git a/test/master_o365collector_test.js b/test/master_o365collector_test.js index 34b590d..d595635 100644 --- a/test/master_o365collector_test.js +++ b/test/master_o365collector_test.js @@ -1,5 +1,5 @@ /* ----------------------------------------------------------------------------- - * @copyright (C) 2017, Alert Logic, Inc + * @copyright (C) 2019, Alert Logic, Inc * @doc * * Unit tests for Master function @@ -16,32 +16,51 @@ var testMock = require('./mock'); var m_o365mgmnt = require('../lib/o365_mgmnt'); var m_azcollect = require('../Master/azcollect'); -var m_o365collector = rewire('../Master/o365collector'); -var m_appsettings = require('../Master/appsettings'); -var m_appstats = require('../Master/appstats'); +var healthchecks = rewire('../Master/healthchecks'); -describe('Master Function o365collector.js Units', function() { +describe('O365 healthcheck tests', function() { var private_checkEnableAuditStreams; var msSubscriptionsStartStub; + var stubErrorFmt; var updateSettingsStub = null; before(function() { - private_checkEnableAuditStreams = m_o365collector.__get__('_checkEnableAuditStreams'); + private_checkEnableAuditStreams = healthchecks.__get__('_checkEnableAuditStreams'); msSubscriptionsStartStub = sinon.stub(m_o365mgmnt, 'subscriptionsStart').callsFake( function fakeFn(contentType, webhook, callback) { - return callback(null); - }); + if(contentType === "Audit.SomeGarbage"){ + return callback('stream not found'); + } else{ + return callback(null); + } + }); + stubErrorFmt = sinon.stub(testMock.context, 'errorStatusFmt'); }); after(function() { msSubscriptionsStartStub.restore(); + stubErrorFmt.restore(); }); beforeEach(function() { msSubscriptionsStartStub.resetHistory(); + stubErrorFmt.resetHistory(); }); afterEach(function() { if (updateSettingsStub) updateSettingsStub.restore(); }); + describe('checkStreams', function(done){ + it('should throw the apprirpiate error when the subscriptionsList returns and error', function(){ + const msSubscriptionsListStub = sinon.stub(m_o365mgmnt, 'subscriptionsList').callsFake(function(callback){ + return "Somthing went wrong with getting the subscription list in a catestrophic way"; + }); + + healthchecks.checkStreams(testMock.context, function(err, result){ + sinon.assert.calledWith(stubErrorFmt, 'O365000002'); + done(); + }); + }); + }); + describe('_checkEnableAuditStreams()', function() { it('enables configured streams', function(done) { process.env.O365_CONTENT_STREAMS = @@ -57,6 +76,22 @@ describe('Master Function o365collector.js Units', function() { return done(); }); }); + + it('throws the correct error when an invalid stream is passed in', function(done) { + process.env.O365_CONTENT_STREAMS = + '["Audit.AzureActiveDirectory", "Audit.Exchange", "Audit.General", "Audit.SomeGarbage"]'; + private_checkEnableAuditStreams(testMock.context, [], function(err, streams){ + if (err) + return done(err); + + sinon.assert.callCount(msSubscriptionsStartStub, 4); + sinon.assert.calledWith(msSubscriptionsStartStub, "Audit.AzureActiveDirectory"); + sinon.assert.calledWith(msSubscriptionsStartStub, "Audit.Exchange"); + sinon.assert.calledWith(msSubscriptionsStartStub, "Audit.General"); + sinon.assert.calledWith(stubErrorFmt,'O365000001'); + return done(); + }); + }); it('checks already enabled streams with proper webhook configs', function(done) { process.env.O365_CONTENT_STREAMS = @@ -198,295 +233,4 @@ describe('Master Function o365collector.js Units', function() { }); }); - describe('O365 collector checkin tests', function() { - it('checks successful OK checkin', function(done) { - process.env.O365_CONTENT_STREAMS = - '["Audit.AzureActiveDirectory", "Audit.General"]'; - var enabledStreams = [ - { - "contentType": "Audit.AzureActiveDirectory", - "status": "enabled", - "webhook": { - "authId": null, - "address": "https://kkuzmin-app-o365.azurewebsites.net/api/o365/webhook", - "expiration": "", - "status": "enabled" - } - }, - { - "contentType": "Audit.General", - "status": "enabled", - "webhook": { - "authId": null, - "address": "https://kkuzmin-app-o365.azurewebsites.net/api/o365/webhook", - "expiration": "", - "status": "disabled" - } - } - ]; - var msSubscriptionsListStub = sinon.stub(m_o365mgmnt, 'subscriptionsList').callsFake( - function fakeFn(callback) { - return callback(null, enabledStreams, null, null); - }); - var expectedStats = [{"Master":{"invocations":20,"errors":1}}]; - var getAppStatsStub = sinon.stub(m_appstats, 'getAppStats').callsFake( - function fakeFn(ts, callback) { - return callback(null, expectedStats); - }); - var azcollectSvc = new m_azcollect.Azcollect('api-endpoint', 'creds'); - sinon.stub(azcollectSvc, 'checkin').resolves([{}]); - - m_o365collector.checkin(testMock.context, testMock.timer, azcollectSvc, - function(err, resp){ - if (err) { - msSubscriptionsListStub.restore(); - getAppStatsStub.restore(); - return done(err); - } else { - sinon.assert.callCount(azcollectSvc.checkin, 1); - sinon.assert.calledWith(azcollectSvc.checkin, - 'o365', process.env.O365_COLLECTOR_ID, 'ok', sinon.match.any, expectedStats); - msSubscriptionsListStub.restore(); - getAppStatsStub.restore(); - return done(); - } - }); - }); - - it('checks successful OK checkin, stats Error', function(done) { - process.env.O365_CONTENT_STREAMS = - '["Audit.AzureActiveDirectory", "Audit.General"]'; - var enabledStreams = [ - { - "contentType": "Audit.AzureActiveDirectory", - "status": "enabled", - "webhook": { - "authId": null, - "address": "https://kkuzmin-app-o365.azurewebsites.net/api/o365/webhook", - "expiration": "", - "status": "enabled" - } - }, - { - "contentType": "Audit.General", - "status": "enabled", - "webhook": { - "authId": null, - "address": "https://kkuzmin-app-o365.azurewebsites.net/api/o365/webhook", - "expiration": "", - "status": "disabled" - } - } - ]; - var msSubscriptionsListStub = sinon.stub(m_o365mgmnt, 'subscriptionsList').callsFake( - function fakeFn(callback) { - return callback(null, enabledStreams, null, null); - }); - var statsError = 'Sample stats error'; - var getAppStatsStub = sinon.stub(m_appstats, 'getAppStats').callsFake( - function fakeFn(ts, callback) { - return callback(statsError); - }); - var azcollectSvc = new m_azcollect.Azcollect('api-endpoint', 'creds'); - sinon.stub(azcollectSvc, 'checkin').resolves([{}]); - - var expectedStats = [{error : `Error getting application stats: ${statsError}`}]; - m_o365collector.checkin(testMock.context, testMock.timer, azcollectSvc, - function(err, resp){ - if (err) { - msSubscriptionsListStub.restore(); - getAppStatsStub.restore(); - return done(err); - } else { - sinon.assert.callCount(azcollectSvc.checkin, 1); - sinon.assert.calledWith(azcollectSvc.checkin, - 'o365', process.env.O365_COLLECTOR_ID, 'ok', sinon.match.any, expectedStats); - msSubscriptionsListStub.restore(); - getAppStatsStub.restore(); - return done(); - } - }); - }); - - - it('checks successful Error checkin during Office subscriptionList error', function(done) { - process.env.O365_CONTENT_STREAMS = - '["Audit.AzureActiveDirectory", "Audit.General"]'; - var enabledStreams = [ - { - "contentType": "Audit.AzureActiveDirectory", - "status": "enabled", - "webhook": { - "authId": null, - "address": "https://kkuzmin-app-o365.azurewebsites.net/api/o365/webhook", - "expiration": "", - "status": "enabled" - } - }, - { - "contentType": "Audit.General", - "status": "enabled", - "webhook": { - "authId": null, - "address": "https://kkuzmin-app-o365.azurewebsites.net/api/o365/webhook", - "expiration": "", - "status": "disabled" - } - } - ]; - var listError = 'Office subscriptionList error'; - var msSubscriptionsListStub = sinon.stub(m_o365mgmnt, 'subscriptionsList').callsFake( - function fakeFn(callback) { - return callback(listError); - }); - const expectedStats = [{"Master":{"invocations":20,"errors":1}}]; - var getAppStatsStub = sinon.stub(m_appstats, 'getAppStats').callsFake( - function fakeFn(ts, callback) { - return callback(null, expectedStats); - }); - var azcollectSvc = new m_azcollect.Azcollect('api-endpoint', 'creds'); - sinon.stub(azcollectSvc, 'checkin').resolves([{}]); - - m_o365collector.checkin(testMock.context, testMock.timer, azcollectSvc, - function(err, resp){ - if (err) { - msSubscriptionsListStub.restore(); - getAppStatsStub.restore(); - return done(err); - } else { - sinon.assert.callCount(azcollectSvc.checkin, 1); - sinon.assert.calledWith(azcollectSvc.checkin, - 'o365', process.env.O365_COLLECTOR_ID, 'error', listError, expectedStats); - msSubscriptionsListStub.restore(); - getAppStatsStub.restore(); - return done(null); - } - }); - }); - - it('checks successful Error checkin during Office subscriptionStart error', function(done) { - process.env.O365_CONTENT_STREAMS = - '["Audit.AzureActiveDirectory", "Audit.General"]'; - var enabledStreams = [ - { - "contentType": "Audit.AzureActiveDirectory", - "status": "enabled", - "webhook": { - "authId": null, - "address": "https://kkuzmin-app-o365.azurewebsites.net/api/o365/webhook", - "expiration": "", - "status": "enabled" - } - }, - { - "contentType": "Audit.General", - "status": "enabled", - "webhook": { - "authId": null, - "address": "https://kkuzmin-app-o365.azurewebsites.net/api/o365/webhook", - "expiration": "", - "status": "disabled" - } - } - ]; - var startError = 'Office subscriptionStart error'; - var msSubscriptionsListStub = sinon.stub(m_o365mgmnt, 'subscriptionsList').callsFake( - function fakeFn(callback) { - return callback(null, enabledStreams, null, null); - }); - const expectedStats = [{"Master":{"invocations":20,"errors":1}}]; - var getAppStatsStub = sinon.stub(m_appstats, 'getAppStats').callsFake( - function fakeFn(ts, callback) { - return callback(null, expectedStats); - }); - msSubscriptionsStartStub.restore(); - var msSubscriptionsStartErrorStub = sinon.stub(m_o365mgmnt, 'subscriptionsStart').callsFake( - function fakeFn(contentType, webhook, callback) { - return callback(startError); - }); - var azcollectSvc = new m_azcollect.Azcollect('api-endpoint', 'creds'); - sinon.stub(azcollectSvc, 'checkin').resolves([{}]); - - m_o365collector.checkin(testMock.context, testMock.timer, azcollectSvc, - function(err, resp){ - if (err) { - msSubscriptionsListStub.restore(); - msSubscriptionsStartErrorStub.restore(); - getAppStatsStub.restore(); - return done(err); - } else { - sinon.assert.callCount(azcollectSvc.checkin, 1); - sinon.assert.calledWith(azcollectSvc.checkin, - 'o365', process.env.O365_COLLECTOR_ID, 'error', startError, expectedStats); - msSubscriptionsListStub.restore(); - msSubscriptionsStartErrorStub.restore(); - getAppStatsStub.restore(); - return done(null); - } - }); - }); - }); - - describe('O365 collector register tests', function() { - it('checks collector and host id are reused if already registered', function(done) { - process.env.O365_COLLECTOR_ID = 'existing-collector-id'; - process.env.O365_HOST_ID = 'existing-collector-id'; - updateSettingsStub = sinon.stub(m_appsettings, 'updateAppsettings').callsFake( - function fakeFn(settings, callback) { - return callback(null, settings); - }); - var azcollectSvc = new m_azcollect.Azcollect('api-endpoint', 'creds'); - sinon.stub(azcollectSvc, 'register_o365').resolves({ - source : { - id : 'new-source-id', - host : { - id : 'new-host-id' - } - } - }); - - m_o365collector.checkRegister(testMock.context, testMock.timer, azcollectSvc, - function(err, resp){ - if (err) { - return done(err); - } else { - sinon.assert.callCount(updateSettingsStub, 0); - sinon.assert.callCount(azcollectSvc.register_o365, 0); - return done(); - } - }); - }); - - it('checks updateSettings is called during registration', function(done) { - process.env.O365_COLLECTOR_ID = null; - updateSettingsStub = sinon.stub(m_appsettings, 'updateAppsettings').callsFake( - function fakeFn(settings, callback) { - return callback(null, settings); - }); - var azcollectSvc = new m_azcollect.Azcollect('api-endpoint', 'creds'); - sinon.stub(azcollectSvc, 'register_o365').resolves({ - source : { - id : 'new-source-id', - host : { - id : 'new-host-id' - } - } - }); - - m_o365collector.checkRegister(testMock.context, testMock.timer, azcollectSvc, - function(err, resp){ - if (err) { - return done(err); - } else { - var expectedSettings = { - O365_COLLECTOR_ID: 'new-source-id', - O365_HOST_ID: 'new-host-id' - }; - sinon.assert.callCount(updateSettingsStub, 1); - sinon.assert.calledWith(updateSettingsStub, expectedSettings); - return done(); - } - }); - }); - }); }); diff --git a/test/mock.js b/test/mock.js index 7e278f1..6727d41 100644 --- a/test/mock.js +++ b/test/mock.js @@ -51,6 +51,13 @@ var context = { done: function () { console.log('Test response:'); }, + errorStatusFmt: function(code, message) { + return { + status: 'error', + error_code: code, + details: [message] + }; + }, res: null }; diff --git a/test/o365webhook_content_test.js b/test/o365webhook_content_test.js index 2fda122..a0163bc 100644 --- a/test/o365webhook_content_test.js +++ b/test/o365webhook_content_test.js @@ -14,38 +14,33 @@ var sinon = require('sinon'); var testMock = require('./mock'); var m_o365mgmnt = require('../lib/o365_mgmnt'); -var m_ingest = require('../O365WebHook/ingest'); var m_o365content = rewire('../O365WebHook/o365content'); - +const AlAzureCollector = require('al-azure-collector-js').AlAzureCollector; describe('O365WebHook Function o365content.js units.', function() { - var ingestSendStub; var clock; - + before(function() { clock = sinon.useFakeTimers(); - ingestSendStub = sinon.stub(m_ingest.Ingest.prototype, 'sendO365Data'); }); after(function() { - ingestSendStub.restore(); clock.restore(); }); - beforeEach(function() { - ingestSendStub.resetHistory(); - }); - afterEach(function() { - }); describe('processNotifications()', function() { - it('batch content is successfully fetched and sent to Ingest', function(done) { + it('batch content is successfully fetched', function(done) { process.env.O365_COLLECTOR_ID = 'o365-collector-id'; var private_MAX_BATCH_MESSAGES = m_o365content.__set__('MAX_BATCH_MESSAGES', 2); const expectedCompressed = '789ced543d6f1c45180e1f4dce02640322b284582d148974b39ed999fd3a1a2e760296631cf98e0411216b76e6ddbbf1eded1cbb73c176e48a0689825f40454587e89152f20bf807f4fc0466f68ed80e29ac5414de62747bf33ceffbecfbf174fe7aafb3aa691c21a1cb1284d1355272edbb573aefc2916a8caa46176fde79fbfbc33f7f7fe3ae7afae11f1ffcf074f5f5d77e79f4d38d6bfe7e27eebc596ac1cb83b16e4cc5a7b0f6d1ba3f99cc4fa6aa427c36432e4dc04fe6357c0b79a30c344105a673a373dd310eccf10cd656d6afb78883625eadffbcb2faeb8fbf7d75cd3eefafd8c35f71212c79acf524fcbbf3c4efcf66a512dc285d6d4bbfe70bc67286538a689e63c4324150ce1289b2844144452e0b2afcae3fe4f508cca6ae0c1c99969817a9a4a1a488084211636982788143946299f330a4314ba36744bff7e8893fb472fd1eeefadb5b969f6449c15216a39c639bb89089fdc528925c90222d628a09f54fbfb6e8cad47c70dc18982e149324e1981224716c13276986d2844a7bf02263491c6618fb2d0dea05ad69793c032178c150ca8ac4f2044359665f0b9aa5592e499a65dcf2faae6fdbb3be9435348de5a54940080d923888d27fef5fa2105f34503fab807bd98163cb2618536c9f3ec9ee92db847ec2cbb69dbc84da947aa444a0aba912b56e746102a1a736d63e34f3d20c0c3773a77030170240826caf84aee5221189bafe5e3de2953a396bf825e5eecda06e4996e2d4ded3a311c8edca55d6856161c448c47394c69021c6c1ce4d460a141399b054863c25a1c56eda2f7151866a6a05f9212629c214856448921ec63dea400fa06eda4ca4eb3fd4f5a4d4dca5e8bb3ad872abc7b0a5ea769f8e5dc8524165b6efffb7357bf9a145b5f22e355d8b3eb4f8e5d65daafa2fd275e7b1d5b4ac7ad7bf7364a0b2fdb85f6b5b46a3a06917e0012fe7ae0abbfa449525df8802ecdddce542554637e38f3d37b2a567fff0f606de971ec107d94174cb732b0b0f21df516623a2494063efe6ce67c3dd7b5daf5413f03e0531d1b7bccd71ada7b0611d030794865140e2d01bf082d76a49b3da3fe76d1fdc77f74756b27fda3dd3459e03cccdd842966eb10b66ace505fc9e43843d77ea5a9dc0197d1fbe9943b328c879463ba976abce01cf26790b0c57e505fcb09e9f8bba0330db85811a55ed243a7368b7f105ee92038f44ce841d36965a5b0386f2308c50ca4511c6219504789b69418b96b4cb8fc119972eb9cf6fb2d577da7bebb0d1d5461b2c70567cfb55125db9f4954b5fb9f4954b5fb9f4ffd9a5ff0103b89f11'; - ingestSendStub.resolves('ok'); var msGetContentStub = sinon.stub(m_o365mgmnt, 'getContent'); msGetContentStub.callsFake( function fakeFn(contentUri, callback) { return callback(null, testMock.o365Content); }); + + const azureCollectorProcessLogs = sinon.stub(AlAzureCollector.prototype, 'processLog'); + azureCollectorProcessLogs.callsFake((slice, parseFun, config, callback) => { + callback(null, {skipped:0, processed: slice.length}); + }); m_o365content.processNotifications(testMock.context, testMock.webhookNotifications, function(err) { @@ -53,27 +48,6 @@ describe('O365WebHook Function o365content.js units.', function() { return done(err); sinon.assert.callCount(msGetContentStub, 2); - sinon.assert.callCount(ingestSendStub, 2); - //sinon.assert.calledWith(ingestSendStub, new Buffer(expectedCompressed, 'hex')); - msGetContentStub.restore(); - done(); - }); - }); - - it('Ingest send error', function(done) { - process.env.O365_COLLECTOR_ID = 'o365-collector-id'; - var private_MAX_BATCH_MESSAGES = m_o365content.__set__('MAX_BATCH_MESSAGES', 2); - ingestSendStub.rejects(new Error('StatusCodeError: 503')); - var msGetContentStub = sinon.stub(m_o365mgmnt, 'getContent'); - msGetContentStub.callsFake( - function fakeFn(contentUri, callback) { - return callback(null, testMock.o365Content); - }); - m_o365content.processNotifications(testMock.context, testMock.webhookNotifications, - function(err) { - sinon.assert.callCount(msGetContentStub, 2); - sinon.assert.callCount(ingestSendStub, 1); - assert.equal(err, 'Unable to send to Ingest. Error: StatusCodeError: 503'); msGetContentStub.restore(); done(); }); @@ -82,7 +56,6 @@ describe('O365WebHook Function o365content.js units.', function() { it('content fetch error', function(done) { process.env.O365_COLLECTOR_ID = 'o365-collector-id'; var private_MAX_BATCH_MESSAGES = m_o365content.__set__('MAX_BATCH_MESSAGES', 2); - ingestSendStub.resolves('ok'); const expectedError = 'Fetch error'; var msGetContentStub = sinon.stub(m_o365mgmnt, 'getContent'); msGetContentStub.callsFake( @@ -92,7 +65,6 @@ describe('O365WebHook Function o365content.js units.', function() { m_o365content.processNotifications(testMock.context, testMock.webhookNotifications, function(err) { sinon.assert.callCount(msGetContentStub, 1); - sinon.assert.callCount(ingestSendStub, 0); assert.equal(err, expectedError); msGetContentStub.restore(); done(); From d745be06e233f6402c1cff2f993a36f2dc5e16e2 Mon Sep 17 00:00:00 2001 From: Joe Dragovich Date: Fri, 22 Mar 2019 16:35:25 +0000 Subject: [PATCH 05/12] add more linting rules and fix the existing violations --- .jshintrc | 14 +++++- Master/azcollect.js | 69 ------------------------------ local_dev/master_local_dev.js | 2 +- local_dev/o365webhook_local_dev.js | 2 +- local_dev/updater_local_dev.js | 2 +- test/master_o365collector_test.js | 5 --- test/mock.js | 45 ------------------- test/o365webhook_content_test.js | 3 -- 8 files changed, 16 insertions(+), 126 deletions(-) delete mode 100644 Master/azcollect.js diff --git a/.jshintrc b/.jshintrc index 2b6f469..a68fc5d 100644 --- a/.jshintrc +++ b/.jshintrc @@ -1,3 +1,15 @@ { - "esversion": 6 + "esversion": 6, + "shadow": "outer", + "undef": true, + "unused": "vars", + "node": true, + "predef": [ + "it", + "describe", + "before", + "after", + "beforeEach", + "afterEach" + ] } diff --git a/Master/azcollect.js b/Master/azcollect.js deleted file mode 100644 index 66b6459..0000000 --- a/Master/azcollect.js +++ /dev/null @@ -1,69 +0,0 @@ -/* ----------------------------------------------------------------------------- - * @copyright (C) 2017, Alert Logic, Inc - * @doc - * - * The module for communicating with Alertlogic Azcollect service. - * - * @end - * ----------------------------------------------------------------------------- - */ - -const m_alServiceC = require('../lib/al_servicec'); -const m_version = require('./version'); - -/** - * @class - * HTTPS client for Alertlogic Azcollect service. - * - * @constructor - * @param {string} apiEndpoint - Alertlogic API hostname. - * @param {Object} aisCreds - Alertlogic API credentials. - * @param {string} [aisCreds.access_key_id] - Aertlogic API access key id. - * @param {string} [aisCreds.secret_key] - Alertlogic API secret key. - * - */ -class Azcollect extends m_alServiceC.AlServiceC { - constructor(apiEndpoint, aimsCreds) { - super(apiEndpoint, 'azcollect', 'v1', - aimsCreds, process.env.TMP); - } - - _o365RegisterBody() { - let o365AuditStreams = JSON.parse(process.env.O365_CONTENT_STREAMS); - let registerParams = {}; - let commonParams = { - version : '1.0.0', - web_app_name : process.env.WEBSITE_SITE_NAME, - app_tenant_id : process.env.APP_TENANT_ID, - app_resource_group : process.env.APP_RESOURCE_GROUP, - subscription_id : process.env.APP_SUBSCRIPTION_ID, - client_id : process.env.CUSTOMCONNSTR_APP_CLIENT_ID, - client_secret : process.env.CUSTOMCONNSTR_APP_CLIENT_SECRET - }; - let configParams = { - config : { - type : 'o365', - office_tenant_id : process.env.APP_TENANT_ID, - content_streams: o365AuditStreams - }}; - return Object.assign({}, commonParams , configParams); - } - - register_o365() { - let regBody = this._o365RegisterBody(); - return this.post(`/register/o365`, {body: regBody}); - } - - checkin(collectorType, collectorId, statusVal, descriptionVal, stats) { - let statusBody = { - type : collectorType, - version : m_version.getVersion(), - status : statusVal, - description : descriptionVal, - statistics : stats - }; - return this.post(`/checkin/${collectorType}/${collectorId}`, {body: statusBody}); - } -} - -exports.Azcollect = Azcollect; diff --git a/local_dev/master_local_dev.js b/local_dev/master_local_dev.js index 6abd5b4..308288b 100644 --- a/local_dev/master_local_dev.js +++ b/local_dev/master_local_dev.js @@ -1,6 +1,6 @@ const util = require('util'); -var devConfig = require('./dev_config'); +require('./dev_config'); var azureFunction = require('../Master/index'); diff --git a/local_dev/o365webhook_local_dev.js b/local_dev/o365webhook_local_dev.js index 30478bf..7775469 100644 --- a/local_dev/o365webhook_local_dev.js +++ b/local_dev/o365webhook_local_dev.js @@ -1,6 +1,6 @@ const util = require('util'); -var devConfig = require('./dev_config'); +require('./dev_config'); var azureFunction = require('../O365WebHook/index'); // Local development query and body params diff --git a/local_dev/updater_local_dev.js b/local_dev/updater_local_dev.js index c31d407..3b2178a 100644 --- a/local_dev/updater_local_dev.js +++ b/local_dev/updater_local_dev.js @@ -1,6 +1,6 @@ const util = require('util'); -var devConfig = require('./dev_config'); +require('./dev_config'); var azureFunction = require('../Updater/index'); diff --git a/test/master_o365collector_test.js b/test/master_o365collector_test.js index d595635..ed0741a 100644 --- a/test/master_o365collector_test.js +++ b/test/master_o365collector_test.js @@ -15,7 +15,6 @@ var sinon = require('sinon'); var testMock = require('./mock'); var m_o365mgmnt = require('../lib/o365_mgmnt'); -var m_azcollect = require('../Master/azcollect'); var healthchecks = rewire('../Master/healthchecks'); describe('O365 healthcheck tests', function() { @@ -50,10 +49,6 @@ describe('O365 healthcheck tests', function() { describe('checkStreams', function(done){ it('should throw the apprirpiate error when the subscriptionsList returns and error', function(){ - const msSubscriptionsListStub = sinon.stub(m_o365mgmnt, 'subscriptionsList').callsFake(function(callback){ - return "Somthing went wrong with getting the subscription list in a catestrophic way"; - }); - healthchecks.checkStreams(testMock.context, function(err, result){ sinon.assert.calledWith(stubErrorFmt, 'O365000002'); done(); diff --git a/test/mock.js b/test/mock.js index 6727d41..ca8c358 100644 --- a/test/mock.js +++ b/test/mock.js @@ -9,8 +9,6 @@ */ const util = require('util'); -const fs = require('fs'); -const m_alUtil = require('../lib/al_util'); process.env.WEBSITE_HOSTNAME = 'kkuzmin-app-o365.azurewebsites.net'; process.env.WEBSITE_SITE_NAME = 'kkuzmin-app-o365.azurewebsites.net'; @@ -104,49 +102,6 @@ var allEnabledStreams = [ } ]; -var twoOldEnabledStreams = [ - { - "contentType": "Audit.AzureActiveDirectory", - "status": "enabled", - "webhook": { - "authId": null, - "address": "https://old-app.azurewebsites.net/api/o365/webhook", - "expiration": "", - "status": "enabled" - } - }, - { - "contentType": "Audit.Exchange", - "status": "enabled", - "webhook": { - "authId": null, - "address": "https://old-app-o365.azurewebsites.net/api/o365/webhook", - "expiration": "", - "status": "enabled" - } - }, - { - "contentType": "Audit.General", - "status": "enabled", - "webhook": { - "authId": null, - "address": "https://kkuzmin-app-o365.azurewebsites.net/api/o365/webhook", - "expiration": "", - "status": "enabled" - } - }, - { - "contentType": "Audit.SharePoint", - "status": "enabled", - "webhook": { - "authId": null, - "address": "https://kkuzmin-app-o365.azurewebsites.net/api/o365/webhook", - "expiration": "", - "status": "enabled" - } - } -]; - var timer = { isPastDue: false, last: '2017-08-03T13:30:00', diff --git a/test/o365webhook_content_test.js b/test/o365webhook_content_test.js index a0163bc..1f2c227 100644 --- a/test/o365webhook_content_test.js +++ b/test/o365webhook_content_test.js @@ -29,8 +29,6 @@ describe('O365WebHook Function o365content.js units.', function() { describe('processNotifications()', function() { it('batch content is successfully fetched', function(done) { process.env.O365_COLLECTOR_ID = 'o365-collector-id'; - var private_MAX_BATCH_MESSAGES = m_o365content.__set__('MAX_BATCH_MESSAGES', 2); - const expectedCompressed = '789ced543d6f1c45180e1f4dce02640322b284582d148974b39ed999fd3a1a2e760296631cf98e0411216b76e6ddbbf1eded1cbb73c176e48a0689825f40454587e89152f20bf807f4fc0466f68ed80e29ac5414de62747bf33ceffbecfbf174fe7aafb3aa691c21a1cb1284d1355272edbb573aefc2916a8caa46176fde79fbfbc33f7f7fe3ae7afae11f1ffcf074f5f5d77e79f4d38d6bfe7e27eebc596ac1cb83b16e4cc5a7b0f6d1ba3f99cc4fa6aa427c36432e4dc04fe6357c0b79a30c344105a673a373dd310eccf10cd656d6afb78883625eadffbcb2faeb8fbf7d75cd3eefafd8c35f71212c79acf524fcbbf3c4efcf66a512dc285d6d4bbfe70bc67286538a689e63c4324150ce1289b2844144452e0b2afcae3fe4f508cca6ae0c1c99969817a9a4a1a488084211636982788143946299f330a4314ba36744bff7e8893fb472fd1eeefadb5b969f6449c15216a39c639bb89089fdc528925c90222d628a09f54fbfb6e8cad47c70dc18982e149324e1981224716c13276986d2844a7bf02263491c6618fb2d0dea05ad69793c032178c150ca8ac4f2044359665f0b9aa5592e499a65dcf2faae6fdbb3be9435348de5a54940080d923888d27fef5fa2105f34503fab807bd98163cb2618536c9f3ec9ee92db847ec2cbb69dbc84da947aa444a0aba912b56e746102a1a736d63e34f3d20c0c3773a77030170240826caf84aee5221189bafe5e3de2953a396bf825e5eecda06e4996e2d4ded3a311c8edca55d6856161c448c47394c69021c6c1ce4d460a141399b054863c25a1c56eda2f7151866a6a05f9212629c214856448921ec63dea400fa06eda4ca4eb3fd4f5a4d4dca5e8bb3ad872abc7b0a5ea769f8e5dc8524165b6efffb7357bf9a145b5f22e355d8b3eb4f8e5d65daafa2fd275e7b1d5b4ac7ad7bf7364a0b2fdb85f6b5b46a3a06917e0012fe7ae0abbfa449525df8802ecdddce542554637e38f3d37b2a567fff0f606de971ec107d94174cb732b0b0f21df516623a2494063efe6ce67c3dd7b5daf5413f03e0531d1b7bccd71ada7b0611d030794865140e2d01bf082d76a49b3da3fe76d1fdc77f74756b27fda3dd3459e03cccdd842966eb10b66ace505fc9e43843d77ea5a9dc0197d1fbe9943b328c879463ba976abce01cf26790b0c57e505fcb09e9f8bba0330db85811a55ed243a7368b7f105ee92038f44ce841d36965a5b0386f2308c50ca4511c6219504789b69418b96b4cb8fc119972eb9cf6fb2d577da7bebb0d1d5461b2c70567cfb55125db9f4954b5fb9f4954b5fb9f4ffd9a5ff0103b89f11'; var msGetContentStub = sinon.stub(m_o365mgmnt, 'getContent'); msGetContentStub.callsFake( function fakeFn(contentUri, callback) { @@ -55,7 +53,6 @@ describe('O365WebHook Function o365content.js units.', function() { it('content fetch error', function(done) { process.env.O365_COLLECTOR_ID = 'o365-collector-id'; - var private_MAX_BATCH_MESSAGES = m_o365content.__set__('MAX_BATCH_MESSAGES', 2); const expectedError = 'Fetch error'; var msGetContentStub = sinon.stub(m_o365mgmnt, 'getContent'); msGetContentStub.callsFake( From 5f643190a3d3987b49aeb49a2f2769419e37f767 Mon Sep 17 00:00:00 2001 From: Joe Dragovich Date: Tue, 26 Mar 2019 12:02:44 +0000 Subject: [PATCH 06/12] add old config vars to checkin --- Master/index.js | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Master/index.js b/Master/index.js index 47c011a..7acaace 100644 --- a/Master/index.js +++ b/Master/index.js @@ -15,9 +15,11 @@ const { AlAzureMaster } = require('al-azure-collector-js'); const { checkStreams } = require('./healthchecks.js'); //get the old o365 collector parameters if they exist -const g_aimsCreds = {}; -if(process.env.CUSTOMCONNSTR_APP_CI_ACCESS_KEY_ID) g_aimsCreds.aimsKeyId = process.env.CUSTOMCONNSTR_APP_CI_ACCESS_KEY_ID; -if(process.env.CUSTOMCONNSTR_APP_CI_SECRET_KEY) g_aimsCreds.aimsKeySecret = process.env.CUSTOMCONNSTR_APP_CI_SECRET_KEY; +const collectorKeys = {}; +if(process.env.CUSTOMCONNSTR_APP_CI_ACCESS_KEY_ID) collectorKeys.aimsKeyId = process.env.CUSTOMCONNSTR_APP_CI_ACCESS_KEY_ID; +if(process.env.CUSTOMCONNSTR_APP_CI_SECRET_KEY) collectorKeys.aimsKeySecret = process.env.CUSTOMCONNSTR_APP_CI_SECRET_KEY; +if(process.env.CUSTOMCONNSTR_O365_HOST_ID) collectorKeys.hostId = process.env.CUSTOMCONNSTR_O365_HOST_ID; +if(process.env.CUSTOMCONNSTR_O365_COLLECTOR_ID) collectorKeys.sourceId = process.env.CUSTOMCONNSTR_O365_COLLECTOR_ID; const APP_FUNCTIONS = ['Master', 'Updater', 'O365WebHook']; @@ -25,7 +27,7 @@ module.exports = function (context, AlertlogicMasterTimer) { const healthFuns = [ checkStreams ]; - const master = new AlAzureMaster(context, 'o365', pkg.version, healthFuns, null, g_aimsCreds, {}, APP_FUNCTIONS); + const master = new AlAzureMaster(context, 'o365', pkg.version, healthFuns, null, collectorKeys, {}, APP_FUNCTIONS); async.waterfall([ function(asyncCallback) { return master.register(_o365RegisterBody(), asyncCallback); From 19d7eb9a9c2040dc50ee6625619e993f550bf925 Mon Sep 17 00:00:00 2001 From: Joe Dragovich Date: Tue, 26 Mar 2019 15:06:55 +0000 Subject: [PATCH 07/12] correct mispelled variable --- Master/index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Master/index.js b/Master/index.js index 7acaace..44496b3 100644 --- a/Master/index.js +++ b/Master/index.js @@ -18,8 +18,8 @@ const { checkStreams } = require('./healthchecks.js'); const collectorKeys = {}; if(process.env.CUSTOMCONNSTR_APP_CI_ACCESS_KEY_ID) collectorKeys.aimsKeyId = process.env.CUSTOMCONNSTR_APP_CI_ACCESS_KEY_ID; if(process.env.CUSTOMCONNSTR_APP_CI_SECRET_KEY) collectorKeys.aimsKeySecret = process.env.CUSTOMCONNSTR_APP_CI_SECRET_KEY; -if(process.env.CUSTOMCONNSTR_O365_HOST_ID) collectorKeys.hostId = process.env.CUSTOMCONNSTR_O365_HOST_ID; -if(process.env.CUSTOMCONNSTR_O365_COLLECTOR_ID) collectorKeys.sourceId = process.env.CUSTOMCONNSTR_O365_COLLECTOR_ID; +if(process.env.O365_HOST_ID) collectorKeys.hostId = process.env.O365_HOST_ID; +if(process.env.O365_COLLECTOR_ID) collectorKeys.sourceId = process.env.O365_COLLECTOR_ID; const APP_FUNCTIONS = ['Master', 'Updater', 'O365WebHook']; From 7bc3048bcf42acf8742d2cc1569e5556c4f9c7c9 Mon Sep 17 00:00:00 2001 From: Joe Dragovich Date: Wed, 27 Mar 2019 15:34:06 +0000 Subject: [PATCH 08/12] bump version number --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index d650802..c58a5ce 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "azure_collector", - "version": "1.1.0", + "version": "1.1.1", "dependencies": { "async": "^2.6.2", "azure": "^2.0.0-preview", From cd93f5bff61c90662b31ca98f07d4f3500754164 Mon Sep 17 00:00:00 2001 From: JDragovichAlertLogic <47750771+JDragovichAlertLogic@users.noreply.github.com> Date: Fri, 12 Apr 2019 14:37:25 +0100 Subject: [PATCH 09/12] update collector to use NPM library repos (#55) * update collector to use NPM library repos * bump version --- Master/index.js | 2 +- O365WebHook/formatO365Log.js | 2 +- O365WebHook/o365content.js | 2 +- Updater/index.js | 2 +- package.json | 5 +++-- test/o365webhook_content_test.js | 2 +- 6 files changed, 8 insertions(+), 7 deletions(-) diff --git a/Master/index.js b/Master/index.js index 44496b3..75c5b56 100644 --- a/Master/index.js +++ b/Master/index.js @@ -11,7 +11,7 @@ const async = require('async'); const pkg = require('../package.json'); -const { AlAzureMaster } = require('al-azure-collector-js'); +const { AlAzureMaster } = require('@alertlogic/al-azure-collector-js'); const { checkStreams } = require('./healthchecks.js'); //get the old o365 collector parameters if they exist diff --git a/O365WebHook/formatO365Log.js b/O365WebHook/formatO365Log.js index fdaed09..a2e8f83 100644 --- a/O365WebHook/formatO365Log.js +++ b/O365WebHook/formatO365Log.js @@ -8,7 +8,7 @@ * ---------------------------------------------------------------------------- */ -const Parse = require('al-collector-js').Parse; +const Parse = require('@alertlogic/al-collector-js').Parse; module.exports = function(item) { //Paths from https://docs.microsoft.com/en-us/office/office-365-management-api/office-365-management-activity-api-schema#common-schema diff --git a/O365WebHook/o365content.js b/O365WebHook/o365content.js index d081666..3fed64f 100644 --- a/O365WebHook/o365content.js +++ b/O365WebHook/o365content.js @@ -14,7 +14,7 @@ const async = require('async'); const pkg = require('../package.json'); const m_o365mgmnt = require('../lib/o365_mgmnt'); -const AlAzureCollector = require('al-azure-collector-js').AlAzureCollector; +const AlAzureCollector = require('@alertlogic/al-azure-collector-js').AlAzureCollector; const formatO365Log = require('./formatO365Log'); // One O365 content message is about 1KB. diff --git a/Updater/index.js b/Updater/index.js index d05cdaf..d422ab4 100644 --- a/Updater/index.js +++ b/Updater/index.js @@ -10,7 +10,7 @@ * ----------------------------------------------------------------------------- */ -const AlAzureUpdater = require('al-azure-collector-js').AlAzureUpdater; +const AlAzureUpdater = require('@alertlogic/al-azure-collector-js').AlAzureUpdater; module.exports = function (context, AlertlogicUpdaterTimer) { const updater = new AlAzureUpdater(); diff --git a/package.json b/package.json index c58a5ce..0844b23 100644 --- a/package.json +++ b/package.json @@ -1,11 +1,12 @@ { "name": "azure_collector", - "version": "1.1.1", + "version": "1.1.2", "dependencies": { "async": "^2.6.2", "azure": "^2.0.0-preview", "path": "^0.12.7", - "al-azure-collector-js": "git://github.com/alertlogic/al-azure-collector-js#master", + "@alertlogic/al-azure-collector-js": "^1.1.1", + "@alertlogic/al-collector-js": "^1.1.2", "request-promise-native": "^1.0.4" }, "scripts": { diff --git a/test/o365webhook_content_test.js b/test/o365webhook_content_test.js index 1f2c227..683274e 100644 --- a/test/o365webhook_content_test.js +++ b/test/o365webhook_content_test.js @@ -15,7 +15,7 @@ var sinon = require('sinon'); var testMock = require('./mock'); var m_o365mgmnt = require('../lib/o365_mgmnt'); var m_o365content = rewire('../O365WebHook/o365content'); -const AlAzureCollector = require('al-azure-collector-js').AlAzureCollector; +const AlAzureCollector = require('@alertlogic/al-azure-collector-js').AlAzureCollector; describe('O365WebHook Function o365content.js units.', function() { var clock; From 665761503e045b4cec5c1dc49c5166aecd87b043 Mon Sep 17 00:00:00 2001 From: kkuzmin Date: Wed, 24 Apr 2019 11:19:12 +0100 Subject: [PATCH 10/12] Add optionale parsed properties only if they exist (#56) * Add optionale parsed properties only if they exist * Address comments --- O365WebHook/formatO365Log.js | 16 ++++++++++------ test/formatO365Log_test.js | 26 +++++++++++++++++++++----- 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/O365WebHook/formatO365Log.js b/O365WebHook/formatO365Log.js index a2e8f83..087eddc 100644 --- a/O365WebHook/formatO365Log.js +++ b/O365WebHook/formatO365Log.js @@ -29,15 +29,19 @@ module.exports = function(item) { let creationTime = Parse.getMsgTs(item, creationTimePaths); let messageTypeId = Parse.getMsgTypeId(item, messageTypeIdPaths); - - return { + let formattedMsg = { messageTs: creationTime.sec, priority: 11, progName: 'o365webhook', - pid: undefined, message: message, - messageType: 'json/azure.o365', - messageTypeId: `${messageTypeId}`, - messageTsUs: creationTime.usec ? creationTime.usec : undefined + messageType: 'json/azure.o365' }; + + if (messageTypeId) { + formattedMsg.messageTypeId = `${messageTypeId}`; + } + if (creationTime.usec) { + formattedMsg.messageTsUs = creationTime.usec; + } + return formattedMsg; }; diff --git a/test/formatO365Log_test.js b/test/formatO365Log_test.js index 369b7c9..6b868ca 100644 --- a/test/formatO365Log_test.js +++ b/test/formatO365Log_test.js @@ -3,22 +3,38 @@ const formatO365Log = require('../O365WebHook/formatO365Log'); const mock = require('./mock'); describe('formatO365 units', function(){ - it('Formats a O365 log correctly', function(done){ - const logRecord = mock.o365Content[0]; + it('Formats a O365 log correctly, no optional properties', function(done){ + let logRecord = Object.assign({}, mock.o365Content[0]); + delete logRecord.RecordType; const formattedRecord = formatO365Log(logRecord); const expectedRecord = { messageTs: new Date(logRecord.CreationTime).getTime() / 1000, priority: 11, progName: 'o365webhook', - pid: undefined, + message: JSON.stringify(logRecord), + messageType: 'json/azure.o365' + }; + + assert.deepEqual(formattedRecord, expectedRecord); + done(); + }); + + it('Formats a O365 log correctly, with optional properties', function(done){ + let logRecord = Object.assign({}, mock.o365Content[0]); + logRecord.CreationTime = "2018-03-21T17:00:32.125Z"; + const formattedRecord = formatO365Log(logRecord); + + const expectedRecord = { + messageTs: 1521651632, + priority: 11, + progName: 'o365webhook', message: JSON.stringify(logRecord), messageType: 'json/azure.o365', messageTypeId: `${logRecord.RecordType}`, - messageTsUs: undefined + messageTsUs: 125000 }; - assert.deepEqual(formattedRecord, expectedRecord); done(); }); From 11f697bc681ab11d2850e4f635f9396f5aa01386 Mon Sep 17 00:00:00 2001 From: JDragovichAlertLogic <47750771+JDragovichAlertLogic@users.noreply.github.com> Date: Wed, 24 Apr 2019 17:01:36 +0100 Subject: [PATCH 11/12] add checking for null and undefined (#57) * add checking for null and undefined * add NaN check * remove checking for usec per comments * bump al-collector-js to 1.2.3 * bump al-collector-js to 1.2.4 --- O365WebHook/formatO365Log.js | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/O365WebHook/formatO365Log.js b/O365WebHook/formatO365Log.js index 087eddc..768cea8 100644 --- a/O365WebHook/formatO365Log.js +++ b/O365WebHook/formatO365Log.js @@ -37,7 +37,7 @@ module.exports = function(item) { messageType: 'json/azure.o365' }; - if (messageTypeId) { + if (messageTypeId !== undefined && messageTypeId !== null) { formattedMsg.messageTypeId = `${messageTypeId}`; } if (creationTime.usec) { diff --git a/package.json b/package.json index 0844b23..6192d03 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "azure": "^2.0.0-preview", "path": "^0.12.7", "@alertlogic/al-azure-collector-js": "^1.1.1", - "@alertlogic/al-collector-js": "^1.1.2", + "@alertlogic/al-collector-js": "^1.2.4", "request-promise-native": "^1.0.4" }, "scripts": { From 0f62627496ffaa580120344769f264e2462ad30f Mon Sep 17 00:00:00 2001 From: Joe Dragovich Date: Thu, 25 Apr 2019 16:04:17 +0100 Subject: [PATCH 12/12] bump al-azure-collector-js to 1.1.3 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 6192d03..9d991e3 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "async": "^2.6.2", "azure": "^2.0.0-preview", "path": "^0.12.7", - "@alertlogic/al-azure-collector-js": "^1.1.1", + "@alertlogic/al-azure-collector-js": "^1.1.3", "@alertlogic/al-collector-js": "^1.2.4", "request-promise-native": "^1.0.4" },