diff --git a/Master/endpoints.js b/Master/endpoints.js index 1514c57..2a39cf0 100644 --- a/Master/endpoints.js +++ b/Master/endpoints.js @@ -16,8 +16,8 @@ const m_appSettings = require('./appsettings'); exports.checkUpdate = function (context, AlertlogicMasterTimer, callback) { if (process.env.APP_INGEST_ENDPOINT && process.env.APP_AZCOLLECT_ENDPOINT) { - context.log('DEBUG: Reuse Ingest endpoint', process.env.APP_INGEST_ENDPOINT); - context.log('DEBUG: Reuse Azcollect endpoint', process.env.APP_AZCOLLECT_ENDPOINT); + context.log.verbose('Reuse Ingest endpoint', process.env.APP_INGEST_ENDPOINT); + context.log.verbose('Reuse Azcollect endpoint', process.env.APP_AZCOLLECT_ENDPOINT); return callback(null); } else { // Endpoint settings do not exist. Update them. diff --git a/Master/index.js b/Master/index.js index 601a3e5..05b95be 100644 --- a/Master/index.js +++ b/Master/index.js @@ -28,7 +28,7 @@ module.exports = function (context, AlertlogicMasterTimer) { if (endpointsError) { return asyncCallback(endpointsError); } - context.log('INFO: Alertlogic endpoints updated.'); + context.log.info('Alertlogic endpoints updated.'); return asyncCallback(null); }); }, @@ -41,7 +41,7 @@ module.exports = function (context, AlertlogicMasterTimer) { if (azcollectError) { return asyncCallback(azcollectError); } - context.log('INFO: O365 source registered', collectorId); + context.log.info('O365 source registered', collectorId); return asyncCallback(null, azcollectSvc); }); }, @@ -52,15 +52,15 @@ module.exports = function (context, AlertlogicMasterTimer) { if (azcollectError) { return asyncCallback(`Checkin failed ${azcollectError}`); } - context.log('INFO: O365 source checkin OK', checkinResp); + context.log.info('O365 source checkin OK', checkinResp); return asyncCallback(null); }); } ], function(error, results) { if (error) { - context.log('ERROR: Master error ', error); + context.log.error('Master error ', error); } - context.done(); + context.done(error); }); }; diff --git a/Master/o365collector.js b/Master/o365collector.js index f7faf38..986fa33 100644 --- a/Master/o365collector.js +++ b/Master/o365collector.js @@ -17,13 +17,17 @@ const m_o365mgmnt = require('../lib/o365_mgmnt'); exports.checkRegister = function (context, AlertlogicMasterTimer, azcollectSvc, callback) { - if (process.env.O365_COLLECTOR_ID) { - context.log('DEBUG: Reuse collector id', process.env.O365_COLLECTOR_ID); + if (process.env.O365_COLLECTOR_ID && process.env.O365_HOST_ID) { + context.log.verbose('Reuse collector id', process.env.O365_COLLECTOR_ID); return callback(null, process.env.O365_COLLECTOR_ID); } else { // Collector is not registered. azcollectSvc.register_o365().then(resp => { - m_appSettings.updateAppsettings({O365_COLLECTOR_ID: resp.source.id}, + let newSettings = { + O365_COLLECTOR_ID: resp.source.id, + O365_HOST_ID: resp.source.host.id + }; + m_appSettings.updateAppsettings(newSettings, function(settingsError) { if (settingsError) { return callback(settingsError); @@ -80,20 +84,20 @@ exports.checkin = function (context, AlertlogicMasterTimer, azcollectSvc, callba var _checkEnableAuditStreams = function(context, listedStreams, callback) { try { let o365AuditStreams = JSON.parse(process.env.O365_CONTENT_STREAMS); + // TODO: take webhook path from O365Webhook/function.json + let webhookURL = 'https://' + process.env.WEBSITE_HOSTNAME + + '/api/o365/webhook'; async.map(o365AuditStreams, function(stream, asyncCallback) { let currentStream = listedStreams.find( obj => obj.contentType === stream); if (currentStream && currentStream.status === 'enabled' && currentStream.webhook && - currentStream.webhook.status === 'enabled') { - context.log('DEBUG: Stream already enabled', stream); + currentStream.webhook.status === 'enabled' && + currentStream.webhook.address === webhookURL) { + context.log.verbose('Stream already enabled', stream); return asyncCallback(null, stream); } else { - // TODO: take webhook path from O365Webhook/function.json - let webhookURL = 'https://' + - process.env.WEBSITE_HOSTNAME + - '/api/o365/webhook'; let webhook = { webhook : { address : webhookURL, expiration : "" diff --git a/O365WebHook/index.js b/O365WebHook/index.js index 4545426..bcc85d4 100644 --- a/O365WebHook/index.js +++ b/O365WebHook/index.js @@ -25,12 +25,12 @@ module.exports = function (context, event) { return m_o365content.processNotifications(context, eventBody, function(err) { if (err) { - context.log(`ERROR: ${err}`); + context.log.error(`${err}`); context.res.headers = {}; context.res.status = 500; - context.done(); + context.done(err); } else { - context.log('Debug: Success!'); + context.log.info('OK!'); context.done(); } }); diff --git a/O365WebHook/ingest.js b/O365WebHook/ingest.js index 0f4261e..f312842 100644 --- a/O365WebHook/ingest.js +++ b/O365WebHook/ingest.js @@ -39,7 +39,7 @@ class Ingest extends m_alServiceC.AlServiceC { }, body : data }; - return this.post(`/data/o365msgs`, payload); + return this.post(`/data/aicspmsgs`, payload); } } diff --git a/O365WebHook/ingest_proto.js b/O365WebHook/ingest_proto.js index 0b8f5c9..7b0bd23 100644 --- a/O365WebHook/ingest_proto.js +++ b/O365WebHook/ingest_proto.js @@ -13,6 +13,7 @@ const protobuf = require('protobufjs'); const async = require('async'); const Long = require('long'); const path = require('path'); +const crypto = require('crypto'); // FIXME - protobuf load // We have to load PROTO_DEF every invocation. Maybe the solution can to to use @@ -20,7 +21,7 @@ const path = require('path'); module.exports.load = function(context, callback) { protobuf.load(getCommonProtoPath(), function(err, root) { if (err) - context.log('Error: Unable to load proto files.', err); + context.log.error('Unable to load proto files.', err); callback(err, root); }); @@ -33,7 +34,7 @@ module.exports.setMessage = function(context, root, content, callback) { }, function(err, result) { if (err) - context.log('Error: Unable to build messages.'); + context.log.error('Unable to build messages.'); callback(err, result); } @@ -43,20 +44,28 @@ module.exports.setMessage = function(context, root, content, callback) { module.exports.setHostMetadata = function(context, root, content, callback) { var hostmetaType = root.lookupType('host_metadata.metadata'); - + var hostmetaData = getHostmeta(context, root); + var meta = { + hostUuid : process.env.O365_HOST_ID, + data : hostmetaData, + dataChecksum : new Buffer('') + }; + var sha = crypto.createHash('sha1'); + var hashPayload = hostmetaType.encode(meta).finish(); + hashValue = sha.update(hashPayload).digest(); + var metadataPayload = { - // FIXME - we need to calculate checksum properly - dataChecksum: new Buffer.from([234,104,231,10,12,60,139,208,204,230, - 236,248,60,113,61,93,52,49,18,194]), - timestamp: Math.floor(Date.now() / 1000), - data: dummyMetadataDict(context, root) + hostUuid : process.env.O365_HOST_ID, + dataChecksum : hashValue, + timestamp : Math.floor(Date.now() / 1000), + data : hostmetaData }; build(hostmetaType, metadataPayload, function(err, buf) { if (err) - context.log('Error: Unable to build host_metadata.'); + context.log.error('Unable to build host_metadata.'); - callback(err, buf); + return callback(err, buf); }); }; @@ -72,9 +81,9 @@ module.exports.setBatch = function(context, root, metadata, messages, callback) build(batchType, batchPayload, function(err, buf) { if (err) - context.log('Error: Unable to build collected_batch.'); + context.log.error('Unable to build collected_batch.'); - callback(err, buf); + return callback(err, buf); }); }; @@ -88,16 +97,16 @@ module.exports.setBatchList = function(context, root, batches, callback) { build(batchListType, batchListPayload, function(err, buf) { if (err) - context.log('Error: Unable to build collected_batch_list.'); + context.log.error('Unable to build collected_batch_list.'); - callback(err, buf); + return callback(err, buf); }); }; module.exports.encode = function(context, root, batchList, callback) { var batchListType = root.lookupType('common_proto.collected_batch_list'); var buf = batchListType.encode(batchList).finish(); - callback(null, buf); + return callback(null, buf); }; @@ -110,7 +119,7 @@ function build(type, payload, callback) { var payloadCreated = type.create(payload); - callback(null, payloadCreated); + return callback(null, payloadCreated); } @@ -139,44 +148,31 @@ function parseMessage(context, root, memo, content, callback) { build(messageType, messagePayload, function(err, buf) { if (err) - context.log('Error: Unable to build collected_message.'); + context.log.error('Unable to build collected_message.'); memo.push(buf); - callback(err, memo); + return callback(err, memo); }); } -// TODO - Fill Metadata dictionary with some dummy content. -// FIXME - we need to use some real data in metadata -function dummyMetadataDict(context, root) { +function getHostmeta(context, root) { var dictType = root.lookupType('alc_dict.dict'); var elemType = root.lookupType('alc_dict.elem'); var valueType = root.lookupType('alc_dict.value'); - var val1 = {str: 'standalone'}; - var valPayload1 = buildSync(valueType, val1); - - var val2 = {str: '454712-mnimn2.syd.intensive.int'}; - var valPayload2 = buildSync(valueType, val2); - - var elem1 = { + var hostTypeElem = { key: 'host_type', - value: val1 + value: {str: 'azure_fun'} }; - var elemPayload1 = buildSync(elemType, elem1); - - var elem2 = { + var localHostnameElem = { key: 'local_hostname', - value: val2 + value: {str: process.env.WEBSITE_HOSTNAME} }; - var elemPayload2 = buildSync(elemType, elem2); - var dict = { - elem: [elem1, elem2] + elem: [localHostnameElem, hostTypeElem] }; - var dictPayload = buildSync(dictType, dict); - - return dictPayload; + + return buildSync(dictType, dict); } diff --git a/O365WebHook/o365content.js b/O365WebHook/o365content.js index 94581c3..13ec585 100644 --- a/O365WebHook/o365content.js +++ b/O365WebHook/o365content.js @@ -83,7 +83,7 @@ function parseContent(context, parsedContent, callback) { var creationTime; if (item.CreationTime == undefined) { - context.log('WARNING: Unable to parse CreationTime from content.'); + context.log.warn('Unable to parse CreationTime from content.'); creationTime = Math.floor(Date.now() / 1000); } else { @@ -105,7 +105,7 @@ function parseContent(context, parsedContent, callback) { if (err) { return callback(`Content parsing failure. ${err}`); } else { - context.log('DEBUG: parsedData: ', result); + context.log.verbose('parsedData: ', result); return callback(null, result); } } @@ -152,6 +152,9 @@ function sendToIngest(context, content, callback) { if (err) { return callback(`Unable to compress. ${err}`); } else { + if (compressed.byteLength > 700000) + context.log.warn(`Compressed log batch length`, + `(${compressed.byteLength}) exceeds maximum allowed value.`); return g_ingestc.sendO365Data(compressed) .then(resp => { return callback(null, resp); diff --git a/PostDeploymentActions/updateMasterTimer.ps1 b/PostDeploymentActions/updateMasterTimer.ps1 new file mode 100755 index 0000000..456aa2c --- /dev/null +++ b/PostDeploymentActions/updateMasterTimer.ps1 @@ -0,0 +1,8 @@ +$date = Get-Date +$min = ($date.Minute + 1) % 15 +$sec = $date.Second +$new_schedule = "$sec $min-59/15 * * * *" +Write-Output "Updating Master timer trigger with ($new_schedule)." +$master_function = Get-Content '..\\wwwroot\\Master\\function.json' -raw | ConvertFrom-Json +$master_function.bindings | % {if($_.name -eq 'AlertlogicMasterTimer'){$_.schedule=$new_schedule}} +$master_function | ConvertTo-Json | set-content '..\\wwwroot\\Master\\function.json' \ No newline at end of file diff --git a/PostDeploymentActions/updateUpdaterTimer.ps1 b/PostDeploymentActions/updateUpdaterTimer.ps1 new file mode 100755 index 0000000..af874c4 --- /dev/null +++ b/PostDeploymentActions/updateUpdaterTimer.ps1 @@ -0,0 +1,9 @@ +$randH = Get-Random -minimum 0 -maximum 11 +$randM = Get-Random -minimum 0 -maximum 59 +$randS = Get-Random -minimum 0 -maximum 59 +$randH12 = $randH + 12 +$new_schedule = "$randS $randM $randH,$randH12 * * *" +Write-Output "Updating Updater timer trigger with ($new_schedule)". +$master_function = Get-Content '..\\wwwroot\\Updater\\function.json' -raw | ConvertFrom-Json +$master_function.bindings | % {if($_.name -eq 'AlertlogicUpdaterTimer'){$_.schedule=$new_schedule}} +$master_function | ConvertTo-Json | set-content '..\\wwwroot\\Updater\\function.json' \ No newline at end of file diff --git a/README.md b/README.md index fbf1689..d4e8953 100644 --- a/README.md +++ b/README.md @@ -1,23 +1,23 @@ -# o365-collector +# azure-collector Alert Logic Office 365 Log Collector # Overview -This repo contains Azure Web application Node.js source code and an ARM template for setting up a data collector in Azure which will collect and forward Office 365 log data to the Alert Logic Cloud Defender Log Manager (LM) feature. +This repository contains Azure Web application Node.js source code and an ARM template for setting up a data collector in Azure which will collect and forward Office 365 log data to the Alert Logic Cloud Defender Log Manager (LM) feature. # Installation Installation requires the following steps: 1. Register a new O365 web application in O365 portal for collecting O365 logs. -1. Set up the required Active Directory security permissions for the application to authorize it to read threat intelligence data and activity reports for your orgaization. +1. Set up the required Active Directory security permissions for the application to authorize it to read threat intelligence data and activity reports for your organization. 1. Create an Access Key that will allow the application to connect to the Alert Logic Cloud Defender and Cloud Insight backend. 1. Download and deploy a custom ARM template to Microsoft Azure to create functions for collecting and managing O365 log data 1. Verify that installation was successful using Alertlogic CloudDefender UI. -### Register a New O365 Web Application in O365 +## Register a New O365 Web Application in O365 In order to install O365 Log collector: @@ -32,21 +32,26 @@ In order to install O365 Log collector: 1. After application is created select it and make a note of `Application Id`, for example, `a261478c-84fb-42f9-84c2-de050a4babe3` -### Set Up the Required Active Directory Security Permissions +## Set Up the Required Active Directory Security Permissions 1. On the `Settings` panel and select `Required permissions` and click `+Add` 1. Hit `Select an API` and chose `Office 365 Management APIs`, click `Select` 1. In `Application permissions` select `Read service health information for your organization`, `Read activity data for your organization`, `Read threat intelligence data for your organization` and `Read activity reports for your organization`. Click `Select` and `Done` buttons. -1. On `Required permissions` panel click `Required permissions` button and confirm the selection. **Note**, only AD tenant admin can grant permisions to an Azure AD application. +1. On `Required permissions` panel click `Required permissions` button and confirm the selection. **Note**, only AD tenant admin can grant permissions to an Azure AD application. 1. On the `Settings` panel of the application and select `Keys`. 1. Enter key `Description` and `Duration` and click `Save`. **Note**, please save the key value, it is needed later during template deployment. +1. Save the `Application ID` and `Service Principal Id` for use below. To get the `Service Principal Id`, navigate to the `Registered App` blade, +click on the link under `Managed application in local directory`. Then click `Properties`. The `Service Principal Id` +is labeled `Object ID` on the properties page. **Caution** This is not the same `Object ID` listed in the `Properties` blade reached +by clicking `Settings` or `All Settings` from the `Registered app`. It is also not the `Object ID` shown on the `Registered app` +blade itself. -### Create an Alert Logic Access Key +## Create an Alert Logic Access Key -Login and get an authentication token from the Alert Logic Cloud Insight product [AIMS API](https://console.product.dev.alertlogic.com/api/aims/). For example, from the command line use [curl](https://en.wikipedia.org/wiki/CURL) as follows (where `` is your CloudInsight user and `` is your CloudInsight password): +Login and get an authentication token from the Alert Logic Cloud Insight product [AIMS API](https://console.product.dev.alertlogic.com/api/aims/). From the command line use [curl](https://en.wikipedia.org/wiki/CURL) as follows (where `` is your CloudInsight user and `` is your CloudInsight password): ``` -curl -X POST -v -u ':' https://api.product.dev.alertlogic.com/aims/v1/authenticate +curl -X POST -v -u ':' https://api.global-services.global.alertlogic.com/aims/v1/authenticate ``` Make a note of the following fields returned in the response: @@ -55,10 +60,10 @@ Make a note of the following fields returned in the response: * ACCOUNT ID * TOKEN -Use the authentication token returned in the response to create access keys for the Azure application deployed in the next section. For example, issue the following curl command (where `` is the auth token, `` is the account id, and `` is the user id returned above): +Use the authentication token returned in the response to create access keys for the Azure application deployed in the next section. Issue the following curl command (where `` is the auth token, `` is the account id, and `` is the user id returned above): ``` -curl -X POST -H "x-aims-auth-token: " https://api.product.dev.alertlogic.com/aims/v1//users//access_keys +curl -X POST -H "x-aims-auth-token: " https://api.global-services.global.alertlogic.com/aims/v1//users//access_keys ``` An example of a successful response is: @@ -69,53 +74,73 @@ An example of a successful response is: Make a note of the `access_key_id` and `secret_key` values for use in the deployment steps below. +**Note:** Only five access keys can be created per user. If you get a "limit exceeded" response you will need to +delete some keys in order to create new ones. Use the following command to delete access keys: -### Download and Deploy the Custom ARM Template in an Azure Subscription +``` +curl -X POST -H "x-aims-auth-token: " https://api.global-services.global.alertlogic.com/aims/v1//users//access_keys/ +``` + +## Function deployment + +Log into [Azure portal](https://portal.azure.com). **Note**, In order to perform steps below you should have an active Azure subscription, to find out visit [Azure subscriptions blade](https://portal.azure.com/#blade/Microsoft_Azure_Billing/SubscriptionsBlade) + +### Deploy via the Custom ARM Template in an Azure Subscription -1. **TODO: it is possible to use URI deployment without downloading a file.** Download an ARM [template](https://github.com/alertlogic/o365-collector/blob/master/template.json) -1. Log into [Azure portal](https://portal.azure.com). **Note**, In order to perform steps below you should have an acive Azure subscription, to find out visit [Azure subscriptions blade](https://portal.azure.com/#blade/Microsoft_Azure_Billing/SubscriptionsBlade) -1. Go to [Customer Deployment](https://portal.azure.com/#create/Microsoft.Template) page. Type in `deploy` in a seach query located on top of Azure Web UI and select `Deploy a custom template`. +1. Download an ARM [template](https://github.com/alertlogic/azure-collector/raw/master/template.json) +1. Go to [Customer Deployment](https://portal.azure.com/#create/Microsoft.Template) page. Type in `deploy` in a search query located on top of Azure Web UI and select `Deploy a custom template`. 1. Click `Build your own template in the editor` and load the file previously downloaded on step 1 above. 1. Click `Save` button. 2. Fill in required template parameters and click the `Purchase` button to start a deployment. I.e.: - - `APP_TENANT_ID` - The GUID of the tenant e.g. `alazurealertlogic.onmicrosoft.com` - - `CUSTOMCONNSTR_APP_CLIENT_ID` - The GUID of your application that created the subscription. -You can obtain it from _Azure_ -> _AD_ -> _App registrations_ -> _Your app name_ - - `CUSTOMCONNSTR_APP_CLIENT_SECRET` - A secret key of your application from _App Registrations_. - - `CUSTOMCONNSTR_APP_CI_ACCESS_KEY_ID` - `access_key_id` returned from AIMs [above](#create_an_alert_logic_access_key). - - `CUSTOMCONNSTR_APP_CI_SECRET_KEY`- `secret_key` returned from AIMs [above](#create_an_alert_logic_access_key). - -1. Once deployment is finished go to `Resource groups` blade and select a resource group used for the deployment on step 3 above. + - `Name` - Any name + - `Storage Name` - Any Storage Account name (that does not currently exist) + - `Alertlogic Access Key Id` - `access_key_id` returned from AIMs [above](#create_an_alert_logic_access_key) + - `Alertlogic Secret Key` - `secret_key` returned from AIMs [above](#create_an_alert_logic_access_key) + - `Alertlogic API endpoint` - usually `api.global-services.global.alertlogic.com` + - `Alertlogic Data Residency` - usually `default` + - `Office365 Content Streams` - The list of streams you would like to collect. Valid values are: + - ["Audit.AzureActiveDirectory","Audit.Exchange","Audit.SharePoint","Audit.General", "DLP.All"] + - `Office365 Tenant Id` - The GUID of the tenant e.g. `alazurealertlogic.onmicrosoft.com` + - `Service Principal Id` - The `Object ID` of the application that created the subscription. + You can obtain it from _Azure_ -> _AD_ -> _App registrations_ -> _Your app name_ -> Link under +_Managed application in local directory_ -> _Properties_ -> _Object ID_ + - `App Client Id` - The GUID of your application that created the subscription. + You can obtain it from _Azure_ -> _AD_ -> _App registrations_ -> _Your app name_ + - `App Client Secret` - The secret key of your application from _App Registrations_ + - `Repository URL` - must be `https://github.com/alertlogic/azure-collector.git` + - `Repository Branch` - should usually be `master` + +### Deploy via Azure CLI + +You can use either [Azure Cloud Shell](https://docs.microsoft.com/en-gb/azure/cloud-shell/quickstart#start-cloud-shell) or +local installation of [Azure CLI](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli?view=azure-cli-latest). + +1. Create a resource group with name "AlertLogicCollect" in location "Central US" by executing following command + ``` + az group create --name AlertLogicCollect --location "Central US" + ``` +1. Once created go to `Resource groups` blade and select the resource group. 1. Select `Access Control (IAM)` and add `Website Contributor` role to AD application identity created above. - -### Verify the Installation - -1. Log into Alertlogic CloudDefender and navigate into `Log Manager -> Sources` page. Check new O365 log source (with a name provided on step 15) has been created and source status is `ok`. - -## Using Azure CLI to deploy a template - -1. Follow the installation steps up to (but not including) [Download and Deploy the Custom ARM Template in an Azure Subscription](#download_and_deploy_the_custom_arm_template_in_an_azure_subscription) above. -1. Download [ARM template](template.json) locally -1. Create a resource group with name "ResourceGroupName" in location "West US" by executing following command - -``` -az group create -n ResourceGroupName -l "West US" -``` - -Deploy a template with the application name "ApplicationName" using following command, during its execution enter required parameters when asked - -``` -az group deployment create -g ResourceGroupName -n ApplicationName --template-file template.json -``` +1. Deploy a template by using following command, during its execution enter required parameters when asked + ``` + az group deployment create \ + --name AlertLogicCollector \ + --resource-group AlertLogicCollect \ + --template-uri "https://raw.githubusercontent.com/alertlogic/azure-collector/master/template.json" + ``` Wait until it is deployed successfully. +## Verify the Installation + +1. Go to `Azure Apps` and choose your function. The last log under `Functions-> Master-> Monitor` should have OK status and should not contain any error messages. +1. Log into Alertlogic CloudDefender and navigate into `Log Manager -> Sources` page. Check new O365 log source (with a name provided during `az group deployment create` above) has been created and source status is `ok`. # How It Works ## Master Function -The `Master` function is a timer trigger function which is responcible for: +The `Master` function is a timer trigger function which is responsible for: - registering the Azure web app In Alertlogic backend; - reporting health-checks to the backed; - performing log source configuration updates, which happen via Alertlogic UI. @@ -154,24 +179,24 @@ The `Collector` function exposes an HTTP API endpoint `https:///o365/w ] ``` -A notification contains a link to the actual lo data which is retrieved by the `Collector`, wrapped into a protobuf structure [TBD link]() and is sent into Alertlogic Ingest service. +A notification contains a link to the actual data which is retrieved by the `Collector`, wrapped into a protobuf structure [TBD link]() and is sent into Alertlogic Ingest service. # Local Development -1. Clone repo `git clone git@github.com:alertlogic/o365-collector.git` -1. `cd o365-collector` +1. Clone repo `git clone git@github.com:alertlogic/azure-collector.git` +1. `cd azure-collector` 1. Run `./local_dev/setup.sh` 1. Edit `./local_dev/dev_config.js` 1. Run Master function locally: `npm run local-master` 1. Run Updater function locally: `npm run local-updater` 1. Run O365WebHook function locally: `npm run local-o365webhook` -1. Run `npm test` in order to perform code analisys. +1. Run `npm test` in order to perform code analysis. Please use the following [code style](https://github.com/airbnb/javascript) as much as possible. ## Setting environment in dev_config.js -- `process.env.APP_TENANT_ID` - The GUID of the tenant ie. 'alazurealertlogic.onmicrosoft.com' +- `process.env.APP_TENANT_ID` - The GUID of the tenant i.e. 'alazurealertlogic.onmicrosoft.com' - `process.env.CUSTOMCONNSTR_APP_CLIENT_ID` - The GUID of your application that created the subscription. You can obtain it from _Azure_ -> _AD_ -> _App registrations_ -> _Your app name_ - `process.env.CUSTOMCONNSTR_APP_CLIENT_SECRET` - A secret key of your application from _App Registrations_. @@ -182,8 +207,6 @@ You can obtain it from _Azure_ -> _AD_ -> _App registrations_ -> _Your app name_ # Known Issues/ Open Questions - Sometimes deployments fail after siteSync action. We need better updater to handle that in order not to wait for 12 hours for the next update attempt. -- Put correct metadata into log batches. -- Initial Azure Function deployment may take up to 45 minutes. # Useful Links diff --git a/Updater/index.js b/Updater/index.js index f60c85f..c31c556 100644 --- a/Updater/index.js +++ b/Updater/index.js @@ -23,16 +23,16 @@ module.exports = function (context, AlertlogicUpdaterTimer) { requestNewToken(context, creds, function(tokenError, adToken) { if (tokenError) { - context.log('Error getting AD token: ', + context.log.error('Error getting AD token: ', tokenError.statusCode, tokenError.statusMessage); - context.done(); + context.done(tokenError); } else { siteSync(context, adToken, function(syncError) { if (syncError) { - context.log('Site sync failed: ', syncError); - context.done(); + context.log.error('Site sync failed: ', syncError); + context.done(syncError); } else { - context.log('Site sync OK'); + context.log.info('Site sync OK'); context.done(); } }); diff --git a/azure_doc.md b/azure_doc.md deleted file mode 100644 index 940327a..0000000 --- a/azure_doc.md +++ /dev/null @@ -1,32 +0,0 @@ -# Azure unofficial doc -This document should contain clarification of official documentation and behavior of Azure in practice. - -## Office 365 Management Activity API reference -The documentation about [Office 365 API](https://msdn.microsoft.com/en-us/office-365/office-365-management-activity-api-reference) -contains several either mistakes or behavior we could not observer. - -### Notification failure and retry -The notification shall retry delivery in case of failure. If it encounter excessive failures it shall increase the time between retries. -That may potentially lead to disabling the webhook. However we could not observer any message redelivery neither disabling the webhook. -The webhook was invoked 8 times and contained 19 messages (contents) in total. - -### Endpoint with startTime and endTime -Either both must be present or omitted and the difference between each of them can not be larger than 24h. It is incorrect and it doesn't -work with both parameters. It worked with `startTime` only and the difference between `startTime` and now can be more than 24h. - - -## Azure Function - -### Return status -There should be at least three ways how to specify return status (code) according this -(doc)[https://docs.microsoft.com/en-us/azure/azure-functions/functions-reference-node]. However the only way which works is: -``` -context.res.headers = {}; -context.res.status = 500; -``` - -### Status in Invocation log -The status (either red cross or green tick) most likely indicates whether the function crashes or finishes with `context.done()`. It doesn't -reflect return status code. - - diff --git a/local_dev/master_local_dev.js b/local_dev/master_local_dev.js index eee3bca..6abd5b4 100644 --- a/local_dev/master_local_dev.js +++ b/local_dev/master_local_dev.js @@ -1,3 +1,5 @@ +const util = require('util'); + var devConfig = require('./dev_config'); var azureFunction = require('../Master/index'); @@ -39,10 +41,19 @@ var debugContext = { bindings: { req }, - log: function () { - var util = require('util'); - var val = util.format.apply(null, arguments); - console.log(val); + log: { + error : function() { + return console.log('ERROR:', util.format.apply(null, arguments)); + }, + warn : function() { + return console.log('WARNING:', util.format.apply(null, arguments)); + }, + info : function() { + return console.log('INFO:', util.format.apply(null, arguments)); + }, + verbose : function() { + return console.log('VERBOSE:', util.format.apply(null, arguments)); + } }, done: function () { console.log('Response:', this.res); diff --git a/local_dev/o365webhook_local_dev.js b/local_dev/o365webhook_local_dev.js index d16dc03..30478bf 100644 --- a/local_dev/o365webhook_local_dev.js +++ b/local_dev/o365webhook_local_dev.js @@ -1,3 +1,5 @@ +const util = require('util'); + var devConfig = require('./dev_config'); var azureFunction = require('../O365WebHook/index'); @@ -31,10 +33,19 @@ var debugContext = { bindings: { req }, - log: function () { - var util = require('util'); - var val = util.format.apply(null, arguments); - console.log(val); + log: { + error : function() { + return console.log('ERROR:', util.format.apply(null, arguments)); + }, + warn : function() { + return console.log('WARNING:', util.format.apply(null, arguments)); + }, + info : function() { + return console.log('INFO:', util.format.apply(null, arguments)); + }, + verbose : function() { + return console.log('VERBOSE:', util.format.apply(null, arguments)); + } }, done: function () { console.log('Response:', this.res); @@ -59,24 +70,6 @@ var debugEvent = { // "notificationSent": "2017-08-13T23:03:56.050Z", // "contentExpiration": "2017-08-20T16:44:49.279Z" // } - // { - // "contentType": "Audit.AzureActiveDirectory", - // "contentId": "20170815224757748004716$20170815224757748004716$audit_azureactivedirectory$Audit_AzureActiveDirectory$IsFromNotification", - // "contentUri": "https://manage.office.com/api/v1.0/bf8d32d3-1c13-4487-af02-80dba2236485/activity/feed/audit/20170815224757748004716$20170815224757748004716$audit_azureactivedirectory$Audit_AzureActiveDirectory$IsFromNotification", - // "notificationStatus": "Failed", - // "contentCreated": "2017-08-17T12:31:25.653Z", - // "notificationSent": "2017-08-17T12:31:25.653Z", - // "contentExpiration": "2017-08-22T22:47:57.748Z" - // }, - { - "contentType": "Audit.Exchange", - "contentId": "20170905123301777014849$20170905123301777014849$audit_exchange$Audit_Exchange$IsFromNotification", - "contentUri": "https://manage.office.com/api/v1.0/bf8d32d3-1c13-4487-af02-80dba2236485/activity/feed/audit/20170905123301777014849$20170905123301777014849$audit_exchange$Audit_Exchange$IsFromNotification", - "notificationStatus": "Succeeded", - "contentCreated": "2017-09-05T12:37:27.163Z", - "notificationSent": "2017-09-05T12:37:27.163Z", - "contentExpiration": "2017-09-12T12:33:01.777Z" - } ] }; diff --git a/local_dev/updater_local_dev.js b/local_dev/updater_local_dev.js index 3581890..c31d407 100644 --- a/local_dev/updater_local_dev.js +++ b/local_dev/updater_local_dev.js @@ -1,3 +1,5 @@ +const util = require('util'); + var devConfig = require('./dev_config'); var azureFunction = require('../Updater/index'); @@ -39,10 +41,19 @@ var debugContext = { bindings: { req }, - log: function () { - var util = require('util'); - var val = util.format.apply(null, arguments); - console.log(val); + log: { + error : function() { + return console.log('ERROR:', util.format.apply(null, arguments)); + }, + warn : function() { + return console.log('WARNING:', util.format.apply(null, arguments)); + }, + info : function() { + return console.log('INFO:', util.format.apply(null, arguments)); + }, + verbose : function() { + return console.log('VERBOSE:', util.format.apply(null, arguments)); + } }, done: function () { console.log('Response:', this.res); diff --git a/package.json b/package.json index 39a64ec..61897ef 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "azure_collector", - "version": "0.0.1", + "version": "1.0.1", "dependencies": { "async": "*", "azure": "^2.0.0-preview", @@ -16,10 +16,13 @@ "local-ad": "node ./local_dev/ad_token_local_dev.js", "local-version": "node ./local_dev/version_local_dev.js", "lint": "jshint --exclude \"./node_modules/*\" **/*.js", - "test": "npm run lint" + "test": "npm run lint && mocha" }, "devDependencies": { "jshint": "^2.9.5", - "pre-commit": "^1.2.2" + "mocha": "^3.5.3", + "pre-commit": "^1.2.2", + "rewire": "^2.5.2", + "sinon": "^3.3.0" } } diff --git a/template.json b/template.json index a2386ec..ac94579 100644 --- a/template.json +++ b/template.json @@ -3,7 +3,8 @@ "contentVersion": "1.0.0.0", "parameters": { "Name": { - "type": "String" + "type": "String", + "defaultValue": "AlertLogicCollector" }, "Storage Name": { "type": "String" @@ -36,6 +37,9 @@ "Office365 Tenant Id": { "type": "String" }, + "Service Principal Id": { + "type": "String" + }, "App Client Id": { "type": "String" }, @@ -44,7 +48,7 @@ }, "Repository URL": { "type": "String", - "defaultValue": "https://github.com/alertlogic/o365-collector.git" + "defaultValue": "https://github.com/alertlogic/azure-collector.git" }, "Repository Branch": { "type": "String", @@ -54,8 +58,11 @@ "variables": { "location": "[resourceGroup().location]", "resourceGroupName": "[resourceGroup().name]", + "resourceGroupId": "[resourceGroup().id]", + "roleAssignmentId": "[guid(uniqueString( resourceGroup().id, deployment().name ))]", "subscriptionId": "[split(subscription().id, '/')[2]]", - "tenantId": "[subscription().tenantId]" + "tenantId": "[subscription().tenantId]", + "contributor": "[concat('/subscriptions/', subscription().subscriptionId, '/providers/Microsoft.Authorization/roleDefinitions/', 'b24988ac-6180-42a0-ab88-20f7382dd24c')]" }, "resources": [ { @@ -80,6 +87,14 @@ "name": "FUNCTIONS_EXTENSION_VERSION", "value": "~1" }, + { + "name": "SCM_USE_FUNCPACK", + "value": "1" + }, + { + "name": "SCM_POST_DEPLOYMENT_ACTIONS_PATH", + "value": "PostDeploymentActions" + }, { "name": "WEBSITE_CONTENTAZUREFILECONNECTIONSTRING", "value": "[concat('DefaultEndpointsProtocol=https;AccountName=',parameters('Storage Name'),';AccountKey=',listKeys(resourceId('Microsoft.Storage/storageAccounts', parameters('Storage Name')), '2015-06-15').key1)]" @@ -173,6 +188,16 @@ "properties": { "accountType": "Standard_LRS" } + }, + { + "type": "Microsoft.Authorization/roleAssignments", + "name": "[variables('roleAssignmentId')]", + "apiVersion": "2015-07-01", + "properties": { + "roleDefinitionId": "[variables('contributor')]", + "principalId": "[parameters('Service Principal Id')]", + "scope": "[variables('resourceGroupId')]" + } } ] } diff --git a/test/master_test.js b/test/master_test.js new file mode 100644 index 0000000..9839161 --- /dev/null +++ b/test/master_test.js @@ -0,0 +1,61 @@ +/* ----------------------------------------------------------------------------- + * @copyright (C) 2017, Alert Logic, Inc + * @doc + * + * Unit tests for Master function + * + * @end + * ----------------------------------------------------------------------------- + */ + +var assert = require('assert'); +var rewire = require('rewire'); +var sinon = require('sinon'); + +var testMock = require('./mock'); +var m_o365mgmnt = require('../lib/o365_mgmnt'); + +var o365collector = rewire('../Master/o365collector'); + +describe('Master Function Units', function() { + var private_checkEnableAuditStreams; + var subscriptionsStartStub; + + before(function() { + private_checkEnableAuditStreams = o365collector.__get__('_checkEnableAuditStreams'); + subscriptionsStartStub = sinon.stub(m_o365mgmnt, 'subscriptionsStart').callsFake( + function fakeFn(contentType, webhook, callback) { + return callback(null); + }); + }); + after(function() { + subscriptionsStartStub.restore(); + }); + + describe('_checkEnableAuditStreams()', function() { + it('checks already enabled streams with proper webhook configs', function(done) { + process.env.O365_CONTENT_STREAMS = + '["Audit.AzureActiveDirectory", "Audit.Exchange", "Audit.SharePoint", "Audit.General"]'; + private_checkEnableAuditStreams(testMock.context, testMock.allEnabledStreams, function(err, streams){ + if (err) + return done(err); + + assert.equal(4, streams.length); + assert.equal(0, subscriptionsStartStub.callCount); + done(); + }); + }); + + it('checks subscriptionStart is called for already enabled webhooks if web app is being reinstalled', function(done) { + process.env.O365_CONTENT_STREAMS = + '["Audit.AzureActiveDirectory", "Audit.Exchange", "Audit.SharePoint", "Audit.General"]'; + private_checkEnableAuditStreams(testMock.context, testMock.oneOldEnabledStream, function(err, streams){ + if (err) + return done(err); + + assert.equal(1, subscriptionsStartStub.callCount); + done(); + }); + }); + }); +}); diff --git a/test/mock.js b/test/mock.js new file mode 100644 index 0000000..8db594e --- /dev/null +++ b/test/mock.js @@ -0,0 +1,134 @@ +/* ----------------------------------------------------------------------------- + * @copyright (C) 2017, Alert Logic, Inc + * @doc + * + * Mocks for unit testing. + * + * @end + * ----------------------------------------------------------------------------- + */ + +const util = require('util'); + +process.env.WEBSITE_HOSTNAME = 'kkuzmin-app-o365.azurewebsites.net'; +process.env.O365_CONTENT_STREAMS = '["Audit.AzureActiveDirectory", "Audit.Exchange", "Audit.SharePoint", "Audit.General"]'; +process.env.TMP = '/tmp/'; +process.env.APP_SUBSCRIPTION_ID = 'subscription-id'; +process.env.CUSTOMCONNSTR_APP_CLIENT_ID = 'client-id'; +process.env.CUSTOMCONNSTR_APP_CLIENT_SECRET = 'client-secret'; +process.env.APP_TENANT_ID = 'test.onmicrosoft.com'; +process.env.O365_TENANT_ID = 'test.onmicrosoft.com'; + +var context = { + invocationId: 'ID', + bindings: { + }, + log: { + error : function() { + return console.log('ERROR:', util.format.apply(null, arguments)); + }, + warn : function() { + return console.log('WARNING:', util.format.apply(null, arguments)); + }, + info : function() { + return console.log('INFO:', util.format.apply(null, arguments)); + }, + verbose : function() { + return console.log('VERBOSE:', util.format.apply(null, arguments)); + } + }, + done: function () { + console.log('Test response:'); + }, + res: null +}; + +var allEnabledStreams = [ + { + "contentType": "Audit.AzureActiveDirectory", + "status": "enabled", + "webhook": { + "authId": null, + "address": "https://kkuzmin-app-o365.azurewebsites.net/api/o365/webhook", + "expiration": "", + "status": "enabled" + } + }, + { + "contentType": "Audit.Exchange", + "status": "enabled", + "webhook": { + "authId": null, + "address": "https://kkuzmin-app-o365.azurewebsites.net/api/o365/webhook", + "expiration": "", + "status": "enabled" + } + }, + { + "contentType": "Audit.General", + "status": "enabled", + "webhook": { + "authId": null, + "address": "https://kkuzmin-app-o365.azurewebsites.net/api/o365/webhook", + "expiration": "", + "status": "enabled" + } + }, + { + "contentType": "Audit.SharePoint", + "status": "enabled", + "webhook": { + "authId": null, + "address": "https://kkuzmin-app-o365.azurewebsites.net/api/o365/webhook", + "expiration": "", + "status": "enabled" + } + } +]; + +var oneOldEnabledStream = [ + { + "contentType": "Audit.AzureActiveDirectory", + "status": "enabled", + "webhook": { + "authId": null, + "address": "https://old-app.azurewebsites.net/api/o365/webhook", + "expiration": "", + "status": "enabled" + } + }, + { + "contentType": "Audit.Exchange", + "status": "enabled", + "webhook": { + "authId": null, + "address": "https://kkuzmin-app-o365.azurewebsites.net/api/o365/webhook", + "expiration": "", + "status": "enabled" + } + }, + { + "contentType": "Audit.General", + "status": "enabled", + "webhook": { + "authId": null, + "address": "https://kkuzmin-app-o365.azurewebsites.net/api/o365/webhook", + "expiration": "", + "status": "enabled" + } + }, + { + "contentType": "Audit.SharePoint", + "status": "enabled", + "webhook": { + "authId": null, + "address": "https://kkuzmin-app-o365.azurewebsites.net/api/o365/webhook", + "expiration": "", + "status": "enabled" + } + } +]; + +exports.allEnabledStreams = allEnabledStreams; +exports.oneOldEnabledStream = oneOldEnabledStream; +exports.context = context;