From 824f0991585b9405069191122c1bb288bd0068c7 Mon Sep 17 00:00:00 2001 From: Jeremy Deppen Date: Sun, 2 May 2021 23:43:51 -0400 Subject: [PATCH 01/22] added cloudformation template to bootstrap infra creation --- deploy/cloudformation.yaml | 297 +++++++++++++++++++++++++++++++++++++ 1 file changed, 297 insertions(+) create mode 100644 deploy/cloudformation.yaml diff --git a/deploy/cloudformation.yaml b/deploy/cloudformation.yaml new file mode 100644 index 00000000..3d9131e6 --- /dev/null +++ b/deploy/cloudformation.yaml @@ -0,0 +1,297 @@ +--- + AWSTemplateFormatVersion: '2010-09-09' + + Description: Bucket Antivirus Quickstart Template + + Parameters: + + AVBucketType: + Type: String + Description: Specifies if the bucket to hold the AV deinitions should be "public" or "private". Only choose "public" if other accounts need to access this bucket." + Default: "private" + AllowedValues: + - "public" + - "private" + + SourceBucket: + Type: String + Description: Name of the source bucket whose objects will be scanned. If more than one source bucket, the others will have to be manually added to the AV Scanner Policy after creation. + Default: "" + AllowedPattern : ".+" + + Conditions: + publicBucket: !Equals [ !Ref AVBucketType, "public" ] + + Resources: + + S3BucketAVDefinitions: + Type: AWS::S3::Bucket + Properties: + BucketName: !Join # Append the CloudFormation StackId for unique bucket naming + - "-" + - - "antivirus-definitions" + - !Select + - 0 + - !Split + - "-" + - !Select + - 2 + - !Split + - "/" + - !Ref "AWS::StackId" + BucketEncryption: + ServerSideEncryptionConfiguration: + - ServerSideEncryptionByDefault: + SSEAlgorithm: AES256 + AccessControl: BucketOwnerFullControl + PublicAccessBlockConfiguration: + BlockPublicAcls: !If [ publicBucket, false, true ] + BlockPublicPolicy: !If [ publicBucket, false, true ] + IgnorePublicAcls: !If [ publicBucket, false, true ] + RestrictPublicBuckets: !If [ publicBucket, false, true ] + Tags: + - Key: Service + Value: bucket-antivirus + VersioningConfiguration: + Status: Suspended + + S3BucketPolicyAVDefinitions: + Type: AWS::S3::BucketPolicy + Condition: publicBucket + Properties: + Bucket: !Ref S3BucketAVDefinitions + PolicyDocument: + Statement: + - Sid: AllowPublic + Action: + - s3:GetObject + - s3:GetObjectTagging + Effect: Allow + Principal: + AWS: + - "*" + Resource: + - !Sub [ "arn:aws:s3:::${BucketName}/*", { BucketName: !Ref S3BucketAVDefinitions } ] + + IamRoleAVDefinitions: + Type: 'AWS::IAM::Role' + Properties: + RoleName: AVDefinitionsLambdaRole + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - 'sts:AssumeRole' + Tags: + - Key: Service + Value: bucket-antivirus + + IamRoleAVScanner: + Type: 'AWS::IAM::Role' + Properties: + RoleName: AVScannerLambdaRole + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - 'sts:AssumeRole' + Tags: + - Key: Service + Value: bucket-antivirus + + IamPolicyAVDefinitions: + Type: AWS::IAM::Policy + DependsOn: + - S3BucketAVDefinitions + - IamRoleAVDefinitions + Properties: + PolicyName: AVDefinitionsLambdaPolicy + Roles: + - !Ref IamRoleAVDefinitions + PolicyDocument: + Version: "2012-10-17" + Statement: + - Sid: WriteCloudWatchLogs + Effect: Allow + Action: + - "logs:CreateLogGroup" + - "logs:CreateLogStream" + - "logs:PutLogEvents" + Resource: "*" + - Sid: S3GetAndPutWithTagging + Effect: Allow + Action: + - "s3:GetObject" + - "s3:GetObjectTagging" + - "s3:PutObject" + - "s3:PutObjectTagging" + - "s3:PutObjectVersionTagging" + Resource: + - !Sub [ "arn:aws:s3:::${BucketName}/*", { BucketName: !Ref S3BucketAVDefinitions } ] + - Sid: S3HeadObject + Effect: Allow + Action: + - "s3:ListBucket" + Resource: + - !Sub [ "arn:aws:s3:::${BucketName}/*", { BucketName: !Ref S3BucketAVDefinitions } ] + - !Sub [ "arn:aws:s3:::${BucketName}", { BucketName: !Ref S3BucketAVDefinitions } ] + + IamPolicyAVScanner: + Type: AWS::IAM::Policy + DependsOn: + - S3BucketAVDefinitions + - IamRoleAVScanner + Properties: + PolicyName: AVScannerLambdaPolicy + Roles: + - !Ref IamRoleAVScanner + PolicyDocument: + Version: "2012-10-17" + Statement: + - Sid: WriteCloudWatchLogs + Effect: Allow + Action: + - "logs:CreateLogGroup" + - "logs:CreateLogStream" + - "logs:PutLogEvents" + Resource: "*" + - Sid: S3AVScan + Effect: Allow + Action: + - "s3:GetObject" + - "s3:GetObjectTagging" + - "s3:GetObjectVersion" + - "s3:PutObjectTagging" + - "s3:PutObjectVersionTagging" + Resource: + - !Sub [ "arn:aws:s3:::${SourceBucketName}/*", { SourceBucketName: !Ref SourceBucket } ] + - Sid: S3AVDefinitions + Effect: Allow + Action: + - "s3:GetObject" + - "s3:GetObjectTagging" + Resource: + - !Sub [ "arn:aws:s3:::${BucketName}/*", { BucketName: !Ref S3BucketAVDefinitions } ] + - Sid: KmsDecrypt + Effect: Allow + Action: + - "kms:Decrypt" + Resource: + - !Sub [ "arn:aws:s3:::${SourceBucketName}/*", { SourceBucketName: !Ref SourceBucket } ] + - Sid: SNSPublic + Effect: Allow + Action: + - "sns:Publish" + Resource: + - "arn:aws:sns:::" + - "arn:aws:sns:::" + - Sid: S3HeadObject + Effect: Allow + Action: + - "s3:ListBucket" + Resource: + - !Sub [ "arn:aws:s3:::${BucketName}/*", { BucketName: !Ref S3BucketAVDefinitions } ] + - !Sub [ "arn:aws:s3:::${BucketName}", { BucketName: !Ref S3BucketAVDefinitions } ] + + LambdaAVUpdateDefinitions: + Type: AWS::Lambda::Function + DependsOn: + - S3BucketAVDefinitions + Properties: + FunctionName: avUpdateDefinitions + Description: LambdaFunction to update the AntiVirus definitions in the AV Definitions bucket. + Runtime: python3.7 + Code: + ZipFile: | + import json + def lambda_handler(event, context): + return { + 'statusCode': 200, 'body': json.dumps('Hello from Lambda!') + } + Handler: "update.lambda_handler" + MemorySize: 1024 + Timeout: 300 + Role: !GetAtt [ IamRoleAVDefinitions, Arn ] + Environment: + Variables: + AV_DEFINITION_S3_BUCKET: !Ref S3BucketAVDefinitions + Tags: + - Key: Service + Value: bucket-antivirus + + LambdaAVUpdateDefinitionsSchedule: + Type: "AWS::Events::Rule" + DependsOn: + - LambdaAVUpdateDefinitions + Properties: + Name: LambdaAVUpdateDefinitionsSchedule + Description: A schedule for the AV Update Definitions Lambda function. + ScheduleExpression: rate(3 hours) + State: ENABLED + Targets: + - Arn: !Sub ${LambdaAVUpdateDefinitions.Arn} + Id: LambdaAVUpdateDefinitionsSchedule + + LambdaAVUpdateDefinitionsSchedulePermission: + Type: "AWS::Lambda::Permission" + DependsOn: + - LambdaAVUpdateDefinitionsSchedule + Properties: + Action: 'lambda:InvokeFunction' + FunctionName: !Sub ${LambdaAVUpdateDefinitions.Arn} + Principal: 'events.amazonaws.com' + SourceArn: !Sub ${LambdaAVUpdateDefinitionsSchedule.Arn} + + LambdaAVScanner: + Type: AWS::Lambda::Function + Properties: + FunctionName: avScanner + Description: LambdaFunction to scan newly uploaded objects in S3. + Runtime: python3.7 + Code: + ZipFile: | + import json + def lambda_handler(event, context): + return { + 'statusCode': 200, 'body': json.dumps('Hello from Lambda!') + } + Handler: "scan.lambda_handler" + MemorySize: 1400 + Timeout: 300 + Role: !GetAtt [ IamRoleAVScanner, Arn ] + Environment: + Variables: + AV_DEFINITION_S3_BUCKET: !Ref S3BucketAVDefinitions + Tags: + - Key: Service + Value: bucket-antivirus + + + + Outputs: + + S3BucketAvDefinitions: + Value: !Ref S3BucketAVDefinitions + Description: S3 Bucket for the AV Definitions + + LambdaAVUpdateDefinitions: + Value: !Ref LambdaAVUpdateDefinitions + Description: Lambda function to update the Antivirus Definitions in its respective bucket + + LambdaAVScanner: + Value: !Ref LambdaAVScanner + Description: Lambda function to scan newly created S3 objects + + IamRoleAVScanner: + Value: !Ref IamRoleAVScanner + Description: IAM Role used by the Lambda Scanner function. Edit its policy to add/change source S3 buckets, and also to enable SNS functionality if desired + + \ No newline at end of file From a97b50de7335d334f6af33c7388d6a5204c52f9b Mon Sep 17 00:00:00 2001 From: Jeremy Deppen Date: Mon, 3 May 2021 08:33:55 -0400 Subject: [PATCH 02/22] bumping av-scanner lambda memory up --- deploy/cloudformation.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/cloudformation.yaml b/deploy/cloudformation.yaml index 3d9131e6..70fc4d6c 100644 --- a/deploy/cloudformation.yaml +++ b/deploy/cloudformation.yaml @@ -264,7 +264,7 @@ 'statusCode': 200, 'body': json.dumps('Hello from Lambda!') } Handler: "scan.lambda_handler" - MemorySize: 1400 + MemorySize: 1500 Timeout: 300 Role: !GetAtt [ IamRoleAVScanner, Arn ] Environment: From bfd645d12b081a78652f6a31ca94b1d15b87173f Mon Sep 17 00:00:00 2001 From: Jeremy Deppen Date: Mon, 3 May 2021 11:33:46 -0400 Subject: [PATCH 03/22] updated readme for cloudformation create --- README.md | 210 +++++------------------------------------------------- 1 file changed, 16 insertions(+), 194 deletions(-) diff --git a/README.md b/README.md index e0ce5cb6..05e9e668 100644 --- a/README.md +++ b/README.md @@ -34,207 +34,29 @@ or INFECTED, along with the date and time of the scan. ### Build from Source -To build the archive to upload to AWS Lambda, run `make`. The build process is completed using +To build the archive to upload to AWS Lambda, run `make all`. The build process is completed using the [amazonlinux](https://hub.docker.com/_/amazonlinux/) [Docker](https://www.docker.com) image. The resulting archive will be built at `build/lambda.zip`. This file will be uploaded to AWS for both Lambda functions below. -### AV Definition Bucket +### Create Relevant AWS Infra via CloudFormation +Use CloudFormation with the `cloudformation.yaml` located in the `deploy/` directory to quickly spin up the AWS infra needed to run this project. CloudFormation will create: +* An S3 bucket that will store AntiVirus definitions. +* A Lambda Function called `avUpdateDefinitions` that will update the AV Definitions in the S3 Bucket every 3 hours. +This function accesses the user’s above S3 Bucket to download updated definitions using `freshclam`. +* A Lambda Function called `avScanner` that is triggered on each new S3 object creation which scans the object and tags it appropriately. It is created with `1600mb` of memory which should be enough, however if you start to see timeouts, this memory may have to be bumped up. In the past, we recommended using `1024mb`, but that has started causing Lambda timeouts and bumping this memory has resolved it. -Create an s3 bucket to store current antivirus definitions. This -provides the fastest download speeds for the scanner. This bucket can -be kept as private. +Running CloudFormation, it will ask for 2 inputs for this stack: +1. BucketType: `private` (default) or `public`. This is applied to the S3 bucket that stores the AntiVirus definitions. We recommend to only use `public` when other AWS accounts need access to this bucket. +2. SourceBucket: [a non-empty string]. The name (do not include `s3://`) of the S3 bucket that will have its objects scanned. _Note - this is just used to create the IAM Policy, you can add/change source buckets later via the IAM Policy that CloudFormation outputs_ -To allow public access, useful for other accounts, -add the following policy to the bucket. +After the Stack has successfully created, there are 3 manual processes that still have to be done: +1. Upload the `build/lambda.zip` file that was created by running `make all` to the `avUpdateDefinitions` and `avScanner` Lambda functions via the Lambda Console. +2. To trigger the Scanner function on new S3 objects, go to the `avScanner` Lambda function console, navigate to `Configuration` -> `Trigger` -> `Add Trigger` -> Search for S3, and choose your bucket(s) and select `All object create events`, then click `Add`. _Note - if you chose more than 1 bucket as the source, or chose a different bucket than the Source Bucket in the CloudFormation parameter, you will have to also edit the IAM Role to reflect these new buckets (see next section)_ +3. Navigate to the `avUpdateDefinitions` Lambda function and manually trigger the function to get the initial Clam definitions in the bucket (instead of waiting for the 3 hour trigger to happen). Do this by clicking the `Test` section, and then clicking the orange `test` button. The function should take a few seconds to execute, and when finished you should see the `clam_defs` in the `av-definitions` S3 bucket. -```json -{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "AllowPublic", - "Effect": "Allow", - "Principal": "*", - "Action": [ - "s3:GetObject", - "s3:GetObjectTagging" - ], - "Resource": "arn:aws:s3:::/*" - } - ] -} -``` - -### Definition Update Lambda - -This function accesses the user’s ClamAV instance to download -updated definitions using `freshclam`. It is recommended to run -this every 3 hours to stay protected from the latest threats. - -1. Create the archive using the method in the - [Build from Source](#build-from-source) section. -2. From the AWS Lambda Dashboard, click **Create function** -3. Choose **Author from scratch** on the *Create function* page -4. Name your function `bucket-antivirus-update` when prompted on the -*Configure function* step. -5. Set *Runtime* to `Python 3.7` -6. Create a new role name `bucket-antivirus-update` that uses the -following policy document - - ```json - { - "Version":"2012-10-17", - "Statement":[ - { - "Sid":"WriteCloudWatchLogs", - "Effect":"Allow", - "Action":[ - "logs:CreateLogGroup", - "logs:CreateLogStream", - "logs:PutLogEvents" - ], - "Resource":"*" - }, - { - "Sid":"s3GetAndPutWithTagging", - "Action":[ - "s3:GetObject", - "s3:GetObjectTagging", - "s3:PutObject", - "s3:PutObjectTagging", - "s3:PutObjectVersionTagging" - ], - "Effect":"Allow", - "Resource":[ - "arn:aws:s3:::/*" - ] - }, - { - "Sid": "s3HeadObject", - "Effect": "Allow", - "Action": "s3:ListBucket", - "Resource": [ - "arn:aws:s3:::/*", - "arn:aws:s3:::" - ] - } - ] - } - ``` - -7. Click next to go to the Configuration page -8. Add a trigger from the left of **CloudWatch Event** using `rate(3 hours)` -for the **Schedule expression**. Be sure to check **Enable trigger** -9. Choose **Upload a ZIP file** for *Code entry type* and select the archive -downloaded in step 1. -10. Add a single environment variable named `AV_DEFINITION_S3_BUCKET` -and set its value to the name of the bucket created to store your AV -definitions. -11. Set *Lambda handler* to `update.lambda_handler` -12. Under *Basic Settings*, set *Timeout* to **5 minutes** and *Memory* to -**1024** -13. Save and test your function. If prompted for test data, just use -the default provided. - -### AV Scanner Lambda - -1. Create the archive using the method in the - [Build from Source](#build-from-source) section. -2. From the AWS Lambda Dashboard, click **Create function** -3. Choose **Author from scratch** on the *Create function* page -4. Name your function `bucket-antivirus-function` -5. Set *Runtime* to `Python 3.7` -6. Create a new role name `bucket-antivirus-function` that uses the -following policy document - - ```json - { - "Version":"2012-10-17", - "Statement":[ - { - "Sid":"WriteCloudWatchLogs", - "Effect":"Allow", - "Action":[ - "logs:CreateLogGroup", - "logs:CreateLogStream", - "logs:PutLogEvents" - ], - "Resource":"*" - }, - { - "Sid":"s3AntiVirusScan", - "Action":[ - "s3:GetObject", - "s3:GetObjectTagging", - "s3:GetObjectVersion", - "s3:PutObjectTagging", - "s3:PutObjectVersionTagging" - ], - "Effect":"Allow", - "Resource": [ - "arn:aws:s3:::/*", - "arn:aws:s3:::/*" - ] - }, - { - "Sid":"s3AntiVirusDefinitions", - "Action":[ - "s3:GetObject", - "s3:GetObjectTagging" - ], - "Effect":"Allow", - "Resource": [ - "arn:aws:s3:::/*" - ] - }, - { - "Sid":"kmsDecrypt", - "Action":[ - "kms:Decrypt" - ], - "Effect":"Allow", - "Resource": [ - "arn:aws:s3:::/*", - "arn:aws:s3:::/*" - ] - }, - { - "Sid":"snsPublish", - "Action": [ - "sns:Publish" - ], - "Effect":"Allow", - "Resource": [ - "arn:aws:sns:::", - "arn:aws:sns:::" - ] - }, - { - "Sid":"s3HeadObject", - "Effect":"Allow", - "Action":"s3:ListBucket", - "Resource":[ - "arn:aws:s3:::/*", - "arn:aws:s3:::" - ] - } - ] - } - ``` - -7. Click *next* to head to the Configuration page -8. Add a new trigger of type **S3 Event** using `ObjectCreate(all)`. -9. Choose **Upload a ZIP file** for *Code entry type* and select the archive -created in step 1. -10. Set *Lambda handler* to `scan.lambda_handler` -11. Add a single environment variable named `AV_DEFINITION_S3_BUCKET` -and set its value to the name of the bucket created to store your AV -definitions. If your bucket is `s3://my-bucket`, the value should be `my-bucket`. -12. Under *Basic settings*, set *Timeout* to **5 minutes** and *Memory* to -**1024** -13. Save the function. Testing is easiest performed by uploading a -file to the bucket configured as the trigger in step 4. +#### Adding or Changing Source Buckets +Changing or adding Source Buckets is done by editing the `AVScannerLambdaRole` IAM Role. More specifically, the `S3AVScan` and `KmsDecrypt` parts of that IAM Role's policy. ### S3 Events From a494d9a619dd9e4a643cd596bc4934601c0a47d0 Mon Sep 17 00:00:00 2001 From: Jeremy Deppen Date: Mon, 3 May 2021 11:35:08 -0400 Subject: [PATCH 04/22] updated readme for cloudformation create --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 05e9e668..33cf68e8 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ Use CloudFormation with the `cloudformation.yaml` located in the `deploy/` direc * An S3 bucket that will store AntiVirus definitions. * A Lambda Function called `avUpdateDefinitions` that will update the AV Definitions in the S3 Bucket every 3 hours. This function accesses the user’s above S3 Bucket to download updated definitions using `freshclam`. -* A Lambda Function called `avScanner` that is triggered on each new S3 object creation which scans the object and tags it appropriately. It is created with `1600mb` of memory which should be enough, however if you start to see timeouts, this memory may have to be bumped up. In the past, we recommended using `1024mb`, but that has started causing Lambda timeouts and bumping this memory has resolved it. +* A Lambda Function called `avScanner` that is triggered on each new S3 object creation which scans the object and tags it appropriately. It is created with `1600mb` of memory which should be enough, however if you start to see function timeouts, this memory may have to be bumped up. In the past, we recommended using `1024mb`, but that has started causing Lambda timeouts and bumping this memory has resolved it. Running CloudFormation, it will ask for 2 inputs for this stack: 1. BucketType: `private` (default) or `public`. This is applied to the S3 bucket that stores the AntiVirus definitions. We recommend to only use `public` when other AWS accounts need access to this bucket. From a689f3a936b4d39d80c9c44fe3db9ba5fefc4451 Mon Sep 17 00:00:00 2001 From: Jeremy Deppen Date: Mon, 3 May 2021 11:38:45 -0400 Subject: [PATCH 05/22] updated readme for cloudformation create --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 33cf68e8..dfc75afe 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ Running CloudFormation, it will ask for 2 inputs for this stack: After the Stack has successfully created, there are 3 manual processes that still have to be done: 1. Upload the `build/lambda.zip` file that was created by running `make all` to the `avUpdateDefinitions` and `avScanner` Lambda functions via the Lambda Console. -2. To trigger the Scanner function on new S3 objects, go to the `avScanner` Lambda function console, navigate to `Configuration` -> `Trigger` -> `Add Trigger` -> Search for S3, and choose your bucket(s) and select `All object create events`, then click `Add`. _Note - if you chose more than 1 bucket as the source, or chose a different bucket than the Source Bucket in the CloudFormation parameter, you will have to also edit the IAM Role to reflect these new buckets (see next section)_ +2. To trigger the Scanner function on new S3 objects, go to the `avScanner` Lambda function console, navigate to `Configuration` -> `Trigger` -> `Add Trigger` -> Search for S3, and choose your bucket(s) and select `All object create events`, then click `Add`. _Note - if you chose more than 1 bucket as the source, or chose a different bucket than the Source Bucket in the CloudFormation parameter, you will have to also edit the IAM Role to reflect these new buckets (see "Adding or Changing Source Buckets")_ 3. Navigate to the `avUpdateDefinitions` Lambda function and manually trigger the function to get the initial Clam definitions in the bucket (instead of waiting for the 3 hour trigger to happen). Do this by clicking the `Test` section, and then clicking the orange `test` button. The function should take a few seconds to execute, and when finished you should see the `clam_defs` in the `av-definitions` S3 bucket. #### Adding or Changing Source Buckets From 26f45eb02c15914236fcd91e97a00b6c6a36d69e Mon Sep 17 00:00:00 2001 From: Jeremy Deppen Date: Mon, 3 May 2021 12:21:34 -0400 Subject: [PATCH 06/22] updated pre commit hooks for cloudformation --- .pre-commit-config.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 806c3de2..bd15c799 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,10 +13,13 @@ repos: - repo: git://github.com/pre-commit/pre-commit-hooks rev: v2.2.3 hooks: + - id: cfn-python-lint + files: deploy/ - id: check-ast - id: check-json - id: check-merge-conflict - id: check-yaml + exlude: deploy/ - id: debug-statements - id: detect-private-key - id: fix-encoding-pragma From 34425a0066c81317597694dcb8e7302d5a6e3b9b Mon Sep 17 00:00:00 2001 From: Jeremy Deppen Date: Mon, 3 May 2021 12:25:39 -0400 Subject: [PATCH 07/22] updated pre commit hooks for cloudformation --- .pre-commit-config.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bd15c799..d85a21fe 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,8 +13,6 @@ repos: - repo: git://github.com/pre-commit/pre-commit-hooks rev: v2.2.3 hooks: - - id: cfn-python-lint - files: deploy/ - id: check-ast - id: check-json - id: check-merge-conflict @@ -31,3 +29,9 @@ repos: hooks: - id: markdownlint entry: markdownlint --ignore .github/*.md + + - repo: github.com/aws-cloudformation/cfn-python-lint + rev: stable + hooks: + - id: cfn-python-lint + files: deploy/ From a984c6b96bd1a3ac4355e1898f3cbd5345aab56d Mon Sep 17 00:00:00 2001 From: Jeremy Deppen Date: Mon, 3 May 2021 12:26:57 -0400 Subject: [PATCH 08/22] updated pre commit hooks for cloudformation --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d85a21fe..7f70e533 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -30,7 +30,7 @@ repos: - id: markdownlint entry: markdownlint --ignore .github/*.md - - repo: github.com/aws-cloudformation/cfn-python-lint + - repo: git://github.com/aws-cloudformation/cfn-python-lint rev: stable hooks: - id: cfn-python-lint From 58110becaf9ba2f85776a5004e6c4506a15c5826 Mon Sep 17 00:00:00 2001 From: Jeremy Deppen Date: Mon, 3 May 2021 12:35:38 -0400 Subject: [PATCH 09/22] bumping pre-commit version to latest --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index a527b641..564f9829 100644 --- a/Makefile +++ b/Makefile @@ -41,7 +41,7 @@ archive: clean ## Create the archive for AWS lambda .PHONY: pre_commit_install ## Ensure that pre-commit hook is installed and kept up to date pre_commit_install: .git/hooks/pre-commit ## Ensure pre-commit is installed .git/hooks/pre-commit: /usr/local/bin/pre-commit - pip install pre-commit==1.18.3 + pip install pre-commit==1.12.1 pre-commit install pre-commit install-hooks From 7583865b062918cda26b4ba42d5982435219e9e9 Mon Sep 17 00:00:00 2001 From: Jeremy Deppen Date: Mon, 3 May 2021 12:36:30 -0400 Subject: [PATCH 10/22] bumping pre-commit version to latest --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 564f9829..b65c4275 100644 --- a/Makefile +++ b/Makefile @@ -41,7 +41,7 @@ archive: clean ## Create the archive for AWS lambda .PHONY: pre_commit_install ## Ensure that pre-commit hook is installed and kept up to date pre_commit_install: .git/hooks/pre-commit ## Ensure pre-commit is installed .git/hooks/pre-commit: /usr/local/bin/pre-commit - pip install pre-commit==1.12.1 + pip install pre-commit==2.12.1 pre-commit install pre-commit install-hooks From b773c9eb9495f07ba89acfc540f003c8df41e54d Mon Sep 17 00:00:00 2001 From: Jeremy Deppen Date: Mon, 3 May 2021 12:37:49 -0400 Subject: [PATCH 11/22] bumping pre-commit version to latest --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 477a8e7b..1f677cf9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -47,7 +47,7 @@ jobs: - restore_cache: keys: - pre-commit-dot-cache-{{ checksum ".pre-commit-config.yaml" }} - - run: sudo pip install pre-commit==1.18.3 + - run: sudo pip install pre-commit==2.12.1 - run: pre-commit install-hooks - save_cache: From 4a3b0e59e5ac53cd100dffce76b88e140fc17359 Mon Sep 17 00:00:00 2001 From: Jeremy Deppen Date: Mon, 3 May 2021 12:40:38 -0400 Subject: [PATCH 12/22] bumping pre-commit version to latest --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7f70e533..a16c8614 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,7 +17,7 @@ repos: - id: check-json - id: check-merge-conflict - id: check-yaml - exlude: deploy/ + exclude: deploy/ - id: debug-statements - id: detect-private-key - id: fix-encoding-pragma From bee01c16c0fd2b871afc4cd31a95dc67e6e1f101 Mon Sep 17 00:00:00 2001 From: Jeremy Deppen Date: Mon, 3 May 2021 12:45:18 -0400 Subject: [PATCH 13/22] bumping pre-commit version to latest --- .pre-commit-config.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a16c8614..4bb31e83 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,7 +31,6 @@ repos: entry: markdownlint --ignore .github/*.md - repo: git://github.com/aws-cloudformation/cfn-python-lint - rev: stable hooks: - id: cfn-python-lint files: deploy/ From 6751dc5189613de14124eadcd48395e40ec2e8d5 Mon Sep 17 00:00:00 2001 From: Jeremy Deppen Date: Mon, 3 May 2021 12:47:12 -0400 Subject: [PATCH 14/22] modifying python black version --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4bb31e83..49e71444 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ repos: - repo: https://github.com/ambv/black - rev: stable + rev: 19.3b0 hooks: - id: black language_version: python3.7 From 4cfab739f0303f70168d7a4f974947476e5b33e5 Mon Sep 17 00:00:00 2001 From: Jeremy Deppen Date: Mon, 3 May 2021 12:49:18 -0400 Subject: [PATCH 15/22] modifying cloudformation lint version --- .pre-commit-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 49e71444..a6cdf395 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,6 +31,7 @@ repos: entry: markdownlint --ignore .github/*.md - repo: git://github.com/aws-cloudformation/cfn-python-lint + rev: v0.49.0 hooks: - id: cfn-python-lint files: deploy/ From 015a0fc30b9bfe7be7da18e248feffb82f223e3c Mon Sep 17 00:00:00 2001 From: Jeremy Deppen Date: Mon, 3 May 2021 12:58:08 -0400 Subject: [PATCH 16/22] linted cloudformation template --- deploy/cloudformation.yaml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/deploy/cloudformation.yaml b/deploy/cloudformation.yaml index 70fc4d6c..fccda579 100644 --- a/deploy/cloudformation.yaml +++ b/deploy/cloudformation.yaml @@ -109,9 +109,6 @@ IamPolicyAVDefinitions: Type: AWS::IAM::Policy - DependsOn: - - S3BucketAVDefinitions - - IamRoleAVDefinitions Properties: PolicyName: AVDefinitionsLambdaPolicy Roles: @@ -146,9 +143,6 @@ IamPolicyAVScanner: Type: AWS::IAM::Policy - DependsOn: - - S3BucketAVDefinitions - - IamRoleAVScanner Properties: PolicyName: AVScannerLambdaPolicy Roles: @@ -203,8 +197,6 @@ LambdaAVUpdateDefinitions: Type: AWS::Lambda::Function - DependsOn: - - S3BucketAVDefinitions Properties: FunctionName: avUpdateDefinitions Description: LambdaFunction to update the AntiVirus definitions in the AV Definitions bucket. From d6ace6cf2fc55601c4e50c49418c5996002d2462 Mon Sep 17 00:00:00 2001 From: Jeremy Deppen Date: Mon, 3 May 2021 13:14:56 -0400 Subject: [PATCH 17/22] linted cloudformation template --- README.md | 11 ++++++++--- deploy/cloudformation.yaml | 4 +--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index dfc75afe..97a07104 100644 --- a/README.md +++ b/README.md @@ -40,22 +40,27 @@ the [amazonlinux](https://hub.docker.com/_/amazonlinux/) [Docker](https://www.do uploaded to AWS for both Lambda functions below. ### Create Relevant AWS Infra via CloudFormation + Use CloudFormation with the `cloudformation.yaml` located in the `deploy/` directory to quickly spin up the AWS infra needed to run this project. CloudFormation will create: -* An S3 bucket that will store AntiVirus definitions. -* A Lambda Function called `avUpdateDefinitions` that will update the AV Definitions in the S3 Bucket every 3 hours. + +- An S3 bucket that will store AntiVirus definitions. +- A Lambda Function called `avUpdateDefinitions` that will update the AV Definitions in the S3 Bucket every 3 hours. This function accesses the user’s above S3 Bucket to download updated definitions using `freshclam`. -* A Lambda Function called `avScanner` that is triggered on each new S3 object creation which scans the object and tags it appropriately. It is created with `1600mb` of memory which should be enough, however if you start to see function timeouts, this memory may have to be bumped up. In the past, we recommended using `1024mb`, but that has started causing Lambda timeouts and bumping this memory has resolved it. +- A Lambda Function called `avScanner` that is triggered on each new S3 object creation which scans the object and tags it appropriately. It is created with `1600mb` of memory which should be enough, however if you start to see function timeouts, this memory may have to be bumped up. In the past, we recommended using `1024mb`, but that has started causing Lambda timeouts and bumping this memory has resolved it. Running CloudFormation, it will ask for 2 inputs for this stack: + 1. BucketType: `private` (default) or `public`. This is applied to the S3 bucket that stores the AntiVirus definitions. We recommend to only use `public` when other AWS accounts need access to this bucket. 2. SourceBucket: [a non-empty string]. The name (do not include `s3://`) of the S3 bucket that will have its objects scanned. _Note - this is just used to create the IAM Policy, you can add/change source buckets later via the IAM Policy that CloudFormation outputs_ After the Stack has successfully created, there are 3 manual processes that still have to be done: + 1. Upload the `build/lambda.zip` file that was created by running `make all` to the `avUpdateDefinitions` and `avScanner` Lambda functions via the Lambda Console. 2. To trigger the Scanner function on new S3 objects, go to the `avScanner` Lambda function console, navigate to `Configuration` -> `Trigger` -> `Add Trigger` -> Search for S3, and choose your bucket(s) and select `All object create events`, then click `Add`. _Note - if you chose more than 1 bucket as the source, or chose a different bucket than the Source Bucket in the CloudFormation parameter, you will have to also edit the IAM Role to reflect these new buckets (see "Adding or Changing Source Buckets")_ 3. Navigate to the `avUpdateDefinitions` Lambda function and manually trigger the function to get the initial Clam definitions in the bucket (instead of waiting for the 3 hour trigger to happen). Do this by clicking the `Test` section, and then clicking the orange `test` button. The function should take a few seconds to execute, and when finished you should see the `clam_defs` in the `av-definitions` S3 bucket. #### Adding or Changing Source Buckets + Changing or adding Source Buckets is done by editing the `AVScannerLambdaRole` IAM Role. More specifically, the `S3AVScan` and `KmsDecrypt` parts of that IAM Role's policy. ### S3 Events diff --git a/deploy/cloudformation.yaml b/deploy/cloudformation.yaml index fccda579..6c3fd1a3 100644 --- a/deploy/cloudformation.yaml +++ b/deploy/cloudformation.yaml @@ -284,6 +284,4 @@ IamRoleAVScanner: Value: !Ref IamRoleAVScanner - Description: IAM Role used by the Lambda Scanner function. Edit its policy to add/change source S3 buckets, and also to enable SNS functionality if desired - - \ No newline at end of file + Description: IAM Role used by the Lambda Scanner function. Edit its policy to add/change source S3 buckets, and also to enable SNS functionality if desired \ No newline at end of file From d56d97eb482dd3000d2f434950018dcf9b5a5950 Mon Sep 17 00:00:00 2001 From: Jeremy Deppen Date: Mon, 3 May 2021 13:17:46 -0400 Subject: [PATCH 18/22] linted precommit file --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a6cdf395..a881ee5d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -34,4 +34,4 @@ repos: rev: v0.49.0 hooks: - id: cfn-python-lint - files: deploy/ + files: deploy/ \ No newline at end of file From ee9b5a1bbee7a67b1f0e9e1fe0ea1927f386aa10 Mon Sep 17 00:00:00 2001 From: Jeremy Deppen Date: Mon, 3 May 2021 13:21:24 -0400 Subject: [PATCH 19/22] linted precommit file --- .circleci/config.yml | 2 +- Makefile | 2 +- README.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1f677cf9..85f04ad2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -93,4 +93,4 @@ workflows: - build: requires: - pre_commit_test - - test + - test \ No newline at end of file diff --git a/Makefile b/Makefile index b65c4275..1c839af0 100644 --- a/Makefile +++ b/Makefile @@ -63,4 +63,4 @@ scan: ./build/lambda.zip ## Run scan function locally .PHONY: update update: ./build/lambda.zip ## Run update function locally - scripts/run-update-lambda + scripts/run-update-lambda \ No newline at end of file diff --git a/README.md b/README.md index 97a07104..afcb9503 100644 --- a/README.md +++ b/README.md @@ -240,4 +240,4 @@ limitations under the License. ClamAV is released under the [GPL Version 2 License](https://github.com/vrtadmin/clamav-devel/blob/master/COPYING) and all [source for ClamAV](https://github.com/vrtadmin/clamav-devel) is available -for download on Github. +for download on Github. \ No newline at end of file From 2abe9d2033ccae9be7ec50b213bb5b44bfe534eb Mon Sep 17 00:00:00 2001 From: Jeremy Deppen Date: Mon, 3 May 2021 13:25:52 -0400 Subject: [PATCH 20/22] linted readme file --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index afcb9503..97a07104 100644 --- a/README.md +++ b/README.md @@ -240,4 +240,4 @@ limitations under the License. ClamAV is released under the [GPL Version 2 License](https://github.com/vrtadmin/clamav-devel/blob/master/COPYING) and all [source for ClamAV](https://github.com/vrtadmin/clamav-devel) is available -for download on Github. \ No newline at end of file +for download on Github. From 1b087b268c776c2e1c9688fd855b9dfe91cd0769 Mon Sep 17 00:00:00 2001 From: Jeremy Deppen Date: Mon, 3 May 2021 13:32:22 -0400 Subject: [PATCH 21/22] linted files --- .pre-commit-config.yaml | 2 +- README.md | 4 +-- deploy/cloudformation.yaml | 68 +++++++++++++++++++------------------- 3 files changed, 37 insertions(+), 37 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a881ee5d..ea714cac 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,7 +4,7 @@ repos: rev: 19.3b0 hooks: - id: black - language_version: python3.7 + language_version: python3.8 exclude: > (?x)^( scripts/gen-docs-index| diff --git a/README.md b/README.md index 97a07104..23a60b59 100644 --- a/README.md +++ b/README.md @@ -44,8 +44,8 @@ the [amazonlinux](https://hub.docker.com/_/amazonlinux/) [Docker](https://www.do Use CloudFormation with the `cloudformation.yaml` located in the `deploy/` directory to quickly spin up the AWS infra needed to run this project. CloudFormation will create: - An S3 bucket that will store AntiVirus definitions. -- A Lambda Function called `avUpdateDefinitions` that will update the AV Definitions in the S3 Bucket every 3 hours. -This function accesses the user’s above S3 Bucket to download updated definitions using `freshclam`. +- A Lambda Function called `avUpdateDefinitions` that will update the AV Definitions in the S3 Bucket every 3 hours. +This function accesses the user’s above S3 Bucket to download updated definitions using `freshclam`. - A Lambda Function called `avScanner` that is triggered on each new S3 object creation which scans the object and tags it appropriately. It is created with `1600mb` of memory which should be enough, however if you start to see function timeouts, this memory may have to be bumped up. In the past, we recommended using `1024mb`, but that has started causing Lambda timeouts and bumping this memory has resolved it. Running CloudFormation, it will ask for 2 inputs for this stack: diff --git a/deploy/cloudformation.yaml b/deploy/cloudformation.yaml index 6c3fd1a3..91fe1457 100644 --- a/deploy/cloudformation.yaml +++ b/deploy/cloudformation.yaml @@ -1,10 +1,10 @@ --- AWSTemplateFormatVersion: '2010-09-09' - + Description: Bucket Antivirus Quickstart Template - + Parameters: - + AVBucketType: Type: String Description: Specifies if the bucket to hold the AV deinitions should be "public" or "private". Only choose "public" if other accounts need to access this bucket." @@ -23,7 +23,7 @@ publicBucket: !Equals [ !Ref AVBucketType, "public" ] Resources: - + S3BucketAVDefinitions: Type: AWS::S3::Bucket Properties: @@ -54,7 +54,7 @@ Value: bucket-antivirus VersioningConfiguration: Status: Suspended - + S3BucketPolicyAVDefinitions: Type: AWS::S3::BucketPolicy Condition: publicBucket @@ -70,7 +70,7 @@ Principal: AWS: - "*" - Resource: + Resource: - !Sub [ "arn:aws:s3:::${BucketName}/*", { BucketName: !Ref S3BucketAVDefinitions } ] IamRoleAVDefinitions: @@ -109,86 +109,86 @@ IamPolicyAVDefinitions: Type: AWS::IAM::Policy - Properties: + Properties: PolicyName: AVDefinitionsLambdaPolicy Roles: - !Ref IamRoleAVDefinitions - PolicyDocument: + PolicyDocument: Version: "2012-10-17" Statement: - Sid: WriteCloudWatchLogs - Effect: Allow - Action: + Effect: Allow + Action: - "logs:CreateLogGroup" - "logs:CreateLogStream" - "logs:PutLogEvents" Resource: "*" - Sid: S3GetAndPutWithTagging - Effect: Allow - Action: + Effect: Allow + Action: - "s3:GetObject" - "s3:GetObjectTagging" - "s3:PutObject" - "s3:PutObjectTagging" - "s3:PutObjectVersionTagging" - Resource: + Resource: - !Sub [ "arn:aws:s3:::${BucketName}/*", { BucketName: !Ref S3BucketAVDefinitions } ] - Sid: S3HeadObject - Effect: Allow - Action: + Effect: Allow + Action: - "s3:ListBucket" - Resource: + Resource: - !Sub [ "arn:aws:s3:::${BucketName}/*", { BucketName: !Ref S3BucketAVDefinitions } ] - !Sub [ "arn:aws:s3:::${BucketName}", { BucketName: !Ref S3BucketAVDefinitions } ] IamPolicyAVScanner: Type: AWS::IAM::Policy - Properties: + Properties: PolicyName: AVScannerLambdaPolicy Roles: - !Ref IamRoleAVScanner - PolicyDocument: + PolicyDocument: Version: "2012-10-17" Statement: - Sid: WriteCloudWatchLogs - Effect: Allow - Action: + Effect: Allow + Action: - "logs:CreateLogGroup" - "logs:CreateLogStream" - "logs:PutLogEvents" Resource: "*" - Sid: S3AVScan - Effect: Allow - Action: + Effect: Allow + Action: - "s3:GetObject" - "s3:GetObjectTagging" - "s3:GetObjectVersion" - "s3:PutObjectTagging" - "s3:PutObjectVersionTagging" - Resource: + Resource: - !Sub [ "arn:aws:s3:::${SourceBucketName}/*", { SourceBucketName: !Ref SourceBucket } ] - - Sid: S3AVDefinitions + - Sid: S3AVDefinitions Effect: Allow Action: - "s3:GetObject" - "s3:GetObjectTagging" Resource: - !Sub [ "arn:aws:s3:::${BucketName}/*", { BucketName: !Ref S3BucketAVDefinitions } ] - - Sid: KmsDecrypt + - Sid: KmsDecrypt Effect: Allow Action: - "kms:Decrypt" Resource: - !Sub [ "arn:aws:s3:::${SourceBucketName}/*", { SourceBucketName: !Ref SourceBucket } ] - - Sid: SNSPublic - Effect: Allow + - Sid: SNSPublic + Effect: Allow Action: - "sns:Publish" Resource: - "arn:aws:sns:::" - "arn:aws:sns:::" - Sid: S3HeadObject - Effect: Allow + Effect: Allow Action: - "s3:ListBucket" Resource: @@ -231,7 +231,7 @@ Targets: - Arn: !Sub ${LambdaAVUpdateDefinitions.Arn} Id: LambdaAVUpdateDefinitionsSchedule - + LambdaAVUpdateDefinitionsSchedulePermission: Type: "AWS::Lambda::Permission" DependsOn: @@ -265,11 +265,11 @@ Tags: - Key: Service Value: bucket-antivirus - - - + + + Outputs: - + S3BucketAvDefinitions: Value: !Ref S3BucketAVDefinitions Description: S3 Bucket for the AV Definitions @@ -280,7 +280,7 @@ LambdaAVScanner: Value: !Ref LambdaAVScanner - Description: Lambda function to scan newly created S3 objects + Description: Lambda function to scan newly created S3 objects IamRoleAVScanner: Value: !Ref IamRoleAVScanner From edc2f5224895ab0a822c7322576c63ac5901d5fe Mon Sep 17 00:00:00 2001 From: Jeremy Deppen Date: Mon, 3 May 2021 13:32:42 -0400 Subject: [PATCH 22/22] python version --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ea714cac..a881ee5d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,7 +4,7 @@ repos: rev: 19.3b0 hooks: - id: black - language_version: python3.8 + language_version: python3.7 exclude: > (?x)^( scripts/gen-docs-index|