release - 0.9.1 (cloud-custodian#5564)

.codecov.yml

@@ -11,7 +11,7 @@ coverage:
   status:
     patch:
       default:
-        threshold: "10%"
+        threshold: "5%"
         only_pulls: true
     project:
       azure:

Makefile

@@ -1,3 +1,4 @@
+SELF_MAKE := $(lastword $(MAKEFILE_LIST))
 
 PKG_SET = tools/c7n_gcp tools/c7n_azure tools/c7n_kube tools/c7n_mailer tools/c7n_logexporter tools/c7n_policystream tools/c7n_trailcreator tools/c7n_org tools/c7n_sphinxext
 
@@ -9,6 +10,28 @@ install-poetry:
 	poetry install
 	for pkg in $(PKG_SET); do cd $$pkg && poetry install && cd ../..; done
 
+pkg-rebase:
+	rm -f poetry.lock
+	for pkg in $(PKG_SET); do cd $$pkg && echo $$pkg && rm -f poetry.lock && cd ../..; done
+
+	rm -f setup.py
+	for pkg in $(PKG_SET); do cd $$pkg && echo $$pkg && rm -f setup.py && cd ../..; done
+
+	rm -f requirements.txt
+	for pkg in $(PKG_SET); do cd $$pkg && echo $$pkg && rm -f requirements.txt && cd ../..; done
+
+	@$(MAKE) -f $(SELF_MAKE) pkg-update
+	git add poetry.lock
+	for pkg in $(PKG_SET); do cd $$pkg && echo $$pkg && git add poetry.lock && cd ../..; done
+
+	@$(MAKE) -f $(SELF_MAKE) pkg-gen-setup
+	git add setup.py
+	for pkg in $(PKG_SET); do cd $$pkg && echo $$pkg && git add setup.py && cd ../..; done
+
+	@$(MAKE) -f $(SELF_MAKE) pkg-gen-requirements
+	git add requirements.txt
+	for pkg in $(PKG_SET); do cd $$pkg && echo $$pkg && git add requirements.txt && cd ../..; done
+
 pkg-update:
 	poetry update
 	for pkg in $(PKG_SET); do cd $$pkg && echo $$pkg && poetry update && cd ../..; done
@@ -30,15 +53,17 @@ pkg-gen-requirements:
 	poetry export --dev --without-hashes -f requirements.txt > requirements.txt
 	for pkg in $(PKG_SET); do cd $$pkg && poetry export --without-hashes -f requirements.txt > requirements.txt && cd ../..; done
 
-pkg-publish-wheel:
-# clean up any artifacts first
-	rm -f dist/*
-	for pkg in $(PKG_SET); do cd $$pkg && rm -f dist/* && cd ../..; done
+pkg-increment:
 # increment versions
 	poetry version patch
 	for pkg in $(PKG_SET); do cd $$pkg && poetry version patch && cd ../..; done
 # generate setup
 	@$(MAKE) pkg-gen-setup
+
+pkg-publish-wheel:
+# clean up any artifacts first
+	rm -f dist/*
+	for pkg in $(PKG_SET); do cd $$pkg && rm -f dist/* && cd ../..; done
 # generate sdist
 	python setup.py bdist_wheel
 	for pkg in $(PKG_SET); do cd $$pkg && python setup.py bdist_wheel && cd ../..; done
@@ -49,6 +74,9 @@ pkg-publish-wheel:
 	twine upload -r testpypi dist/*
 	for pkg in $(PKG_SET); do cd $$pkg && twine upload -r testpypi dist/* && cd ../..; done
 
+test-poetry:
+	. test.env && poetry run pytest -n auto tests tools
+
 test:
 	./bin/tox -e py38
 

docs/source/quickstart/index.rst

@@ -44,6 +44,24 @@ To install Cloud Custodian, just run::
   (custodian) $ pip install c7n_gcp   #Install GCP Package
 
 
+Docker
+++++++
+
+To install via docker, just run::
+
+  $ docker pull cloudcustodian/c7n
+
+You'll need to export cloud provider credentials to the container
+when executing. One example, if your using environment variables for provider
+credentials::
+
+  $ docker run -it \
+    -v $(pwd)/output:/home/custodian/output \
+    -v $(pwd)/policy.yml:/home/custodian/policy.yml \
+    --env-file <(env | grep "^AWS\|^AZURE\|^GOOGLE") \
+       cloudcustodian/c7n run -v -s /home/custodian/output /home/custodian/policy.yml
+
+
 .. _explore-cc:
 
 Explore Cloud Custodian

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "c7n"
-version = "0.9.0"
+version = "0.9.1"
 description = "Cloud Custodian - Policy Rules Engine"
 authors = ["Cloud Custodian Project"]
 readme = "README.md"

requirements.txt

@@ -5,8 +5,8 @@ atomicwrites==1.3.0; sys_platform == "win32"
 attrs==19.3.0
 aws-xray-sdk==2.5.0
 bleach==3.1.4
-boto3==1.12.46
-botocore==1.15.46
+boto3==1.12.47
+botocore==1.15.47
 certifi==2020.4.5.1
 cffi==1.14.0; sys_platform == "linux"
 chardet==3.0.4
@@ -22,7 +22,7 @@ flake8==3.7.9
 future==0.18.2
 idna==2.9
 importlib-metadata==1.6.0
-importlib-resources==1.4.0; python_version < "3.7"
+importlib-resources==1.5.0; python_version < "3.7"
 jeepney==0.4.3; sys_platform == "linux"
 jmespath==0.9.5
 jsonpatch==1.25
@@ -49,7 +49,7 @@ pyrsistent==0.16.0
 pytest==5.3.5
 pytest-cov==2.8.1
 pytest-forked==1.1.3
-pytest-sugar==0.9.2
+pytest-sugar==0.9.3
 pytest-xdist==1.31.0
 python-dateutil==2.8.1
 pywin32-ctypes==0.2.0; sys_platform == "win32"

setup.py

@@ -28,7 +28,7 @@
 
 setup_kwargs = {
     'name': 'c7n',
-    'version': '0.9.0',
+    'version': '0.9.1',
     'description': 'Cloud Custodian - Policy Rules Engine',
     'long_description': 'Cloud Custodian\n=================\n\n<center><img src="https://cloudcustodian.io/img/logo_capone_devex_cloud_custodian.svg" alt="Cloud Custodian Logo" width="200px" height="200px" align_center/></center>\n\n---\n\n[![](https://badges.gitter.im/cloud-custodian/cloud-custodian.svg)](https://gitter.im/cloud-custodian/cloud-custodian?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)\n[![CI](https://github.com/cloud-custodian/cloud-custodian/workflows/CI/badge.svg?event=push)](https://github.com/cloud-custodian/cloud-custodian/actions?query=workflow%3ACI+branch%3Amaster+event%3Apush)\n[![](https://dev.azure.com/cloud-custodian/cloud-custodian/_apis/build/status/Custodian%20-%20CI?branchName=master)](https://dev.azure.com/cloud-custodian/cloud-custodian/_build)\n[![](https://img.shields.io/badge/license-Apache%202-blue.svg)](https://www.apache.org/licenses/LICENSE-2.0)\n[![](https://codecov.io/gh/cloud-custodian/cloud-custodian/branch/master/graph/badge.svg)](https://codecov.io/gh/cloud-custodian/cloud-custodian)\n[![](https://requires.io/github/cloud-custodian/cloud-custodian/requirements.svg?branch=master)](https://requires.io/github/cloud-custodian/cloud-custodian/requirements/?branch=master)\n\nCloud Custodian is a rules engine for managing public cloud accounts and\nresources. It allows users to define policies to enable a well managed\ncloud infrastructure, that\\\'s both secure and cost optimized. It\nconsolidates many of the adhoc scripts organizations have into a\nlightweight and flexible tool, with unified metrics and reporting.\n\nCustodian can be used to manage AWS, Azure, and GCP environments by\nensuring real time compliance to security policies (like encryption and\naccess requirements), tag policies, and cost management via garbage\ncollection of unused resources and off-hours resource management.\n\nCustodian policies are written in simple YAML configuration files that\nenable users to specify policies on a resource type (EC2, ASG, Redshift,\nCosmosDB, PubSub Topic) and are constructed from a vocabulary of filters\nand actions.\n\nIt integrates with the cloud native serverless capabilities of each\nprovider to provide for real time enforcement of policies with builtin\nprovisioning. Or it can be run as a simple cron job on a server to\nexecute against large existing fleets.\n\nCloud Custodian was originally developed at CapitalOne (by @kapilt et\nal), but CapitalOne does not materially contribute or support this\nproject, nor do they have any active maintainers. They represent just\none of the thousands of users of this project. Like many opensource\nprojects, development is lead by the community of hundreds of\ncontributors and several cloud providers have dedicated teams working\non Custodian.\n\n"[Engineering the Next Generation of Cloud\nGovernance](https://cloudrumblings.io/cloud-adoption-engineering-the-next-generation-of-cloud-governance-21fb1a2eff60)"\nby \\@drewfirment\n\nFeatures\n--------\n\n-   Comprehensive support for public cloud services and resources with a\n    rich library of actions and filters to build policies with.\n-   Supports arbitrary filtering on resources with nested boolean\n    conditions.\n-   Dry run any policy to see what it would do.\n-   Automatically provisions serverless functions and event sources (\n    AWS CloudWatchEvents, AWS Config Rules, Azure EventGrid, GCP\n    AuditLog & Pub/Sub, etc)\n-   Cloud provider native metrics outputs on resources that matched a\n    policy\n-   Structured outputs into cloud native object storage of which\n    resources matched a policy.\n-   Intelligent cache usage to minimize api calls.\n-   Supports multi-account/subscription/project usage.\n-   Battle-tested - in production on some very large cloud environments.\n\nLinks\n-----\n\n-   [Homepage](http://cloudcustodian.io)\n-   [Docs](http://cloudcustodian.io/docs/index.html)\n-   [Developer Install](https://cloudcustodian.io/docs/developer/installing.html)\n-   [Presentations](https://www.google.com/search?q=cloud+custodian&source=lnms&tbm=vid)\n\nQuick Install\n-------------\n\n```shell\n$ python3 -m venv custodian\n$ source custodian/bin/activate\n(custodian) $ pip install c7n\n```\n\n\nUsage\n-----\n\nThe first step to using Cloud Custodian is writing a YAML file\ncontaining the policies that you want to run. Each policy specifies\nthe resource type that the policy will run on, a set of filters which\ncontrol resources will be affected by this policy, actions which the policy\nwith take on the matched resources, and a mode which controls which\nhow the policy will execute.\n\nThe best getting started guides are the cloud provider specific tutorials.\n\n - [AWS Getting Started](https://cloudcustodian.io/docs/aws/gettingstarted.html)\n - [Azure Getting Started](https://cloudcustodian.io/docs/azure/gettingstarted.html)\n - [GCP Getting Started](https://cloudcustodian.io/docs/gcp/gettingstarted.html)\n\nAs a quick walk through, below are some sample policies for AWS resources.\n\n  1. will enforce that no S3 buckets have cross-account access enabled.\n  1. will terminate any newly launched EC2 instance that do not have an encrypted EBS volume.\n  1. will tag any EC2 instance that does not have the follow tags\n     "Environment", "AppId", and either "OwnerContact" or "DeptID" to\n     be stopped in four days.\n\n```yaml\npolicies:\n - name: s3-cross-account\n   description: |\n     Checks S3 for buckets with cross-account access and\n     removes the cross-account access.\n   resource: aws.s3\n   region: us-east-1\n   filters:\n     - type: cross-account\n   actions:\n     - type: remove-statements\n       statement_ids: matched\n\n - name: ec2-require-non-public-and-encrypted-volumes\n   resource: aws.ec2\n   description: |\n    Provision a lambda and cloud watch event target\n    that looks at all new instances and terminates those with\n    unencrypted volumes.\n   mode:\n    type: cloudtrail\n    role: CloudCustodian-QuickStart\n    events:\n      - RunInstances\n   filters:\n    - type: ebs\n      key: Encrypted\n      value: false\n   actions:\n    - terminate\n\n - name: tag-compliance\n   resource: aws.ec2\n   description: |\n     Schedule a resource that does not meet tag compliance policies to be stopped in four days. Note a separate policy using the`marked-for-op` filter is required to actually stop the instances after four days.\n   filters:\n    - State.Name: running\n    - "tag:Environment": absent\n    - "tag:AppId": absent\n    - or:\n      - "tag:OwnerContact": absent\n      - "tag:DeptID": absent\n   actions:\n    - type: mark-for-op\n      op: stop\n      days: 4\n```\n\nYou can validate, test, and run Cloud Custodian with the example policy with these commands:\n\n```shell\n# Validate the configuration (note this happens by default on run)\n$ custodian validate policy.yml\n\n# Dryrun on the policies (no actions executed) to see what resources\n# match each policy.\n$ custodian run --dryrun -s out policy.yml\n\n# Run the policy\n$ custodian run -s out policy.yml\n```\n\nYou can run Cloud Custodian via Docker as well:\n\n```shell\n# Download the image\n$ docker pull cloudcustodian/c7n\n$ mkdir output\n\n# Run the policy\n#\n# This will run the policy using only the environment variables for authentication\n$ docker run -it \\\n  -v $(pwd)/output:/home/custodian/output \\\n  -v $(pwd)/policy.yml:/home/custodian/policy.yml \\\n  --env-file <(env | grep "^AWS\\|^AZURE\\|^GOOGLE") \\\n  cloudcustodian/c7n run -v -s /home/custodian/output /home/custodian/policy.yml\n\n# Run the policy (using AWS\'s generated credentials from STS)\n#\n# NOTE: We mount the ``.aws/credentials`` and ``.aws/config`` directories to\n# the docker container to support authentication to AWS using the same credentials\n# credentials that are available to the local user if authenticating with STS.\n\n$ docker run -it \\\n  -v $(pwd)/output:/home/custodian/output \\\n  -v $(pwd)/policy.yml:/home/custodian/policy.yml \\\n  -v $(cd ~ && pwd)/.aws/credentials:/home/custodian/.aws/credentials \\\n  -v $(cd ~ && pwd)/.aws/config:/home/custodian/.aws/config \\\n  --env-file <(env | grep "^AWS") \\\n  cloudcustodian/c7n run -v -s /home/custodian/output /home/custodian/policy.yml\n```\n\nThe [custodian cask\ntool](https://cloudcustodian.io/docs/tools/cask.html) is a go binary\nthat provides a transparent front end to docker that mirors the regular\ncustodian cli, but automatically takes care of mounting volumes.\n\nConsult the documentation for additional information, or reach out on gitter.\n\nCloud Provider Specific Help\n----------------------------\n\nFor specific instructions for AWS, Azure, and GCP, visit the relevant getting started page.\n\n- [AWS](https://cloudcustodian.io/docs/aws/gettingstarted.html)\n- [Azure](https://cloudcustodian.io/docs/azure/gettingstarted.html)\n- [GCP](https://cloudcustodian.io/docs/gcp/gettingstarted.html)\n\nGet Involved\n------------\n\n-   [Gitter](https://gitter.im/cloud-custodian/cloud-custodian)\n-   [GitHub](https://github.com/cloud-custodian/cloud-custodian)\n-   [Mailing List](https://groups.google.com/forum/#!forum/cloud-custodian)\n-   [Reddit](https://reddit.com/r/cloudcustodian)\n-   [StackOverflow](https://stackoverflow.com/questions/tagged/cloudcustodian)\n\nAdditional Tools\n----------------\n\nThe Custodian project also develops and maintains a suite of additional\ntools here\n<https://github.com/cloud-custodian/cloud-custodian/tree/master/tools>:\n\n- [**_Org_:**](https://cloudcustodian.io/docs/tools/c7n-org.html) Multi-account policy execution.\n\n- [**_PolicyStream_:**](https://cloudcustodian.io/docs/tools/c7n-policystream.html) Git history as stream of logical policy changes.\n\n- [**_Salactus_:**](https://cloudcustodian.io/docs/tools/c7n-salactus.html) Scale out s3 scanning.\n\n- [**_Mailer_:**](https://cloudcustodian.io/docs/tools/c7n-mailer.html) A reference implementation of sending messages to users to notify them.\n\n- [**_Trail Creator_:**](https://cloudcustodian.io/docs/tools/c7n-trailcreator.html) Retroactive tagging of resources creators from CloudTrail\n\n- **_TrailDB_:** Cloudtrail indexing and time series generation for dashboarding.\n\n- [**_LogExporter_:**](https://cloudcustodian.io/docs/tools/c7n-logexporter.html) Cloud watch log exporting to s3\n\n- [**_Cask_:**](https://cloudcustodian.io/docs/tools/cask.html) Easy custodian exec via docker\n\n- [**_Guardian_:**](https://cloudcustodian.io/docs/tools/c7n-guardian.html) Automated multi-account Guard Duty setup\n\n- [**_Omni SSM_:**](https://cloudcustodian.io/docs/tools/omnissm.html) EC2 Systems Manager Automation\n\n- **_Sentry_:** Cloudwatch Log parsing for python tracebacks to integrate with\n    <https://sentry.io/welcome/>\n\n- [**_Mugc_:**](https://github.com/cloud-custodian/cloud-custodian/tree/master/tools/ops#mugc) A utility used to clean up Cloud Custodian Lambda policies that are deployed in an AWS environment.\n\nContributing\n------------\n\nSee <https://cloudcustodian.io/docs/contribute.html>\n\n\nCode of Conduct\n---------------\n\nThis project adheres to the [Open Code of Conduct](https://developer.capitalone.com/resources/code-of-conduct). By\nparticipating, you are expected to honor this code.\n\n',
     'long_description_content_type': 'text/markdown',

test.env

@@ -0,0 +1,10 @@
+export AWS_DEFAULT_REGION=us-east-1
+export AWS_ACCESS_KEY_ID=foo
+export AWS_SECRET_ACCESS_KEY=bar
+export AWS_SESSION_TOKEN=fake
+export C7N_TEST_RUN=true
+export AZURE_ACCESS_TOKEN=fake_token
+export AZURE_SUBSCRIPTION_ID=ea42f556-5106-4743-99b0-c129bfa71a47
+export GOOGLE_CLOUD_PROJECT=custodian-1291
+export GOOGLE_APPLICATION_CREDENTIALS=tools/c7n_gcp/tests/data/credentials.json
+export C7N_VALIDATE=true