s3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "!pip3 install --upgrade pip\n",
    "!pip3 install boto3\n",
    "!pip3 install pyyaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from functools import reduce\n",
    "import os\n",
    "import boto3\n",
    "from botocore.client import Config\n",
    "from botocore.exceptions import ClientError\n",
    "import yaml\n",
    "from datetime import datetime, time, timedelta\n",
    "import json\n",
    "from pprint import pprint\n",
    "from pprint import PrettyPrinter\n",
    "from statistics import mean\n",
    "import sys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "print(sys.version_info)\n",
    "print(sys.version_info[:3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "config = yaml.load(open('config.yaml', 'r'))\n",
    "#print(config)\n",
    "aws_region = config['aws']['regions'][0]\n",
    "account = config['aws']['accounts'][1]['name']\n",
    "profile = config['aws']['accounts'][1]['profile']\n",
    "account_number = config['aws']['accounts'][1]['account-number']\n",
    "\n",
    "print((\"%s - %s - %s - %s\" % (str(account_number),account,profile,aws_region)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def err_chk(fnCall, errormsg):\n",
    "        errStr = \"{} returned error: {}\".format(fnCall, e)\n",
    "        print(errStr)\n",
    "        return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "session = boto3.session.Session(profile_name=profile,region_name=aws_region)\n",
    "print(session)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "s3Client = session.client('s3', config=Config(signature_version='s3v4'))\n",
    "print(s3Client)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "buckets = s3Client.list_buckets()\n",
    "print((\"%s buckets\" % (len(buckets.get('Buckets')))))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "pp = PrettyPrinter() #indent=1)\n",
    "pp.pprint(buckets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "tags": []
   },
   "outputs": [],
   "source": [
    "location = s3Client.get_bucket_location(Bucket='<BUCKET_NAME>')\n",
    "pp.pprint(location)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[]\n"
     ]
    }
   ],
   "source": [
    "denied = []\n",
    "\n",
    "for bucket in buckets.get('Buckets'):\n",
    "    bucketName = bucket.get('Name')\n",
    "    try:\n",
    "        head = s3Client.head_bucket(Bucket=bucketName)\n",
    "    except Exception as inst:\n",
    "        #print(type(inst))    # the exception instance\n",
    "        #print(inst.args[0])     # arguments stored in .args\n",
    "        if \"403\" in inst.args[0]:\n",
    "            denied.append(bucketName)\n",
    "            \n",
    "print(denied)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "try:\n",
    "    tags = s3Client.get_bucket_tagging(Bucket='<BUCKET_NAME>')\n",
    "    print(tags)\n",
    "    pp.pprint(tags)\n",
    "except ClientError as e:\n",
    "    err_chk(\"Tags could not be retrieved\", e)\n",
    "    #return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "tags": []
   },
   "outputs": [],
   "source": [
    "try:\n",
    "    policy = s3Client.get_bucket_policy(Bucket='<BUCKET_NAME>')\n",
    "    pp.pprint(policy)\n",
    "except ClientError as e:\n",
    "    err_chk(\"Policy could not be retrieved\", e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "tags": []
   },
   "outputs": [],
   "source": [
    "try:\n",
    "    acl = s3Client.get_bucket_acl(Bucket='<BUCKET_NAME>')\n",
    "    pp.pprint(acl)\n",
    "except ClientError as e:\n",
    "    err_chk(\"ACL could not be retrieved\",e)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "tags": []
   },
   "outputs": [],
   "source": [
    "try:\n",
    "    notifications = s3Client.get_bucket_notification(Bucket='<BUCKET_NAME>')\n",
    "    pp.pprint(notifications)\n",
    "except ClientError as e:\n",
    "    err_chk(\"Notifications could not be retrieved\",e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def tag_value_exists(tagSet, tagKey, tagValue):\n",
    "    for tag in tagSet.get('TagSet'):\n",
    "        if tag.get('Key') == tagKey:\n",
    "            if tag.get('Value') == tagValue:\n",
    "                return True\n",
    "        \n",
    "    return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "tags": []
   },
   "outputs": [],
   "source": [
    "tagged = {}\n",
    "failed = []\n",
    "\n",
    "for bucket in buckets.get('Buckets'):\n",
    "    #print(bucket)\n",
    "    bucketName = bucket.get('Name')\n",
    "    #print(bucketName)\n",
    "\n",
    "    \n",
    "    #access = s3Client.head_bucket(Bucket=bucketName)\n",
    "    #print(access)\n",
    "    \n",
    "    if not bucketName in denied:\n",
    "        try:\n",
    "            tagSet = s3Client.get_bucket_tagging(Bucket=bucketName)\n",
    "            if tag_value_exists(tagSet, 'TAG_KEY', 'TAG_VALUE'):\n",
    "                print(bucketName)\n",
    "                tagged[bucketName] = tagSet\n",
    "        except:\n",
    "            failed.append(bucketName)\n",
    "\n",
    "print(tagged)\n",
    "print(failed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "tags": []
   },
   "outputs": [],
   "source": [
    "pp.pprint(tagged)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "tags": []
   },
   "outputs": [],
   "source": [
    "try:\n",
    "    objects = s3Client.list_objects(Bucket='<BUCKET_NAME>')\n",
    "    print(objects)\n",
    "    contents = objects.get('Contents')\n",
    "    if contents != None:\n",
    "        print(contents)\n",
    "        print(len(contents))\n",
    "    pp.pprint(objects)\n",
    "except ClientError as e:\n",
    "    err_chk(\"ACL could not be retrieved\",e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3-final"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}