diff --git a/README.md b/README.md index a6fada0..473ac1b 100644 --- a/README.md +++ b/README.md @@ -15,50 +15,50 @@ Follow the instructions below to send logs stored on AWS S3 to Logentries. ## Deploy the script on AWS Lambda 1. Create a new Lambda function - ![Create Function](https://raw.githubusercontent.com/omgapuppy/le_lambda/master/doc/step1.png) + ![Create Function](https://raw.githubusercontent.com/logentries/le_lambda/master/doc/step1.png) 2. Choose the Python blueprint for S3 objects - ![Choose Blueprint](https://raw.githubusercontent.com/omgapuppy/le_lambda/master/doc/step2.png) + ![Choose Blueprint](https://raw.githubusercontent.com/logentries/le_lambda/master/doc/step2.png) 3. Configure event sources: * Select S3 as event source type * Choose the bucket log files are being stored in * Set event type "Object Created (All)" - ![Create Function](https://raw.githubusercontent.com/omgapuppy/le_lambda/master/doc/step3.png) + ![Create Function](https://raw.githubusercontent.com/logentries/le_lambda/master/doc/step3.png) 4. Configure function: * Give your function a name * Set runtime to Python 2.7 - ![Create Function](https://raw.githubusercontent.com/omgapuppy/le_lambda/master/doc/step4.png) + ![Create Function](https://raw.githubusercontent.com/logentries/le_lambda/master/doc/step4.png) 5. Edit code: - * Select "Edit code inline" - * Copy the contents of ```le_lambda.py``` - * Replace **all** code in the editor window + * Edit the contents of ```le_lambda.py``` * Replace values of ```log_token``` and ```debug_token``` with tokens obtained earlier. + * Create a .ZIP file, containing the updated ```le_lambda.py``` and ```le_certs.pem``` + * Choose "Upload a .ZIP file" in AWS Lambda and upload the archive created in previous step - ![Create Function](https://raw.githubusercontent.com/omgapuppy/le_lambda/master/doc/step5.png) + ![Create Function](https://raw.githubusercontent.com/logentries/le_lambda/master/doc/step5.png) 6. Lambda function handler and role - * Leave the "Handler" value to default + * Change the "Handler" value to ```le_lambda.lambda_handler``` * Create a new S3 execution role (your IAM user must have sufficient permissions to create & assign new roles) - ![Create Function](https://raw.githubusercontent.com/omgapuppy/le_lambda/master/doc/step6.png) + ![Create Function](https://raw.githubusercontent.com/logentries/le_lambda/master/doc/step6.png) 7. Allocate resources: * Set memory to 1536 MB (script only runs for seconds at a time) * Set timeout to a high value, just below of log file creation frequency * Below example is configured for ELB logs written every 5 minutes - ![Create Function](https://raw.githubusercontent.com/omgapuppy/le_lambda/master/doc/step7.png) + ![Create Function](https://raw.githubusercontent.com/logentries/le_lambda/master/doc/step7.png) 8. Enable function: * Select "Enable now" * Click "Create function" - ![Create Function](https://raw.githubusercontent.com/omgapuppy/le_lambda/master/doc/step8.png) + ![Create Function](https://raw.githubusercontent.com/logentries/le_lambda/master/doc/step8.png) - ![Create Function](https://raw.githubusercontent.com/omgapuppy/le_lambda/master/doc/step9.png) + ![Create Function](https://raw.githubusercontent.com/logentries/le_lambda/master/doc/step9.png) diff --git a/doc/step5.png b/doc/step5.png index 17b0745..a9f3b97 100644 Binary files a/doc/step5.png and b/doc/step5.png differ diff --git a/le_certs.pem b/le_certs.pem new file mode 100644 index 0000000..b7aa258 --- /dev/null +++ b/le_certs.pem @@ -0,0 +1,79 @@ +-----BEGIN CERTIFICATE----- +MIIDVDCCAjygAwIBAgIDAjRWMA0GCSqGSIb3DQEBBQUAMEIxCzAJBgNVBAYTAlVT +MRYwFAYDVQQKEw1HZW9UcnVzdCBJbmMuMRswGQYDVQQDExJHZW9UcnVzdCBHbG9i +YWwgQ0EwHhcNMDIwNTIxMDQwMDAwWhcNMjIwNTIxMDQwMDAwWjBCMQswCQYDVQQG +EwJVUzEWMBQGA1UEChMNR2VvVHJ1c3QgSW5jLjEbMBkGA1UEAxMSR2VvVHJ1c3Qg +R2xvYmFsIENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA2swYYzD9 +9BcjGlZ+W988bDjkcbd4kdS8odhM+KhDtgPpTSEHCIjaWC9mOSm9BXiLnTjoBbdq +fnGk5sRgprDvgOSJKA+eJdbtg/OtppHHmMlCGDUUna2YRpIuT8rxh0PBFpVXLVDv +iS2Aelet8u5fa9IAjbkU+BQVNdnARqN7csiRv8lVK83Qlz6cJmTM386DGXHKTubU +1XupGc1V3sjs0l44U+VcT4wt/lAjNvxm5suOpDkZALeVAjmRCw7+OC7RHQWa9k0+ +bw8HHa8sHo9gOeL6NlMTOdReJivbPagUvTLrGAMoUgRx5aszPeE4uwc2hGKceeoW +MPRfwCvocWvk+QIDAQABo1MwUTAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBTA +ephojYn7qwVkDBF9qn1luMrMTjAfBgNVHSMEGDAWgBTAephojYn7qwVkDBF9qn1l +uMrMTjANBgkqhkiG9w0BAQUFAAOCAQEANeMpauUvXVSOKVCUn5kaFOSPeCpilKIn +Z57QzxpeR+nBsqTP3UEaBU6bS+5Kb1VSsyShNwrrZHYqLizz/Tt1kL/6cdjHPTfS +tQWVYrmm3ok9Nns4d0iXrKYgjy6myQzCsplFAMfOEVEiIuCl6rYVSAlk6l5PdPcF +PseKUgzbFbS9bZvlxrFUaKnjaZC2mqUPuLk/IH2uSrW4nOQdtqvmlKXBx4Ot2/Un +hw4EbNX/3aBd7YdStysVAq45pmp06drE57xNNB6pXE0zX5IJL4hmXXeXxx12E6nV +5fEWCRE11azbJHFwLJhWC9kXtNHjUStedejV0NxPNO3CBWaAocvmMw== +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIF2DCCA8CgAwIBAgIQTKr5yttjb+Af907YWwOGnTANBgkqhkiG9w0BAQwFADCB +hTELMAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4G +A1UEBxMHU2FsZm9yZDEaMBgGA1UEChMRQ09NT0RPIENBIExpbWl0ZWQxKzApBgNV +BAMTIkNPTU9ETyBSU0EgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMTAwMTE5 +MDAwMDAwWhcNMzgwMTE4MjM1OTU5WjCBhTELMAkGA1UEBhMCR0IxGzAZBgNVBAgT +EkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UEBxMHU2FsZm9yZDEaMBgGA1UEChMR +Q09NT0RPIENBIExpbWl0ZWQxKzApBgNVBAMTIkNPTU9ETyBSU0EgQ2VydGlmaWNh +dGlvbiBBdXRob3JpdHkwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCR +6FSS0gpWsawNJN3Fz0RndJkrN6N9I3AAcbxT38T6KhKPS38QVr2fcHK3YX/JSw8X +pz3jsARh7v8Rl8f0hj4K+j5c+ZPmNHrZFGvnnLOFoIJ6dq9xkNfs/Q36nGz637CC +9BR++b7Epi9Pf5l/tfxnQ3K9DADWietrLNPtj5gcFKt+5eNu/Nio5JIk2kNrYrhV +/erBvGy2i/MOjZrkm2xpmfh4SDBF1a3hDTxFYPwyllEnvGfDyi62a+pGx8cgoLEf +Zd5ICLqkTqnyg0Y3hOvozIFIQ2dOciqbXL1MGyiKXCJ7tKuY2e7gUYPDCUZObT6Z ++pUX2nwzV0E8jVHtC7ZcryxjGt9XyD+86V3Em69FmeKjWiS0uqlWPc9vqv9JWL7w +qP/0uK3pN/u6uPQLOvnoQ0IeidiEyxPx2bvhiWC4jChWrBQdnArncevPDt09qZah +SL0896+1DSJMwBGB7FY79tOi4lu3sgQiUpWAk2nojkxl8ZEDLXB0AuqLZxUpaVIC +u9ffUGpVRr+goyhhf3DQw6KqLCGqR84onAZFdr+CGCe01a60y1Dma/RMhnEw6abf +Fobg2P9A3fvQQoh/ozM6LlweQRGBY84YcWsr7KaKtzFcOmpH4MN5WdYgGq/yapiq +crxXStJLnbsQ/LBMQeXtHT1eKJ2czL+zUdqnR+WEUwIDAQABo0IwQDAdBgNVHQ4E +FgQUu69+Aj36pvE8hI6t7jiY7NkyMtQwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB +/wQFMAMBAf8wDQYJKoZIhvcNAQEMBQADggIBAArx1UaEt65Ru2yyTUEUAJNMnMvl +wFTPoCWOAvn9sKIN9SCYPBMtrFaisNZ+EZLpLrqeLppysb0ZRGxhNaKatBYSaVqM +4dc+pBroLwP0rmEdEBsqpIt6xf4FpuHA1sj+nq6PK7o9mfjYcwlYRm6mnPTXJ9OV +2jeDchzTc+CiR5kDOF3VSXkAKRzH7JsgHAckaVd4sjn8OoSgtZx8jb8uk2Intzna +FxiuvTwJaP+EmzzV1gsD41eeFPfR60/IvYcjt7ZJQ3mFXLrrkguhxuhoqEwWsRqZ +CuhTLJK7oQkYdQxlqHvLI7cawiiFwxv/0Cti76R7CZGYZ4wUAc1oBmpjIXUDgIiK +boHGhfKppC3n9KUkEEeDys30jXlYsQab5xoq2Z0B15R97QNKyvDb6KkBPvVWmcke +jkk9u+UJueBPSZI9FoJAzMxZxuY67RIuaTxslbH9qh17f4a+Hg4yRvv7E491f0yL +S0Zj/gA0QHDBw7mh3aZw4gSzQbzpgJHqZJx64SIDqZxubw5lT2yHh17zbqD5daWb +QOhTsiedSrnAdyGN/4fy3ryM7xfft0kL0fJuMAsaDk527RH89elWsn2/x20Kk4yl +0MC2Hb46TpSi125sC8KKfPog88Tk5c0NqMuRkrF8hey1FGlmDoLnzc7ILaZRfyHB +NVOFBkpdn627G190 +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIENjCCAx6gAwIBAgIBATANBgkqhkiG9w0BAQUFADBvMQswCQYDVQQGEwJTRTEU +MBIGA1UEChMLQWRkVHJ1c3QgQUIxJjAkBgNVBAsTHUFkZFRydXN0IEV4dGVybmFs +IFRUUCBOZXR3b3JrMSIwIAYDVQQDExlBZGRUcnVzdCBFeHRlcm5hbCBDQSBSb290 +MB4XDTAwMDUzMDEwNDgzOFoXDTIwMDUzMDEwNDgzOFowbzELMAkGA1UEBhMCU0Ux +FDASBgNVBAoTC0FkZFRydXN0IEFCMSYwJAYDVQQLEx1BZGRUcnVzdCBFeHRlcm5h +bCBUVFAgTmV0d29yazEiMCAGA1UEAxMZQWRkVHJ1c3QgRXh0ZXJuYWwgQ0EgUm9v +dDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALf3GjPm8gAELTngTlvt +H7xsD821+iO2zt6bETOXpClMfZOfvUq8k+0DGuOPz+VtUFrWlymUWoCwSXrbLpX9 +uMq/NzgtHj6RQa1wVsfwTz/oMp50ysiQVOnGXw94nZpAPA6sYapeFI+eh6FqUNzX +mk6vBbOmcZSccbNQYArHE504B4YCqOmoaSYYkKtMsE8jqzpPhNjfzp/haW+710LX +a0Tkx63ubUFfclpxCDezeWWkWaCUN/cALw3CknLa0Dhy2xSoRcRdKn23tNbE7qzN +E0S3ySvdQwAl+mG5aWpYIxG3pzOPVnVZ9c0p10a3CitlttNCbxWyuHv77+ldU9U0 +WicCAwEAAaOB3DCB2TAdBgNVHQ4EFgQUrb2YejS0Jvf6xCZU7wO94CTLVBowCwYD +VR0PBAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wgZkGA1UdIwSBkTCBjoAUrb2YejS0 +Jvf6xCZU7wO94CTLVBqhc6RxMG8xCzAJBgNVBAYTAlNFMRQwEgYDVQQKEwtBZGRU +cnVzdCBBQjEmMCQGA1UECxMdQWRkVHJ1c3QgRXh0ZXJuYWwgVFRQIE5ldHdvcmsx +IjAgBgNVBAMTGUFkZFRydXN0IEV4dGVybmFsIENBIFJvb3SCAQEwDQYJKoZIhvcN +AQEFBQADggEBALCb4IUlwtYj4g+WBpKdQZic2YR5gdkeWxQHIzZlj7DYd7usQWxH +YINRsPkyPef89iYTx4AWpb9a/IfPeHmJIZriTAcKhjW88t5RxNKWt9x+Tu5w/Rw5 +6wwCURQtjr0W4MHfRnXnJK3s9EK0hZNwEGe6nQY1ShjTK3rMUUKhemPR5ruhxSvC +Nr4TDea9Y355e6cJDUCrat2PisP29owaQgVR1EX1n6diIWgVIEM8med8vSTYqZEX +c4g/VhsxOBi0cQ+azcgOno4uG+GMmIPLHzHxREzGBHNJdmAPx/i9F4BrLunMTA5a +mnkPIAou1Z5jJh5VkpTYghdae9C8x49OhgQ= +-----END CERTIFICATE----- diff --git a/le_lambda.py b/le_lambda.py index 348dba3..ae0a5e1 100644 --- a/le_lambda.py +++ b/le_lambda.py @@ -1,8 +1,10 @@ import boto3 import socket +import ssl import datetime import re import csv + print('Loading function') s3 = boto3.client('s3') @@ -25,18 +27,19 @@ def lambda_handler(event, context): host = 'data.logentries.com' - port = 80 - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + port = 20000 + s_ = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s = ssl.wrap_socket(s_, ca_certs='le_certs.pem', cert_reqs=ssl.CERT_REQUIRED) s.connect((host, port)) tokens = [] - if validate_uuid4(debug_token) is True: + if validate_uuid(debug_token) is True: tokens.append(debug_token) - if validate_uuid4(lambda_token) is True: + if validate_uuid(lambda_token) is True: tokens.append(lambda_token) else: pass - if validate_uuid4(log_token) is False: + if validate_uuid(log_token) is False: for token in tokens: s.sendall('%s %s\n' % (token, "{}: log token not present for username={}" .format(str(datetime.datetime.utcnow()), username))) @@ -50,11 +53,11 @@ def lambda_handler(event, context): body = response['Body'] data = body.read() for token in tokens: - s.sendall('%s %s\n' % (token, "username={} downloaded file={} from bucket={}." + s.sendall('%s %s\n' % (token, "username='{}' downloaded file='{}' from bucket='{}'." .format(username, key, bucket))) lines = data.split("\n") for token in tokens: - s.sendall('%s %s\n' % (token, "Beginning to send lines={} start_time={}." + s.sendall('%s %s\n' % (token, "Beginning to send lines='{}' start_time='{}'." .format(str(len(lines)), str(datetime.datetime.utcnow())))) if validate_elb_log(str(key)) is True: # timestamp elb client:port backend:port request_processing_time backend_processing_time @@ -80,21 +83,21 @@ def lambda_handler(event, context): for line in lines: s.sendall('%s %s\n' % (log_token, line)) for token in tokens: - s.sendall('%s %s\n' % (token, "username={} finished sending log data end_time={}" + s.sendall('%s %s\n' % (token, "username='{}' finished sending log data end_time='{}'" .format(username, str(datetime.datetime.utcnow())))) except Exception as e: for token in tokens: print e - s.sendall('%s %s\n' % (token, 'Error getting username={} file={} from bucket={}. Make sure ' - 'they exist and your bucket is in the same region as this function.' + s.sendall('%s %s\n' % (token, "Error getting username='{}' file='{}' from bucket='{}'. Make sure " + "they exist and your bucket is in the same region as this function." .format(username, key, bucket))) finally: s.close() -def validate_uuid4(uuid): +def validate_uuid(uuid_string): regex = re.compile('^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$', re.I) - match = regex.match(uuid) + match = regex.match(uuid_string) return bool(match)