Commit e1367f17 authored by Cameron McFarland's avatar Cameron McFarland

Adding lambda function for firehose.

parent 0a83a4f7
from __future__ import print_function
import base64
print('Loading function')
def lambda_handler(event, context):
output = []
for record in event['records']:
print(record['recordId'])
payload = base64.b64decode(record['data'])
processed = payload + "\n"
output_record = {
'recordId': record['recordId'],
'result': 'Ok',
'data': base64.b64encode(processed)
}
output.append(output_record)
print('Successfully processed {} records.'.format(len(event['records'])))
return {'records': output}
......@@ -30,6 +30,10 @@ data "template_file" "iam_policy_enricher" {
template = "${file("${path.module}/templates/iam_policy_enricher.json")}"
}
data "template_file" "iam_policy_lambda" {
template = "${file("${path.module}/templates/iam_policy_lambda.json")}"
}
data "template_file" "iam_role_collector" {
template = "${file("${path.module}/templates/iam_role_collector.json")}"
}
......@@ -38,6 +42,10 @@ data "template_file" "iam_role_enricher" {
template = "${file("${path.module}/templates/iam_role_enricher.json")}"
}
data "template_file" "iam_role_lambda" {
template = "${file("${path.module}/templates/iam_role_lambda.json")}"
}
// Policies
resource "aws_iam_policy" "snowplow_collector_policy" {
description = "Policy the allows the collector to access other AWS services such as Kinesis."
......@@ -55,6 +63,14 @@ resource "aws_iam_policy" "snowplow_enricher_policy" {
policy = "${data.template_file.iam_policy_enricher.rendered}"
}
resource "aws_iam_policy" "snowplow_lambda_policy" {
description = ""
name = "AWSLambdaBasicExecutionRole-b1df0a33-ac33-47d3-930b-e8e0bf9443ef"
path = "/service-role/"
policy = "${data.template_file.iam_policy_lambda.rendered}"
}
// Roles
resource "aws_iam_role" "snowplow_collector_role" {
name = "snowplow-collector-role"
......@@ -74,6 +90,16 @@ resource "aws_iam_role" "snowplow_enricher_role" {
}
}
resource "aws_iam_role" "snowplow_lambda_role" {
name = ""
assume_role_policy = "${data.template_file.iam_role_lambda.rendered}"
path = "/service-role/"
tags = {
environment = "SnowPlow"
}
}
// Role Policy Attachments
resource "aws_iam_role_policy_attachment" "collector_role_policy_attachment" {
role = "${aws_iam_role.snowplow_collector_role.name}"
......@@ -85,6 +111,11 @@ resource "aws_iam_role_policy_attachment" "enricher_role_policy_attachment" {
policy_arn = "${aws_iam_policy.snowplow_enricher_policy.arn}"
}
resource "aws_iam_role_policy_attachment" "lambda_role_policy_attachment" {
role = "${aws_iam_role.snowplow_lambda_role.name}"
policy_arn = "${aws_iam_policy.snowplow_lambda_policy.arn}"
}
// VPC
resource "aws_vpc" "snowplow_vpc" {
cidr_block = "10.32.0.0/16"
......@@ -509,3 +540,21 @@ resource "aws_lb_listener" "snowplow_collector_lb_listener" {
target_group_arn = "${aws_lb_target_group.snowplow_collector_lb_target_group.arn}"
}
}
// Lambda Function
resource "aws_lambda_function" "snowplow_event_formatter_lambda_function" {
description = "This adds a newline to the end of each record."
function_name = "SnowPlowFirehoseFormatter"
handler = "lambda_function.lambda_handler"
role = "${aws_iam_role.snowplow_lambda_role.arn}"
runtime = "python2.7"
filename = "lambda/lambda_function_payload.zip"
source_code_hash = "${base64sha256(file("lambda/lambda_function_payload.zip"))}"
timeout = "60"
tags = {
environment = "SnowPlow"
"lambda-console:blueprint" = "kinesis-firehose-process-record-python"
}
}
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": "logs:CreateLogGroup",
"Resource": "arn:aws:logs:us-east-1:855262394183:*"
},
{
"Effect": "Allow",
"Action": [
"logs:CreateLogStream",
"logs:PutLogEvents"
],
"Resource": [
"arn:aws:logs:us-east-1:855262394183:log-group:/aws/lambda/SnowPlowFirehoseFormatter:*"
]
}
]
}
{
"Version":"2012-10-17",
"Statement":[
{
"Effect":"Allow",
"Principal":{
"Service":"lambda.amazonaws.com"
},
"Action":"sts:AssumeRole"
}
]
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment