Commit 1a075635 authored by Cameron McFarland's avatar Cameron McFarland

Adding firehose delivery streams, role and policies.

parent e1367f17
......@@ -37,6 +37,11 @@ do that on their own.
* SnowplowEnrich-gitlab-us-east-1
* SnowplowS3Loader-gitlab-us-east-1
## Firehose
The data format conversion sections of the Firehose deliver configs are
placeholder configs since data format conversion is disabled. Terraform still
seems to want to configure everything if it's disabled.
## Launch Config Changes and Production Instances
Updating the launch config will apply to new systems coming up in the
auto-scaling group. But existing EC2 instances won't be changed. You will
......
......@@ -30,6 +30,14 @@ data "template_file" "iam_policy_enricher" {
template = "${file("${path.module}/templates/iam_policy_enricher.json")}"
}
data "template_file" "iam_policy_firehose_enriched_bad" {
template = "${file("${path.module}/templates/iam_policy_firehose_enriched_bad.json")}"
}
data "template_file" "iam_policy_firehose_enriched_good" {
template = "${file("${path.module}/templates/iam_policy_firehose_enriched_good.json")}"
}
data "template_file" "iam_policy_lambda" {
template = "${file("${path.module}/templates/iam_policy_lambda.json")}"
}
......@@ -46,6 +54,14 @@ data "template_file" "iam_role_lambda" {
template = "${file("${path.module}/templates/iam_role_lambda.json")}"
}
data "template_file" "iam_role_firehose_delivery" {
template = "${file("${path.module}/templates/iam_role_firehose_delivery.json")}"
}
data "template_file" "snowplow_s3_bucket_policy" {
template = "${file("${path.module}/templates/gitlab-com-snowplow-events.policy.json")}"
}
// Policies
resource "aws_iam_policy" "snowplow_collector_policy" {
description = "Policy the allows the collector to access other AWS services such as Kinesis."
......@@ -71,6 +87,18 @@ resource "aws_iam_policy" "snowplow_lambda_policy" {
policy = "${data.template_file.iam_policy_lambda.rendered}"
}
resource "aws_iam_role_policy" "snowplow_firehose_enriched_bad_policy" {
name = "firehose_enriched_bad"
policy = "${data.template_file.iam_policy_firehose_enriched_bad.rendered}"
role = "${aws_iam_role.snowplow_firehose_delivery_role.id}"
}
resource "aws_iam_role_policy" "snowplow_firehose_enriched_good_policy" {
name = "firehose_enriched_good"
policy = "${data.template_file.iam_policy_firehose_enriched_good.rendered}"
role = "${aws_iam_role.snowplow_firehose_delivery_role.id}"
}
// Roles
resource "aws_iam_role" "snowplow_collector_role" {
name = "snowplow-collector-role"
......@@ -100,6 +128,18 @@ resource "aws_iam_role" "snowplow_lambda_role" {
}
}
resource "aws_iam_role" "snowplow_firehose_delivery_role" {
name = ""
assume_role_policy = "${data.template_file.iam_role_lambda.rendered}"
path = "/"
assume_role_policy = "${data.template_file.iam_role_firehose_delivery.rendered}"
tags = {
environment = "SnowPlow"
}
}
// Role Policy Attachments
resource "aws_iam_role_policy_attachment" "collector_role_policy_attachment" {
role = "${aws_iam_role.snowplow_collector_role.name}"
......@@ -116,6 +156,21 @@ resource "aws_iam_role_policy_attachment" "lambda_role_policy_attachment" {
policy_arn = "${aws_iam_policy.snowplow_lambda_policy.arn}"
}
// S3 Buckets
resource "aws_s3_bucket" "snowplow_s3_bucket" {
bucket = "gitlab-com-snowplow-events"
tags = {
environment = "SnowPlow"
}
}
resource "aws_s3_bucket_policy" "snowplow_s3_bucket_policy" {
bucket = "${aws_s3_bucket.snowplow_s3_bucket.id}"
policy = "${data.template_file.snowplow_s3_bucket_policy.rendered}"
}
// VPC
resource "aws_vpc" "snowplow_vpc" {
cidr_block = "10.32.0.0/16"
......@@ -558,3 +613,84 @@ resource "aws_lambda_function" "snowplow_event_formatter_lambda_function" {
"lambda-console:blueprint" = "kinesis-firehose-process-record-python"
}
}
// Firehose
resource "aws_kinesis_firehose_delivery_stream" "snowplow_enriched_bad_firehose" {
destination = "extended_s3"
name = "SnowPlowEnrichedBad"
// Terraform seems to be bad at this?
// https://github.com/terraform-providers/terraform-provider-aws/issues/6053
lifecycle {
ignore_changes = [
"extended_s3_configuration.0.data_format_conversion_configuration",
"extended_s3_configuration.0.data_format_conversion_configuration.0.enabled",
]
}
extended_s3_configuration {
bucket_arn = "${aws_s3_bucket.snowplow_s3_bucket.arn}"
role_arn = "${aws_iam_role.snowplow_firehose_delivery_role.arn}"
compression_format = "GZIP"
prefix = "enriched-bad/"
error_output_prefix = "enriched-bad/"
s3_backup_mode = "Disabled"
processing_configuration {
enabled = "true"
processors {
type = "Lambda"
parameters {
parameter_name = "LambdaArn"
parameter_value = "${aws_lambda_function.snowplow_event_formatter_lambda_function.arn}:$LATEST"
}
}
}
}
tags = {
environment = "SnowPlow"
}
}
resource "aws_kinesis_firehose_delivery_stream" "snowplow_enriched_good_firehose" {
destination = "extended_s3"
name = "SnowPlowEnrichedGood"
// Terraform seems to be bad at this?
// https://github.com/terraform-providers/terraform-provider-aws/issues/6053
lifecycle {
ignore_changes = [
"extended_s3_configuration.0.data_format_conversion_configuration",
"extended_s3_configuration.0.data_format_conversion_configuration.0.enabled",
]
}
extended_s3_configuration {
bucket_arn = "${aws_s3_bucket.snowplow_s3_bucket.arn}"
role_arn = "${aws_iam_role.snowplow_firehose_delivery_role.arn}"
compression_format = "GZIP"
prefix = "output/"
error_output_prefix = "output/"
s3_backup_mode = "Disabled"
processing_configuration {
enabled = "true"
processors {
type = "Lambda"
parameters {
parameter_name = "LambdaArn"
parameter_value = "${aws_lambda_function.snowplow_event_formatter_lambda_function.arn}:$LATEST"
}
}
}
}
tags = {
environment = "SnowPlow"
}
}
{
"Version" : "2012-10-17",
"Id" : "Policy1560181228695",
"Statement" : [
{
"Sid" : "Stmt1560181207940",
"Effect" : "Allow",
"Principal" : {
"AWS" : "arn:aws:iam::855262394183:user/datateam-snowplow-ro"
},
"Action" : "s3:ListBucket",
"Resource" : "arn:aws:s3:::gitlab-com-snowplow-events"
},
{
"Sid" : "Stmt1560181227007",
"Effect" : "Allow",
"Principal" : {
"AWS" : "arn:aws:iam::855262394183:user/datateam-snowplow-ro"
},
"Action" : "s3:GetObject",
"Resource" : "arn:aws:s3:::gitlab-com-snowplow-events/*"
}
]
}
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "",
"Effect": "Allow",
"Action": [
"glue:GetTableVersions"
],
"Resource": "*"
},
{
"Sid": "",
"Effect": "Allow",
"Action": [
"s3:AbortMultipartUpload",
"s3:GetBucketLocation",
"s3:GetObject",
"s3:ListBucket",
"s3:ListBucketMultipartUploads",
"s3:PutObject"
],
"Resource": [
"arn:aws:s3:::gitlab-com-snowplow-events",
"arn:aws:s3:::gitlab-com-snowplow-events/*",
"arn:aws:s3:::%FIREHOSE_BUCKET_NAME%",
"arn:aws:s3:::%FIREHOSE_BUCKET_NAME%/*"
]
},
{
"Sid": "",
"Effect": "Allow",
"Action": [
"lambda:InvokeFunction",
"lambda:GetFunctionConfiguration"
],
"Resource": "arn:aws:lambda:us-east-1:855262394183:function:SnowPlowFirehoseFormatter:$LATEST"
},
{
"Sid": "",
"Effect": "Allow",
"Action": [
"logs:PutLogEvents"
],
"Resource": [
"arn:aws:logs:us-east-1:855262394183:log-group:/aws/kinesisfirehose/SnowPlowEnrichedBad:log-stream:*"
]
},
{
"Sid": "",
"Effect": "Allow",
"Action": [
"kinesis:DescribeStream",
"kinesis:GetShardIterator",
"kinesis:GetRecords"
],
"Resource": "arn:aws:kinesis:us-east-1:855262394183:stream/snowplow-enriched-bad"
},
{
"Effect": "Allow",
"Action": [
"kms:Decrypt"
],
"Resource": [
"arn:aws:kms:us-east-1:855262394183:key/%SSE_KEY_ID%"
],
"Condition": {
"StringEquals": {
"kms:ViaService": "kinesis.us-east-1.amazonaws.com"
},
"StringLike": {
"kms:EncryptionContext:aws:kinesis:arn": "arn:aws:kinesis:us-east-1:855262394183:stream/snowplow-enriched-bad"
}
}
}
]
}
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "",
"Effect": "Allow",
"Action": [
"glue:GetTableVersions"
],
"Resource": "*"
},
{
"Sid": "",
"Effect": "Allow",
"Action": [
"s3:AbortMultipartUpload",
"s3:GetBucketLocation",
"s3:GetObject",
"s3:ListBucket",
"s3:ListBucketMultipartUploads",
"s3:PutObject"
],
"Resource": [
"arn:aws:s3:::gitlab-com-snowplow-events",
"arn:aws:s3:::gitlab-com-snowplow-events/*",
"arn:aws:s3:::%FIREHOSE_BUCKET_NAME%",
"arn:aws:s3:::%FIREHOSE_BUCKET_NAME%/*"
]
},
{
"Sid": "",
"Effect": "Allow",
"Action": [
"lambda:InvokeFunction",
"lambda:GetFunctionConfiguration"
],
"Resource": "arn:aws:lambda:us-east-1:855262394183:function:SnowPlowFirehoseFormatter:$LATEST"
},
{
"Sid": "",
"Effect": "Allow",
"Action": [
"logs:PutLogEvents"
],
"Resource": [
"arn:aws:logs:us-east-1:855262394183:log-group:/aws/kinesisfirehose/SnowPlowEnrichedGood:log-stream:*"
]
},
{
"Sid": "",
"Effect": "Allow",
"Action": [
"kinesis:DescribeStream",
"kinesis:GetShardIterator",
"kinesis:GetRecords"
],
"Resource": "arn:aws:kinesis:us-east-1:855262394183:stream/snowplow-enriched-good"
},
{
"Effect": "Allow",
"Action": [
"kms:Decrypt"
],
"Resource": [
"arn:aws:kms:us-east-1:855262394183:key/%SSE_KEY_ID%"
],
"Condition": {
"StringEquals": {
"kms:ViaService": "kinesis.us-east-1.amazonaws.com"
},
"StringLike": {
"kms:EncryptionContext:aws:kinesis:arn": "arn:aws:kinesis:us-east-1:855262394183:stream/snowplow-enriched-good"
}
}
}
]
}
{
"Version" : "2012-10-17",
"Statement" : [
{
"Sid" : "",
"Effect" : "Allow",
"Principal" : {
"Service" : "firehose.amazonaws.com"
},
"Action" : "sts:AssumeRole",
"Condition" : {
"StringEquals" : {
"sts:ExternalId":"855262394183"
}
}
}
]
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment