Commit 2946efc4 authored by Cameron McFarland's avatar Cameron McFarland

Adding a new firehose to capture bad raw events to disk for examination.

parent 234178fc
...@@ -34,6 +34,10 @@ data "template_file" "iam_policy_firehose_enriched_bad" { ...@@ -34,6 +34,10 @@ data "template_file" "iam_policy_firehose_enriched_bad" {
template = "${file("${path.module}/templates/iam_policy_firehose_enriched_bad.json")}" template = "${file("${path.module}/templates/iam_policy_firehose_enriched_bad.json")}"
} }
data "template_file" "iam_policy_firehose_raw_bad" {
template = "${file("${path.module}/templates/iam_policy_firehose_raw_bad.json")}"
}
data "template_file" "iam_policy_firehose_enriched_good" { data "template_file" "iam_policy_firehose_enriched_good" {
template = "${file("${path.module}/templates/iam_policy_firehose_enriched_good.json")}" template = "${file("${path.module}/templates/iam_policy_firehose_enriched_good.json")}"
} }
...@@ -93,6 +97,12 @@ resource "aws_iam_role_policy" "snowplow_firehose_enriched_bad_policy" { ...@@ -93,6 +97,12 @@ resource "aws_iam_role_policy" "snowplow_firehose_enriched_bad_policy" {
role = "${aws_iam_role.snowplow_firehose_delivery_role.id}" role = "${aws_iam_role.snowplow_firehose_delivery_role.id}"
} }
resource "aws_iam_role_policy" "snowplow_firehose_raw_bad_policy" {
name = "firehose_raw_bad"
policy = "${data.template_file.iam_policy_firehose_raw_bad.rendered}"
role = "${aws_iam_role.snowplow_firehose_delivery_role.id}"
}
resource "aws_iam_role_policy" "snowplow_firehose_enriched_good_policy" { resource "aws_iam_role_policy" "snowplow_firehose_enriched_good_policy" {
name = "firehose_enriched_good" name = "firehose_enriched_good"
policy = "${data.template_file.iam_policy_firehose_enriched_good.rendered}" policy = "${data.template_file.iam_policy_firehose_enriched_good.rendered}"
...@@ -368,7 +378,7 @@ resource "aws_security_group" "snowplow_security_group" { ...@@ -368,7 +378,7 @@ resource "aws_security_group" "snowplow_security_group" {
// Kinesis Streams // Kinesis Streams
resource "aws_kinesis_stream" "snowplow_raw_good" { resource "aws_kinesis_stream" "snowplow_raw_good" {
name = "snowplow-raw-good" name = "snowplow-raw-good"
shard_count = 1 shard_count = 4
retention_period = 48 retention_period = 48
shard_level_metrics = [ shard_level_metrics = [
...@@ -426,8 +436,6 @@ resource "aws_kinesis_stream" "snowplow_enriched_good" { ...@@ -426,8 +436,6 @@ resource "aws_kinesis_stream" "snowplow_enriched_good" {
} }
} }
// Kinesis Firehose
// EC2 Launch Configs // EC2 Launch Configs
data "aws_ami" "amazonlinux2" { data "aws_ami" "amazonlinux2" {
most_recent = true most_recent = true
...@@ -629,6 +637,11 @@ resource "aws_kinesis_firehose_delivery_stream" "snowplow_enriched_bad_firehose" ...@@ -629,6 +637,11 @@ resource "aws_kinesis_firehose_delivery_stream" "snowplow_enriched_bad_firehose"
destination = "extended_s3" destination = "extended_s3"
name = "SnowPlowEnrichedBad" name = "SnowPlowEnrichedBad"
kinesis_source_configuration {
kinesis_stream_arn = "${aws_kinesis_stream.snowplow_enriched_bad.arn}"
role_arn = "${aws_iam_role.snowplow_firehose_delivery_role.arn}"
}
// Terraform seems to be bad at this? // Terraform seems to be bad at this?
// https://github.com/terraform-providers/terraform-provider-aws/issues/6053 // https://github.com/terraform-providers/terraform-provider-aws/issues/6053
lifecycle { lifecycle {
...@@ -669,6 +682,11 @@ resource "aws_kinesis_firehose_delivery_stream" "snowplow_enriched_good_firehose ...@@ -669,6 +682,11 @@ resource "aws_kinesis_firehose_delivery_stream" "snowplow_enriched_good_firehose
destination = "extended_s3" destination = "extended_s3"
name = "SnowPlowEnrichedGood" name = "SnowPlowEnrichedGood"
kinesis_source_configuration {
kinesis_stream_arn = "${aws_kinesis_stream.snowplow_enriched_good.arn}"
role_arn = "${aws_iam_role.snowplow_firehose_delivery_role.arn}"
}
// Terraform seems to be bad at this? // Terraform seems to be bad at this?
// https://github.com/terraform-providers/terraform-provider-aws/issues/6053 // https://github.com/terraform-providers/terraform-provider-aws/issues/6053
lifecycle { lifecycle {
...@@ -704,3 +722,48 @@ resource "aws_kinesis_firehose_delivery_stream" "snowplow_enriched_good_firehose ...@@ -704,3 +722,48 @@ resource "aws_kinesis_firehose_delivery_stream" "snowplow_enriched_good_firehose
environment = "SnowPlow" environment = "SnowPlow"
} }
} }
resource "aws_kinesis_firehose_delivery_stream" "snowplow_raw_bad_firehose" {
destination = "extended_s3"
name = "SnowPlowRawBad"
kinesis_source_configuration {
kinesis_stream_arn = "${aws_kinesis_stream.snowplow_raw_bad.arn}"
role_arn = "${aws_iam_role.snowplow_firehose_delivery_role.arn}"
}
// Terraform seems to be bad at this?
// https://github.com/terraform-providers/terraform-provider-aws/issues/6053
lifecycle {
ignore_changes = [
"extended_s3_configuration.0.data_format_conversion_configuration",
"extended_s3_configuration.0.data_format_conversion_configuration.0.enabled",
]
}
extended_s3_configuration {
bucket_arn = "${aws_s3_bucket.snowplow_s3_bucket.arn}"
role_arn = "${aws_iam_role.snowplow_firehose_delivery_role.arn}"
compression_format = "GZIP"
prefix = "raw-bad/"
error_output_prefix = "raw-bad/"
s3_backup_mode = "Disabled"
processing_configuration {
enabled = "true"
processors {
type = "Lambda"
parameters {
parameter_name = "LambdaArn"
parameter_value = "${aws_lambda_function.snowplow_event_formatter_lambda_function.arn}:$LATEST"
}
}
}
}
tags = {
environment = "SnowPlow"
}
}
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "",
"Effect": "Allow",
"Action": [
"glue:GetTableVersions"
],
"Resource": "*"
},
{
"Sid": "",
"Effect": "Allow",
"Action": [
"s3:AbortMultipartUpload",
"s3:GetBucketLocation",
"s3:GetObject",
"s3:ListBucket",
"s3:ListBucketMultipartUploads",
"s3:PutObject"
],
"Resource": [
"arn:aws:s3:::gitlab-com-snowplow-events",
"arn:aws:s3:::gitlab-com-snowplow-events/*",
"arn:aws:s3:::%FIREHOSE_BUCKET_NAME%",
"arn:aws:s3:::%FIREHOSE_BUCKET_NAME%/*"
]
},
{
"Sid": "",
"Effect": "Allow",
"Action": [
"lambda:InvokeFunction",
"lambda:GetFunctionConfiguration"
],
"Resource": "arn:aws:lambda:us-east-1:855262394183:function:SnowPlowFirehoseFormatter:$LATEST"
},
{
"Sid": "",
"Effect": "Allow",
"Action": [
"logs:PutLogEvents"
],
"Resource": [
"arn:aws:logs:us-east-1:855262394183:log-group:/aws/kinesisfirehose/SnowPlowRawBad:log-stream:*"
]
},
{
"Sid": "",
"Effect": "Allow",
"Action": [
"kinesis:DescribeStream",
"kinesis:GetShardIterator",
"kinesis:GetRecords"
],
"Resource": "arn:aws:kinesis:us-east-1:855262394183:stream/snowplow-raw-bad"
},
{
"Effect": "Allow",
"Action": [
"kms:Decrypt"
],
"Resource": [
"arn:aws:kms:us-east-1:855262394183:key/%SSE_KEY_ID%"
],
"Condition": {
"StringEquals": {
"kms:ViaService": "kinesis.us-east-1.amazonaws.com"
},
"StringLike": {
"kms:EncryptionContext:aws:kinesis:arn": "arn:aws:kinesis:us-east-1:855262394183:stream/snowplow-raw-bad"
}
}
}
]
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment