main.tf 20.1 KB
Newer Older
1 2
// Configure remote state
terraform {
3 4 5 6 7
  backend "s3" {
    bucket = "gitlab-com-infrastructure"
    key    = "terraform/aws-snowplow/terraform.tfstate"
    region = "us-east-1"
  }
8 9 10 11 12
}

// Use credentials from environment or shared credentials file
provider "aws" {
  region  = "us-east-1"
13
  version = "~> 1.41"
14 15
}

16
// Data Templates
17
data "template_file" "user_data_collector" {
18
  template = "${file("${path.module}/templates/collector-user-data.sh")}"
19 20 21
}

data "template_file" "user_data_enricher" {
22
  template = "${file("${path.module}/templates/enricher-user-data.sh")}"
23 24
}

25 26 27 28 29 30 31 32
data "template_file" "iam_policy_collector" {
  template = "${file("${path.module}/templates/iam_policy_collector.json")}"
}

data "template_file" "iam_policy_enricher" {
  template = "${file("${path.module}/templates/iam_policy_enricher.json")}"
}

33 34 35 36
data "template_file" "iam_policy_firehose_enriched_bad" {
  template = "${file("${path.module}/templates/iam_policy_firehose_enriched_bad.json")}"
}

37 38 39 40
data "template_file" "iam_policy_firehose_raw_bad" {
  template = "${file("${path.module}/templates/iam_policy_firehose_raw_bad.json")}"
}

41 42 43 44
data "template_file" "iam_policy_firehose_enriched_good" {
  template = "${file("${path.module}/templates/iam_policy_firehose_enriched_good.json")}"
}

45 46 47 48
data "template_file" "iam_policy_lambda" {
  template = "${file("${path.module}/templates/iam_policy_lambda.json")}"
}

49 50 51 52 53 54 55 56
data "template_file" "iam_role_collector" {
  template = "${file("${path.module}/templates/iam_role_collector.json")}"
}

data "template_file" "iam_role_enricher" {
  template = "${file("${path.module}/templates/iam_role_enricher.json")}"
}

57 58 59 60
data "template_file" "iam_role_lambda" {
  template = "${file("${path.module}/templates/iam_role_lambda.json")}"
}

61 62 63 64 65 66 67 68
data "template_file" "iam_role_firehose_delivery" {
  template = "${file("${path.module}/templates/iam_role_firehose_delivery.json")}"
}

data "template_file" "snowplow_s3_bucket_policy" {
  template = "${file("${path.module}/templates/gitlab-com-snowplow-events.policy.json")}"
}

69 70 71 72 73 74
data "archive_file" "snowplow_lambda_event_formatter_archive" {
  type        = "zip"
  source_file = "${path.module}/lambda/lambda_function.py"
  output_path = "${path.module}/lambda/lambda_function_payload.zip"
}

75 76
// Policies
resource "aws_iam_policy" "snowplow_collector_policy" {
Cameron McFarland's avatar
Cameron McFarland committed
77
  description = "Policy the allows the collector to access other AWS services such as Kinesis."
78 79
  name        = "snowplow-collector-policy"
  path        = "/"
Cameron McFarland's avatar
Cameron McFarland committed
80

81
  policy = "${data.template_file.iam_policy_collector.rendered}"
82 83 84
}

resource "aws_iam_policy" "snowplow_enricher_policy" {
Cameron McFarland's avatar
Cameron McFarland committed
85
  description = ""
86 87
  name        = "snowplow-enricher-policy"
  path        = "/"
Cameron McFarland's avatar
Cameron McFarland committed
88

89
  policy = "${data.template_file.iam_policy_enricher.rendered}"
90 91
}

92 93 94 95 96 97 98 99
resource "aws_iam_policy" "snowplow_lambda_policy" {
  description = ""
  name        = "AWSLambdaBasicExecutionRole-b1df0a33-ac33-47d3-930b-e8e0bf9443ef"
  path        = "/service-role/"

  policy = "${data.template_file.iam_policy_lambda.rendered}"
}

100 101 102 103 104 105
resource "aws_iam_role_policy" "snowplow_firehose_enriched_bad_policy" {
  name   = "firehose_enriched_bad"
  policy = "${data.template_file.iam_policy_firehose_enriched_bad.rendered}"
  role   = "${aws_iam_role.snowplow_firehose_delivery_role.id}"
}

106 107 108 109 110 111
resource "aws_iam_role_policy" "snowplow_firehose_raw_bad_policy" {
  name   = "firehose_raw_bad"
  policy = "${data.template_file.iam_policy_firehose_raw_bad.rendered}"
  role   = "${aws_iam_role.snowplow_firehose_delivery_role.id}"
}

112 113 114 115 116 117
resource "aws_iam_role_policy" "snowplow_firehose_enriched_good_policy" {
  name   = "firehose_enriched_good"
  policy = "${data.template_file.iam_policy_firehose_enriched_good.rendered}"
  role   = "${aws_iam_role.snowplow_firehose_delivery_role.id}"
}

Cameron McFarland's avatar
Cameron McFarland committed
118 119
// Roles
resource "aws_iam_role" "snowplow_collector_role" {
120 121
  name               = "snowplow-collector-role"
  assume_role_policy = "${data.template_file.iam_role_collector.rendered}"
Cameron McFarland's avatar
Cameron McFarland committed
122 123 124 125 126 127 128

  tags = {
    environment = "SnowPlow"
  }
}

resource "aws_iam_role" "snowplow_enricher_role" {
129 130
  name               = "snowplow-enricher-role"
  assume_role_policy = "${data.template_file.iam_role_enricher.rendered}"
Cameron McFarland's avatar
Cameron McFarland committed
131 132 133 134 135 136

  tags = {
    environment = "SnowPlow"
  }
}

137 138 139 140 141 142 143 144 145 146
resource "aws_iam_role" "snowplow_lambda_role" {
  name               = ""
  assume_role_policy = "${data.template_file.iam_role_lambda.rendered}"
  path               = "/service-role/"

  tags = {
    environment = "SnowPlow"
  }
}

147 148 149 150 151 152 153 154 155 156 157 158
resource "aws_iam_role" "snowplow_firehose_delivery_role" {
  name               = ""
  assume_role_policy = "${data.template_file.iam_role_lambda.rendered}"
  path               = "/"

  assume_role_policy = "${data.template_file.iam_role_firehose_delivery.rendered}"

  tags = {
    environment = "SnowPlow"
  }
}

Cameron McFarland's avatar
Cameron McFarland committed
159 160 161 162 163 164 165 166 167 168 169
// Role Policy Attachments
resource "aws_iam_role_policy_attachment" "collector_role_policy_attachment" {
  role       = "${aws_iam_role.snowplow_collector_role.name}"
  policy_arn = "${aws_iam_policy.snowplow_collector_policy.arn}"
}

resource "aws_iam_role_policy_attachment" "enricher_role_policy_attachment" {
  role       = "${aws_iam_role.snowplow_enricher_role.name}"
  policy_arn = "${aws_iam_policy.snowplow_enricher_policy.arn}"
}

170 171 172 173 174
resource "aws_iam_role_policy_attachment" "lambda_role_policy_attachment" {
  role       = "${aws_iam_role.snowplow_lambda_role.name}"
  policy_arn = "${aws_iam_policy.snowplow_lambda_policy.arn}"
}

175 176 177 178 179 180 181 182 183 184 185 186 187 188 189
// S3 Buckets
resource "aws_s3_bucket" "snowplow_s3_bucket" {
  bucket = "gitlab-com-snowplow-events"

  tags = {
    environment = "SnowPlow"
  }
}

resource "aws_s3_bucket_policy" "snowplow_s3_bucket_policy" {
  bucket = "${aws_s3_bucket.snowplow_s3_bucket.id}"

  policy = "${data.template_file.snowplow_s3_bucket_policy.rendered}"
}

190 191 192 193 194 195
resource "aws_s3_bucket_notification" "snowplow_bucket_notifications" {
  bucket = "gitlab-com-snowplow-events"

  queue {
    queue_arn     = "arn:aws:sqs:us-east-1:730570900080:sf-snowpipe-AIDAJ5LKTH5PBBNSL6UC2-14u9uWhgmVmEsNRfTA5B9w"
    events        = ["s3:ObjectCreated:*"]
196
    filter_prefix = ""
197 198 199
  }
}

200 201
// VPC
resource "aws_vpc" "snowplow_vpc" {
Cameron McFarland's avatar
Cameron McFarland committed
202 203
  cidr_block = "10.32.0.0/16"

204
  tags = {
205
    Name        = "SnowPlow VPC"
206 207 208 209 210
    environment = "SnowPlow"
  }
}

// Subnet
211
resource "aws_subnet" "snowplow_subnet_1" {
212 213
  vpc_id            = "${aws_vpc.snowplow_vpc.id}"
  cidr_block        = "10.32.2.0/24"
214
  availability_zone = "us-east-1a"
Cameron McFarland's avatar
Cameron McFarland committed
215

216
  tags = {
217
    Name        = "SnowPlow Subnet 1"
218 219 220
    environment = "SnowPlow"
  }
}
221

222
resource "aws_subnet" "snowplow_subnet_2" {
223 224
  vpc_id            = "${aws_vpc.snowplow_vpc.id}"
  cidr_block        = "10.32.1.0/24"
225
  availability_zone = "us-east-1e"
Cameron McFarland's avatar
Cameron McFarland committed
226

227
  tags = {
228
    Name        = "SnowPlow Subnet 2"
229 230 231 232
    environment = "SnowPlow"
  }
}

233
resource "aws_subnet" "snowplow_subnet_3" {
234 235
  vpc_id            = "${aws_vpc.snowplow_vpc.id}"
  cidr_block        = "10.32.0.0/24"
236
  availability_zone = "us-east-1b"
Cameron McFarland's avatar
Cameron McFarland committed
237

238
  tags = {
239
    Name        = "SnowPlow Subnet 3"
240 241 242 243
    environment = "SnowPlow"
  }
}

244 245 246 247 248 249
resource "aws_subnet" "snowplow_subnet_4" {
  vpc_id            = "${aws_vpc.snowplow_vpc.id}"
  cidr_block        = "10.32.3.0/24"
  availability_zone = "us-east-1c"

  tags = {
250
    Name        = "SnowPlow Subnet 4"
251 252 253 254
    environment = "SnowPlow"
  }
}

255 256 257 258
// Internet Gateway
resource "aws_internet_gateway" "snowplow_gw" {
  vpc_id = "${aws_vpc.snowplow_vpc.id}"

259
  tags = {
260
    Name        = "SnowPlow Gateway"
261 262 263 264
    environment = "SnowPlow"
  }
}

265 266 267 268
// Routing Tables
resource "aws_route_table" "snowplow_route_table" {
  vpc_id = "${aws_vpc.snowplow_vpc.id}"

269
  tags = {
270
    Name        = "SnowPlow Routing Table"
271 272 273
    environment = "SnowPlow"
  }
}
274

Cameron McFarland's avatar
Cameron McFarland committed
275
resource "aws_main_route_table_association" "snowplow_main_route_table_association" {
276
  vpc_id         = "${aws_vpc.snowplow_vpc.id}"
Cameron McFarland's avatar
Cameron McFarland committed
277 278 279 280
  route_table_id = "${aws_route_table.snowplow_route_table.id}"
}

resource "aws_route" "snowplow_route" {
281
  route_table_id         = "${aws_route_table.snowplow_route_table.id}"
Cameron McFarland's avatar
Cameron McFarland committed
282
  destination_cidr_block = "0.0.0.0/0"
283
  gateway_id             = "${aws_internet_gateway.snowplow_gw.id}"
Cameron McFarland's avatar
Cameron McFarland committed
284 285 286 287
}

resource "aws_route_table_association" "snowplow_route_table_association_1" {
  route_table_id = "${aws_route_table.snowplow_route_table.id}"
288
  subnet_id      = "${aws_subnet.snowplow_subnet_1.id}"
Cameron McFarland's avatar
Cameron McFarland committed
289 290 291 292
}

resource "aws_route_table_association" "snowplow_route_table_association_2" {
  route_table_id = "${aws_route_table.snowplow_route_table.id}"
293
  subnet_id      = "${aws_subnet.snowplow_subnet_2.id}"
Cameron McFarland's avatar
Cameron McFarland committed
294 295 296 297
}

resource "aws_route_table_association" "snowplow_route_table_association_3" {
  route_table_id = "${aws_route_table.snowplow_route_table.id}"
298
  subnet_id      = "${aws_subnet.snowplow_subnet_3.id}"
Cameron McFarland's avatar
Cameron McFarland committed
299 300
}

301 302 303 304 305
resource "aws_route_table_association" "snowplow_route_table_association_4" {
  route_table_id = "${aws_route_table.snowplow_route_table.id}"
  subnet_id      = "${aws_subnet.snowplow_subnet_4.id}"
}

306 307
// Security Groups
resource "aws_security_group" "snowplow_security_group" {
Cameron McFarland's avatar
Cameron McFarland committed
308 309
  description = "For snowplow stuff"
  name        = "SnowPlow"
310
  vpc_id      = "${aws_vpc.snowplow_vpc.id}"
311

312 313 314 315 316 317 318 319 320 321 322 323 324 325
  egress {
    from_port = "0"
    to_port   = "0"
    protocol  = "-1"

    cidr_blocks = [
      "0.0.0.0/0",
    ]

    ipv6_cidr_blocks = [
      "::/0",
    ]
  }

326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379
  ingress {
    from_port   = 8000
    to_port     = 8000
    protocol    = "tcp"
    description = "Collector"

    cidr_blocks = [
      "0.0.0.0/0",
    ]

    ipv6_cidr_blocks = [
      "::/0",
    ]
  }

  ingress {
    from_port   = 8080
    to_port     = 8080
    protocol    = "tcp"
    description = "Enricher"

    cidr_blocks = [
      "0.0.0.0/0",
    ]

    ipv6_cidr_blocks = [
      "::/0",
    ]
  }

  ingress {
    from_port   = 22
    to_port     = 22
    protocol    = "tcp"
    description = "SSH for Admin IPv4"

    cidr_blocks = [
      "0.0.0.0/0",
    ]
  }

  ingress {
    from_port   = 22
    to_port     = 22
    protocol    = "tcp"
    description = "SSH for Admin IPv6"

    ipv6_cidr_blocks = [
      "::/0",
    ]
  }

  tags = {
    environment = "SnowPlow"
Cameron McFarland's avatar
Cameron McFarland committed
380
    Name        = "SnowPlow"
381 382 383
  }
}

384
// Kinesis Streams
385
resource "aws_kinesis_stream" "snowplow_raw_good" {
386
  name             = "snowplow-raw-good"
387
  shard_count      = 8
388
  retention_period = 48
Cameron McFarland's avatar
Cameron McFarland committed
389

390 391 392 393
  shard_level_metrics = [
    "IncomingBytes",
    "OutgoingBytes",
  ]
394

395 396 397 398 399 400
  tags = {
    environment = "SnowPlow"
  }
}

resource "aws_kinesis_stream" "snowplow_raw_bad" {
401 402
  name             = "snowplow-raw-bad"
  shard_count      = 1
403
  retention_period = 48
Cameron McFarland's avatar
Cameron McFarland committed
404

405 406 407 408 409 410 411 412 413 414
  shard_level_metrics = [
    "IncomingBytes",
    "OutgoingBytes",
  ]

  tags = {
    environment = "SnowPlow"
  }
}

415
resource "aws_kinesis_stream" "snowplow_enriched_bad" {
416 417
  name             = "snowplow-enriched-bad"
  shard_count      = 1
418
  retention_period = 48
Cameron McFarland's avatar
Cameron McFarland committed
419

420 421 422 423
  shard_level_metrics = [
    "IncomingBytes",
    "OutgoingBytes",
  ]
424

425 426 427 428 429 430
  tags = {
    environment = "SnowPlow"
  }
}

resource "aws_kinesis_stream" "snowplow_enriched_good" {
431
  name             = "snowplow-enriched-good"
432
  shard_count      = 2
433
  retention_period = 48
Cameron McFarland's avatar
Cameron McFarland committed
434

435 436 437 438 439 440 441 442 443 444
  shard_level_metrics = [
    "IncomingBytes",
    "OutgoingBytes",
  ]

  tags = {
    environment = "SnowPlow"
  }
}

445
// EC2 Launch Configs
446 447 448 449
data "aws_ami" "amazonlinux2" {
  most_recent = true

  filter {
450
    name   = "name"
451 452 453 454
    values = ["amzn2-ami-hvm-*-x86_64-gp2"]
  }

  filter {
455
    name   = "virtualization-type"
456 457 458 459 460 461 462
    values = ["hvm"]
  }

  owners = ["137112412989"] # Amazon Images
}

resource "aws_launch_configuration" "snowplow_collector_launch_config" {
463
  image_id                    = "${data.aws_ami.amazonlinux2.id}"
464
  instance_type               = "t3.micro"
465
  associate_public_ip_address = "true"
466
  enable_monitoring           = "false"
467
  iam_instance_profile        = "${aws_iam_role.snowplow_collector_role.id}"
Cameron McFarland's avatar
Cameron McFarland committed
468
  key_name                    = "snowplow"
469 470

  security_groups = [
471
    "${aws_security_group.snowplow_security_group.id}",
472 473
  ]

474
  user_data = "${data.template_file.user_data_collector.rendered}"
475 476 477 478 479 480 481 482 483 484 485

  lifecycle {
    create_before_destroy = true
  }
}

resource "aws_launch_configuration" "snowplow_enricher_launch_config" {
  image_id                    = "${data.aws_ami.amazonlinux2.id}"
  instance_type               = "t2.micro"
  associate_public_ip_address = "true"
  enable_monitoring           = "false"
486
  iam_instance_profile        = "${aws_iam_role.snowplow_enricher_role.id}"
Cameron McFarland's avatar
Cameron McFarland committed
487
  key_name                    = "snowplow"
488 489

  security_groups = [
490
    "${aws_security_group.snowplow_security_group.id}",
491 492
  ]

493
  user_data = "${data.template_file.user_data_enricher.rendered}"
494 495 496 497 498 499

  lifecycle {
    create_before_destroy = true
  }
}

500 501 502 503 504 505 506
// EC2 Target Group
resource "aws_lb_target_group" "snowplow_collector_lb_target_group" {
  name     = "SnowPlowNLBTargetGroup"
  port     = 8000
  protocol = "TCP"
  vpc_id   = "${aws_vpc.snowplow_vpc.id}"

507 508 509 510 511 512 513 514 515
  health_check {
    interval            = "30"
    path                = "/health"
    protocol            = "HTTP"
    timeout             = "6"
    healthy_threshold   = "3"
    unhealthy_threshold = "3"
  }

516 517 518 519 520
  tags = {
    environment = "SnowPlow"
  }
}

521 522
// EC2 Auto Scaling Groups
resource "aws_autoscaling_group" "snowplow_collector_autoscaling_group" {
523
  launch_configuration = "${aws_launch_configuration.snowplow_collector_launch_config.id}"
Cameron McFarland's avatar
Cameron McFarland committed
524
  max_size             = "24"
Cameron McFarland's avatar
Cameron McFarland committed
525
  min_size             = "0"
Cameron McFarland's avatar
Cameron McFarland committed
526
  desired_capacity     = "12"
527

528 529
  target_group_arns = [
    "${aws_lb_target_group.snowplow_collector_lb_target_group.id}",
530 531
  ]

532 533 534 535 536 537 538
  vpc_zone_identifier = [
    "${aws_subnet.snowplow_subnet_1.id}",
    "${aws_subnet.snowplow_subnet_2.id}",
    "${aws_subnet.snowplow_subnet_3.id}",
    "${aws_subnet.snowplow_subnet_4.id}",
  ]

539 540 541 542 543 544 545 546 547
  enabled_metrics = [
    "GroupStandbyInstances",
    "GroupTotalInstances",
    "GroupPendingInstances",
    "GroupTerminatingInstances",
    "GroupDesiredCapacity",
    "GroupInServiceInstances",
    "GroupMinSize",
    "GroupMaxSize",
548 549
  ]

550 551 552 553 554 555 556 557 558 559 560 561 562 563
  tag {
    key                 = "environment"
    value               = "SnowPlow"
    propagate_at_launch = true
  }

  tag {
    key                 = "Name"
    value               = "SnowPlowAutoCollector"
    propagate_at_launch = true
  }
}

resource "aws_autoscaling_group" "snowplow_enricher_autoscaling_group" {
564
  launch_configuration = "${aws_launch_configuration.snowplow_enricher_launch_config.id}"
Cameron McFarland's avatar
Cameron McFarland committed
565
  max_size             = "12"
Cameron McFarland's avatar
Cameron McFarland committed
566
  min_size             = "0"
567
  desired_capacity     = "3"
568

569 570 571 572 573 574 575
  vpc_zone_identifier = [
    "${aws_subnet.snowplow_subnet_1.id}",
    "${aws_subnet.snowplow_subnet_2.id}",
    "${aws_subnet.snowplow_subnet_3.id}",
    "${aws_subnet.snowplow_subnet_4.id}",
  ]

576 577 578 579 580 581 582 583 584 585 586
  enabled_metrics = [
    "GroupStandbyInstances",
    "GroupTotalInstances",
    "GroupPendingInstances",
    "GroupTerminatingInstances",
    "GroupDesiredCapacity",
    "GroupInServiceInstances",
    "GroupMinSize",
    "GroupMaxSize",
  ]

587 588 589 590 591 592 593 594 595 596 597 598 599
  tag {
    key                 = "environment"
    value               = "SnowPlow"
    propagate_at_launch = true
  }

  tag {
    key                 = "Name"
    value               = "SnowPlowAutoEnricher"
    propagate_at_launch = true
  }
}

600 601 602 603 604 605 606 607 608 609
// EC2 Load Balancer
resource "aws_lb" "snowplow_lb" {
  name               = "SnowPlowNLB"
  internal           = false
  load_balancer_type = "network"

  subnets = [
    "${aws_subnet.snowplow_subnet_1.id}",
    "${aws_subnet.snowplow_subnet_2.id}",
    "${aws_subnet.snowplow_subnet_3.id}",
610
    "${aws_subnet.snowplow_subnet_4.id}",
611 612 613 614 615 616 617
  ]

  tags = {
    environment = "SnowPlow"
  }
}

618
// EC2 Load Balancer Listener
619 620 621 622 623 624 625 626 627 628 629 630
resource "aws_lb_listener" "snowplow_collector_lb_listener" {
  load_balancer_arn = "${aws_lb.snowplow_lb.arn}"
  port              = "443"
  protocol          = "TLS"
  ssl_policy        = "ELBSecurityPolicy-2016-08"
  certificate_arn   = "arn:aws:acm:us-east-1:855262394183:certificate/1d954bfe-8b8d-46f2-a678-5a3e53cc6225"

  default_action {
    type             = "forward"
    target_group_arn = "${aws_lb_target_group.snowplow_collector_lb_target_group.arn}"
  }
}
631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648

// Lambda Function
resource "aws_lambda_function" "snowplow_event_formatter_lambda_function" {
  description   = "This adds a newline to the end of each record."
  function_name = "SnowPlowFirehoseFormatter"
  handler       = "lambda_function.lambda_handler"
  role          = "${aws_iam_role.snowplow_lambda_role.arn}"
  runtime       = "python2.7"

  filename         = "lambda/lambda_function_payload.zip"
  source_code_hash = "${base64sha256(file("lambda/lambda_function_payload.zip"))}"
  timeout          = "60"

  tags = {
    environment                = "SnowPlow"
    "lambda-console:blueprint" = "kinesis-firehose-process-record-python"
  }
}
649 650 651 652 653 654

// Firehose
resource "aws_kinesis_firehose_delivery_stream" "snowplow_enriched_bad_firehose" {
  destination = "extended_s3"
  name        = "SnowPlowEnrichedBad"

655 656 657 658 659
  kinesis_source_configuration {
    kinesis_stream_arn = "${aws_kinesis_stream.snowplow_enriched_bad.arn}"
    role_arn           = "${aws_iam_role.snowplow_firehose_delivery_role.arn}"
  }

660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699
  // Terraform seems to be bad at this?
  // https://github.com/terraform-providers/terraform-provider-aws/issues/6053
  lifecycle {
    ignore_changes = [
      "extended_s3_configuration.0.data_format_conversion_configuration",
      "extended_s3_configuration.0.data_format_conversion_configuration.0.enabled",
    ]
  }

  extended_s3_configuration {
    bucket_arn          = "${aws_s3_bucket.snowplow_s3_bucket.arn}"
    role_arn            = "${aws_iam_role.snowplow_firehose_delivery_role.arn}"
    compression_format  = "GZIP"
    prefix              = "enriched-bad/"
    error_output_prefix = "enriched-bad/"
    s3_backup_mode      = "Disabled"

    processing_configuration {
      enabled = "true"

      processors {
        type = "Lambda"

        parameters {
          parameter_name  = "LambdaArn"
          parameter_value = "${aws_lambda_function.snowplow_event_formatter_lambda_function.arn}:$LATEST"
        }
      }
    }
  }

  tags = {
    environment = "SnowPlow"
  }
}

resource "aws_kinesis_firehose_delivery_stream" "snowplow_enriched_good_firehose" {
  destination = "extended_s3"
  name        = "SnowPlowEnrichedGood"

700 701 702 703 704
  kinesis_source_configuration {
    kinesis_stream_arn = "${aws_kinesis_stream.snowplow_enriched_good.arn}"
    role_arn           = "${aws_iam_role.snowplow_firehose_delivery_role.arn}"
  }

705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739
  // Terraform seems to be bad at this?
  // https://github.com/terraform-providers/terraform-provider-aws/issues/6053
  lifecycle {
    ignore_changes = [
      "extended_s3_configuration.0.data_format_conversion_configuration",
      "extended_s3_configuration.0.data_format_conversion_configuration.0.enabled",
    ]
  }

  extended_s3_configuration {
    bucket_arn          = "${aws_s3_bucket.snowplow_s3_bucket.arn}"
    role_arn            = "${aws_iam_role.snowplow_firehose_delivery_role.arn}"
    compression_format  = "GZIP"
    prefix              = "output/"
    error_output_prefix = "output/"
    s3_backup_mode      = "Disabled"

    processing_configuration {
      enabled = "true"

      processors {
        type = "Lambda"

        parameters {
          parameter_name  = "LambdaArn"
          parameter_value = "${aws_lambda_function.snowplow_event_formatter_lambda_function.arn}:$LATEST"
        }
      }
    }
  }

  tags = {
    environment = "SnowPlow"
  }
}
740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784

resource "aws_kinesis_firehose_delivery_stream" "snowplow_raw_bad_firehose" {
  destination = "extended_s3"
  name        = "SnowPlowRawBad"

  kinesis_source_configuration {
    kinesis_stream_arn = "${aws_kinesis_stream.snowplow_raw_bad.arn}"
    role_arn           = "${aws_iam_role.snowplow_firehose_delivery_role.arn}"
  }

  // Terraform seems to be bad at this?
  // https://github.com/terraform-providers/terraform-provider-aws/issues/6053
  lifecycle {
    ignore_changes = [
      "extended_s3_configuration.0.data_format_conversion_configuration",
      "extended_s3_configuration.0.data_format_conversion_configuration.0.enabled",
    ]
  }

  extended_s3_configuration {
    bucket_arn          = "${aws_s3_bucket.snowplow_s3_bucket.arn}"
    role_arn            = "${aws_iam_role.snowplow_firehose_delivery_role.arn}"
    compression_format  = "GZIP"
    prefix              = "raw-bad/"
    error_output_prefix = "raw-bad/"
    s3_backup_mode      = "Disabled"

    processing_configuration {
      enabled = "true"

      processors {
        type = "Lambda"

        parameters {
          parameter_name  = "LambdaArn"
          parameter_value = "${aws_lambda_function.snowplow_event_formatter_lambda_function.arn}:$LATEST"
        }
      }
    }
  }

  tags = {
    environment = "SnowPlow"
  }
}