Commit 32daca8b authored by Cameron McFarland's avatar Cameron McFarland

Adding more TF resources that should match some of the items in AWS right now.

parent 79d7cb2a
......@@ -17,6 +17,7 @@ Going to probably need an elastic IP for the snowplow endpoint.
IAM Policies and Roles:
Need roles/policies to allow proper access to the collectors, enrichers and s3 loaders.
ECS Tasks need a IAM role to allow access to things? Yes.
Making a bunch of policies is great, but they were for EC2, not ECS.
Kinesis Streams:
snowplow-raw-good
......@@ -36,9 +37,6 @@ Mount an EBS volume? Not sure how this works yet.
The default config (of which there isn't one in the pre-built images) does not support env variables.
Switch to EC2 and a dynamic LB? Why? Why not?
I switched to EC2, I made the snowplow operator user and policy and groups.
I also created an EC2 role to allow EC2 to access kinesis. This seems to be working. At least I can see a config working quickly.
Collector:
I made two kinesis streams with 1 shard each to start with. snowplow-good and snowplow-bad
OMG it worked.
......@@ -56,3 +54,15 @@ Testing an event: curl http://34.227.92.217:8000/i\?e\=pv
S3Loader:
"I realize now my folly: the app name needs to be different between the enricher and loader ergo the 2 dynamoDB tables were conflicting. Everything makes so much sense now…"
ECS:
Needs us to run EC2 instances to run docker on. WTF.
https://docs.aws.amazon.com/AmazonECS/latest/developerguide/ECS_GetStarted_Fargate.html
EC2 Autoscaling Stuff:
Need three target groups. These define the health checks we need to use to determine if something is healthy.
Need three auto-scaling groups.
Last steps:
Are we using the right SSH key?
Did we clean up everything we made for testing?
......@@ -9,6 +9,136 @@ provider "aws" {
version = "~> 1.41"
}
// Policies
resource "aws_iam_policy" "snowplow_collector_policy" {
description = "Policy that allows the collector to access other AWS services such as Kinesis and Cloudwatch."
name = "snowplow-collector-policy"
path = "/"
policy = <<EOF
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "",
"Effect": "Allow",
"Action": [
"cloudwatch:PutMetricData"
],
"Resource": [
"*"
]
},
{
"Sid": "",
"Effect": "Allow",
"Action": [
"kinesis:*"
],
"Resource": [
"*"
]
}
]
}
EOF
}
resource "aws_iam_policy" "snowplow_enricher_policy" {
description = "Policy that allows the enricher to access other AWS services such as Kinesis and Cloudwatch."
name = "snowplow-enricher-policy"
path = "/"
policy = <<EOF
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "",
"Effect": "Allow",
"Action": [
"cloudwatch:PutMetricData"
],
"Resource": [
"*"
]
},
{
"Sid": "",
"Effect": "Allow",
"Action": [
"dynamodb:*"
],
"Resource": [
"arn:aws:dynamodb:us-east-1:855262394183:table/SnowplowEnrich-gitlab-us-east-1"
]
},
{
"Sid": "",
"Effect": "Allow",
"Action": [
"kinesis:*"
],
"Resource": [
"*"
]
}
]
}
EOF
}
resource "aws_iam_policy" "snowplow_s3loader_policy" {
description = "Policy that allows the s3loader to access other AWS services such as Kinesis and Cloudwatch."
name = "snowplow-s3loader-policy"
path = "/"
policy = <<EOF
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "",
"Effect": "Allow",
"Action": [
"cloudwatch:PutMetricData"
],
"Resource": [
"*"
]
},
{
"Sid": "",
"Effect": "Allow",
"Action": [
"s3:*"
],
"Resource": [
"arn:aws:s3:::gitlab-com-snowplow-test-one/*"
]
},
{
"Sid": "",
"Effect": "Allow",
"Action": [
"dynamodb:*"
],
"Resource": [
"arn:aws:dynamodb:us-east-1:855262394183:table/SnowplowS3Loader-gitlab-us-east-1"
]
},
{
"Sid": "",
"Effect": "Allow",
"Action": [
"kinesis:*"
],
"Resource": [
"*"
]
}
]
}
EOF
}
// VPC
resource "aws_vpc" "snowplow_vpc" {
cidr_block = "10.32.0.0/16"
......@@ -19,45 +149,80 @@ resource "aws_vpc" "snowplow_vpc" {
}
// Subnet
resource "aws_subnet" "snowplow_subnet" {
resource "aws_subnet" "snowplow_subnet_1" {
vpc_id = "${aws_vpc.snowplow_vpc.id}"
cidr_block = "10.32.0.0/24"
cidr_block = "10.32.2.0/24"
availability_zone = "us-east-1a"
tags = {
Name = "SnowPlow Subnet 1"
environment = "SnowPlow"
}
}
resource "aws_subnet" "snowplow_subnet_2" {
vpc_id = "${aws_vpc.snowplow_vpc.id}"
cidr_block = "10.32.1.0/24"
availability_zone = "us-east-1e"
tags = {
Name = "SnowPlow Subnet"
Name = "SnowPlow Subnet 2"
environment = "SnowPlow"
}
}
// ECR SnowPlow Collector Repository
resource "aws_ecr_repository" "snowplow_ecr_collector_repo" {
name = "SnowPlow Collector"
resource "aws_subnet" "snowplow_subnet_3" {
vpc_id = "${aws_vpc.snowplow_vpc.id}"
cidr_block = "10.32.0.0/24"
availability_zone = "us-east-1b"
tags = {
Name = "SnowPlow Subnet 3"
environment = "SnowPlow"
}
}
resource "aws_ecr_repository" "snowplow_ecr_enricher_repo" {
name = "SnowPlow Enricher"
// Internet Gateway
resource "aws_internet_gateway" "snowplow_gw" {
vpc_id = "${aws_vpc.snowplow_vpc.id}"
tags = {
Name = "SnowPlow Gateway"
environment = "SnowPlow"
}
}
// ECS Cluster
resource "aws_ecs_cluster" "snowplow_ecs_cluster" {
name = "SnowPlow"
// Routing Tables
resource "aws_route_table" "snowplow_route_table" {
vpc_id = "${aws_vpc.snowplow_vpc.id}"
route {
cidr_block = "0.0.0.0/0"
gateway_id = "${aws_internet_gateway.snowplow_gw.id}"
}
tags = {
Name = "SnowPlow Routing Table"
environment = "SnowPlow"
}
}
// Kinesis Streams
resource "aws_kinesis_stream" "snowplow_good_stream" {
name = "SnowPlow-Good-Stream"
resource "aws_kinesis_stream" "snowplow_raw_good" {
name = "snowplow-raw-good"
shard_count = 1
retention_period = 48
shard_level_metrics = [
"IncomingBytes",
"OutgoingBytes",
]
tags = {
environment = "SnowPlow"
}
}
resource "aws_kinesis_stream" "snowplow_raw_bad" {
name = "snowplow-raw-bad"
shard_count = 1
retention_period = 48
shard_level_metrics = [
"IncomingBytes",
"OutgoingBytes",
......@@ -68,11 +233,38 @@ resource "aws_kinesis_stream" "snowplow_good_stream" {
}
}
resource "aws_kinesis_stream" "snowplow_bad_stream" {
name = "SnowPlow-Bad-Stream"
resource "aws_kinesis_stream" "snowplow_enriched_bad" {
name = "snowplow-enriched-bad"
shard_count = 1
retention_period = 48
shard_level_metrics = [
"IncomingBytes",
"OutgoingBytes",
]
tags = {
environment = "SnowPlow"
}
}
resource "aws_kinesis_stream" "snowplow_enriched_good" {
name = "snowplow-enriched-good"
shard_count = 1
retention_period = 48
shard_level_metrics = [
"IncomingBytes",
"OutgoingBytes",
]
tags = {
environment = "SnowPlow"
}
}
resource "aws_kinesis_stream" "snowplow_s3loader_bad" {
name = "snowplow-s3loader-bad"
shard_count = 1
retention_period = 48
shard_level_metrics = [
"IncomingBytes",
"OutgoingBytes",
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment