Commit 3981eb58 authored by Cameron McFarland's avatar Cameron McFarland

Removing s3loader configs since firehose does what we need.

parent 4f11fe96
......@@ -22,10 +22,6 @@ data "template_file" "user_data_enricher" {
template = "${file("${path.module}/templates/enricher-user-data.sh")}"
}
data "template_file" "user_data_s3loader" {
template = "${file("${path.module}/templates/s3loader-user-data.sh")}"
}
data "template_file" "iam_policy_collector" {
template = "${file("${path.module}/templates/iam_policy_collector.json")}"
}
......@@ -34,10 +30,6 @@ data "template_file" "iam_policy_enricher" {
template = "${file("${path.module}/templates/iam_policy_enricher.json")}"
}
data "template_file" "iam_policy_s3loader" {
template = "${file("${path.module}/templates/iam_policy_s3loader.json")}"
}
data "template_file" "iam_role_collector" {
template = "${file("${path.module}/templates/iam_role_collector.json")}"
}
......@@ -46,10 +38,6 @@ data "template_file" "iam_role_enricher" {
template = "${file("${path.module}/templates/iam_role_enricher.json")}"
}
data "template_file" "iam_role_s3loader" {
template = "${file("${path.module}/templates/iam_role_s3loader.json")}"
}
// Policies
resource "aws_iam_policy" "snowplow_collector_policy" {
description = "Policy the allows the collector to access other AWS services such as Kinesis."
......@@ -67,14 +55,6 @@ resource "aws_iam_policy" "snowplow_enricher_policy" {
policy = "${data.template_file.iam_policy_enricher.rendered}"
}
resource "aws_iam_policy" "snowplow_s3loader_policy" {
description = "Policy to provide AWS access to snowplow s3loader instances such as S3, DynamoDB, and Kinesis."
name = "snowplow-s3loader-policy"
path = "/"
policy = "${data.template_file.iam_policy_s3loader.rendered}"
}
// Roles
resource "aws_iam_role" "snowplow_collector_role" {
name = "snowplow-collector-role"
......@@ -94,15 +74,6 @@ resource "aws_iam_role" "snowplow_enricher_role" {
}
}
resource "aws_iam_role" "snowplow_s3loader_role" {
name = "snowplow-s3loader-role"
assume_role_policy = "${data.template_file.iam_role_s3loader.rendered}"
tags = {
environment = "SnowPlow"
}
}
// Role Policy Attachments
resource "aws_iam_role_policy_attachment" "collector_role_policy_attachment" {
role = "${aws_iam_role.snowplow_collector_role.name}"
......@@ -114,11 +85,6 @@ resource "aws_iam_role_policy_attachment" "enricher_role_policy_attachment" {
policy_arn = "${aws_iam_policy.snowplow_enricher_policy.arn}"
}
resource "aws_iam_role_policy_attachment" "s3loader_role_policy_attachment" {
role = "${aws_iam_role.snowplow_s3loader_role.name}"
policy_arn = "${aws_iam_policy.snowplow_s3loader_policy.arn}"
}
// VPC
resource "aws_vpc" "snowplow_vpc" {
cidr_block = "10.32.0.0/16"
......@@ -334,21 +300,6 @@ resource "aws_kinesis_stream" "snowplow_enriched_good" {
}
}
resource "aws_kinesis_stream" "snowplow_s3loader_bad" {
name = "snowplow-s3loader-bad"
shard_count = 1
retention_period = 48
shard_level_metrics = [
"IncomingBytes",
"OutgoingBytes",
]
tags = {
environment = "SnowPlow"
}
}
// EC2 Launch Configs
data "aws_ami" "amazonlinux2" {
most_recent = true
......@@ -404,25 +355,6 @@ resource "aws_launch_configuration" "snowplow_enricher_launch_config" {
}
}
resource "aws_launch_configuration" "snowplow_s3loader_launch_config" {
image_id = "${data.aws_ami.amazonlinux2.id}"
instance_type = "t2.micro"
associate_public_ip_address = "true"
enable_monitoring = "false"
iam_instance_profile = "${aws_iam_role.snowplow_s3loader_role.id}"
key_name = "cmcfarland"
security_groups = [
"${aws_security_group.snowplow_security_group.id}",
]
user_data = "${data.template_file.user_data_s3loader.rendered}"
lifecycle {
create_before_destroy = true
}
}
// EC2 Target Group
resource "aws_lb_target_group" "snowplow_collector_lb_target_group" {
name = "SnowPlowNLBTargetGroup"
......@@ -500,25 +432,6 @@ resource "aws_autoscaling_group" "snowplow_enricher_autoscaling_group" {
}
}
resource "aws_autoscaling_group" "snowplow_s3loader_autoscaling_group" {
launch_configuration = "${aws_launch_configuration.snowplow_s3loader_launch_config.id}"
max_size = "1"
min_size = "0"
desired_capacity = "0"
tag {
key = "environment"
value = "SnowPlow"
propagate_at_launch = true
}
tag {
key = "Name"
value = "SnowPlowAutoS3Loader"
propagate_at_launch = true
}
}
// EC2 Load Balancer
resource "aws_lb" "snowplow_lb" {
name = "SnowPlowNLB"
......
#!/bin/bash
## This has, so far, been written to run on Amazon Linux 2 AMI.
## Install Java 1.8
yum -y install java-1.8.0-openjdk lzop lzo-devel
## Set up user, group, and install location
groupadd snowplow
adduser --system --gid snowplow snowplow
mkdir -p /snowplow
mkdir -p /snowplow/config
mkdir -p /snowplow/logs
## Install SnowPlow Kinesis Collector
mkdir -p /tmp/build
cd /tmp/build
wget -q http://dl.bintray.com/snowplow/snowplow-generic/snowplow_s3_loader_0.6.0.zip
unzip -d /snowplow/bin snowplow_s3_loader_0.6.0.zip
cd /tmp
rm -rf /tmp/build
## Need a logging config
cat > /snowplow/bin/log4j.properties <<EOX
log4j.rootLogger=INFO, FILE
log4j.appender.FILE=org.apache.log4j.FileAppender
log4j.appender.FILE.File=/snowplow/logs/log.out
log4j.appender.FILE.ImmediateFlush=true
log4j.appender.FILE.Threshold=debug
log4j.appender.FILE.Append=false
log4j.appender.FILE.layout=org.apache.log4j.PatternLayout
EOX
## Need to copy in a config
cat > /snowplow/config/s3loader.hocon <<EOF
source = "kinesis"
sink = "kinesis"
aws {
accessKey = "iam"
secretKey = "iam"
}
nsq {
channelName = "noidea"
host = "snowplownsq.gitlab.com"
port = 8085
lookupPort = 8090
}
kinesis {
initialPosition = "LATEST"
maxRecords = 5
region = "us-east-1"
appName = "SnowplowS3Loader-gitlab-us-east-1"
}
streams {
inStreamName = "snowplow-enriched-good"
outStreamName = "snowplow-s3loader-bad"
buffer {
byteLimit = 16384
recordLimit = 1000
timeLimit = 10000
}
}
s3 {
region = "us-east-1"
bucket = "gitlab-com-snowplow-test-one/s3loader"
format = "gzip"
maxTimeout = 120000000
}
EOF
chown -R snowplow:snowplow /snowplow
## Star the collector service
su snowplow -g snowplow -c '/usr/bin/java -Dlog4j.configuration=file:/snowplow/bin/log4j.properties -jar /snowplow/bin/snowplow-s3-loader-0.6.0.jar --config /snowplow/config/s3loader.hocon'
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment