Commit bc85d30b authored by John Jarvis's avatar John Jarvis

refactoring sidekiq.

parent a94a4dd0
......@@ -206,7 +206,6 @@ module "pg-bouncer" {
public_ports = "${var.public_ports["pgb"]}"
region = "${var.region}"
# source = "../../modules/google/generic-sv"
source = "../../modules/google/generic-sv"
tier = "db"
vpc = "${module.network.self_link}"
......@@ -286,21 +285,36 @@ module "redis-cache" {
##################################
module "sidekiq" {
bootstrap_version = 3
chef_provision = "${var.chef_provision}"
chef_run_list = "\"role[${var.environment}-base-be-sidekiq-besteffort]\""
dns_zone_name = "${var.dns_zone_name}"
environment = "${var.environment}"
ip_cidr_range = "${var.subnetworks["sidekiq"]}"
machine_type = "${var.machine_types["sidekiq-besteffort"]}"
name = "sidekiq-besteffort"
node_count = "${var.node_count["sidekiq-besteffort"]}"
project = "${var.project}"
public_ports = "${var.public_ports["sidekiq"]}"
region = "${var.region}"
source = "../../modules/google/generic-sv"
tier = "sv"
vpc = "${module.network.self_link}"
bootstrap_version = 3
chef_provision = "${var.chef_provision}"
chef_run_list = "\"role[${var.environment}-base-be-sidekiq-besteffort]\""
dns_zone_name = "${var.dns_zone_name}"
environment = "${var.environment}"
ip_cidr_range = "${var.subnetworks["sidekiq"]}"
machine_type = "${var.machine_types["sidekiq-besteffort"]}"
name = "sidekiq"
project = "${var.project}"
public_ports = "${var.public_ports["sidekiq"]}"
region = "${var.region}"
sidekiq_asap_count = "${var.node_count["sidekiq-asap"]}"
sidekiq_asap_instance_type = "${var.machine_types["sidekiq-asap"]}"
sidekiq_besteffort_count = "${var.node_count["sidekiq-besteffort"]}"
sidekiq_besteffort_instance_type = "${var.machine_types["sidekiq-besteffort"]}"
sidekiq_traces_count = "${var.node_count["sidekiq-traces"]}"
sidekiq_traces_instance_type = "${var.machine_types["sidekiq-traces"]}"
sidekiq_elasticsearch_count = "${var.node_count["sidekiq-elasticsearch"]}"
sidekiq_elasticsearch_instance_type = "${var.machine_types["sidekiq-elasticsearch"]}"
sidekiq_pages_count = "${var.node_count["sidekiq-pages"]}"
sidekiq_pages_instance_type = "${var.machine_types["sidekiq-pages"]}"
sidekiq_pipeline_count = "${var.node_count["sidekiq-pipeline"]}"
sidekiq_pipeline_instance_type = "${var.machine_types["sidekiq-pipeline"]}"
sidekiq_pullmirror_count = "${var.node_count["sidekiq-pullmirror"]}"
sidekiq_pullmirror_instance_type = "${var.machine_types["sidekiq-pullmirror"]}"
sidekiq_realtime_count = "${var.node_count["sidekiq-realtime"]}"
sidekiq_realtime_instance_type = "${var.machine_types["sidekiq-realtime"]}"
source = "../../modules/google/generic-sv-sidekiq"
tier = "sv"
vpc = "${module.network.self_link}"
}
##################################
......
......@@ -85,23 +85,31 @@ variable "machine_types" {
type = "map"
default = {
"api" = "n1-standard-8"
"consul" = "n1-standard-4"
"deploy" = "n1-standard-2"
"runner" = "n1-standard-2"
"db" = "n1-highmem-32"
"fe-lb" = "n1-standard-4"
"geodb" = "n1-highmem-32"
"git" = "n1-standard-16"
"pgb" = "n1-standard-4"
"mailroom" = "n1-standard-4"
"monitoring" = "n1-standard-8"
"redis" = "n1-highmem-2"
"redis-cache" = "n1-highmem-16"
"registry" = "n1-standard-2"
"sidekiq-besteffort" = "n1-standard-64"
"stor" = "n1-standard-4"
"web" = "n1-standard-16"
"api" = "n1-standard-8"
"consul" = "n1-standard-4"
"deploy" = "n1-standard-2"
"runner" = "n1-standard-2"
"db" = "n1-highmem-32"
"fe-lb" = "n1-standard-4"
"geodb" = "n1-highmem-32"
"git" = "n1-standard-16"
"pgb" = "n1-standard-4"
"mailroom" = "n1-standard-4"
"monitoring" = "n1-standard-8"
"redis" = "n1-highmem-2"
"redis-cache" = "n1-highmem-16"
"registry" = "n1-standard-2"
"sidekiq-asap" = "n1-standard-8"
"sidekiq-besteffort" = "n1-standard-64"
"sidekiq-elasticsearch" = "n1-standard-8"
"sidekiq-pages" = "n1-standard-8"
"sidekiq-pipeline" = "n1-standard-8"
"sidekiq-pullmirror" = "n1-standard-8"
"sidekiq-realtime" = "n1-standard-8"
"sidekiq-traces" = "n1-standard-8"
"stor" = "n1-standard-4"
"web" = "n1-standard-16"
}
}
......@@ -109,26 +117,33 @@ variable "node_count" {
type = "map"
default = {
"api" = 2
"deploy" = 1
"runner" = 1
"artifacts" = 1
"consul" = 3
"db" = 1
"fe-lb" = 2
"geodb" = 1
"git" = 2
"lfs" = 1
"mailroom" = 2
"pages" = 1
"pgb" = 1
"redis" = 1
"redis-cache" = 1
"registry" = 2
"share" = 1
"sidekiq-besteffort" = 3
"stor" = 16
"web" = 2
"api" = 1
"deploy" = 1
"runner" = 1
"artifacts" = 1
"consul" = 3
"db" = 1
"fe-lb" = 2
"geodb" = 1
"git" = 1
"lfs" = 1
"mailroom" = 2
"pages" = 1
"pgb" = 1
"redis" = 1
"redis-cache" = 1
"registry" = 2
"share" = 1
"sidekiq-asap" = 2
"sidekiq-besteffort" = 3
"sidekiq-elasticsearch" = 2
"sidekiq-pages" = 2
"sidekiq-pipeline" = 2
"sidekiq-pullmirror" = 2
"sidekiq-realtime" = 2
"sidekiq-traces" = 2
"stor" = 16
"web" = 1
}
}
......
resource "google_compute_instance" "sidekiq_besteffort" {
count = "${var.sidekiq_besteffort_count}"
name = "${format("%v-besteffort-%02d-%v-%v", var.name, count.index + 1, var.tier, var.environment)}"
machine_type = "${var.sidekiq_besteffort_instance_type}"
metadata = {
"CHEF_URL" = "${var.chef_provision.["server_url"]}"
"CHEF_VERSION" = "${var.chef_provision.["version"]}"
"CHEF_NODE_NAME" = "${format("%v-%02d.%v.%v.%v", var.name, count.index + 1, var.tier, var.environment, var.dns_zone_name)}"
"CHEF_ENVIRONMENT" = "${var.environment}"
"CHEF_RUN_LIST" = "\"role[${var.environment}-base-be-sidekiq-besteffort]\""
"CHEF_DNS_ZONE_NAME" = "${var.dns_zone_name}"
"CHEF_PROJECT" = "${var.project}"
"shutdown-script" = "${file("${path.module}/../../../scripts/google/teardown-v1.sh")}"
}
metadata_startup_script = "${file("${path.module}/../../../scripts/google/bootstrap-v${var.bootstrap_version}.sh")}"
project = "${var.project}"
zone = "${var.zone != "" ? var.zone : data.google_compute_zones.available.names[(count.index + 1) % length(data.google_compute_zones.available.names)]}"
service_account {
// this should be the instance under which the instance should be running, rather than the one creating it...
email = "[email protected]"
// all the defaults plus cloudkms to access kms
scopes = [
"https://www.googleapis.com/auth/cloud.useraccounts.readonly",
"https://www.googleapis.com/auth/devstorage.read_only",
"https://www.googleapis.com/auth/logging.write",
"https://www.googleapis.com/auth/monitoring.write",
"https://www.googleapis.com/auth/pubsub",
"https://www.googleapis.com/auth/service.management.readonly",
"https://www.googleapis.com/auth/servicecontrol",
"https://www.googleapis.com/auth/trace.append",
"https://www.googleapis.com/auth/cloudkms",
"https://www.googleapis.com/auth/compute.readonly",
]
}
scheduling {
preemptible = "${var.preemptible}"
}
boot_disk {
auto_delete = true
initialize_params {
image = "${var.os_boot_image}"
size = "${var.os_disk_size}"
type = "${var.os_disk_type}"
}
}
network_interface {
subnetwork = "${google_compute_subnetwork.sidekiq.name}"
access_config = {}
}
labels {
environment = "${var.environment}"
pet_name = "${var.name}-besteffort"
}
tags = [
"${var.name}",
"${var.name}-besteffort",
"${var.environment}",
]
}
output "sidekiq_besteffort_self_link" {
value = "${google_compute_instance.sidekiq_besteffort.*.self_link}"
}
data "google_compute_zones" "available" {
region = "${var.region}"
status = "UP"
}
resource "google_compute_firewall" "public" {
count = "${length(var.public_ports) > 0 ? 1 : 0}"
name = "${format("%v-%v", var.name, var.environment)}"
network = "${var.vpc}"
allow {
protocol = "tcp"
ports = ["${var.public_ports}"]
}
source_ranges = ["0.0.0.0/0"]
target_tags = ["${var.name}"]
}
resource "google_compute_instance" "sidekiq_pages" {
count = "${var.sidekiq_pages_count}"
name = "${format("%v-pages-%02d-%v-%v", var.name, count.index + 1, var.tier, var.environment)}"
machine_type = "${var.sidekiq_pages_instance_type}"
metadata = {
"CHEF_URL" = "${var.chef_provision.["server_url"]}"
"CHEF_VERSION" = "${var.chef_provision.["version"]}"
"CHEF_NODE_NAME" = "${format("%v-%02d.%v.%v.%v", var.name, count.index + 1, var.tier, var.environment, var.dns_zone_name)}"
"CHEF_ENVIRONMENT" = "${var.environment}"
"CHEF_RUN_LIST" = "\"role[${var.environment}-base-be-sidekiq-pages]\""
"CHEF_DNS_ZONE_NAME" = "${var.dns_zone_name}"
"CHEF_PROJECT" = "${var.project}"
"shutdown-script" = "${file("${path.module}/../../../scripts/google/teardown-v1.sh")}"
}
metadata_startup_script = "${file("${path.module}/../../../scripts/google/bootstrap-v${var.bootstrap_version}.sh")}"
project = "${var.project}"
zone = "${var.zone != "" ? var.zone : data.google_compute_zones.available.names[(count.index + 1) % length(data.google_compute_zones.available.names)]}"
service_account {
// this should be the instance under which the instance should be running, rather than the one creating it...
email = "[email protected]"
// all the defaults plus cloudkms to access kms
scopes = [
"https://www.googleapis.com/auth/cloud.useraccounts.readonly",
"https://www.googleapis.com/auth/devstorage.read_only",
"https://www.googleapis.com/auth/logging.write",
"https://www.googleapis.com/auth/monitoring.write",
"https://www.googleapis.com/auth/pubsub",
"https://www.googleapis.com/auth/service.management.readonly",
"https://www.googleapis.com/auth/servicecontrol",
"https://www.googleapis.com/auth/trace.append",
"https://www.googleapis.com/auth/cloudkms",
"https://www.googleapis.com/auth/compute.readonly",
]
}
scheduling {
preemptible = "${var.preemptible}"
}
boot_disk {
auto_delete = true
initialize_params {
image = "${var.os_boot_image}"
size = "${var.os_disk_size}"
type = "${var.os_disk_type}"
}
}
network_interface {
subnetwork = "${google_compute_subnetwork.sidekiq.name}"
access_config = {}
}
labels {
environment = "${var.environment}"
pet_name = "${var.name}-pages"
}
tags = [
"${var.name}",
"${var.name}-pages",
"${var.environment}",
]
}
output "sidekiq_pages_self_link" {
value = "${google_compute_instance.sidekiq_pages.*.self_link}"
}
resource "google_compute_instance" "sidekiq_pipeline" {
count = "${var.sidekiq_pipeline_count}"
name = "${format("%v-pipeline-%02d-%v-%v", var.name, count.index + 1, var.tier, var.environment)}"
machine_type = "${var.sidekiq_pipeline_instance_type}"
metadata = {
"CHEF_URL" = "${var.chef_provision.["server_url"]}"
"CHEF_VERSION" = "${var.chef_provision.["version"]}"
"CHEF_NODE_NAME" = "${format("%v-%02d.%v.%v.%v", var.name, count.index + 1, var.tier, var.environment, var.dns_zone_name)}"
"CHEF_ENVIRONMENT" = "${var.environment}"
"CHEF_RUN_LIST" = "\"role[${var.environment}-base-be-sidekiq-pipeline]\""
"CHEF_DNS_ZONE_NAME" = "${var.dns_zone_name}"
"CHEF_PROJECT" = "${var.project}"
"shutdown-script" = "${file("${path.module}/../../../scripts/google/teardown-v1.sh")}"
}
metadata_startup_script = "${file("${path.module}/../../../scripts/google/bootstrap-v${var.bootstrap_version}.sh")}"
project = "${var.project}"
zone = "${var.zone != "" ? var.zone : data.google_compute_zones.available.names[(count.index + 1) % length(data.google_compute_zones.available.names)]}"
service_account {
// this should be the instance under which the instance should be running, rather than the one creating it...
email = "[email protected]"
// all the defaults plus cloudkms to access kms
scopes = [
"https://www.googleapis.com/auth/cloud.useraccounts.readonly",
"https://www.googleapis.com/auth/devstorage.read_only",
"https://www.googleapis.com/auth/logging.write",
"https://www.googleapis.com/auth/monitoring.write",
"https://www.googleapis.com/auth/pubsub",
"https://www.googleapis.com/auth/service.management.readonly",
"https://www.googleapis.com/auth/servicecontrol",
"https://www.googleapis.com/auth/trace.append",
"https://www.googleapis.com/auth/cloudkms",
"https://www.googleapis.com/auth/compute.readonly",
]
}
scheduling {
preemptible = "${var.preemptible}"
}
boot_disk {
auto_delete = true
initialize_params {
image = "${var.os_boot_image}"
size = "${var.os_disk_size}"
type = "${var.os_disk_type}"
}
}
network_interface {
subnetwork = "${google_compute_subnetwork.sidekiq.name}"
access_config = {}
}
labels {
environment = "${var.environment}"
pet_name = "${var.name}-pipeline"
}
tags = [
"${var.name}",
"${var.name}-pipeline",
"${var.environment}",
]
}
output "sidekiq_pipeline_self_link" {
value = "${google_compute_instance.sidekiq_pipeline.*.self_link}"
}
resource "google_compute_instance" "sidekiq_pullmirror" {
count = "${var.sidekiq_pullmirror_count}"
name = "${format("%v-pullmirror-%02d-%v-%v", var.name, count.index + 1, var.tier, var.environment)}"
machine_type = "${var.sidekiq_pullmirror_instance_type}"
metadata = {
"CHEF_URL" = "${var.chef_provision.["server_url"]}"
"CHEF_VERSION" = "${var.chef_provision.["version"]}"
"CHEF_NODE_NAME" = "${format("%v-%02d.%v.%v.%v", var.name, count.index + 1, var.tier, var.environment, var.dns_zone_name)}"
"CHEF_ENVIRONMENT" = "${var.environment}"
"CHEF_RUN_LIST" = "\"role[${var.environment}-base-be-sidekiq-pullmirror]\""
"CHEF_DNS_ZONE_NAME" = "${var.dns_zone_name}"
"CHEF_PROJECT" = "${var.project}"
"shutdown-script" = "${file("${path.module}/../../../scripts/google/teardown-v1.sh")}"
}
metadata_startup_script = "${file("${path.module}/../../../scripts/google/bootstrap-v${var.bootstrap_version}.sh")}"
project = "${var.project}"
zone = "${var.zone != "" ? var.zone : data.google_compute_zones.available.names[(count.index + 1) % length(data.google_compute_zones.available.names)]}"
service_account {
// this should be the instance under which the instance should be running, rather than the one creating it...
email = "[email protected]"
// all the defaults plus cloudkms to access kms
scopes = [
"https://www.googleapis.com/auth/cloud.useraccounts.readonly",
"https://www.googleapis.com/auth/devstorage.read_only",
"https://www.googleapis.com/auth/logging.write",
"https://www.googleapis.com/auth/monitoring.write",
"https://www.googleapis.com/auth/pubsub",
"https://www.googleapis.com/auth/service.management.readonly",
"https://www.googleapis.com/auth/servicecontrol",
"https://www.googleapis.com/auth/trace.append",
"https://www.googleapis.com/auth/cloudkms",
"https://www.googleapis.com/auth/compute.readonly",
]
}
scheduling {
preemptible = "${var.preemptible}"
}
boot_disk {
auto_delete = true
initialize_params {
image = "${var.os_boot_image}"
size = "${var.os_disk_size}"
type = "${var.os_disk_type}"
}
}
network_interface {
subnetwork = "${google_compute_subnetwork.sidekiq.name}"
access_config = {}
}
labels {
environment = "${var.environment}"
pet_name = "${var.name}-pullmirror"
}
tags = [
"${var.name}",
"${var.name}-pullmirror",
"${var.environment}",
]
}
output "sidekiq_pullmirror_self_link" {
value = "${google_compute_instance.sidekiq_pullmirror.*.self_link}"
}
resource "google_compute_instance" "sidekiq_realtime" {
count = "${var.sidekiq_realtime_count}"
name = "${format("%v-realtime-%02d-%v-%v", var.name, count.index + 1, var.tier, var.environment)}"
machine_type = "${var.sidekiq_realtime_instance_type}"
metadata = {
"CHEF_URL" = "${var.chef_provision.["server_url"]}"
"CHEF_VERSION" = "${var.chef_provision.["version"]}"
"CHEF_NODE_NAME" = "${format("%v-%02d.%v.%v.%v", var.name, count.index + 1, var.tier, var.environment, var.dns_zone_name)}"
"CHEF_ENVIRONMENT" = "${var.environment}"
"CHEF_RUN_LIST" = "\"role[${var.environment}-base-be-sidekiq-realtime]\""
"CHEF_DNS_ZONE_NAME" = "${var.dns_zone_name}"
"CHEF_PROJECT" = "${var.project}"
"shutdown-script" = "${file("${path.module}/../../../scripts/google/teardown-v1.sh")}"
}
metadata_startup_script = "${file("${path.module}/../../../scripts/google/bootstrap-v${var.bootstrap_version}.sh")}"
project = "${var.project}"
zone = "${var.zone != "" ? var.zone : data.google_compute_zones.available.names[(count.index + 1) % length(data.google_compute_zones.available.names)]}"
service_account {
// this should be the instance under which the instance should be running, rather than the one creating it...
email = "[email protected]"
// all the defaults plus cloudkms to access kms
scopes = [
"https://www.googleapis.com/auth/cloud.useraccounts.readonly",
"https://www.googleapis.com/auth/devstorage.read_only",
"https://www.googleapis.com/auth/logging.write",
"https://www.googleapis.com/auth/monitoring.write",
"https://www.googleapis.com/auth/pubsub",
"https://www.googleapis.com/auth/service.management.readonly",
"https://www.googleapis.com/auth/servicecontrol",
"https://www.googleapis.com/auth/trace.append",
"https://www.googleapis.com/auth/cloudkms",
"https://www.googleapis.com/auth/compute.readonly",
]
}
scheduling {
preemptible = "${var.preemptible}"
}
boot_disk {
auto_delete = true
initialize_params {
image = "${var.os_boot_image}"
size = "${var.os_disk_size}"
type = "${var.os_disk_type}"
}
}
network_interface {
subnetwork = "${google_compute_subnetwork.sidekiq.name}"
access_config = {}
}
labels {
environment = "${var.environment}"
pet_name = "${var.name}-realtime"
}
tags = [
"${var.name}",
"${var.name}-realtime",
"${var.environment}",
]
}
output "sidekiq_realtime_self_link" {
value = "${google_compute_instance.sidekiq_realtime.*.self_link}"
}
resource "google_compute_subnetwork" "sidekiq" {
name = "${format("%v-%v", var.name, var.environment)}"
network = "${var.vpc}"
project = "${var.project}"
region = "${var.region}"
ip_cidr_range = "${var.ip_cidr_range}"
private_ip_google_access = true
}
resource "google_compute_instance" "sidekiq_traces" {
count = "${var.sidekiq_traces_count}"
name = "${format("%v-traces-%02d-%v-%v", var.name, count.index + 1, var.tier, var.environment)}"
machine_type = "${var.sidekiq_traces_instance_type}"
metadata = {
"CHEF_URL" = "${var.chef_provision.["server_url"]}"
"CHEF_VERSION" = "${var.chef_provision.["version"]}"
"CHEF_NODE_NAME" = "${format("%v-%02d.%v.%v.%v", var.name, count.index + 1, var.tier, var.environment, var.dns_zone_name)}"
"CHEF_ENVIRONMENT" = "${var.environment}"
"CHEF_RUN_LIST" = "\"role[${var.environment}-base-be-sidekiq-traces]\""
"CHEF_DNS_ZONE_NAME" = "${var.dns_zone_name}"
"CHEF_PROJECT" = "${var.project}"
"shutdown-script" = "${file("${path.module}/../../../scripts/google/teardown-v1.sh")}"
}
metadata_startup_script = "${file("${path.module}/../../../scripts/google/bootstrap-v${var.bootstrap_version}.sh")}"
project = "${var.project}"
zone = "${var.zone != "" ? var.zone :