Commit 543f4058 authored by John Jarvis's avatar John Jarvis

Adding pubsubbeat

parent a14fd625
......@@ -722,6 +722,41 @@ module "consul" {
service_account_email = "${var.service_account_email}"
}
##################################
#
# Pubsubbeats
#
# Machines for running the beats
# that consume logs from pubsub
# and send them to elastic cloud
#
# You must have a chef role with the
# following format:
# role[<env>-infra-pubsubbeat-<beat_name>]
#
##################################
module "pubsubbeat" {
bootstrap_version = 3
names = "${var.pubsubbeats["names"]}"
machine_types = "${var.pubsubbeats["machine_types"]}"
chef_provision = "${var.chef_provision}"
chef_run_list = "\"role[gprd-infra-consul]\""
dns_zone_name = "${var.dns_zone_name}"
environment = "${var.environment}"
ip_cidr_range = "${var.subnetworks["pubsubbeat"]}"
project = "${var.project}"
public_ports = "${var.public_ports["pubsubbeat"]}"
region = "${var.region}"
health_check = "tcp"
service_port = 22
source = "../../modules/google/pubsubbeat"
tier = "inf"
vpc = "${module.network.self_link}"
service_account_email = "${var.service_account_email}"
}
##################################
#
# Monitoring
......@@ -1040,13 +1075,3 @@ module "bastion" {
service_account_email = "${var.service_account_email}"
}
##################################
#
# Pubsub for logging
#
##################################
resource "google_pubsub_topic" "mytopic" {
name = "${var.environment}-logging"
}
......@@ -83,6 +83,19 @@ variable "tcp_lbs_bastion" {
}
#######################
# pubsubbeat config
#######################
variable "pubsubbeats" {
type = "map"
default = {
"names" = ["gitaly", "haproxy", "pages", "postgres", "production", "system", "workhorse"]
"machine_types" = ["n1-standard-1", "n1-standard-1", "n1-standard-1", "n1-standard-1", "n1-standard-1", "n1-standard-1", "n1-standard-1"]
}
}
######################
variable "base_chef_run_list" {
default = "\"role[gitlab]\",\"recipe[gitlab_users::default]\",\"recipe[gitlab_sudo::default]\",\"recipe[gitlab-server::bashrc]\""
......@@ -121,6 +134,7 @@ variable "public_ports" {
"geodb" = []
"git" = []
"mailroom" = []
"pubsubbeat" = [22]
"redis" = []
"redis-cache" = []
"registry" = []
......@@ -262,6 +276,7 @@ variable "subnetworks" {
"runner" = "10.218.4.0/24"
"console" = "10.218.5.0/24"
"monitoring" = "10.219.1.0/24"
"pubsubbeat" = "10.219.2.0/24"
"registry" = "10.220.10.0/23"
"mailroom" = "10.220.14.0/23"
"api" = "10.220.2.0/23"
......
data "google_compute_zones" "available" {
region = "${var.region}"
status = "UP"
}
resource "google_compute_firewall" "public" {
count = "${length(var.public_ports) > 0 ? 1 : 0}"
name = "${format("%v-%v", var.name, var.environment)}"
network = "${var.vpc}"
allow {
protocol = "tcp"
ports = ["${var.public_ports}"]
}
source_ranges = ["0.0.0.0/0"]
target_tags = ["${var.name}"]
}
resource "google_compute_backend_service" "default" {
count = "${length(data.google_compute_zones.available.names)}"
name = "${format("%v-%v-%v", var.environment, var.name, data.google_compute_zones.available.names[count.index])}"
protocol = "${var.backend_protocol}"
port_name = "${var.name}"
backend {
group = "${google_compute_instance_group.default.*.self_link[count.index]}"
}
health_checks = ["${var.health_check == "http" ? google_compute_health_check.http.self_link : google_compute_health_check.tcp.self_link }"]
}
resource "google_compute_health_check" "tcp" {
name = "${format("%v-%v-tcp", var.environment, var.name)}"
tcp_health_check {
port = "${var.service_port}"
}
}
resource "google_compute_health_check" "http" {
name = "${format("%v-%v-http", var.environment, var.name)}"
http_health_check {
port = "${var.service_port}"
request_path = "${var.service_path}"
}
}
# Add one instance group per zone
# and only select the appropriate instances
# for each one
resource "google_compute_instance_group" "default" {
count = "${length(data.google_compute_zones.available.names)}"
name = "${format("%v-%v-%v", var.environment, var.name, data.google_compute_zones.available.names[count.index])}"
description = "Instance group for monitoring VM."
zone = "${data.google_compute_zones.available.names[count.index]}"
named_port {
name = "${var.name}"
port = "${var.service_port}"
}
# This filters the full set of instances to only ones for the appropriate zone.
instances = ["${matchkeys(google_compute_instance.default.*.self_link, google_compute_instance.default.*.zone, list(data.google_compute_zones.available.names[count.index]))}"]
}
resource "google_compute_instance" "default" {
allow_stopping_for_update = "${var.allow_stopping_for_update}"
count = "${length(var.names)}"
name = "${format("%v-%v-%v-%v", var.name, var.names[count.index], var.tier, var.environment)}"
machine_type = "${var.machine_types[count.index]}"
metadata = {
"CHEF_URL" = "${var.chef_provision.["server_url"]}"
"CHEF_VERSION" = "${var.chef_provision.["version"]}"
"CHEF_NODE_NAME" = "${format("%v-%v.%v.%v.%v", var.name, var.names[count.index], var.tier, var.environment, var.dns_zone_name)}"
"CHEF_ENVIRONMENT" = "${var.environment}"
"CHEF_INIT_RUN_LIST" = "${var.chef_init_run_list}"
"CHEF_RUN_LIST" = "\"role[${var.environment}-infra-${var.name}]\""
"CHEF_DNS_ZONE_NAME" = "${var.dns_zone_name}"
"CHEF_PROJECT" = "${var.project}"
"CHEF_BOOTSTRAP_BUCKET" = "${var.chef_provision.["bootstrap_bucket"]}"
"CHEF_BOOTSTRAP_KEYRING" = "${var.chef_provision.["bootstrap_keyring"]}"
"CHEF_BOOTSTRAP_KEY" = "${var.chef_provision.["bootstrap_key"]}"
"block-project-ssh-keys" = "${var.block_project_ssh_keys}"
"enable-oslogin" = "${var.enable_oslogin}"
"shutdown-script" = "${file("${path.module}/../../../scripts/google/teardown-v1.sh")}"
}
metadata_startup_script = "${file("${path.module}/../../../scripts/google/bootstrap-v${var.bootstrap_version}.sh")}"
project = "${var.project}"
zone = "${var.zone != "" ? var.zone : data.google_compute_zones.available.names[(count.index + 1) % length(data.google_compute_zones.available.names)]}"
service_account {
// this should be the instance under which the instance should be running, rather than the one creating it...
email = "${var.service_account_email}"
// all the defaults plus cloudkms to access kms
scopes = [
"https://www.googleapis.com/auth/cloud.useraccounts.readonly",
"https://www.googleapis.com/auth/devstorage.read_only",
"https://www.googleapis.com/auth/logging.write",
"https://www.googleapis.com/auth/monitoring.write",
"https://www.googleapis.com/auth/pubsub",
"https://www.googleapis.com/auth/service.management.readonly",
"https://www.googleapis.com/auth/servicecontrol",
"https://www.googleapis.com/auth/trace.append",
"https://www.googleapis.com/auth/cloudkms",
"https://www.googleapis.com/auth/compute.readonly",
]
}
scheduling {
preemptible = "${var.preemptible}"
}
boot_disk {
auto_delete = true
initialize_params {
image = "${var.os_boot_image}"
size = "${var.os_disk_size}"
type = "${var.os_disk_type}"
}
}
network_interface {
subnetwork = "${google_compute_subnetwork.subnetwork.name}"
access_config = {}
}
labels {
environment = "${var.environment}"
pet_name = "${var.name}-${var.names[count.index]}"
}
tags = [
"${var.name}",
"${var.name}-${var.names[count.index]}",
"${var.environment}",
]
}
output "instances_self_link" {
value = "${google_compute_instance.default.*.self_link}"
}
output "instance_groups_self_link" {
value = "${google_compute_instance_group.default.*.self_link}"
}
output "instance_public_ips" {
value = "${google_compute_instance.default.*.network_interface.0.access_config.0.assigned_nat_ip}"
}
resource "google_compute_subnetwork" "subnetwork" {
count = "${length(var.names) > 0 ? 1 : 0}"
name = "${format("%v-%v", var.name, var.environment)}"
network = "${var.vpc}"
project = "${var.project}"
region = "${var.region}"
ip_cidr_range = "${var.ip_cidr_range}"
private_ip_google_access = true
}
##################################
#
# Pubsub topics for logging
# one per pubsub host
#
##################################
resource "google_pubsub_topic" "mytopic" {
count = "${length(var.names)}"
name = "${format("%v-%v-%v-%v", var.name, var.names[count.index], var.tier, var.environment)}"
}
variable "names" {
description = "Names of the indexes, one per pubsubbeat"
type = "list"
}
variable "machine_types" {
description = "Machine types for pubsubbeats"
type = "list"
}
variable "allow_stopping_for_update" {
description = "wether Terraform is allowed to stop the instance to update its properties"
default = "false"
}
variable "backend_protocol" {
default = "HTTP"
}
variable "health_check" {
default = "http"
}
variable "service_port" {}
variable "service_path" {
default = "/"
}
variable "block_project_ssh_keys" {
type = "string"
description = "Whether to block project level SSH keys"
default = "TRUE"
}
variable "bootstrap_version" {
description = "version of the bootstrap script"
default = 1
}
variable "chef_init_run_list" {
type = "string"
default = ""
description = "run_list for the node in chef that are ran on the first boot only"
}
variable "chef_provision" {
type = "map"
description = "Configuration details for chef server"
}
variable "chef_run_list" {
type = "string"
description = "run_list for the node in chef"
}
variable "dns_zone_name" {
type = "string"
description = "The GCP name of the DNS zone to use for this environment"
}
variable "enable_oslogin" {
type = "string"
description = "Whether to enable OS Login GCP feature"
# Note: setting this to TRUE breaks chef!
# https://gitlab.com/gitlab-com/gitlab-com-infrastructure/merge_requests/297#note_66690562
default = "FALSE"
}
variable "environment" {
type = "string"
description = "The environment name"
}
variable "ip_cidr_range" {
type = "string"
description = "The IP range"
}
variable "name" {
default = "pubsub"
}
variable "os_boot_image" {
type = "string"
description = "The OS image to boot"
default = "ubuntu-os-cloud/ubuntu-1604-xenial-v20180122"
}
variable "os_disk_size" {
type = "string"
description = "The OS disk size in GiB"
default = 20
}
variable "os_disk_type" {
type = "string"
description = "The OS disk type"
default = "pd-standard"
}
variable "preemptible" {
type = "string"
description = "Use preemptible instances for this pet"
default = "false"
}
variable "project" {
type = "string"
description = "The project name"
}
variable "public_ports" {
type = "list"
description = "The list of ports that should be publicly reachable"
default = []
}
variable "region" {
type = "string"
description = "The target region"
}
variable "service_account_email" {
type = "string"
description = "Service account emails under which the instance is running"
}
variable "tier" {
type = "string"
description = "The tier for this service"
}
variable "vpc" {
type = "string"
description = "The target network"
}
variable "zone" {
type = "string"
default = ""
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment