Adding new promtheus server and versioned bootstrap scripts.

environments/gprd/main.tf

@@ -454,6 +454,34 @@ module "consul" {
   vpc            = "${module.network.self_link}"
 }
 
+##################################
+#
+#  Monitoring (prometheus)
+#
+##################################
+
+module "monitoring" {
+  attach_data_disk     = true
+  data_disk_size       = 1000
+  data_disk_type       = "pd-standard"
+  chef_provision       = "${var.chef_provision}"
+  chef_run_list        = "${var.base_chef_run_list}"
+  dns_zone_name        = "${var.dns_zone_name}"
+  environment          = "${var.environment}"
+  ip_cidr_range        = "${var.subnetworks["monitoring"]}"
+  machine_type         = "${var.machine_types["monitoring"]}"
+  name                 = "prometheus"
+  node_count           = "${var.node_count["monitoring"]}"
+  project              = "${var.project}"
+  public_ports         = "${var.public_ports["monitoring"]}"
+  region               = "${var.region}"
+  source               = "../../modules/google/generic-stor"
+  tier                 = "inf"
+  vpc                  = "${module.network.self_link}"
+  persistent_disk_path = "/opt/prometheus"
+  bootstrap_version    = 2
+}
+
 ##################################
 #
 #  VPN connection to Azure

environments/gprd/variables.tf

@@ -22,6 +22,7 @@ variable "public_ports" {
     "sidekiq"     = [22]
     "stor"        = [22]
     "web"         = [22]
+    "monitoring"  = [22]
   }
 }
 
@@ -64,6 +65,7 @@ variable "machine_types" {
     "geodb"              = "n1-highmem-32"
     "git"                = "n1-standard-16"
     "pgb"                = "n1-standard-4"
+    "monitoring"         = "n1-standard-8"
     "redis"              = "n1-highmem-2"
     "redis-cache"        = "n1-highmem-16"
     "sidekiq-besteffort" = "n1-standard-64"
@@ -86,6 +88,7 @@ variable "node_count" {
     "lfs"                = 1
     "pages"              = 1
     "pgb"                = 1
+    "monitoring"         = 1
     "redis"              = 1
     "redis-cache"        = 1
     "share"              = 1
@@ -104,6 +107,7 @@ variable "subnetworks" {
     "db"          = "10.217.1.0/24"
     "fe-lb"       = "10.216.1.0/24"
     "geodb"       = "10.217.6.0/24"
+    "monitoring"  = "10.219.1.0/24"
     "git"         = "10.220.4.0/23"
     "pgb"         = "10.217.4.0/24"
     "redis"       = "10.217.2.0/24"

modules/google/generic-nodisk/instance.tf

@@ -22,7 +22,7 @@ resource "google_compute_instance" "instance_without_attached_disk" {
     "CHEF_PROJECT"       = "${var.project}"
   }
 
-  metadata_startup_script = "${file("${path.module}/../../../scripts/google/generic-pet-bootstrap.sh")}"
+  metadata_startup_script = "${file("${path.module}/../../../scripts/google/bootstrap-v${var.bootstrap_version}.sh")}"
   project                 = "${var.project}"
   zone                    = "${var.zone != "" ? var.zone : data.google_compute_zones.available.names[(count.index + 1) % length(data.google_compute_zones.available.names)]}"
 

modules/google/generic-nodisk/variables.tf

+variable "bootstrap_version" {
+  description = "version of the bootstrap script"
+  default     = 1
+}
+
 variable "chef_provision" {
   type        = "map"
   description = "Configuration details for chef server"

modules/google/generic-pet/instance.tf

@@ -27,7 +27,7 @@ resource "google_compute_instance" "instance_with_attached_disk" {
     "CHEF_PROJECT"       = "${var.project}"
   }
 
-  metadata_startup_script = "${file("${path.module}/../../../scripts/google/generic-pet-bootstrap.sh")}"
+  metadata_startup_script = "${file("${path.module}/../../../scripts/google/bootstrap-v${var.bootstrap_version}.sh")}"
   project                 = "${var.project}"
   zone                    = "${var.zone != "" ? var.zone : data.google_compute_zones.available.names[(count.index + 1) % length(data.google_compute_zones.available.names)]}"
 
@@ -104,7 +104,7 @@ resource "google_compute_instance" "instance_without_attached_disk" {
     "CHEF_PROJECT"       = "${var.project}"
   }
 
-  metadata_startup_script = "${file("${path.module}/../../../scripts/google/generic-pet-bootstrap.sh")}"
+  metadata_startup_script = "${file("${path.module}/../../../scripts/google/bootstrap-v${var.bootstrap_version}.sh")}"
   project                 = "${var.project}"
   zone                    = "${var.zone != "" ? var.zone : data.google_compute_zones.available.names[(count.index + 1) % length(data.google_compute_zones.available.names)]}"
 

modules/google/generic-pet/variables.tf

+variable "bootstrap_version" {
+  description = "version of the bootstrap script"
+  default     = 1
+}
+
 variable "attach_data_disk" {
   type        = "string"
   description = "Attach a data disk to this machine"

modules/google/generic-stor/instance.tf

@@ -27,16 +27,17 @@ resource "google_compute_instance" "instance_with_attached_disk" {
   machine_type = "${var.machine_type}"
 
   metadata = {
-    "CHEF_URL"           = "${var.chef_provision.["server_url"]}"
-    "CHEF_VERSION"       = "${var.chef_provision.["version"]}"
-    "CHEF_NODE_NAME"     = "${format("%v-%02d.%v.%v.%v", var.name, count.index + 1, var.tier, var.environment, var.dns_zone_name)}"
-    "CHEF_ENVIRONMENT"   = "${var.environment}"
-    "CHEF_RUN_LIST"      = "${var.chef_run_list}"
-    "CHEF_DNS_ZONE_NAME" = "${var.dns_zone_name}"
-    "CHEF_PROJECT"       = "${var.project}"
+    "CHEF_URL"             = "${var.chef_provision.["server_url"]}"
+    "CHEF_VERSION"         = "${var.chef_provision.["version"]}"
+    "CHEF_NODE_NAME"       = "${format("%v-%02d.%v.%v.%v", var.name, count.index + 1, var.tier, var.environment, var.dns_zone_name)}"
+    "CHEF_ENVIRONMENT"     = "${var.environment}"
+    "CHEF_RUN_LIST"        = "${var.chef_run_list}"
+    "CHEF_DNS_ZONE_NAME"   = "${var.dns_zone_name}"
+    "CHEF_PROJECT"         = "${var.project}"
+    "PERSISTENT_DISK_PATH" = "${var.persistent_disk_path}"
   }
 
-  metadata_startup_script = "${file("${path.module}/../../../scripts/google/generic-pet-bootstrap.sh")}"
+  metadata_startup_script = "${file("${path.module}/../../../scripts/google/bootstrap-v${var.bootstrap_version}.sh")}"
   project                 = "${var.project}"
   zone                    = "${var.zone != "" ? var.zone : data.google_compute_zones.available.names[(count.index + 1) % length(data.google_compute_zones.available.names)]}"
 

modules/google/generic-stor/variables.tf

+variable "bootstrap_version" {
+  description = "version of the bootstrap script"
+  default     = 1
+}
+
+variable "persistent_disk_path" {
+  type        = "string"
+  description = "default location for disk mount"
+  default     = "/var/opt/gitlab"
+}
+
 variable "attach_data_disk" {
   type        = "string"
   description = "Attach a data disk to this machine"

modules/google/gke/variables.tf

+variable "bootstrap_version" {
+  description = "version of the bootstrap script"
+  default     = 1
+}
+
 variable "disk_size" {
   type        = "string"
   description = "The size of the disk for the worker nodes"

scripts/google/bootstrap-v2.sh

+#!/bin/bash
+
+exec &> >(tee -a "/tmp/bootstrap.log")
+
+#Pass env variables
+for i in $(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/attributes/" -H "Metadata-Flavor: Google"); do
+  if [[ $i == CHEF* ]] ;
+  then
+    export "$i"="$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/attributes/$i" -H "Metadata-Flavor: Google")"
+  fi
+done
+
+#Lookup consul's service endpoint
+apt-get install jq -y -q
+
+# TODO
+# CONSUL_IP=$(gcloud compute --project=${CHEF_PROJECT} forwarding-rules list --filter="description~inf/${CHEF_ENVIRONMENT}-${CHEF_DNS_ZONE_NAME}-consul-serf" --format="json"|jq ".[0].IPAddress")
+
+# Format persistent disk
+if [[ -L /dev/disk/by-id/google-persistent-disk-1 ]]; then
+    mkfs.ext4 -m 0 -F -E lazy_itable_init=0,lazy_journal_init=0,discard /dev/sdb
+    mkdir -p "${PERSISTENT_DISK_PATH:-/var/opt/gitlab}"
+    mount -o discard,defaults /dev/sdb ${PERSISTENT_DISK_PATH:-/var/opt/gitlab}
+    echo UUID="$(sudo blkid -s UUID -o value /dev/sdb)" ${PERSISTENT_DISK_PATH:-/var/opt/gitlab} ext4 discard,defaults 0 2 | tee -a /etc/fstab
+fi
+
+# Install chef
+
+curl -L https://omnitruck.chef.io/install.sh | sudo bash -s -- -v "${CHEF_VERSION}"
+mkdir /etc/chef
+
+# Get validation.pem from gkms
+gsutil cp gs://gitlab-gprd-chef-boostrap/validation.enc /tmp/validation.enc
+
+gcloud kms decrypt --keyring=gitlab-gprd-bootstrap --location=global --key=gitlab-gprd-bootstrap-validation --plaintext-file=/etc/chef/validation.pem  --ciphertext-file=/tmp/validation.enc
+
+
+rm -f /tmp/validator.enc
+
+# create client.rb
+cat > /etc/chef/client.rb <<-EOF
+chef_server_url  "$CHEF_URL"
+validation_client_name "gitlab-validator"
+log_location   STDOUT
+node_name "$CHEF_NODE_NAME"
+environment "$CHEF_ENVIRONMENT"
+EOF
+
+# create run_list
+cat > /etc/chef/first-run.json <<-EOF
+{
+  "run_list":[ $CHEF_RUN_LIST ]
+}
+EOF
+
+# run chef
+chef-client -j /etc/chef/first-run.json