Implementation of https://gitlab.com/gitlab-com/gl-infra/infrastructure/issues/6511

environments/testbed/.terraform-version

+0.11.11

environments/testbed/gstg-gprd-ops-variables.tf

+../../shared/gstg-gprd-ops/variables.tf
\ No newline at end of file

environments/testbed/gstg-gprd-variables.tf

+../../shared/gstg-gprd/variables.tf
\ No newline at end of file

environments/testbed/main.tf

+## State storage
+terraform {
+  backend "s3" {}
+}
+
+## AWS
+provider "aws" {
+  region = "us-east-1"
+}
+
+variable "gitlab_com_zone_id" {}
+variable "gitlab_net_zone_id" {}
+
+## Google
+
+provider "google" {
+  version = "~> 1.18.0"
+  project = "${var.project}"
+  region  = "${var.region}"
+}
+
+##################################
+#
+#  Network
+#
+#################################
+
+module "network" {
+  source           = "git::ssh://git@ops.gitlab.net/gitlab-com/gl-infra/terraform-modules/google/vpc.git?ref=v1.0.0"
+  project          = "${var.project}"
+  environment      = "${var.environment}"
+  internal_subnets = "${var.internal_subnets}"
+}
+
+##################################
+#
+#  Network Peering
+#
+#################################
+
+resource "google_compute_network_peering" "peering" {
+  count        = "${length(var.peer_networks["names"])}"
+  name         = "peering-${element(var.peer_networks["names"], count.index)}"
+  network      = "${var.network_env}"
+  peer_network = "${element(var.peer_networks["links"], count.index)}"
+}
+
+##################################
+#
+#  Monitoring
+#
+#  Uses the monitoring module, this
+#  creates a single instance behind
+#  a load balancer with identity aware
+#  proxy enabled.
+#
+##################################
+
+resource "google_compute_subnetwork" "monitoring" {
+  ip_cidr_range            = "${var.subnetworks["monitoring"]}"
+  name                     = "${format("monitoring-%v", var.environment)}"
+  network                  = "${module.network.self_link}"
+  private_ip_google_access = true
+  project                  = "${var.project}"
+  region                   = "${var.region}"
+}
+
+###############################################
+#
+# Load balancer and VM for the pre bastion
+#
+###############################################
+
+module "gcp-tcp-lb-bastion" {
+  environment            = "${var.environment}"
+  forwarding_port_ranges = "${var.tcp_lbs_bastion["forwarding_port_ranges"]}"
+  fqdns                  = "${var.lb_fqdns_bastion}"
+  gitlab_zone_id         = "${var.gitlab_com_zone_id}"
+  health_check_ports     = "${var.tcp_lbs_bastion["health_check_ports"]}"
+  instances              = ["${module.bastion.instances_self_link}"]
+  lb_count               = "${length(var.tcp_lbs_bastion["names"])}"
+  name                   = "gcp-tcp-lb-bastion"
+  names                  = "${var.tcp_lbs_bastion["names"]}"
+  project                = "${var.project}"
+  region                 = "${var.region}"
+  session_affinity       = "CLIENT_IP"
+  source                 = "git::ssh://git@ops.gitlab.net/gitlab-com/gl-infra/terraform-modules/google/tcp-lb.git?ref=remove-lifecycle"
+  targets                = ["bastion"]
+}
+
+module "bastion" {
+  bootstrap_version     = "${var.bootstrap_script_version}"
+  chef_provision        = "${var.chef_provision}"
+  chef_run_list         = "\"role[${var.environment}-base-bastion]\""
+  dns_zone_name         = "${var.dns_zone_name}"
+  environment           = "${var.environment}"
+  health_check          = "tcp"
+  ip_cidr_range         = "${var.subnetworks["bastion"]}"
+  machine_type          = "${var.machine_types["bastion"]}"
+  name                  = "bastion"
+  node_count            = "${var.node_count["bastion"]}"
+  project               = "${var.project}"
+  public_ports          = "${var.public_ports["bastion"]}"
+  region                = "${var.region}"
+  service_account_email = "${var.service_account_email}"
+  service_port          = 22
+  source                = "git::ssh://git@ops.gitlab.net/gitlab-com/gl-infra/terraform-modules/google/generic-sv-with-group.git?ref=v1.0.3"
+  tier                  = "inf"
+  use_new_node_name     = true
+  vpc                   = "${module.network.self_link}"
+}
+
+##################################
+#
+#  Google storage buckets
+#
+##################################
+
+module "storage" {
+  environment                       = "${var.environment}"
+  versioning                        = "${var.versioning}"
+  artifact_age                      = "${var.artifact_age}"
+  lfs_object_age                    = "${var.lfs_object_age}"
+  package_repo_age                  = "${var.package_repo_age}"
+  upload_age                        = "${var.upload_age}"
+  storage_log_age                   = "${var.storage_log_age}"
+  storage_class                     = "${var.storage_class}"
+  service_account_email             = "${var.service_account_email}"
+  gcs_service_account_email         = "${var.gcs_service_account_email}"
+  gcs_storage_analytics_group_email = "${var.gcs_storage_analytics_group_email}"
+  source                            = "git::ssh://git@ops.gitlab.net/gitlab-com/gl-infra/terraform-modules/google/storage-buckets.git?ref=v1.1.1"
+}
+
+#######################
+#
+# load balancer for all hosts in this section
+#
+#######################
+
+module "monitoring-lb" {
+  cert_link          = "${var.monitoring_cert_link}"
+  environment        = "${var.environment}"
+  gitlab_net_zone_id = "${var.gitlab_net_zone_id}"
+  hosts              = ["${var.monitoring_hosts["names"]}"]
+  name               = "monitoring-lb"
+  project            = "${var.project}"
+  region             = "${var.region}"
+  service_ports      = ["${var.monitoring_hosts["ports"]}"]
+  source             = "git::ssh://git@ops.gitlab.net/gitlab-com/gl-infra/terraform-modules/google/monitoring-lb.git?ref=v1.0.0"
+  subnetwork_name    = "${google_compute_subnetwork.monitoring.name}"
+  targets            = ["${var.monitoring_hosts["names"]}"]
+  url_map            = "${google_compute_url_map.monitoring-lb.self_link}"
+}
+
+#######################
+module "prometheus" {
+  attach_data_disk      = true
+  bootstrap_version     = "${var.bootstrap_script_version}"
+  chef_provision        = "${var.chef_provision}"
+  chef_run_list         = "\"role[${var.environment}-infra-prometheus]\""
+  data_disk_size        = 100
+  data_disk_type        = "pd-standard"
+  dns_zone_name         = "${var.dns_zone_name}"
+  environment           = "${var.environment}"
+  fw_whitelist_subnets  = "${concat(var.monitoring_whitelist_prometheus["subnets"], var.other_monitoring_subnets)}"
+  fw_whitelist_ports    = "${var.monitoring_whitelist_prometheus["ports"]}"
+  machine_type          = "${var.machine_types["monitoring"]}"
+  name                  = "prometheus"
+  node_count            = "${var.node_count["prometheus"]}"
+  oauth2_client_id      = "${var.oauth2_client_id_monitoring}"
+  oauth2_client_secret  = "${var.oauth2_client_secret_monitoring}"
+  persistent_disk_path  = "/opt/prometheus"
+  project               = "${var.project}"
+  public_ports          = ["22"]
+  region                = "${var.region}"
+  service_account_email = "${var.service_account_email}"
+  service_path          = "/graph"
+  service_port          = "${element(var.monitoring_hosts["ports"], index(var.monitoring_hosts["names"], "prometheus"))}"
+  source                = "git::ssh://git@ops.gitlab.net/gitlab-com/gl-infra/terraform-modules/google/monitoring-with-count.git?ref=v1.0.4"
+  subnetwork_name       = "${google_compute_subnetwork.monitoring.name}"
+  tier                  = "inf"
+  use_external_ip       = true
+  use_new_node_name     = true
+  vpc                   = "${module.network.self_link}"
+}
+
+#######################################################
+#
+# VM for onprem.testbed.gitlab.net
+#
+#######################################################
+
+resource "aws_route53_record" "default" {
+  zone_id = "${var.gitlab_net_zone_id}"
+  name    = "onprem.testbed.gitlab.net"
+  type    = "A"
+  ttl     = "300"
+  records = ["${module.gitlab-onprem.instance_public_ips[0]}"]
+}
+
+module "gitlab-onprem" {
+  backend_protocol      = "HTTPS"
+  bootstrap_version     = "${var.bootstrap_script_version}"
+  chef_provision        = "${var.chef_provision}"
+  chef_run_list         = "\"role[${var.environment}-base-gitlab-onprem]\""
+  data_disk_size        = 1000
+  data_disk_type        = "pd-ssd"
+  dns_zone_name         = "${var.dns_zone_name}"
+  environment           = "${var.environment}"
+  health_check          = "http"
+  health_check_port     = 8887
+  ip_cidr_range         = "${var.subnetworks["gitlab-onprem"]}"
+  machine_type          = "${var.machine_types["gitlab-onprem"]}"
+  name                  = "gitlab-onprem"
+  node_count            = 1
+  persistent_disk_path  = "/var/opt/gitlab"
+  project               = "${var.project}"
+  public_ports          = "${var.public_ports["gitlab-onprem"]}"
+  region                = "${var.region}"
+  service_account_email = "${var.service_account_email}"
+  service_path          = "/-/liveness"
+  service_port          = 443
+  source                = "git::ssh://git@ops.gitlab.net/gitlab-com/gl-infra/terraform-modules/google/monitoring-with-count.git?ref=v1.0.4"
+  tier                  = "inf"
+  use_external_ip       = true
+  use_new_node_name     = true
+  vpc                   = "${module.network.self_link}"
+}
+
+module "sd-exporter" {
+  additional_scopes         = ["https://www.googleapis.com/auth/monitoring"]
+  allow_stopping_for_update = true
+  bootstrap_version         = "${var.bootstrap_script_version}"
+  chef_provision            = "${var.chef_provision}"
+  chef_run_list             = "\"role[${var.environment}-infra-sd-exporter]\""
+  create_backend_service    = false
+  dns_zone_name             = "${var.dns_zone_name}"
+  environment               = "${var.environment}"
+  machine_type              = "${var.machine_types["sd-exporter"]}"
+  name                      = "sd-exporter"
+  node_count                = "${var.node_count["sd-exporter"]}"
+  project                   = "${var.project}"
+  public_ports              = "${var.public_ports["sd-exporter"]}"
+  region                    = "${var.region}"
+  service_account_email     = "${var.service_account_email}"
+  source                    = "git::ssh://git@ops.gitlab.net/gitlab-com/gl-infra/terraform-modules/google/generic-sv-with-group.git?ref=v1.0.2"
+  subnetwork_name           = "${google_compute_subnetwork.monitoring.name}"
+  tier                      = "inf"
+  use_new_node_name         = true
+  vpc                       = "${module.network.self_link}"
+}
+
+##################################
+#
+#  Gitaly node
+#
+##################################
+
+module "gitaly" {
+  bootstrap_version     = "${var.bootstrap_script_version}"
+  chef_provision        = "${var.chef_provision}"
+  chef_run_list         = "\"role[${var.environment}-base-be-gitaly]\""
+  dns_zone_name         = "${var.dns_zone_name}"
+  egress_ports          = "${var.egress_ports}"
+  environment           = "${var.environment}"
+  health_check          = "tcp"
+  ip_cidr_range         = "${var.subnetworks["gitaly"]}"
+  machine_type          = "${var.machine_types["gitaly"]}"
+  name                  = "gitaly"
+  node_count            = "${var.node_count["gitaly"]}"
+  os_disk_type          = "pd-ssd"
+  project               = "${var.project}"
+  public_ports          = "${var.public_ports["gitaly"]}"
+  region                = "${var.region}"
+  service_account_email = "${var.service_account_email}"
+  service_port          = 22
+  source                = "git::ssh://git@ops.gitlab.net/gitlab-com/gl-infra/terraform-modules/google/generic-sv-with-group.git?ref=v1.0.3"
+  tier                  = "sv"
+  use_new_node_name     = true
+  vpc                   = "${module.network.self_link}"
+}
+
+##################################
+#
+#  GKE Cluster for runners
+#
+##################################
+
+# After provisioning you will need to configure
+# the cluster for gitlab-runner. Instructions
+# for this are https://gitlab.com/gitlab-com/runbooks/tree/master/howto/gke-runner
+
+module "gke-runner" {
+  environment           = "${var.environment}"
+  name                  = "gke-runner"
+  vpc                   = "${module.network.self_link}"
+  source                = "git::ssh://git@ops.gitlab.net/gitlab-com/gl-infra/terraform-modules/google/gke.git?ref=v1.0.2"
+  initial_node_count    = 1
+  ip_cidr_range         = "${var.subnetworks["gke-runner"]}"
+  dns_zone_name         = "${var.dns_zone_name}"
+  machine_type          = "${var.machine_types["gke-runner"]}"
+  project               = "${var.project}"
+  region                = "${var.region}"
+  pod_ip_cidr_range     = "${var.subnetworks["gke-runner-pod-cidr"]}"
+  service_ip_cidr_range = "${var.subnetworks["gke-runner-service-cidr"]}"
+}

environments/testbed/testbed-monitoring-url-map.tf

+###########################################################
+# This is specific to the testbed environment
+# and defines the mapping from monitoring hosts to backend
+# services
+
+resource "google_compute_url_map" "monitoring-lb" {
+  name            = "${format("%v-monitoring-lb", var.environment)}"
+  default_service = "${module.prometheus.google_compute_backend_service_self_link}"
+
+  host_rule {
+    hosts        = ["prometheus.testbed.gitlab.net"]
+    path_matcher = "prometheus"
+  }
+
+  path_matcher {
+    name            = "prometheus"
+    default_service = "${module.prometheus.google_compute_backend_service_self_link}"
+
+    path_rule {
+      paths   = ["/*"]
+      service = "${module.prometheus.google_compute_backend_service_self_link}"
+    }
+  }
+}

environments/testbed/variables.tf

+variable "gitlab_io_zone_id" {}
+
+variable "project" {
+  default = "gitlab-testbed"
+}
+
+variable "bootstrap_script_version" {
+  default = 8
+}
+
+variable "region" {
+  default = "us-east1"
+}
+
+variable "environment" {
+  default = "testbed"
+}
+
+variable "dns_zone_name" {
+  default = "gitlab.net"
+}
+
+variable "default_kernel_version" {
+  default = "4.15.0-1015"
+}
+
+variable "oauth2_client_id_monitoring" {
+  default = "test"
+}
+
+variable "oauth2_client_secret_monitoring" {
+  default = "test"
+}
+
+variable "machine_types" {
+  type = "map"
+
+  default = {
+    "bastion"       = "n1-standard-1"
+    "web"           = "n1-standard-16"
+    "monitoring"    = "n1-standard-2"
+    "gitlab-onprem" = "n1-standard-2"
+    "sd-exporter"   = "n1-standard-1"
+    "gke-runner"    = "n1-standard-2"
+    "gitaly"        = "n1-standard-2"
+  }
+}
+
+variable "monitoring_hosts" {
+  type = "map"
+
+  default = {
+    "names" = ["prometheus", "prometheus-app"]
+    "ports" = [9090, 9090]
+  }
+}
+
+variable "service_account_email" {
+  type = "string"
+
+  default = "terraform@gitlab-testbed.iam.gserviceaccount.com"
+}
+
+#############################
+# Default firewall
+# rule for allowing
+# all protocols on all
+# ports
+#
+# 10.240.x.x: all of testbed
+# 10.250.7.x: ops runner
+# 10.250.10.x: chatops runner
+# 10.250.12.x: release runner
+# 10.12.0.0/14: pod address range in gitlab-ops for runners
+###########################
+
+variable "internal_subnets" {
+  type    = "list"
+  default = ["10.240.0.0/13", "10.250.7.0/24", "10.250.10.0/24", "10.250.12.0/24", "10.12.0.0/14"]
+}
+
+variable "other_monitoring_subnets" {
+  type = "list"
+
+  # Left empty for testbed
+  default = []
+}
+
+# The testbed network is allocated
+# 10.240.0.0/13
+#   First IP: 10.240.0.0
+#   Last IP:  10.247.255.255
+# For allocations by project see https://gitlab.com/gitlab-com/runbooks/blob/master/howto/subnet-allocations.md
+
+variable "subnetworks" {
+  type = "map"
+
+  default = {
+    "bastion"                 = "10.240.1.0/24"
+    "monitoring"              = "10.240.3.0/24"
+    "sd-exporter"             = "10.240.6.0/24"
+    "redis"                   = "10.240.7.0/24"
+    "gitlab-onprem"           = "10.240.8.0/24"
+    "gke-runner"              = "10.240.9.0/24"
+    "gitaly"                  = "10.240.10.0/24"
+    "filestore-storage"       = "10.240.128.0/29"
+    "gke-runner-pod-cidr"     = "10.246.0.0/16"
+    "gke-runner-service-cidr" = "10.247.0.0/16"
+  }
+}
+
+##################
+# Network Peering
+##################
+
+variable "network_env" {
+  default = "https://www.googleapis.com/compute/v1/projects/gitlab-testbed/global/networks/testbed"
+}
+
+variable "peer_networks" {
+  type = "map"
+
+  default = {
+    "names" = ["ops"]
+
+    "links" = [
+      "https://www.googleapis.com/compute/v1/projects/gitlab-ops/global/networks/ops",
+    ]
+  }
+}
+
+variable "public_ports" {
+  type = "map"
+
+  default = {
+    "bastion"       = [22]
+    "gitlab-onprem" = [443, 80, 22, 5005]
+    "sd-exporter"   = []
+    "gitaly"        = []
+  }
+}
+
+variable "node_count" {
+  type = "map"
+
+  default = {
+    "bastion"     = 1
+    "web"         = 1
+    "prometheus"  = 1
+    "sd-exporter" = 1
+    "gitaly"      = 1
+  }
+}
+
+variable "chef_provision" {
+  type        = "map"
+  description = "Configuration details for chef server"
+
+  default = {
+    bootstrap_bucket  = "gitlab-testbed-chef-bootstrap"
+    bootstrap_key     = "gitlab-testbed-bootstrap-validation"
+    bootstrap_keyring = "gitlab-testbed-bootstrap"
+
+    server_url    = "https://chef.gitlab.com/organizations/gitlab/"
+    user_name     = "gitlab-ci"
+    user_key_path = ".chef.pem"
+    version       = "12.22.5"
+  }
+}
+
+variable "monitoring_cert_link" {
+  default = "projects/gitlab-testbed/global/sslCertificates/wildcard-testbed-gitlab-net"
+}
+
+variable "data_disk_sizes" {
+  type = "map"
+
+  default = {
+    "web" = "100"
+  }
+}
+
+variable "lb_fqdns" {
+  type    = "list"
+  default = ["onprem.testbed.gitlab.net"]
+}
+
+variable "lb_fqdns_bastion" {
+  type    = "list"
+  default = ["lb-bastion.testbed.gitlab.com"]
+}
+
+variable "lb_fqdns_registry" {
+  type    = "list"
+  default = ["registry.onprem.testbed.gitlab.net"]
+}
+
+variable "tcp_lbs_bastion" {
+  type = "map"
+
+  default = {
+    "names"                  = ["ssh"]
+    "forwarding_port_ranges" = ["22"]
+    "health_check_ports"     = ["80"]
+  }
+}
+
+variable "tcp_lbs" {
+  type = "map"
+
+  default = {
+    "names"                  = ["http", "https", "ssh", "registry"]
+    "forwarding_port_ranges" = ["80", "443", "22", "5000"]
+    "health_check_ports"     = ["8001", "8002", "8003", "5000"]     #Is the healthcheck for registry on 5000, or do we need 8004  here?
+  }
+}
+
+variable "testbed_gitlab_net_cert_link" {
+  default = "projects/gitlab-testbed/global/sslCertificates/testbed-gitlab-net"
+}
+
+variable "gcs_service_account_email" {
+  type    = "string"
+  default = "gitlab-object-storage@gitlab-testbed.iam.gserviceaccount.com"
+}
+
+variable "egress_ports" {
+  type    = "list"
+  default = ["80", "443"]
+}