Implementation of https://gitlab.com/gitlab-com/gl-infra/infrastructure/issues/6511
Showing
environments/testbed/main.tf
0 → 100644
## State storage | ||
terraform { | ||
backend "s3" {} | ||
} | ||
## AWS | ||
provider "aws" { | ||
region = "us-east-1" | ||
} | ||
variable "gitlab_com_zone_id" {} | ||
variable "gitlab_net_zone_id" {} | ||
provider "google" { | ||
version = "~> 1.18.0" | ||
project = "${var.project}" | ||
region = "${var.region}" | ||
} | ||
################################## | ||
# | ||
# Network | ||
# | ||
################################# | ||
module "network" { | ||
source = "git::ssh://[email protected]/gitlab-com/gl-infra/terraform-modules/google/vpc.git?ref=v1.0.0" | ||
project = "${var.project}" | ||
environment = "${var.environment}" | ||
internal_subnets = "${var.internal_subnets}" | ||
} | ||
################################## | ||
# | ||
# Network Peering | ||
# | ||
################################# | ||
resource "google_compute_network_peering" "peering" { | ||
count = "${length(var.peer_networks["names"])}" | ||
name = "peering-${element(var.peer_networks["names"], count.index)}" | ||
network = "${var.network_env}" | ||
peer_network = "${element(var.peer_networks["links"], count.index)}" | ||
} | ||
################################## | ||
# | ||
# Monitoring | ||
# | ||
# Uses the monitoring module, this | ||
# creates a single instance behind | ||
# a load balancer with identity aware | ||
# proxy enabled. | ||
# | ||
################################## | ||
resource "google_compute_subnetwork" "monitoring" { | ||
ip_cidr_range = "${var.subnetworks["monitoring"]}" | ||
name = "${format("monitoring-%v", var.environment)}" | ||
network = "${module.network.self_link}" | ||
private_ip_google_access = true | ||
project = "${var.project}" | ||
region = "${var.region}" | ||
} | ||
############################################### | ||
# | ||
# Load balancer and VM for the pre bastion | ||
# | ||
############################################### | ||
module "gcp-tcp-lb-bastion" { | ||
environment = "${var.environment}" | ||
forwarding_port_ranges = "${var.tcp_lbs_bastion["forwarding_port_ranges"]}" | ||
fqdns = "${var.lb_fqdns_bastion}" | ||
gitlab_zone_id = "${var.gitlab_com_zone_id}" | ||
health_check_ports = "${var.tcp_lbs_bastion["health_check_ports"]}" | ||
instances = ["${module.bastion.instances_self_link}"] | ||
lb_count = "${length(var.tcp_lbs_bastion["names"])}" | ||
name = "gcp-tcp-lb-bastion" | ||
names = "${var.tcp_lbs_bastion["names"]}" | ||
project = "${var.project}" | ||
region = "${var.region}" | ||
session_affinity = "CLIENT_IP" | ||
source = "git::ssh://[email protected]/gitlab-com/gl-infra/terraform-modules/google/tcp-lb.git?ref=remove-lifecycle" | ||
targets = ["bastion"] | ||
} | ||
module "bastion" { | ||
bootstrap_version = "${var.bootstrap_script_version}" | ||
chef_provision = "${var.chef_provision}" | ||
chef_run_list = "\"role[${var.environment}-base-bastion]\"" | ||
dns_zone_name = "${var.dns_zone_name}" | ||
environment = "${var.environment}" | ||
health_check = "tcp" | ||
ip_cidr_range = "${var.subnetworks["bastion"]}" | ||
machine_type = "${var.machine_types["bastion"]}" | ||
name = "bastion" | ||
node_count = "${var.node_count["bastion"]}" | ||
project = "${var.project}" | ||
public_ports = "${var.public_ports["bastion"]}" | ||
region = "${var.region}" | ||
service_account_email = "${var.service_account_email}" | ||
service_port = 22 | ||
source = "git::ssh://[email protected]/gitlab-com/gl-infra/terraform-modules/google/generic-sv-with-group.git?ref=v1.0.3" | ||
tier = "inf" | ||
use_new_node_name = true | ||
vpc = "${module.network.self_link}" | ||
} | ||
################################## | ||
# | ||
# Google storage buckets | ||
# | ||
################################## | ||
module "storage" { | ||
environment = "${var.environment}" | ||
versioning = "${var.versioning}" | ||
artifact_age = "${var.artifact_age}" | ||
lfs_object_age = "${var.lfs_object_age}" | ||
package_repo_age = "${var.package_repo_age}" | ||
upload_age = "${var.upload_age}" | ||
storage_log_age = "${var.storage_log_age}" | ||
storage_class = "${var.storage_class}" | ||
service_account_email = "${var.service_account_email}" | ||
gcs_service_account_email = "${var.gcs_service_account_email}" | ||
gcs_storage_analytics_group_email = "${var.gcs_storage_analytics_group_email}" | ||
source = "git::ssh://[email protected]/gitlab-com/gl-infra/terraform-modules/google/storage-buckets.git?ref=v1.1.1" | ||
} | ||
####################### | ||
# | ||
# load balancer for all hosts in this section | ||
# | ||
####################### | ||
module "monitoring-lb" { | ||
cert_link = "${var.monitoring_cert_link}" | ||
environment = "${var.environment}" | ||
gitlab_net_zone_id = "${var.gitlab_net_zone_id}" | ||
hosts = ["${var.monitoring_hosts["names"]}"] | ||
name = "monitoring-lb" | ||
project = "${var.project}" | ||
region = "${var.region}" | ||
service_ports = ["${var.monitoring_hosts["ports"]}"] | ||
source = "git::ssh://[email protected]/gitlab-com/gl-infra/terraform-modules/google/monitoring-lb.git?ref=v1.0.0" | ||
subnetwork_name = "${google_compute_subnetwork.monitoring.name}" | ||
targets = ["${var.monitoring_hosts["names"]}"] | ||
url_map = "${google_compute_url_map.monitoring-lb.self_link}" | ||
} | ||
####################### | ||
module "prometheus" { | ||
attach_data_disk = true | ||
bootstrap_version = "${var.bootstrap_script_version}" | ||
chef_provision = "${var.chef_provision}" | ||
chef_run_list = "\"role[${var.environment}-infra-prometheus]\"" | ||
data_disk_size = 100 | ||
data_disk_type = "pd-standard" | ||
dns_zone_name = "${var.dns_zone_name}" | ||
environment = "${var.environment}" | ||
fw_whitelist_subnets = "${concat(var.monitoring_whitelist_prometheus["subnets"], var.other_monitoring_subnets)}" | ||
fw_whitelist_ports = "${var.monitoring_whitelist_prometheus["ports"]}" | ||
machine_type = "${var.machine_types["monitoring"]}" | ||
name = "prometheus" | ||
node_count = "${var.node_count["prometheus"]}" | ||
oauth2_client_id = "${var.oauth2_client_id_monitoring}" | ||
oauth2_client_secret = "${var.oauth2_client_secret_monitoring}" | ||
persistent_disk_path = "/opt/prometheus" | ||
project = "${var.project}" | ||
public_ports = ["22"] | ||
region = "${var.region}" | ||
service_account_email = "${var.service_account_email}" | ||
service_path = "/graph" | ||
service_port = "${element(var.monitoring_hosts["ports"], index(var.monitoring_hosts["names"], "prometheus"))}" | ||
source = "git::ssh://[email protected]/gitlab-com/gl-infra/terraform-modules/google/monitoring-with-count.git?ref=v1.0.4" | ||
subnetwork_name = "${google_compute_subnetwork.monitoring.name}" | ||
tier = "inf" | ||
use_external_ip = true | ||
use_new_node_name = true | ||
vpc = "${module.network.self_link}" | ||
} | ||
####################################################### | ||
# | ||
# VM for onprem.testbed.gitlab.net | ||
# | ||
####################################################### | ||
resource "aws_route53_record" "default" { | ||
zone_id = "${var.gitlab_net_zone_id}" | ||
name = "onprem.testbed.gitlab.net" | ||
type = "A" | ||
ttl = "300" | ||
records = ["${module.gitlab-onprem.instance_public_ips[0]}"] | ||
} | ||
module "gitlab-onprem" { | ||
backend_protocol = "HTTPS" | ||
bootstrap_version = "${var.bootstrap_script_version}" | ||
chef_provision = "${var.chef_provision}" | ||
chef_run_list = "\"role[${var.environment}-base-gitlab-onprem]\"" | ||
data_disk_size = 1000 | ||
data_disk_type = "pd-ssd" | ||
dns_zone_name = "${var.dns_zone_name}" | ||
environment = "${var.environment}" | ||
health_check = "http" | ||
health_check_port = 8887 | ||
ip_cidr_range = "${var.subnetworks["gitlab-onprem"]}" | ||
machine_type = "${var.machine_types["gitlab-onprem"]}" | ||
name = "gitlab-onprem" | ||
node_count = 1 | ||
persistent_disk_path = "/var/opt/gitlab" | ||
project = "${var.project}" | ||
public_ports = "${var.public_ports["gitlab-onprem"]}" | ||
region = "${var.region}" | ||
service_account_email = "${var.service_account_email}" | ||
service_path = "/-/liveness" | ||
service_port = 443 | ||
source = "git::ssh://[email protected]/gitlab-com/gl-infra/terraform-modules/google/monitoring-with-count.git?ref=v1.0.4" | ||
tier = "inf" | ||
use_external_ip = true | ||
use_new_node_name = true | ||
vpc = "${module.network.self_link}" | ||
} | ||
module "sd-exporter" { | ||
additional_scopes = ["https://www.googleapis.com/auth/monitoring"] | ||
allow_stopping_for_update = true | ||
bootstrap_version = "${var.bootstrap_script_version}" | ||
chef_provision = "${var.chef_provision}" | ||
chef_run_list = "\"role[${var.environment}-infra-sd-exporter]\"" | ||
create_backend_service = false | ||
dns_zone_name = "${var.dns_zone_name}" | ||
environment = "${var.environment}" | ||
machine_type = "${var.machine_types["sd-exporter"]}" | ||
name = "sd-exporter" | ||
node_count = "${var.node_count["sd-exporter"]}" | ||
project = "${var.project}" | ||
public_ports = "${var.public_ports["sd-exporter"]}" | ||
region = "${var.region}" | ||
service_account_email = "${var.service_account_email}" | ||
source = "git::ssh://[email protected]/gitlab-com/gl-infra/terraform-modules/google/generic-sv-with-group.git?ref=v1.0.2" | ||
subnetwork_name = "${google_compute_subnetwork.monitoring.name}" | ||
tier = "inf" | ||
use_new_node_name = true | ||
vpc = "${module.network.self_link}" | ||
} | ||
################################## | ||
# | ||
# Gitaly node | ||
# | ||
################################## | ||
module "gitaly" { | ||
bootstrap_version = "${var.bootstrap_script_version}" | ||
chef_provision = "${var.chef_provision}" | ||
chef_run_list = "\"role[${var.environment}-base-be-gitaly]\"" | ||
dns_zone_name = "${var.dns_zone_name}" | ||
egress_ports = "${var.egress_ports}" | ||
environment = "${var.environment}" | ||
health_check = "tcp" | ||
ip_cidr_range = "${var.subnetworks["gitaly"]}" | ||
machine_type = "${var.machine_types["gitaly"]}" | ||
name = "gitaly" | ||
node_count = "${var.node_count["gitaly"]}" | ||
os_disk_type = "pd-ssd" | ||
project = "${var.project}" | ||
public_ports = "${var.public_ports["gitaly"]}" | ||
region = "${var.region}" | ||
service_account_email = "${var.service_account_email}" | ||
service_port = 22 | ||
source = "git::ssh://[email protected]/gitlab-com/gl-infra/terraform-modules/google/generic-sv-with-group.git?ref=v1.0.3" | ||
tier = "sv" | ||
use_new_node_name = true | ||
vpc = "${module.network.self_link}" | ||
} | ||
################################## | ||
# | ||
# GKE Cluster for runners | ||
# | ||
################################## | ||
# After provisioning you will need to configure | ||
# the cluster for gitlab-runner. Instructions | ||
# for this are https://gitlab.com/gitlab-com/runbooks/tree/master/howto/gke-runner | ||
module "gke-runner" { | ||
environment = "${var.environment}" | ||
name = "gke-runner" | ||
vpc = "${module.network.self_link}" | ||
source = "git::ssh://[email protected]/gitlab-com/gl-infra/terraform-modules/google/gke.git?ref=v1.0.2" | ||
initial_node_count = 1 | ||
ip_cidr_range = "${var.subnetworks["gke-runner"]}" | ||
dns_zone_name = "${var.dns_zone_name}" | ||
machine_type = "${var.machine_types["gke-runner"]}" | ||
project = "${var.project}" | ||
region = "${var.region}" | ||
pod_ip_cidr_range = "${var.subnetworks["gke-runner-pod-cidr"]}" | ||
service_ip_cidr_range = "${var.subnetworks["gke-runner-service-cidr"]}" | ||
} |
variable "gitlab_io_zone_id" {} | ||
variable "project" { | ||
default = "gitlab-testbed" | ||
} | ||
variable "bootstrap_script_version" { | ||
default = 8 | ||
} | ||
variable "region" { | ||
default = "us-east1" | ||
} | ||
variable "environment" { | ||
default = "testbed" | ||
} | ||
variable "dns_zone_name" { | ||
default = "gitlab.net" | ||
} | ||
variable "default_kernel_version" { | ||
default = "4.15.0-1015" | ||
} | ||
variable "oauth2_client_id_monitoring" { | ||
default = "test" | ||
} | ||
variable "oauth2_client_secret_monitoring" { | ||
default = "test" | ||
} | ||
variable "machine_types" { | ||
type = "map" | ||
default = { | ||
"bastion" = "n1-standard-1" | ||
"web" = "n1-standard-16" | ||
"monitoring" = "n1-standard-2" | ||
"gitlab-onprem" = "n1-standard-2" | ||
"sd-exporter" = "n1-standard-1" | ||
"gke-runner" = "n1-standard-2" | ||
"gitaly" = "n1-standard-2" | ||
} | ||
} | ||
variable "monitoring_hosts" { | ||
type = "map" | ||
default = { | ||
"names" = ["prometheus", "prometheus-app"] | ||
"ports" = [9090, 9090] | ||
} | ||
} | ||
variable "service_account_email" { | ||
type = "string" | ||
default = "[email protected]" | ||
} | ||
############################# | ||
# Default firewall | ||
# rule for allowing | ||
# all protocols on all | ||
# ports | ||
# | ||
# 10.240.x.x: all of testbed | ||
# 10.250.7.x: ops runner | ||
# 10.250.10.x: chatops runner | ||
# 10.250.12.x: release runner | ||
# 10.12.0.0/14: pod address range in gitlab-ops for runners | ||
########################### | ||
variable "internal_subnets" { | ||
type = "list" | ||
default = ["10.240.0.0/13", "10.250.7.0/24", "10.250.10.0/24", "10.250.12.0/24", "10.12.0.0/14"] | ||
} | ||
variable "other_monitoring_subnets" { | ||
type = "list" | ||
# Left empty for testbed | ||
default = [] | ||
} | ||
# The testbed network is allocated | ||
# 10.240.0.0/13 | ||
# First IP: 10.240.0.0 | ||
# Last IP: 10.247.255.255 | ||
# For allocations by project see https://gitlab.com/gitlab-com/runbooks/blob/master/howto/subnet-allocations.md | ||
variable "subnetworks" { | ||
type = "map" | ||
default = { | ||
"bastion" = "10.240.1.0/24" | ||
"monitoring" = "10.240.3.0/24" | ||
"sd-exporter" = "10.240.6.0/24" | ||
"redis" = "10.240.7.0/24" | ||
"gitlab-onprem" = "10.240.8.0/24" | ||
"gke-runner" = "10.240.9.0/24" | ||
"gitaly" = "10.240.10.0/24" | ||
"filestore-storage" = "10.240.128.0/29" | ||
"gke-runner-pod-cidr" = "10.246.0.0/16" | ||
"gke-runner-service-cidr" = "10.247.0.0/16" | ||
} | ||
} | ||
################## | ||
# Network Peering | ||
################## | ||
variable "network_env" { | ||
default = "https://www.googleapis.com/compute/v1/projects/gitlab-testbed/global/networks/testbed" | ||
} | ||
variable "peer_networks" { | ||
type = "map" | ||
default = { | ||
"names" = ["ops"] | ||
"links" = [ | ||
"https://www.googleapis.com/compute/v1/projects/gitlab-ops/global/networks/ops", | ||
] | ||
} | ||
} | ||
variable "public_ports" { | ||
type = "map" | ||
default = { | ||
"bastion" = [22] | ||
"gitlab-onprem" = [443, 80, 22, 5005] | ||
"sd-exporter" = [] | ||
"gitaly" = [] | ||
} | ||
} | ||
variable "node_count" { | ||
type = "map" | ||
default = { | ||
"bastion" = 1 | ||
"web" = 1 | ||
"prometheus" = 1 | ||
"sd-exporter" = 1 | ||
"gitaly" = 1 | ||
} | ||
} | ||
variable "chef_provision" { | ||
type = "map" | ||
description = "Configuration details for chef server" | ||
default = { | ||
bootstrap_bucket = "gitlab-testbed-chef-bootstrap" | ||
bootstrap_key = "gitlab-testbed-bootstrap-validation" | ||
bootstrap_keyring = "gitlab-testbed-bootstrap" | ||
server_url = "https://chef.gitlab.com/organizations/gitlab/" | ||
user_name = "gitlab-ci" | ||
user_key_path = ".chef.pem" | ||
version = "12.22.5" | ||
} | ||
} | ||
variable "monitoring_cert_link" { | ||
default = "projects/gitlab-testbed/global/sslCertificates/wildcard-testbed-gitlab-net" | ||
} | ||
variable "data_disk_sizes" { | ||
type = "map" | ||
default = { | ||
"web" = "100" | ||
} | ||
} | ||
variable "lb_fqdns" { | ||
type = "list" | ||
default = ["onprem.testbed.gitlab.net"] | ||
} | ||
variable "lb_fqdns_bastion" { | ||
type = "list" | ||
default = ["lb-bastion.testbed.gitlab.com"] | ||
} | ||
variable "lb_fqdns_registry" { | ||
type = "list" | ||
default = ["registry.onprem.testbed.gitlab.net"] | ||
} | ||
variable "tcp_lbs_bastion" { | ||
type = "map" | ||
default = { | ||
"names" = ["ssh"] | ||
"forwarding_port_ranges" = ["22"] | ||
"health_check_ports" = ["80"] | ||
} | ||
} | ||
variable "tcp_lbs" { | ||
type = "map" | ||
default = { | ||
"names" = ["http", "https", "ssh", "registry"] | ||
"forwarding_port_ranges" = ["80", "443", "22", "5000"] | ||
"health_check_ports" = ["8001", "8002", "8003", "5000"] #Is the healthcheck for registry on 5000, or do we need 8004 here? | ||
} | ||
} | ||
variable "testbed_gitlab_net_cert_link" { | ||
default = "projects/gitlab-testbed/global/sslCertificates/testbed-gitlab-net" | ||
} | ||
variable "gcs_service_account_email" { | ||
type = "string" | ||
default = "[email protected]" | ||
} | ||
variable "egress_ports" { | ||
type = "list" | ||
default = ["80", "443"] | ||
} |
Please register or sign in to comment