Commit 29f1ed33 authored by John Jarvis's avatar John Jarvis

Merge branch 'jarv/k8-logging' into 'master'

SD->pubsub->ES for logging

See merge request !788
parents d688fb06 8706c034
......@@ -310,6 +310,107 @@ module "gke-pre" {
service_ip_cidr_range = "${var.subnetworks["gke-pre-service-cidr"]}"
}
##################################
#
# Pubsubbeat for GKE
# This adds a single beat for logs
# from GKE
#
# Machines for running the beats
# that consume logs from pubsub
# and send them to elastic cloud
#
# You must have a chef role with the
# following format:
# role[<env>-infra-pubsubbeat-<beat_name>]
#
##################################
module "pubsubbeat" {
bootstrap_version = "${var.bootstrap_script_version}"
chef_provision = "${var.chef_provision}"
dns_zone_name = "${var.dns_zone_name}"
environment = "${var.environment}"
health_check = "tcp"
ip_cidr_range = "${var.subnetworks["pubsubbeat"]}"
machine_types = ["n1-standard-1"]
names = ["gke"]
project = "${var.project}"
public_ports = "${var.public_ports["pubsubbeat"]}"
region = "${var.region}"
service_account_email = "${var.service_account_email}"
service_port = 22
source = "git::ssh://git@ops.gitlab.net/gitlab-com/gl-infra/terraform-modules/google/pubsubbeat.git?ref=v1.0.6"
tier = "inf"
use_new_node_name = true
vpc = "${module.network.self_link}"
}
resource "google_logging_project_sink" "pubsub" {
name = "${var.environment}-pubsub-sink"
# There is only a single pubsub topic for gke
destination = "pubsub.googleapis.com/projects/${var.project}/topics/${module.pubsubbeat.topic_names[0]}"
filter = "resource.labels.cluster_name = pre-gke-pre"
# Use a unique writer (creates a unique service account used for writing)
unique_writer_identity = true
depends_on = ["module.pubsubbeat"]
}
resource "google_project_iam_binding" "log-writer" {
role = "roles/pubsub.publisher"
members = [
"${google_logging_project_sink.pubsub.writer_identity}",
]
}
##################################
#
# Log Proxy
#
#################################
module "proxy-iap" {
environment = "${var.environment}"
source = "git::ssh://git@ops.gitlab.net/gitlab-com/gl-infra/terraform-modules/google/web-iap.git?ref=v1.0.0"
name = "proxy"
project = "${var.project}"
region = "${var.region}"
gitlab_zone_id = "${var.gitlab_net_zone_id}"
cert_link = "${var.monitoring_cert_link}"
backend_service_link = "${module.proxy.google_compute_backend_service_iap_self_link}"
web_ip_fqdn = "log.pre.gitlab.net"
service_ports = ["443", "80", "9090"]
}
module "proxy" {
bootstrap_version = "${var.bootstrap_script_version}"
chef_provision = "${var.chef_provision}"
chef_run_list = "\"role[${var.environment}-infra-proxy]\""
dns_zone_name = "${var.dns_zone_name}"
enable_iap = true
environment = "${var.environment}"
health_check = "http"
service_path = "/app/kibana"
ip_cidr_range = "${var.subnetworks["proxy"]}"
machine_type = "${var.machine_types["proxy"]}"
name = "proxy"
node_count = 1
oauth2_client_id = "${var.oauth2_client_id_log_proxy}"
oauth2_client_secret = "${var.oauth2_client_secret_log_proxy}"
project = "${var.project}"
public_ports = "${var.public_ports["proxy"]}"
region = "${var.region}"
service_account_email = "${var.service_account_email}"
service_port = "9090"
source = "git::ssh://git@ops.gitlab.net/gitlab-com/gl-infra/terraform-modules/google/generic-sv-with-group.git?ref=v1.0.1"
tier = "inf"
use_new_node_name = true
vpc = "${module.network.self_link}"
}
##################################
#
# External HAProxy LoadBalancer
......
## These variables must be set in env secrets!
variable "oauth2_client_id_log_proxy" {}
variable "oauth2_client_secret_log_proxy" {}
variable "oauth2_client_id_monitoring" {}
variable "oauth2_client_secret_monitoring" {}
variable "gitlab_io_zone_id" {}
variable "project" {
......@@ -24,30 +31,6 @@ variable "default_kernel_version" {
default = "4.15.0-1015"
}
variable "oauth2_client_id_dashboards" {
default = "test"
}
variable "oauth2_client_secret_dashboards" {
default = "test"
}
variable "oauth2_client_id_gitlab_pre" {
default = "test"
}
variable "oauth2_client_secret_gitlab_pre" {
default = "test"
}
variable "oauth2_client_id_monitoring" {
default = "test"
}
variable "oauth2_client_secret_monitoring" {
default = "test"
}
variable "machine_types" {
type = "map"
......@@ -68,6 +51,7 @@ variable "machine_types" {
"gitaly" = "n1-standard-1"
"deploy" = "n1-standard-1"
"consul" = "n1-standard-1"
"proxy" = "n1-standard-1"
}
}
......@@ -143,6 +127,8 @@ variable "subnetworks" {
"gke-pre" = "10.232.20.0/24"
"web-puma" = "10.232.21.0/24"
"consul" = "10.232.22.0/24"
"pubsubbeat" = "10.232.23.0/24"
"proxy" = "10.232.24.0/24"
"gke-pre-pod-cidr" = "10.235.0.0/16"
"gke-pre-service-cidr" = "10.236.0.0/16"
......@@ -194,6 +180,8 @@ variable "public_ports" {
"gitaly" = []
"deploy" = []
"consul" = []
"pubsubbeat" = []
"proxy" = []
}
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment