Commit 32ab1508 authored by Kenny Moore's avatar Kenny Moore
Browse files

Merge branch 'BQ_Audit' into 'master'

Bq audit

See merge request !13
parents e620705b 57cd992a
# Local .terraform directories
**/.terraform/*
# .tfstate files
*.tfstate
*.tfstate.*
# Crash log files
crash.log
# Ignore any .tfvars files that are generated automatically for each Terraform run. Most
# .tfvars files are managed as part of configuration and so should be included in
# version control.
#
# Never commit the tfvars files
*.tfvars
# Ignore override files as they are usually used to override resources locally and so
# are not checked in
override.tf
override.tf.json
*_override.tf
*_override.tf.json
# Include override files you do wish to add to version control using negated pattern
#
# !example_override.tf
# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan
# example: *tfplan*
# Don't publish GCP service account creds
vci-dev-tf-admin-a9b805dc141e.json
# Ignore Mac junk
.DS_store
\ No newline at end of file
......@@ -50,7 +50,13 @@ module "audit" {
mcomm_group_email = var.mcomm_group_email
database_function_url = local.database_function_url[var.environment]
shortcode = var.shortcode
environment = var.environment
environment = var.environment
audit_logs_access = var.audit_logs_access
big_query = true
big_query_retention = 14
gcs = true
gcs_storage_class = "COLDLINE"
gcs_expiration_days = 1100
}
resource "google_folder" "customer_folder" {
......
......@@ -3,7 +3,7 @@ locals {
project_id_prefix = "${var.division}-audit-"
project_id_customer = substr(local.short_mcomm,0,30-5-length(local.project_id_prefix))
project_id = "${local.project_id_prefix}${local.project_id_customer}-${random_id.id.hex}"
project_id = "${local.project_id_prefix}${local.project_id_customer}-${random_id.id[0].hex}"
project_name = substr("${var.division} Audit ${local.short_mcomm}",0,30)
......@@ -48,31 +48,85 @@ locals {
log_export_destination = {
//dev = ""
test = {
its = "pubsub.googleapis.com/projects/gcp-at-um-mon/topics/gcp-at-um-logs-test"
campus = "pubsub.googleapis.com/projects/gcp-at-um-mon/topics/gcp-at-um-logs-test"
michigan_medicine = "pubsub.googleapis.com/projects/gcp-at-um-mon/topics/gcp-at-um-logs-test"
hipaa = "pubsub.googleapis.com/projects/gcp-at-um-mon/topics/hipaa-logs-test"
its = "pubsub.googleapis.com/projects/gcp-at-um-test-mon/topics/gcp-at-um-logs-test"
campus = "pubsub.googleapis.com/projects/gcp-at-um-test-mon/topics/gcp-at-um-logs-test"
michigan_medicine = "pubsub.googleapis.com/projects/gcp-at-um-test-mon/topics/gcp-at-um-logs-test"
hipaa = "pubsub.googleapis.com/projects/gcp-at-um-test-mon/topics/hipaa-logs-test"
# "pubsub.googleapis.com/projects/gcp-at-um-mon/topics/hipaa-logs-test"
}
prod = {
its = "pubsub.googleapis.com/projects/gcp-at-um-mon/topics/gcp-at-um-logs-prod"
campus = "pubsub.googleapis.com/projects/gcp-at-um-mon/topics/gcp-at-um-logs-prod"
michigan_medicine = "pubsub.googleapis.com/projects/gcp-at-um-mon/topics/gcp-at-um-logs-prod"
hipaa = "pubsub.googleapis.com/projects/gcp-at-um-mon/topics/hipaa-logs-prod" # "pubsub.googleapis.com/projects/${var.division}-logs-${var.environment}"
}
}
}
big_query_enabled = var.big_query ? 1 : 0
gcs_enabled = var.gcs ? 1 : 0
big_query_or_gcs_enabled = var.gcs || var.big_query ? 1 : 0
gcs_expiration_seconds = (var.gcs_expiration_days - 1) * 24 * 60 * 60
}
resource "google_project" "gcp_project" {
name = local.project_name
project_id = local.project_id
folder_id = var.folder_id
billing_account = var.billing_id
count = local.big_query_or_gcs_enabled
name = local.project_name
project_id = local.project_id
folder_id = var.folder_id
billing_account = var.billing_id
auto_create_network = false
labels = {
"shortcode" = var.shortcode
}
}
module "log_export_bq" {
count = local.big_query_enabled
source = "terraform-google-modules/log-export/google"
version = "5.1.0"
destination_uri = module.destination_bq[0].destination_uri # "${module.destination_bq.destination_uri}"
filter = local.log_filters
log_sink_name = "${var.division}-audit-bq"
parent_resource_id = var.folder_id
parent_resource_type = "folder"
unique_writer_identity = true
include_children = true
}
module "destination_bq" {
count = local.big_query_enabled
source = "terraform-google-modules/log-export/google//modules/bigquery"
project_id = google_project.gcp_project[0].project_id
dataset_name = replace("${var.division}-audit-${local.short_mcomm}-${random_id.id[0].hex}", "-", "_")
description = "Aggregated Log Sink (folder) - HIPAA Customer Logs"
log_sink_writer_identity = module.log_export_bq[0].writer_identity # "${module.log_export_bq.writer_identity}"
location = "US"
expiration_days = var.big_query_retention
}
resource "google_bigquery_dataset_iam_binding" "bq_user" {
count = local.big_query_enabled
project = google_project.gcp_project[0].project_id
dataset_id = module.destination_bq[0].resource_name
role = "roles/bigquery.user"
members = [
# "user:kenmoore@umich.edu",
"group:${var.audit_logs_access}",
"group:${var.mcomm_group_email}"
]
}
# resource "google_bigquery_dataset_iam_binding" "job_user" {
# project = google_project.gcp_project.project_id
# dataset_id = module.destination_bq.resource_name
# role = "roles/bigquery.jobUser"
# members = [
# "user:kenmoore@umich.edu",
# ]
# }
data "google_service_account_id_token" "customer_db_token" {
target_audience = var.database_function_url
}
......@@ -91,8 +145,9 @@ resource "null_resource" "customer_database" {
resource "google_project_iam_member" "project_iam" {
project = google_project.gcp_project.project_id
role = "roles/editor"
count = local.big_query_or_gcs_enabled
project = google_project.gcp_project[0].project_id
role = "roles/viewer"
member = "group:${var.mcomm_group_email}"
}
......@@ -117,40 +172,44 @@ resource "google_project_iam_member" "project_iam" {
# working on lien on audit project
resource "google_resource_manager_lien" "project_lien" {
parent = "projects/${google_project.gcp_project.project_id}"
count = local.big_query_or_gcs_enabled
parent = "projects/${google_project.gcp_project[0].project_id}"
restrictions = ["resourcemanager.projects.delete"]
origin = "Per HIPAA recommendation/requirements; Prevent deletion of ${google_project.gcp_project.project_id}"
reason = "${google_project.gcp_project.project_id} holds the audit logs for HIPAA customer folder id ${var.folder_id}"
origin = "Per HIPAA recommendation/requirements; Prevent deletion of ${google_project.gcp_project[0].project_id}"
reason = "${google_project.gcp_project[0].project_id} holds the audit logs for HIPAA customer folder id ${var.folder_id}"
}
resource "random_id" "id" {
count = local.big_query_or_gcs_enabled
byte_length = 2
}
resource "google_logging_folder_sink" "hipaa_customer_logs" {
name = "${var.division}-audit-${local.short_mcomm}"
count = local.gcs_enabled
name = "${var.division}-audit-${local.short_mcomm}"
description = "Aggregated Log Sink - HIPAA Customer Logs"
folder = var.folder_id
folder = var.folder_id
# include logs from all projects in folder
include_children = true
# send to GCS bucket in audit project
destination = "storage.googleapis.com/${google_storage_bucket.hipaa_customer_logs.name}"
destination = "storage.googleapis.com/${google_storage_bucket.hipaa_customer_logs[0].name}"
filter = local.log_filters
}
resource "google_storage_bucket" "hipaa_customer_logs" {
project = google_project.gcp_project.project_id
name = "${var.division}-audit-${local.short_mcomm}-${random_id.id.hex}"
resource "google_storage_bucket" "hipaa_customer_logs" {
count = local.gcs_enabled
project = google_project.gcp_project[0].project_id
name = "${var.division}-audit-${local.short_mcomm}-${random_id.id[0].hex}"
location = "US-CENTRAL1"
force_destroy = false
uniform_bucket_level_access = true
storage_class = "COLDLINE"
storage_class = var.gcs_storage_class
retention_policy {
retention_period = "95000000" # just over 3 years
retention_period = local.gcs_expiration_seconds # just over 3 years ## <-- needs to be a local
}
lifecycle_rule {
......@@ -163,43 +222,49 @@ resource "google_storage_bucket" "hipaa_customer_logs" {
}
}
lifecycle_rule {
action {
type = "Delete"
}
condition {
age = 1100 # just over 3 x 365 days = 3 years (slightly more than the retention policy)
dynamic "lifecycle_rule" {
for_each = var.gcs_expiration_days == null ? [] : [ var.gcs_expiration_days ]
content {
action {
type = "Delete"
}
condition {
age = var.gcs_expiration_days # just over 3 x 365 days = 3 years (slightly more than the retention policy)
}
}
}
}
resource "google_project_iam_binding" "hipaa_customer_log_writer" {
project = google_project.gcp_project.project_id
count = local.gcs_enabled
project = google_project.gcp_project[0].project_id
role = "roles/storage.objectCreator"
members = [
google_logging_folder_sink.hipaa_customer_logs.writer_identity,
google_logging_folder_sink.hipaa_customer_logs[0].writer_identity,
]
}
resource "google_logging_project_sink" "log_export" {
project = google_project.gcp_project.project_id
# name = "${google_project.gcp_project.project_id}-log-export"
name = "log-export-splunk"
destination = local.log_export_destination[var.environment][var.division]
filter = local.filter_string # if adding filter to match nothing; use in conjunction with aggregated log sink + exclusion filter
exclusions {
name = "Aggregated_Logs_Exclusion"
description = "Excluding logs already captured by aggregated log sink (at folder). Please DO NOT alter/remove."
filter = local.log_filters
}
unique_writer_identity = true
}
# resource "google_logging_project_sink" "log_export" {
# count = local.big_query_or_gcs_enabled
# project = google_project.gcp_project.project_id
# # name = "${google_project.gcp_project.project_id}-log-export"
# name = "log-export-splunk"
# destination = local.log_export_destination[var.environment][var.division]
# filter = local.filter_string # if adding filter to match nothing; use in conjunction with aggregated log sink + exclusion filter
# exclusions {
# name = "Aggregated_Logs_Exclusion"
# description = "Excluding logs already captured by aggregated log sink (at folder). Please DO NOT alter/remove."
# filter = local.log_filters
# }
# unique_writer_identity = true
# }
resource "google_pubsub_topic_iam_member" "publisher" {
project = local.log_export_project
topic = local.log_export_destination[var.environment][var.division]
role = "roles/pubsub.publisher"
member = google_logging_project_sink.log_export.writer_identity
}
# resource "google_pubsub_topic_iam_member" "publisher" {
# count = local.big_query_or_gcs_enabled
# project = local.log_export_project
# topic = local.log_export_destination[var.environment][var.division]
# role = "roles/pubsub.publisher"
# member = google_logging_project_sink.log_export.writer_identity
# }
......@@ -40,4 +40,32 @@ variable "environment" {
condition = var.environment == "dev" || var.environment == "test" || var.environment == "prod"
error_message = "The environment value must be one of \"dev\",\"test\", or \"prod\"."
}
}
\ No newline at end of file
}
variable "audit_logs_access" {
type = string
description = "MCommunity group that will be given permission to GCS bucket and/or BQ dataset"
}
variable "big_query" {
type = bool
}
variable "big_query_retention" {
description = "Number of days to retain BigQuery data in table"
type = number #null
}
variable "gcs" {
type = bool
}
variable "gcs_storage_class" {
type = string
# default = "coldline"
}
variable "gcs_expiration_days" {
type = number
# default = 1100
}
......@@ -75,3 +75,9 @@ variable "customer_is_shared_vpc_admin" {
description = "Should the customer be given Shared VPC Admin permission to the customer folder. Defaults to false."
default = false
}
variable "audit_logs_access" {
type = string
# default = "gcp.admins@umich.edu"
description = "MCommunity group that will be given permission to GCS bucket and/or BQ dataset"
}
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment