Commit d2dfe9f8 authored by Kenny Moore's avatar Kenny Moore
Browse files

Restoring from backup; Shared VPC working

parent 7408483d
......@@ -172,7 +172,7 @@ resource "google_bigquery_dataset" "hipaa_data_bq" {
//give datalab service accounts writer access to the corresponding worker dataset
dynamic "access" {
for_each = compact([ for v in keys(var.datalab_user_list) : var.datalab_user_list[v] == each.value ? google_service_account.datalab_service_account[v].email : "" ])
for_each = compact([ for v in var.datalab_user_list : v["project"] == each.value ? google_service_account.datalab_service_account[v["username"]].email : "" ])
content {
role = google_project_iam_custom_role.BigQueryDataEditor_NO_Export[each.key].id
user_by_email = access.value
......
# enable APIs required for OS patching
resource "google_project_service" "os-config-api" {
for_each = { for v in local.worker_projects : v => v }
project = google_project.hipaa_project[each.key].project_id
service = "osconfig.googleapis.com"
}
resource "google_project_service" "container-analysis-api" {
for_each = { for v in local.worker_projects : v => v }
project = google_project.hipaa_project[each.key].project_id
service = "containeranalysis.googleapis.com"
}
resource "google_project_service" "compute-scanning" {
for_each = { for v in local.worker_projects : v => v }
project = google_project.hipaa_project[each.key].project_id
service = "computescanning.googleapis.com"
}
# set GCE Metadata for OS patching
resource "google_compute_project_metadata" "os_patching" {
for_each = { for v in local.worker_projects : v => v }
project = google_project.hipaa_project[each.key].project_id
metadata = {
enable-guest-attributes = true
enable-osconfig = true
}
}
resource "google_project_iam_binding" "datalab_service_account_iam_binding" {
for_each = { for v in local.worker_projects : v => v }
project = google_project.hipaa_project[each.key].project_id
role = "roles/bigquery.jobUser"
members = compact([ for v in keys(var.datalab_user_list) : var.datalab_user_list[v] == each.value ? "serviceAccount:${google_service_account.datalab_service_account[v].email}" : "" ])
# members = compact([ for v in keys(var.datalab_user_list) : var.datalab_user_list[v] == each.value ? "serviceAccount:${google_service_account.datalab_service_account[v].email}" : "" ])
members = compact([ for v in var.datalab_user_list : v["project"] == each.value ? "serviceAccount:${google_service_account.datalab_service_account[v["username"]].email}" : "" ])
}
resource "google_project_service" "sourcerepo-api" {
......@@ -56,13 +86,13 @@ resource "google_project_iam_binding" "view_compute_instances" {
}
resource "google_service_account" "datalab_service_account" {
for_each = var.datalab_user_list
for_each = { for v in var.datalab_user_list : v["username"] => v["project"] }
account_id = "datalab-${split("@", each.key)[0]}"
project = google_project.hipaa_project[each.value].project_id
}
resource "google_service_account_iam_binding" "datalab_service_account_iam_binding" {
for_each = var.datalab_user_list
for_each = { for v in var.datalab_user_list : v["username"] => v["username"] }
service_account_id = google_service_account.datalab_service_account[each.key].name
role = "roles/iam.serviceAccountUser"
members = [
......@@ -71,7 +101,7 @@ resource "google_service_account_iam_binding" "datalab_service_account_iam_bindi
}
data "google_iam_policy" "datalab_user_to_instance_policy" {
for_each = var.datalab_user_list
for_each = { for v in var.datalab_user_list : v["username"] => v["username"] }
binding {
role = "roles/compute.instanceAdmin.v1"
......@@ -82,33 +112,105 @@ data "google_iam_policy" "datalab_user_to_instance_policy" {
}
resource "google_compute_instance_iam_policy" "datalab_user_to_instance_binding" {
for_each = var.datalab_user_list
for_each = { for v in var.datalab_user_list : v["username"] => v["project"] }
instance_name = "datalab-${split("@", each.key)[0]}"
project = google_project.hipaa_project[each.value].project_id
zone = var.datalab_zone
policy_data = data.google_iam_policy.datalab_user_to_instance_policy[each.key].policy_data
depends_on = [ null_resource.datalab ]
# depends_on = [ null_resource.datalab ]
depends_on = [ google_compute_instance.ai_notebook ]
}
## ## ## ## Needs depends on for IAM policy/ai_notebook
## ## ## ## ai_notebook = GCE Instance until ai_notebook API is out of beta (q3 2020)
#### disk must be 100+ GB - add paramenter ####
resource "google_compute_instance" "ai_notebook" {
for_each = { for v in var.datalab_user_list : v["username"] => v }
project = google_project.hipaa_project[each.value["project"]].project_id
# name = "notebook-${split("@", each.key)[0]}"
name = "datalab-${split("@", each.key)[0]}"
machine_type = each.value["machine_type"]
# machine_type = "f1-micro"
zone = var.datalab_zone
tags = ["foo", "bar"] # https-server
boot_disk {
initialize_params {
image = "deeplearning-platform-release/pytorch-latest-gpu" # var.image
}
}
# // Local SSD disk
# scratch_disk {
# interface = "SCSI"
# }
network_interface {
network = "datalab-network" # var.network (shared or default)
resource "null_resource" "datalab" {
for_each = var.datalab_user_list
triggers = {
user = each.key
project_id = google_project.hipaa_project[each.value].project_id
access_config {
// Ephemeral IP
}
}
metadata = {
proxy-mode = "project_editors" # service_account (perhaps? this is what I found in the UI)
# # proxy-mode = "service_account" ("project_editors", "email")
install-nvidia-driver = "true"
framework = "PyTorch"
shutdown-script = "timeout 30 gcloud compute instances remove-metadata gcloud-notebook-kenmoore-beta-sa --keys=proxy-url --zone var.datalab_zone"
}
# metadata_startup_script = "echo hi > /test.txt"
metadata_startup_script = "sudo apt-get update; sudo apt-get install -y google-osconfig-agent" # This did not work :(
service_account {
# email = google_service_account.datalab_service_account[each.key].email
# scopes = ["userinfo-email", "compute-ro", "storage-ro"]
scopes = ["https://www.googleapis.com/auth/cloud-platform"]
}
scheduling {
on_host_maintenance = "MIGRATE"
}
}
# # would like to adjust to allow networking project (shared vpc)
# module "datalab" {
# # for_each = var.datalab_user_list
# source = "terraform-google-modules/datalab/google//modules/default_instance"
# version = "~> 0.1"
# project_id = google_project.hipaa_project[each.value].project_id
# zone = var.datalab_zone
# # datalab_user_email = google_service_account.datalab_service_account[each.key].email # "google_service_account" "datalab_service_account"
# datalab_user_email =
# network_name = "datalab-network"
# subnet_name = "datalab-network"
# }
# resource "null_resource" "datalab" {
# for_each = var.datalab_user_list
# triggers = {
# user = each.key
# project_id = google_project.hipaa_project[each.value].project_id
# }
# provisioner "local-exec" {
# //command = "datalab create --project ${google_project.hipaa_project[each.value].project_id} --machine-type n1-standard-1 --zone us-central1-a --no-connect --for-user ${each.key} --service-account ${google_service_account.datalab_service_account[each.key].email} datalab-${split("@", each.key)[0]}"
# command = "datalab create --project ${self.triggers.project_id} --machine-type n1-standard-1 --zone ${var.datalab_zone} --no-connect --for-user ${self.triggers.user} --service-account ${google_service_account.datalab_service_account[self.triggers.user].email} datalab-${split("@", self.triggers.user)[0]}"
# }
# provisioner "local-exec" {
# when = destroy
# //command = "datalab delete --quiet --delete-disk --project ${google_project.hipaa_project[each.value].project_id} --zone us-central1-a datalab-${split("@", each.key)[0]}"
# command = "datalab delete --quiet --delete-disk --project ${self.triggers.project_id} --zone ${var.datalab_zone} datalab-${split("@", self.triggers.user)[0]}"
# }
# depends_on = [ google_sourcerepo_repository.datalab-notebooks, google_compute_network.datalab-network, null_resource.install_gcloud_cli ]
# }
provisioner "local-exec" {
//command = "datalab create --project ${google_project.hipaa_project[each.value].project_id} --machine-type n1-standard-1 --zone us-central1-a --no-connect --for-user ${each.key} --service-account ${google_service_account.datalab_service_account[each.key].email} datalab-${split("@", each.key)[0]}"
command = "datalab create --project ${self.triggers.project_id} --machine-type n1-standard-1 --zone ${var.datalab_zone} --no-connect --for-user ${self.triggers.user} --service-account ${google_service_account.datalab_service_account[self.triggers.user].email} datalab-${split("@", self.triggers.user)[0]}"
}
provisioner "local-exec" {
when = destroy
//command = "datalab delete --quiet --delete-disk --project ${google_project.hipaa_project[each.value].project_id} --zone us-central1-a datalab-${split("@", each.key)[0]}"
command = "datalab delete --quiet --delete-disk --project ${self.triggers.project_id} --zone ${var.datalab_zone} datalab-${split("@", self.triggers.user)[0]}"
}
depends_on = [ google_sourcerepo_repository.datalab-notebooks, google_compute_network.datalab-network, null_resource.install_gcloud_cli ]
}
\ No newline at end of file
variable "target_folder" {
default = "folders/931994294999"
}
resource "google_folder_organization_policy" "resource_locations" {
# folder = "folders/931994294999"
folder = var.target_folder
constraint = "gcp.resourceLocations"
list_policy{
allow {
values = ["in:us-locations"]
}
}
}
# {
# "policies": [
# {
# "constraint": "constraints/compute.trustedImageProjects",
# "etag": "BwWjL9PEyhw=",
# "updateTime": "2020-04-13T17:58:15.104Z",
# "listPolicy": {
# "allowedValues": [
# "projects/gcp-at-u-m-image-repo"
# ],
# "inheritFromParent": true
# }
# },
# {
# "constraint": "constraints/gcp.resourceLocations",
# "etag": "BwWjPnW20Fs=",
# "updateTime": "2020-04-14T11:25:41.607Z",
# "listPolicy": {
# "allowedValues": [
# "in:us-locations"
# ],
# "inheritFromParent": true
# }
# }
# ]
# }
......@@ -57,20 +57,20 @@ resource "google_logging_metric" "set_iam_permissions_change" {
}
}
resource "null_resource" "install_gcloud_cli" {
triggers = {
run_time = timestamp()
}
provisioner "local-exec" {
command = <<EOH
if [ `uname` != "Darwin" ]; then
echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key --keyring /usr/share/keyrings/cloud.google.gpg add -
sudo apt-get --assume-yes update && sudo apt-get --assume-yes install google-cloud-sdk google-cloud-sdk-datalab jq
echo $GOOGLE_CLOUD_KEYFILE_JSON > ~/key.json
gcloud auth activate-service-account --key-file ~/key.json
rm ~/key.json
fi
EOH
}
}
# resource "null_resource" "install_gcloud_cli" {
# triggers = {
# run_time = timestamp()
# }
# provisioner "local-exec" {
# command = <<EOH
# if [ `uname` != "Darwin" ]; then
# echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
# curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key --keyring /usr/share/keyrings/cloud.google.gpg add -
# sudo apt-get --assume-yes update && sudo apt-get --assume-yes install google-cloud-sdk google-cloud-sdk-datalab jq
# echo $GOOGLE_CLOUD_KEYFILE_JSON > ~/key.json
# gcloud auth activate-service-account --key-file ~/key.json
# rm ~/key.json
# fi
# EOH
# }
# }
......@@ -29,4 +29,4 @@ resource "google_monitoring_alert_policy" "set_iam_change" {
}
}
notification_channels = [google_monitoring_notification_channel.email.name]
}
}
\ No newline at end of file
locals {
default_cidr = "10.255.0.0/16"
default_regions = ["us-central1", "us-east1", "us-east4", "us-west1"]
default_subnets = {for x in local.default_regions : x => cidrsubnet(local.default_cidr,4,index(local.default_regions, x))}
# pods_cidr = "10.255.128.0/17"
# pods_range = {for x in local.default_regions : x => cidrsubnet(local.pods_cidr,3,index(local.default_regions, x))}
# services_cidr = "10.255.224.0/19"
# services_range = {for x in local.default_regions : x => cidrsubnet(local.services_cidr,3,index(local.default_regions, x))}
network_type = "worker"
}
# Create shared VPC in network project
resource "google_compute_network" "hipaa-shared-vpc" {
project = google_project.hipaa_project["network"].project_id
......@@ -8,4 +19,32 @@ resource "google_compute_network" "hipaa-shared-vpc" {
# Enable VPC Sharing in network project
resource "google_compute_shared_vpc_host_project" "hipaa-shared-vpc" {
project = google_project.hipaa_project["network"].project_id
}
resource "google_compute_shared_vpc_service_project" "hipaa-shared-vpc-worker" {
for_each = { for v in local.worker_projects : v => v }
host_project = google_compute_shared_vpc_host_project.hipaa-shared-vpc.project
service_project = google_project.hipaa_project[each.key].project_id
}
## Thinking about the names for these subnets...
resource "google_compute_subnetwork" "worker_subnet" {
for_each = local.default_subnets
project = google_project.hipaa_project["network"].project_id
name = "${local.network_type}-${each.key}"
region = each.key
ip_cidr_range = each.value
network = google_compute_network.hipaa-shared-vpc.self_link
# secondary_ip_range = var.gke ? [{range_name = "${each.key}-pods", ip_cidr_range = local.pods_range[each.key]},{range_name = "${each.key}-services", ip_cidr_range = local.services_range[each.key]}] : []
}
resource "google_compute_subnetwork_iam_binding" "worker_subnet_binding" {
for_each = local.default_subnets
project = google_project.hipaa_project["network"].project_id
region = each.key
subnetwork = google_compute_subnetwork.worker_subnet[each.key].name # <-- Issue here...created 4 subnets, need to reference for each subnet and apply permission for each worker group
role = "roles/compute.networkUser"
members = [
"group:${local.all_workers_group_email}",
]
}
\ No newline at end of file
......@@ -28,9 +28,10 @@ variable "dependencies" {
}
variable "datalab_user_list" {
type = map(string)
# type = map(string)
type = list(object({username = string, project = string, machine_type = string}))
description = "A map of users to create a datalab instance for. The key is the username and the value is the project id"
default = {}
default = []
}
variable "datalab_zone" {
......@@ -42,7 +43,7 @@ locals {
base_projects = ["audit", "data", "network", "monitor"]
projects_storing_data = concat(["data"], local.worker_projects)
projects_with_lien = ["audit", "data", "network", "monitor"]
worker_projects = [ for x in range(var.worker_count): "worker${x}" ]
worker_projects = [ for x in range(var.worker_count): "worker${x}" ]
bq_enabled_projects = concat(["audit", "data", "monitor"], local.worker_projects)
all_projects = concat(local.base_projects, local.worker_projects)
owners_group_email = "${var.project_prefix}-owners@${var.domain}"
......@@ -51,5 +52,6 @@ locals {
read-only_group_email = "${var.project_prefix}-read-only@${var.domain}"
alerts_group_email = "${var.project_prefix}-alerts@${var.domain}"
worker_project_email = { for x in local.worker_projects : x => "${var.project_prefix}-${x}@${var.domain}" }
datalab_service_account_readers = [ for x in keys(var.datalab_user_list) : google_service_account.datalab_service_account[x].email ]
all_workers_group_email = "${var.project_prefix}-workers@${var.domain}" # Add all worker emails to this MComm Group
datalab_service_account_readers = [ for x in var.datalab_user_list : google_service_account.datalab_service_account[x["username"]].email ]
}
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment