Commit de89b14b authored by Kenny Moore's avatar Kenny Moore
Browse files

Move to AI Platform Notebooks


Co-authored-by: Adam Robinson's avataradarobin@umich.edu <adarobin@umich.edu>
parent 1b7fe43a
......@@ -27,6 +27,12 @@ resource "google_compute_project_metadata" "os_patching" {
}
}
resource "google_project_service" "notebooks_api" {
for_each = { for v in local.worker_projects : v => v }
project = google_project.hipaa_project[each.key].project_id
service = "notebooks.googleapis.com"
}
resource "google_project_iam_binding" "datalab_service_account_iam_binding" {
for_each = { for v in local.worker_projects : v => v }
project = google_project.hipaa_project[each.key].project_id
......@@ -36,6 +42,7 @@ resource "google_project_iam_binding" "datalab_service_account_iam_binding" {
members = compact([ for v in var.datalab_user_list : v["project"] == each.value ? "serviceAccount:${google_service_account.datalab_service_account[v["username"]].email}" : "" ])
}
# DO WE NEED THIS?
resource "google_project_service" "sourcerepo-api" {
for_each = { for v in local.worker_projects : v => v }
project = google_project.hipaa_project[each.key].project_id
......@@ -85,6 +92,15 @@ resource "google_project_iam_binding" "view_compute_instances" {
]
}
resource "google_project_iam_binding" "view_notebooks" {
for_each = { for v in local.worker_projects : v => v }
project = google_project.hipaa_project[each.key].project_id
role = "roles/notebooks.viewer"
members = [
"group:${local.worker_project_email[each.key]}",
]
}
resource "google_service_account" "datalab_service_account" {
for_each = { for v in var.datalab_user_list : v["username"] => v["project"] }
account_id = "datalab-${split("@", each.key)[0]}"
......@@ -100,7 +116,7 @@ resource "google_service_account_iam_binding" "datalab_service_account_iam_bindi
]
}
data "google_iam_policy" "datalab_user_to_instance_policy" {
data "google_iam_policy" "notebook_user_to_instance_policy" {
for_each = { for v in var.datalab_user_list : v["username"] => v["username"] }
binding {
role = "roles/compute.instanceAdmin.v1"
......@@ -111,70 +127,129 @@ data "google_iam_policy" "datalab_user_to_instance_policy" {
}
}
resource "google_compute_instance_iam_policy" "datalab_user_to_instance_binding" {
resource "google_compute_instance_iam_policy" "notebook_user_to_instance_binding" {
for_each = { for v in var.datalab_user_list : v["username"] => v["project"] }
instance_name = "datalab-${split("@", each.key)[0]}"
instance_name = google_notebooks_instance.worker_nb[each.key].name
project = google_project.hipaa_project[each.value].project_id
zone = var.datalab_zone
policy_data = data.google_iam_policy.datalab_user_to_instance_policy[each.key].policy_data
# depends_on = [ null_resource.datalab ]
depends_on = [ google_compute_instance.ai_notebook ]
}
## ## ## ## Needs depends on for IAM policy/ai_notebook
## ## ## ## ai_notebook = GCE Instance until ai_notebook API is out of beta (q3 2020)
#### disk must be 100+ GB - add paramenter ####
resource "google_compute_instance" "ai_notebook" {
for_each = { for v in var.datalab_user_list : v["username"] => v }
project = google_project.hipaa_project[each.value["project"]].project_id
# name = "notebook-${split("@", each.key)[0]}"
name = "datalab-${split("@", each.key)[0]}"
machine_type = each.value["machine_type"]
# machine_type = "f1-micro"
zone = var.datalab_zone
tags = ["foo", "bar"] # https-server
boot_disk {
initialize_params {
image = "deeplearning-platform-release/pytorch-latest-gpu" # var.image
}
zone = data.google_compute_zones.notebooks.names[0]
policy_data = data.google_iam_policy.notebook_user_to_instance_policy[each.key].policy_data
}
# # Create AI Platform Notebooks Service Agent (to replace default)
# resource "google_service_account" "ai_notebooks_agent" {
# for_each = { for v in local.worker_projects : v => v }
# project = google_project.hipaa_project[each.key].project_id
# account_id = "ai-notebooks-agent"
# display_name = "Cloud AI Platform Notebooks Service Account"
# }
# # roles/notebooks.serviceAgent
# resource "google_project_iam_binding" "ai_notebooks_agent" {
# for_each = { for v in local.worker_projects : v => v }
# project = google_project.hipaa_project[each.key].project_id
# role = "roles/notebooks.serviceAgent"
# members = [
# "serviceAccount:${google_service_account.ai_notebooks_agent[each.key].email}",
# ]
# }
resource "google_notebooks_instance" "worker_nb" {
provider = google-beta
for_each = { for v in var.datalab_user_list : v["username"] => v }
project = google_project.hipaa_project[each.value["project"]].project_id
name = "notebook-${split("@", each.key)[0]}"
location = data.google_compute_zones.notebooks.names[0]
network = google_compute_network.hipaa-shared-vpc.id
subnet = google_compute_subnetwork.worker_subnet[var.notebook_region].id
machine_type = each.value["machine_type"]
service_account = google_service_account.datalab_service_account[each.key].email
instance_owners = each.value["username"]
vm_image {
project = "deeplearning-platform-release" # needs a var?
image_family = "tf-latest-cpu" # needs a var?
}
# temporary - waiting on bug fix
lifecycle {
ignore_changes = [
subnet,
network,
instance_owners,
]
}
depends_on = [ google_project_service.notebooks_api, google_compute_subnetwork_iam_binding.worker_subnet_binding ]
}
# resource "google_compute_subnetwork" "worker_subnet" {
# creation_timestamp = "2020-05-21T05:40:54.303-07:00"
# gateway_address = "10.255.0.1"
# id = "projects/hipaa-test-network-706f/regions/us-central1/subnetworks/worker-us-central1"
# ip_cidr_range = "10.255.0.0/20"
# name = "worker-us-central1"
# network = "https://www.googleapis.com/compute/v1/projects/hipaa-test-network-706f/global/networks/hipaa-test-network"
# private_ip_google_access = false
# project = "hipaa-test-network-706f"
# region = "us-central1"
# secondary_ip_range = []
# self_link = "https://www.googleapis.com/compute/v1/projects/hipaa-test-network-706f/regions/us-central1/subnetworks/worker-us-central1"
# }
# resource "google_compute_instance" "ai_notebook" {
# for_each = { for v in var.datalab_user_list : v["username"] => v }
# project = google_project.hipaa_project[each.value["project"]].project_id
# # name = "notebook-${split("@", each.key)[0]}"
# name = "datalab-${split("@", each.key)[0]}"
# machine_type = each.value["machine_type"]
# # machine_type = "f1-micro"
# zone = var.datalab_zone
# tags = ["foo", "bar"] # https-server
# boot_disk {
# initialize_params {
# image = "deeplearning-platform-release/pytorch-latest-gpu" # var.image
# }
# }
# // Local SSD disk
# scratch_disk {
# interface = "SCSI"
# }
network_interface {
network = "datalab-network" # var.network (shared or default)
# network_interface {
# network = "datalab-network" # var.network (shared or default)
access_config {
// Ephemeral IP
}
}
# access_config {
# // Ephemeral IP
# }
# }
metadata = {
proxy-mode = "project_editors" # service_account (perhaps? this is what I found in the UI)
# # proxy-mode = "service_account" ("project_editors", "email")
install-nvidia-driver = "true"
framework = "PyTorch"
shutdown-script = "timeout 30 gcloud compute instances remove-metadata gcloud-notebook-kenmoore-beta-sa --keys=proxy-url --zone var.datalab_zone"
}
# metadata = {
# proxy-mode = "project_editors" # service_account (perhaps? this is what I found in the UI)
# # # proxy-mode = "service_account" ("project_editors", "email")
# install-nvidia-driver = "true"
# framework = "PyTorch"
# shutdown-script = "timeout 30 gcloud compute instances remove-metadata gcloud-notebook-kenmoore-beta-sa --keys=proxy-url --zone var.datalab_zone"
# }
# metadata_startup_script = "echo hi > /test.txt"
metadata_startup_script = "sudo apt-get update; sudo apt-get install -y google-osconfig-agent" # This did not work :(
# # metadata_startup_script = "echo hi > /test.txt"
# metadata_startup_script = "sudo apt-get update; sudo apt-get install -y google-osconfig-agent" # This did not work :(
service_account {
# email = google_service_account.datalab_service_account[each.key].email
# scopes = ["userinfo-email", "compute-ro", "storage-ro"]
scopes = ["https://www.googleapis.com/auth/cloud-platform"]
}
# service_account {
# # email = google_service_account.datalab_service_account[each.key].email
# # scopes = ["userinfo-email", "compute-ro", "storage-ro"]
# scopes = ["https://www.googleapis.com/auth/cloud-platform"]
# }
# scheduling {
# on_host_maintenance = "MIGRATE"
# }
# }
scheduling {
on_host_maintenance = "MIGRATE"
}
}
......
......@@ -39,4 +39,4 @@ resource "google_folder_organization_policy" "resource_locations" {
# }
# }
# ]
# }
# }
\ No newline at end of file
......@@ -44,7 +44,9 @@ resource "google_compute_subnetwork_iam_binding" "worker_subnet_binding" {
region = each.key
subnetwork = google_compute_subnetwork.worker_subnet[each.key].name # <-- Issue here...created 4 subnets, need to reference for each subnet and apply permission for each worker group
role = "roles/compute.networkUser"
members = [
"group:${local.all_workers_group_email}",
]
}
\ No newline at end of file
members = concat(compact([ for v in local.worker_projects : "serviceAccount:service-${google_project.hipaa_project[v].number}@gcp-sa-notebooks.iam.gserviceaccount.com"]), ["group:${local.all_workers_group_email}"])
}
# project_number = "672858288516"
# ai_notebook_agent = "service-672858288516@gcp-sa-notebooks.iam.gserviceaccount.com"
\ No newline at end of file
......@@ -34,11 +34,19 @@ variable "datalab_user_list" {
default = []
}
variable "datalab_zone" {
description = "The zone to create datalab instances in"
default = "us-central1-a"
variable "notebook_region" {
default = "us-central1"
}
data "google_compute_zones" "notebooks" {
project = google_project.hipaa_project["network"].project_id
region = var.notebook_region
}
# variable "datalab_zone" {
# description = "The zone to create datalab instances in"
# default = "us-central1-a"
# }
locals {
base_projects = ["audit", "data", "network", "monitor"]
projects_storing_data = concat(["data"], local.worker_projects)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment