diff --git a/terraform/aws/aws-ec2-instance-windows-server/README.md b/terraform/aws/aws-ec2-instance-windows-server/README.md index a1efdff..3521dd5 100644 --- a/terraform/aws/aws-ec2-instance-windows-server/README.md +++ b/terraform/aws/aws-ec2-instance-windows-server/README.md @@ -9,10 +9,21 @@ This example creates the following: ## Considerations - This example was verified on Windows Server 2022 and Windows Server 2025. -- The userdata script sets the password of the Windows Administrator account to the value of the `windows_admin_password` input variable. AWS does not encrypt userdata. Do not use this method to set a production password. For production use, get the password from a secret store, for example AWS Secrets Manager. +- This example stores the Windows Administrator password and the Tailscale auth key in SSM Parameter Store, as `SecureString` parameters. The instance fetches these values at boot with an IAM role. They do not appear in the userdata script. - The userdata script authenticates the device with a scheduled task. This task runs at instance launch. Allow 1-2 minutes for the device to appear in the Tailscale Admin Console. - Connect to the instance with RDP over Tailscale. Do not connect over the public internet. This example does not open TCP port 3389 to the internet. +## Troubleshooting + +- The userdata script writes a full log to `C:\Windows\Temp\tailscale-user-data.log`. This log shows each step: parameter retrieval, the password set, the scheduled task creation, and the Tailscale connection check. +- The scheduled task `TailscaleUpOnce` runs `tailscale up`. `schtasks` does not capture the output of a task. The script instead redirects the output of `tailscale up` to `C:\Windows\Temp\tailscale-up-task.log`, then copies it into the userdata log above. +- The task deletes itself after it runs. To check its last run result before then, run: + ``` + schtasks /query /tn "TailscaleUpOnce" /v /fo list + ``` +- If Tailscale does not connect, the userdata log ends with the full output of `tailscale status`. Use this to find the reason, for example an expired or invalid auth key. +- You need a way to reach the instance to read these logs. Use RDP over Tailscale if the device did connect, or another connection method of your choice if it did not. + ## To use Follow the documentation to configure the Terraform providers: diff --git a/terraform/aws/aws-ec2-instance-windows-server/main.tf b/terraform/aws/aws-ec2-instance-windows-server/main.tf index 5bc7892..901c625 100644 --- a/terraform/aws/aws-ec2-instance-windows-server/main.tf +++ b/terraform/aws/aws-ec2-instance-windows-server/main.tf @@ -18,6 +18,9 @@ locals { # Use the provided auth key if set, otherwise use the one created below. tailscale_auth_key = coalesce(var.tailscale_auth_key, try(tailscale_tailnet_key.main[0].key, null)) + + windows_admin_password_ssm_parameter_name = "/${local.name}/windows-admin-password" + tailscale_auth_key_ssm_parameter_name = "/${local.name}/tailscale-auth-key" } # Remove this to use your own VPC. @@ -38,6 +41,67 @@ resource "tailscale_tailnet_key" "main" { tags = local.tailscale_acl_tags } +# The default AWS-managed KMS key used to encrypt SecureString parameters. +data "aws_kms_alias" "ssm" { + name = "alias/aws/ssm" +} + +resource "aws_ssm_parameter" "windows_admin_password" { + name = local.windows_admin_password_ssm_parameter_name + type = "SecureString" + value = var.windows_admin_password +} + +resource "aws_ssm_parameter" "tailscale_auth_key" { + name = local.tailscale_auth_key_ssm_parameter_name + type = "SecureString" + value = local.tailscale_auth_key +} + +resource "aws_iam_role" "windows_instance" { + name = local.name + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = "sts:AssumeRole" + Principal = { Service = "ec2.amazonaws.com" } + }, + ] + }) +} + +resource "aws_iam_role_policy" "windows_instance_ssm" { + name = "read-windows-admin-password" + role = aws_iam_role.windows_instance.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = "ssm:GetParameter" + Resource = [ + aws_ssm_parameter.windows_admin_password.arn, + aws_ssm_parameter.tailscale_auth_key.arn, + ] + }, + { + Effect = "Allow" + Action = "kms:Decrypt" + Resource = data.aws_kms_alias.ssm.target_key_arn + }, + ] + }) +} + +resource "aws_iam_instance_profile" "windows_instance" { + name = local.name + role = aws_iam_role.windows_instance.name +} + module "tailscale_aws_ec2_windows" { source = "../internal-modules/aws-ec2-instance-windows-server" @@ -47,14 +111,15 @@ module "tailscale_aws_ec2_windows" { subnet_id = local.subnet_id vpc_security_group_ids = local.security_group_ids + instance_profile_name = aws_iam_instance_profile.windows_instance.name + # Variables for Tailscale resources tailscale_hostname = local.name - tailscale_auth_key = local.tailscale_auth_key - - # Variables for the local Windows account used to run the Tailscale scheduled task - windows_admin_password = var.windows_admin_password depends_on = [ + aws_ssm_parameter.tailscale_auth_key, + aws_ssm_parameter.windows_admin_password, + aws_iam_role_policy.windows_instance_ssm, module.vpc.nat_ids, # remove if using your own VPC otherwise ensure provisioned NAT gateway is available ] } diff --git a/terraform/aws/aws-ec2-instance-windows-server/variables.tf b/terraform/aws/aws-ec2-instance-windows-server/variables.tf index ebd72c3..2188a88 100644 --- a/terraform/aws/aws-ec2-instance-windows-server/variables.tf +++ b/terraform/aws/aws-ec2-instance-windows-server/variables.tf @@ -1,5 +1,5 @@ variable "windows_admin_password" { - description = "Password to set for the Windows Administrator account. Required so the Tailscale scheduled task can authenticate. Must not contain a double quote character." + description = "Password to set for the Windows Administrator account. Stored in SSM Parameter Store as a SecureString, and fetched by the instance at boot. Required so the Tailscale scheduled task can authenticate. Must not contain a double quote character." type = string sensitive = true } diff --git a/terraform/aws/internal-modules/aws-ec2-instance-windows-server/main.tf b/terraform/aws/internal-modules/aws-ec2-instance-windows-server/main.tf index 303e4d1..e7feaee 100644 --- a/terraform/aws/internal-modules/aws-ec2-instance-windows-server/main.tf +++ b/terraform/aws/internal-modules/aws-ec2-instance-windows-server/main.tf @@ -1,12 +1,17 @@ locals { + # Names of the SSM Parameter Store SecureString parameters the instance + # reads at boot. The caller must create parameters with these names. + tailscale_auth_key_ssm_parameter_name = "/${var.tailscale_hostname}/tailscale-auth-key" + windows_admin_password_ssm_parameter_name = "/${var.tailscale_hostname}/windows-admin-password" + windows_install_script = templatefile( "${path.module}/scripts/tailscale-windows.ps1.tftpl", { - tailscale_auth_key = var.tailscale_auth_key, - tailscale_hostname = var.tailscale_hostname, - tailscale_msi_url = var.tailscale_msi_url, - username = var.windows_admin_username, - password = var.windows_admin_password, + auth_key_ssm_parameter_name = local.tailscale_auth_key_ssm_parameter_name, + tailscale_hostname = var.tailscale_hostname, + tailscale_msi_url = var.tailscale_msi_url, + username = var.windows_admin_username, + password_ssm_parameter_name = local.windows_admin_password_ssm_parameter_name, } ) } diff --git a/terraform/aws/internal-modules/aws-ec2-instance-windows-server/scripts/tailscale-windows.ps1.tftpl b/terraform/aws/internal-modules/aws-ec2-instance-windows-server/scripts/tailscale-windows.ps1.tftpl index 4df732b..32bde1b 100644 --- a/terraform/aws/internal-modules/aws-ec2-instance-windows-server/scripts/tailscale-windows.ps1.tftpl +++ b/terraform/aws/internal-modules/aws-ec2-instance-windows-server/scripts/tailscale-windows.ps1.tftpl @@ -44,22 +44,65 @@ Write-Host "Install complete." Write-Host "Removing $Installer" Remove-Item $Installer -Force +# Get the password for the local account below, and the Tailscale auth key, +# from SSM Parameter Store. Keeping these values out of userdata means they +# do not appear in the output of the EC2 DescribeInstanceAttribute API. +function Get-SecureParameter { + param([string]$Name) + try { + $value = (Get-SSMParameter -Name $Name -WithDecryption $true).Value + Write-Host "Successfully retrieved parameter [$Name]." + return $value + } + catch { + Write-Host "Failed to retrieve parameter [$Name]: $_. Exiting." + exit 1 + } +} + +Write-Host "Getting the password for local account [${username}] from SSM Parameter Store" +$password = Get-SecureParameter -Name "${password_ssm_parameter_name}" +if ([string]::IsNullOrEmpty($password)) { + Write-Host "Parameter [${password_ssm_parameter_name}] is empty. Exiting." + exit 1 +} + +Write-Host "Getting the Tailscale auth key from SSM Parameter Store" +$authKey = Get-SecureParameter -Name "${auth_key_ssm_parameter_name}" +if ([string]::IsNullOrEmpty($authKey)) { + Write-Host "Parameter [${auth_key_ssm_parameter_name}] is empty. Exiting." + exit 1 +} + # Set the password for the local account below. schtasks needs the real, # current password for this account to create a task that runs at boot. Write-Host "Setting the password for local account [${username}]" -net user "${username}" "${password}" /active:yes +net user "${username}" "$password" /active:yes +Write-Host "net user exit code: $LASTEXITCODE" # Create a task to authenticate to the tailnet on boot, then run it now so # the device does not have to wait for a reboot to join the tailnet. +# +# schtasks does not capture the output of the program it runs, so the task +# runs a wrapper script that redirects tailscale's own output to a log file. +# That log is the best source of the actual failure reason from tailscale. Write-Host "`n#`n# Creating task to authenticate to tailnet on boot`n#`n" +$taskLogPath = "$env:SystemRoot\Temp\tailscale-up-task.log" +$taskScriptPath = "$env:SystemRoot\Temp\tailscale-up-task.cmd" +$taskScriptContent = "@echo off`r`n`"C:\Program Files\Tailscale\tailscale.exe`" up --unattended --hostname `"${tailscale_hostname}`" --auth-key `"$authKey`" > `"$taskLogPath`" 2>&1`r`n" +Set-Content -Path $taskScriptPath -Value $taskScriptContent -Encoding ASCII + # /create = make a new task # /tn = task name # /tr = executable to run # /sc onstart = run on boot # /ru /rp = user and password to run the command as # /V1 /Z = delete the task after it has run -schtasks /create /tn "TailscaleUpOnce" /tr "'C:\Program Files\Tailscale\tailscale.exe' up --unattended --hostname '${tailscale_hostname}' --auth-key '${tailscale_auth_key}'" /sc onstart /ru "${username}" /rp "${password}" /V1 /Z +schtasks /create /tn "TailscaleUpOnce" /tr "$taskScriptPath" /sc onstart /ru "${username}" /rp "$password" /V1 /Z +Write-Host "schtasks /create exit code: $LASTEXITCODE" + schtasks /run /tn "TailscaleUpOnce" +Write-Host "schtasks /run exit code: $LASTEXITCODE" Write-Host "Waiting for Tailscale to authenticate..." $connected = $false @@ -71,10 +114,25 @@ for ($loop = 1; $loop -le 30; $loop++) { } Start-Sleep -Seconds 2 } + +Write-Host "`n#`n# TailscaleUpOnce task output ($taskLogPath):`n#`n" +if (Test-Path $taskLogPath) { + Get-Content $taskLogPath | ForEach-Object { Write-Host $_ } +} else { + Write-Host "Task log not found. The task may not have run yet." +} + +# Remove the wrapper script now that it has run. It briefly held the auth +# key in plain text, and it is not needed again: this task only ever runs +# once (see /Z above). +Remove-Item $taskScriptPath -Force -ErrorAction SilentlyContinue + if ($connected) { Write-Host "`n#`n# Tailscale status: connected`n#`n" } else { Write-Host "`n#`n# Tailscale status: NOT connected`n#`n" + Write-Host "`n#`n# Full tailscale status output:`n#`n" + & "C:\Program Files\Tailscale\tailscale.exe" status } Write-Host "`n#`n# Complete.`n#`n" diff --git a/terraform/aws/internal-modules/aws-ec2-instance-windows-server/variables-tailscale.tf b/terraform/aws/internal-modules/aws-ec2-instance-windows-server/variables-tailscale.tf index 87dfd51..8472fca 100644 --- a/terraform/aws/internal-modules/aws-ec2-instance-windows-server/variables-tailscale.tf +++ b/terraform/aws/internal-modules/aws-ec2-instance-windows-server/variables-tailscale.tf @@ -1,10 +1,6 @@ # # Variables for Tailscale resources # -variable "tailscale_auth_key" { - description = "Tailscale auth key to authenticate the device" - type = string -} variable "tailscale_hostname" { description = "Hostname to assign to the device" type = string @@ -23,8 +19,3 @@ variable "windows_admin_username" { type = string default = "Administrator" } -variable "windows_admin_password" { - description = "Password to set for `windows_admin_username`. Required so the scheduled task can authenticate as this account. Must not contain a double quote character." - type = string - sensitive = true -}