aws-ec2-instance-windows-server: use ssm parameter store, and adds be…

terraform/aws/aws-ec2-instance-windows-server/README.md

@@ -9,10 +9,21 @@ This example creates the following:
 ## Considerations
 
 - This example was verified on Windows Server 2022 and Windows Server 2025.
-- The userdata script sets the password of the Windows Administrator account to the value of the `windows_admin_password` input variable. AWS does not encrypt userdata. Do not use this method to set a production password. For production use, get the password from a secret store, for example AWS Secrets Manager.
+- This example stores the Windows Administrator password and the Tailscale auth key in SSM Parameter Store, as `SecureString` parameters. The instance fetches these values at boot with an IAM role. They do not appear in the userdata script.
 - The userdata script authenticates the device with a scheduled task. This task runs at instance launch. Allow 1-2 minutes for the device to appear in the Tailscale Admin Console.
 - Connect to the instance with RDP over Tailscale. Do not connect over the public internet. This example does not open TCP port 3389 to the internet.
 
+## Troubleshooting
+
+- The userdata script writes a full log to `C:\Windows\Temp\tailscale-user-data.log`. This log shows each step: parameter retrieval, the password set, the scheduled task creation, and the Tailscale connection check.
+- The scheduled task `TailscaleUpOnce` runs `tailscale up`. `schtasks` does not capture the output of a task. The script instead redirects the output of `tailscale up` to `C:\Windows\Temp\tailscale-up-task.log`, then copies it into the userdata log above.
+- The task deletes itself after it runs. To check its last run result before then, run:
+  ```
+  schtasks /query /tn "TailscaleUpOnce" /v /fo list
+  ```
+- If Tailscale does not connect, the userdata log ends with the full output of `tailscale status`. Use this to find the reason, for example an expired or invalid auth key.
+- You need a way to reach the instance to read these logs. Use RDP over Tailscale if the device did connect, or another connection method of your choice if it did not.
+
 ## To use
 
 Follow the documentation to configure the Terraform providers:

terraform/aws/aws-ec2-instance-windows-server/main.tf

@@ -18,6 +18,9 @@ locals {
 
   # Use the provided auth key if set, otherwise use the one created below.
   tailscale_auth_key = coalesce(var.tailscale_auth_key, try(tailscale_tailnet_key.main[0].key, null))
+
+  windows_admin_password_ssm_parameter_name = "/${local.name}/windows-admin-password"
+  tailscale_auth_key_ssm_parameter_name     = "/${local.name}/tailscale-auth-key"
 }
 
 # Remove this to use your own VPC.
@@ -38,6 +41,67 @@ resource "tailscale_tailnet_key" "main" {
   tags                = local.tailscale_acl_tags
 }
 
+# The default AWS-managed KMS key used to encrypt SecureString parameters.
+data "aws_kms_alias" "ssm" {
+  name = "alias/aws/ssm"
+}
+
+resource "aws_ssm_parameter" "windows_admin_password" {
+  name  = local.windows_admin_password_ssm_parameter_name
+  type  = "SecureString"
+  value = var.windows_admin_password
+}
+
+resource "aws_ssm_parameter" "tailscale_auth_key" {
+  name  = local.tailscale_auth_key_ssm_parameter_name
+  type  = "SecureString"
+  value = local.tailscale_auth_key
+}
+
+resource "aws_iam_role" "windows_instance" {
+  name = local.name
+
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Effect    = "Allow"
+        Action    = "sts:AssumeRole"
+        Principal = { Service = "ec2.amazonaws.com" }
+      },
+    ]
+  })
+}
+
+resource "aws_iam_role_policy" "windows_instance_ssm" {
+  name = "read-windows-admin-password"
+  role = aws_iam_role.windows_instance.id
+
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Effect = "Allow"
+        Action = "ssm:GetParameter"
+        Resource = [
+          aws_ssm_parameter.windows_admin_password.arn,
+          aws_ssm_parameter.tailscale_auth_key.arn,
+        ]
+      },
+      {
+        Effect   = "Allow"
+        Action   = "kms:Decrypt"
+        Resource = data.aws_kms_alias.ssm.target_key_arn
+      },
+    ]
+  })
+}
+
+resource "aws_iam_instance_profile" "windows_instance" {
+  name = local.name
+  role = aws_iam_role.windows_instance.name
+}
+
 module "tailscale_aws_ec2_windows" {
   source = "../internal-modules/aws-ec2-instance-windows-server"
 
@@ -47,14 +111,15 @@ module "tailscale_aws_ec2_windows" {
   subnet_id              = local.subnet_id
   vpc_security_group_ids = local.security_group_ids
 
+  instance_profile_name = aws_iam_instance_profile.windows_instance.name
+
   # Variables for Tailscale resources
   tailscale_hostname = local.name
-  tailscale_auth_key = local.tailscale_auth_key
-
-  # Variables for the local Windows account used to run the Tailscale scheduled task
-  windows_admin_password = var.windows_admin_password
 
   depends_on = [
+    aws_ssm_parameter.tailscale_auth_key,
+    aws_ssm_parameter.windows_admin_password,
+    aws_iam_role_policy.windows_instance_ssm,
     module.vpc.nat_ids, # remove if using your own VPC otherwise ensure provisioned NAT gateway is available
   ]
 }

terraform/aws/aws-ec2-instance-windows-server/variables.tf

@@ -1,5 +1,5 @@
 variable "windows_admin_password" {
-  description = "Password to set for the Windows Administrator account. Required so the Tailscale scheduled task can authenticate. Must not contain a double quote character."
+  description = "Password to set for the Windows Administrator account. Stored in SSM Parameter Store as a SecureString, and fetched by the instance at boot. Required so the Tailscale scheduled task can authenticate. Must not contain a double quote character."
   type        = string
   sensitive   = true
 }

terraform/aws/internal-modules/aws-ec2-instance-windows-server/main.tf

@@ -1,12 +1,17 @@
 locals {
+  # Names of the SSM Parameter Store SecureString parameters the instance
+  # reads at boot. The caller must create parameters with these names.
+  tailscale_auth_key_ssm_parameter_name     = "/${var.tailscale_hostname}/tailscale-auth-key"
+  windows_admin_password_ssm_parameter_name = "/${var.tailscale_hostname}/windows-admin-password"
+
   windows_install_script = templatefile(
     "${path.module}/scripts/tailscale-windows.ps1.tftpl",
     {
-      tailscale_auth_key = var.tailscale_auth_key,
-      tailscale_hostname = var.tailscale_hostname,
-      tailscale_msi_url  = var.tailscale_msi_url,
-      username           = var.windows_admin_username,
-      password           = var.windows_admin_password,
+      auth_key_ssm_parameter_name = local.tailscale_auth_key_ssm_parameter_name,
+      tailscale_hostname          = var.tailscale_hostname,
+      tailscale_msi_url           = var.tailscale_msi_url,
+      username                    = var.windows_admin_username,
+      password_ssm_parameter_name = local.windows_admin_password_ssm_parameter_name,
     }
   )
 }

terraform/aws/internal-modules/aws-ec2-instance-windows-server/scripts/tailscale-windows.ps1.tftpl

@@ -44,22 +44,65 @@ Write-Host "Install complete."
 Write-Host "Removing $Installer"
 Remove-Item $Installer -Force
 
+# Get the password for the local account below, and the Tailscale auth key,
+# from SSM Parameter Store. Keeping these values out of userdata means they
+# do not appear in the output of the EC2 DescribeInstanceAttribute API.
+function Get-SecureParameter {
+    param([string]$Name)
+    try {
+        $value = (Get-SSMParameter -Name $Name -WithDecryption $true).Value
+        Write-Host "Successfully retrieved parameter [$Name]."
+        return $value
+    }
+    catch {
+        Write-Host "Failed to retrieve parameter [$Name]: $_. Exiting."
+        exit 1
+    }
+}
+
+Write-Host "Getting the password for local account [${username}] from SSM Parameter Store"
+$password = Get-SecureParameter -Name "${password_ssm_parameter_name}"
+if ([string]::IsNullOrEmpty($password)) {
+    Write-Host "Parameter [${password_ssm_parameter_name}] is empty. Exiting."
+    exit 1
+}
+
+Write-Host "Getting the Tailscale auth key from SSM Parameter Store"
+$authKey = Get-SecureParameter -Name "${auth_key_ssm_parameter_name}"
+if ([string]::IsNullOrEmpty($authKey)) {
+    Write-Host "Parameter [${auth_key_ssm_parameter_name}] is empty. Exiting."
+    exit 1
+}
+
 # Set the password for the local account below. schtasks needs the real,
 # current password for this account to create a task that runs at boot.
 Write-Host "Setting the password for local account [${username}]"
-net user "${username}" "${password}" /active:yes
+net user "${username}" "$password" /active:yes
+Write-Host "net user exit code: $LASTEXITCODE"
 
 # Create a task to authenticate to the tailnet on boot, then run it now so
 # the device does not have to wait for a reboot to join the tailnet.
+#
+# schtasks does not capture the output of the program it runs, so the task
+# runs a wrapper script that redirects tailscale's own output to a log file.
+# That log is the best source of the actual failure reason from tailscale.
 Write-Host "`n#`n# Creating task to authenticate to tailnet on boot`n#`n"
+$taskLogPath = "$env:SystemRoot\Temp\tailscale-up-task.log"
+$taskScriptPath = "$env:SystemRoot\Temp\tailscale-up-task.cmd"
+$taskScriptContent = "@echo off`r`n`"C:\Program Files\Tailscale\tailscale.exe`" up --unattended --hostname `"${tailscale_hostname}`" --auth-key `"$authKey`" > `"$taskLogPath`" 2>&1`r`n"
+Set-Content -Path $taskScriptPath -Value $taskScriptContent -Encoding ASCII
+
 # /create = make a new task
 # /tn     = task name
 # /tr     = executable to run
 # /sc onstart = run on boot
 # /ru /rp = user and password to run the command as
 # /V1 /Z  = delete the task after it has run
-schtasks /create /tn "TailscaleUpOnce" /tr "'C:\Program Files\Tailscale\tailscale.exe' up --unattended --hostname '${tailscale_hostname}' --auth-key '${tailscale_auth_key}'" /sc onstart /ru "${username}" /rp "${password}" /V1 /Z
+schtasks /create /tn "TailscaleUpOnce" /tr "$taskScriptPath" /sc onstart /ru "${username}" /rp "$password" /V1 /Z
+Write-Host "schtasks /create exit code: $LASTEXITCODE"
+
 schtasks /run /tn "TailscaleUpOnce"
+Write-Host "schtasks /run exit code: $LASTEXITCODE"
 
 Write-Host "Waiting for Tailscale to authenticate..."
 $connected = $false
@@ -71,10 +114,25 @@ for ($loop = 1; $loop -le 30; $loop++) {
     }
     Start-Sleep -Seconds 2
 }
+
+Write-Host "`n#`n# TailscaleUpOnce task output ($taskLogPath):`n#`n"
+if (Test-Path $taskLogPath) {
+    Get-Content $taskLogPath | ForEach-Object { Write-Host $_ }
+} else {
+    Write-Host "Task log not found. The task may not have run yet."
+}
+
+# Remove the wrapper script now that it has run. It briefly held the auth
+# key in plain text, and it is not needed again: this task only ever runs
+# once (see /Z above).
+Remove-Item $taskScriptPath -Force -ErrorAction SilentlyContinue
+
 if ($connected) {
     Write-Host "`n#`n# Tailscale status: connected`n#`n"
 } else {
     Write-Host "`n#`n# Tailscale status: NOT connected`n#`n"
+    Write-Host "`n#`n# Full tailscale status output:`n#`n"
+    & "C:\Program Files\Tailscale\tailscale.exe" status
 }
 
 Write-Host "`n#`n# Complete.`n#`n"

terraform/aws/internal-modules/aws-ec2-instance-windows-server/variables-tailscale.tf

@@ -1,10 +1,6 @@
 #
 # Variables for Tailscale resources
 #
-variable "tailscale_auth_key" {
-  description = "Tailscale auth key to authenticate the device"
-  type        = string
-}
 variable "tailscale_hostname" {
   description = "Hostname to assign to the device"
   type        = string
@@ -23,8 +19,3 @@ variable "windows_admin_username" {
   type        = string
   default     = "Administrator"
 }
-variable "windows_admin_password" {
-  description = "Password to set for `windows_admin_username`. Required so the scheduled task can authenticate as this account. Must not contain a double quote character."
-  type        = string
-  sensitive   = true
-}