feat: stream EC2 build logs to CloudWatch Logs

- Add /${name_prefix}/ec2-builds CloudWatch Log Group (30d retention)
- Grant ec2_builder IAM role logs:CreateLogStream/PutLogEvents on the group
- Add EC2_BUILD_LOG_GROUP env var to process_build Lambda
- Install amazon-cloudwatch-agent in user-data, configure it to tail
  /var/log/build.log and stream to CW with build_id as log stream name

Author: fok666

infrastructure/lambdas/process_build/index.py

@@ -32,6 +32,7 @@
 EC2_INSTANCE_TYPE = os.environ.get("EC2_INSTANCE_TYPE", "c5.xlarge")
 MAX_BUILD_MINUTES = int(os.environ.get("MAX_BUILD_MINUTES", "30"))
 PROJECT_NAME = os.environ.get("PROJECT_NAME", "lambda-layer-builder")
+LOG_GROUP_NAME = os.environ.get("EC2_BUILD_LOG_GROUP", "/lambda-layer-builder/prod/ec2-builds")
 
 
 def handler(event, context):
@@ -67,6 +68,7 @@ def _process_build(message):
         architectures=architectures,
         requirements=requirements,
         single_file=single_file,
+        log_group_name=LOG_GROUP_NAME,
     )
 
     # Pick a random subnet for AZ diversity
@@ -149,7 +151,7 @@ def _update_status(build_id, status, error=None):
         print(f"Failed to update status for {build_id}: {e}")
 
 
-def _generate_user_data(build_id, python_version, architectures, requirements, single_file):
+def _generate_user_data(build_id, python_version, architectures, requirements, single_file, log_group_name):
     """Generate the EC2 user-data bash script for the build."""
     req_escaped = requirements.replace("\\", "\\\\").replace("'", "'\\''")
     arches_str = " ".join(architectures)
@@ -208,12 +210,43 @@ def _generate_user_data(build_id, python_version, architectures, requirements, s
 }}
 trap cleanup EXIT
 
-# --- Install Docker ---
-echo "$(date): Installing Docker..."
-dnf install -y docker git aws-cli 2>/dev/null || yum install -y docker git aws-cli
+# --- Install Docker and CloudWatch Agent ---
+echo "$(date): Installing Docker and CloudWatch Agent..."
+dnf install -y docker git aws-cli amazon-cloudwatch-agent 2>/dev/null || yum install -y docker git aws-cli
 systemctl start docker
 systemctl enable docker
 
+# --- Configure CloudWatch Logs streaming ---
+# Stream /var/log/build.log to CloudWatch; each build gets its own log stream.
+echo "$(date): Configuring CloudWatch Logs streaming..."
+mkdir -p /opt/aws/amazon-cloudwatch-agent/etc
+cat > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json << 'CWEOF'
+{{
+  "logs": {{
+    "logs_collected": {{
+      "files": {{
+        "collect_list": [
+          {{
+            "file_path": "/var/log/build.log",
+            "log_group_name": "{log_group_name}",
+            "log_stream_name": "{build_id}",
+            "timezone": "UTC",
+            "timestamp_format": "%Y-%m-%dT%H:%M:%S"
+          }}
+        ]
+      }}
+    }}
+  }}
+}}
+CWEOF
+
+/opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \
+    -a fetch-config -m ec2 \
+    -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json \
+    -s 2>/dev/null \
+  && echo "$(date): CloudWatch streaming active → {log_group_name}/{build_id}" \
+  || echo "$(date): WARNING: CloudWatch agent failed to start"
+
 # Enable QEMU for cross-architecture builds
 docker run --rm --privileged multiarch/qemu-user-static --reset -p yes 2>/dev/null || true
 

infrastructure/terraform/ec2.tf

@@ -6,6 +6,17 @@
 # Each instance self-terminates after build completion or timeout.
 # =============================================================================
 
+# CloudWatch Log Group for EC2 build output
+# Each build writes to its own log stream (stream name = build_id).
+resource "aws_cloudwatch_log_group" "ec2_builds" {
+  name              = "/${local.name_prefix}/ec2-builds"
+  retention_in_days = 30
+
+  tags = {
+    Name = "${local.name_prefix}-ec2-builds"
+  }
+}
+
 # Pre-create the EC2 Spot service-linked role so Lambda doesn't need to create it at runtime.
 # This role is account-global and only needs to exist once.
 # Terraform will silently import it if it already exists.

infrastructure/terraform/iam.tf

@@ -246,6 +246,16 @@ resource "aws_iam_role_policy" "ec2_builder" {
           "ec2:DescribeInstances",
         ]
         Resource = "*"
+      },
+      {
+        Effect = "Allow"
+        Action = [
+          "logs:CreateLogGroup",
+          "logs:CreateLogStream",
+          "logs:PutLogEvents",
+          "logs:DescribeLogStreams",
+        ]
+        Resource = "${aws_cloudwatch_log_group.ec2_builds.arn}:*"
       }
     ]
   })

infrastructure/terraform/lambda.tf

@@ -82,6 +82,7 @@ resource "aws_lambda_function" "process_build" {
       EC2_INSTANCE_TYPE    = var.ec2_instance_type
       MAX_BUILD_MINUTES    = tostring(var.ec2_max_build_time_minutes)
       PROJECT_NAME         = var.project_name
+      EC2_BUILD_LOG_GROUP  = aws_cloudwatch_log_group.ec2_builds.name
     }
   }