fix: stabilize component CI and runtime profile recording

.github/workflows/component-tests.yaml

@@ -11,7 +11,7 @@ permissions: read-all
 
 jobs:
   build-and-push-image:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout code
         uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
@@ -27,7 +27,7 @@ jobs:
           sudo apt-get update
           sudo apt-get install -y jq curl
           IG_ARCH=amd64
-          IG_VERSION=$(curl -s https://api.github.com/repos/inspektor-gadget/inspektor-gadget/releases/latest | jq -r .tag_name)
+          IG_VERSION=v0.57.0
           echo "Installing IG version: ${IG_VERSION}"
           curl -sL https://github.com/inspektor-gadget/inspektor-gadget/releases/download/${IG_VERSION}/ig-linux-${IG_ARCH}-${IG_VERSION}.tar.gz | sudo tar -C /usr/local/bin -xzf - ig
           sudo chmod +x /usr/local/bin/ig
@@ -69,11 +69,13 @@ jobs:
       image_repo: ${{ steps.build-and-push-image.outputs.image_repo }}
 
   component-tests:
-    runs-on: ubuntu-latest
+    # Ubuntu 24.04 kernel 7.0.0-1017-azure loses syscall BPF events on all but one CPU.
+    # Follow IG CI until https://github.com/actions/runner-images/issues/15003 is fixed.
+    runs-on: ubuntu-22.04
     needs: build-and-push-image
-    continue-on-error: true
-    timeout-minutes: 30
+    timeout-minutes: 45
     strategy:
+      fail-fast: false
       matrix:
         test: [
           Test_01_BasicAlertTest,
@@ -121,42 +123,45 @@ jobs:
         with:
           name: node-agent-image
 
-      - name: Load fork PR image
-        if: ${{ github.event.pull_request.head.repo.full_name != github.repository }}
-        run: |
-          docker load -i node-agent-image.tar
+      - name: Set up native Kubernetes node
+        timeout-minutes: 22
+        run: ./tests/scripts/setup-native-component-cluster.sh
 
-      - name: Set up Kind
+      - name: Capture native runtime diagnostics
+        timeout-minutes: 2
         run: |
-          curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.31.0/kind-linux-amd64
-          chmod +x ./kind
-          ./kind create cluster
-          curl -LO "https://dl.k8s.io/release/v1.35.0/bin/linux/amd64/kubectl"
-          chmod +x ./kubectl
-          sudo mv ./kubectl /usr/local/bin/kubectl
-
-      - name: Load fork PR image into Kind
+          mkdir -p diagnostics
+          {
+            timeout --kill-after=1s 10s minikube version || true
+            timeout --kill-after=1s 10s docker version || true
+            timeout --kill-after=1s 10s containerd --version || true
+            timeout --kill-after=1s 10s runc --version || true
+            uname -a
+            readlink /proc/1/ns/pid
+          } 2>&1 | tee diagnostics/cluster-runtime.txt
+
+      - name: Load fork PR image into native runtime
         if: ${{ github.event.pull_request.head.repo.full_name != github.repository }}
-        run: |
-          ./kind load docker-image \
-            ${{ needs.build-and-push-image.outputs.image_repo }}:${{ needs.build-and-push-image.outputs.image_tag }}
+        timeout-minutes: 3
+        run: sudo minikube image load "${PWD}/node-agent-image.tar"
 
       - name: Install Helm and Kubectl
         run: |
           curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
           chmod 700 get_helm.sh
-          sudo ./get_helm.sh
+          sudo ./get_helm.sh --version v3.22.1
 
       - name: Install Prometheus and Node Exporter
         run: |
           helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
           helm repo update
-          helm upgrade --install prometheus prometheus-community/kube-prometheus-stack --set grafana.enabled=false --namespace monitoring --create-namespace --set prometheus.prometheusSpec.podMonitorSelectorNilUsesHelmValues=false,prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false --set prometheus.prometheusSpec.maximumStartupDurationSeconds=300 --wait --timeout 5m
+          helm upgrade --install prometheus prometheus-community/kube-prometheus-stack --version 92.11.0 --set grafana.enabled=false --namespace monitoring --create-namespace --set prometheus.prometheusSpec.podMonitorSelectorNilUsesHelmValues=false,prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false --set prometheus.prometheusSpec.maximumStartupDurationSeconds=300 --wait --timeout 5m
           # Check that the prometheus pod is running
           kubectl wait --for=condition=Ready pod -l app.kubernetes.io/name=prometheus -n monitoring --timeout=300s
       - name: Install Node Agent Chart
+        env:
+          STORAGE_TAG: v0.0.354
         run: |
-          STORAGE_TAG=$(./tests/scripts/storage-tag.sh)
           echo "Storage tag that will be used: ${STORAGE_TAG}"
           helm upgrade --install kubescape ./tests/chart --set clusterName=`kubectl config current-context` --set nodeAgent.image.tag=${{ needs.build-and-push-image.outputs.image_tag }} --set nodeAgent.image.repository=${{ needs.build-and-push-image.outputs.image_repo }} --set storage.image.tag=${STORAGE_TAG} -n kubescape --create-namespace --wait --timeout 10m --debug
           # Check that the node-agent pod is running
@@ -170,7 +175,7 @@ jobs:
           CGO_ENABLED: 0
         uses: actions/setup-go@7b8cf10d4e4a01d4992d18a89f4d7dc5a3e6d6f4 # v4
         with:
-          go-version: "1.27"
+          go-version: "1.27.2"
       - name: Download test dependencies
         working-directory: tests
         run: |
@@ -189,12 +194,95 @@ jobs:
           sudo sh -c "ulimit -l unlimited"
       - name: Run test
         run: |
-          cd tests && go test -v ./... -run ${{ matrix.test }} --timeout=20m --tags=component
-      - name: Print node agent & storage logs
+          repetitions=1
+          if [ "${{ matrix.test }}" = "Test_01_BasicAlertTest" ]; then
+            repetitions=3
+          fi
+          cd tests && go test -v ./... -run '^${{ matrix.test }}$' -count="$repetitions" --timeout=20m --tags=component
+      - name: Collect component test diagnostics
         if: always()
+        timeout-minutes: 5
         run: |
-          echo "Node agent logs"
-          kubectl logs $(kubectl get pods -n kubescape -o name | grep node-agent) -n kubescape -c node-agent
-          echo "-----------------------------------------"
-          echo "Storage logs"
-          kubectl logs $(kubectl get pods -n kubescape -o name | grep storage) -n kubescape
+          mkdir -p diagnostics
+          collect() {
+            timeout --kill-after=1s 10s kubectl --request-timeout=8s "$@"
+          }
+          collect_goroutines() (
+            namespace=$1
+            pod=$2
+            prefix="diagnostics/${namespace}-${pod}-goroutines"
+            timeout --kill-after=1s 8s kubectl --request-timeout=5s \
+              -n "$namespace" port-forward --address=127.0.0.1 "pod/$pod" :6060 \
+              > "${prefix}-port-forward.log" 2>&1 &
+            forward_pid=$!
+            trap 'kill "$forward_pid" 2>/dev/null || true; wait "$forward_pid" 2>/dev/null || true' EXIT
+            trap 'exit 0' INT TERM
+            port=""
+            for attempt in 1 2 3 4 5 6; do
+              port=$(sed -n 's/^Forwarding from 127\.0\.0\.1:\([0-9]*\) -> 6060$/\1/p' "${prefix}-port-forward.log")
+              [ -z "$port" ] || break
+              kill -0 "$forward_pid" 2>/dev/null || break
+              sleep 0.5
+            done
+            if [ -n "$port" ]; then
+              curl --noproxy '*' --fail --silent --show-error --connect-timeout 1 --max-time 4 \
+                "http://127.0.0.1:${port}/debug/pprof/goroutine?debug=2" \
+                > "${prefix}.txt" 2> "${prefix}-error.txt" || true
+            else
+              echo "Profiler port-forward did not become ready" > "${prefix}-error.txt"
+            fi
+          )
+          # Retain runtime PID identity to diagnose process-tree attribution.
+          {
+            ps -e -o pid=,comm= | awk '$2 ~ /^containerd-shim/ { print $1 }' | head -64 | while read -r shim_pid; do
+              echo "Host shim: ${shim_pid}"
+              awk '/^(Name|Pid|PPid|NSpid):/' "/proc/${shim_pid}/status" || true
+            done
+            readlink /proc/1/ns/pid || true
+          } > diagnostics/shim-pid-namespaces.txt 2>&1
+          collect get pods -A -o json > diagnostics/pods.json 2> diagnostics/pods-error.txt || true
+          # Preserve agent/storage logs before querying optional aggregated APIs.
+          jq -r '[.items[] | select(.metadata.namespace == "kubescape") |
+            select(.metadata.name | test("^(node-agent|storage)-"))] | .[:4][] |
+            [.metadata.namespace, .metadata.name] | @tsv' diagnostics/pods.json > diagnostics/core-pods.txt || true
+          while read -r namespace pod; do
+            [ -n "$pod" ] || continue
+            collect logs -n "$namespace" "$pod" --all-containers=true --timestamps=true --limit-bytes=10485760 > "diagnostics/${namespace}-${pod}.log" 2>&1 || true
+            collect logs -n "$namespace" "$pod" --all-containers=true --timestamps=true --limit-bytes=10485760 --previous > "diagnostics/${namespace}-${pod}-previous.log" 2>&1 || true
+            if [[ "$pod" == node-agent-* ]]; then
+              collect_goroutines "$namespace" "$pod"
+            fi
+          done < diagnostics/core-pods.txt
+          collect get --raw /api/v1/namespaces/kubescape/services/node-agent:prometheus/proxy/metrics > diagnostics/node-agent-metrics.txt 2>&1 || true
+          collect get events -A --sort-by=.metadata.creationTimestamp > diagnostics/events.txt 2>&1 || true
+          # Storage's default LIST contains metadata only; request real payloads.
+          collect get --raw '/apis/spdx.softwarecomposition.kubescape.io/v1beta1/containerprofiles?resourceVersion=fullSpec' > diagnostics/containerprofiles.spdx.softwarecomposition.kubescape.io.yaml 2>&1 || true
+          for resource in \
+            rules.kubescape.io \
+            runtimerulealertbindings.kubescape.io; do
+            collect get "$resource" -A -o yaml > "diagnostics/${resource}.yaml" 2>&1 || true
+          done
+          # Cap workload logs and prioritize failed or terminated test containers.
+          jq -r '[.items[] | select(.metadata.namespace | startswith("node-agent-test-"))] |
+            sort_by(if .status.phase == "Failed" then 0
+              elif any(.status.containerStatuses[]?; .state.terminated != null) then 1
+              else 2 end) | .[:4][] |
+            [.metadata.namespace, .metadata.name] | @tsv' diagnostics/pods.json > diagnostics/test-pods.txt || true
+          while read -r namespace pod; do
+            [ -n "$pod" ] || continue
+            collect logs -n "$namespace" "$pod" --all-containers=true --timestamps=true --limit-bytes=1048576 > "diagnostics/${namespace}-${pod}.log" 2>&1 || true
+            collect logs -n "$namespace" "$pod" --all-containers=true --timestamps=true --limit-bytes=1048576 --previous > "diagnostics/${namespace}-${pod}-previous.log" 2>&1 || true
+          done < diagnostics/test-pods.txt
+          for resource in \
+            applicationprofiles.spdx.softwarecomposition.kubescape.io \
+            networkneighborhoods.spdx.softwarecomposition.kubescape.io; do
+            collect get "$resource" -A -o yaml > "diagnostics/${resource}.yaml" 2>&1 || true
+          done
+
+      - name: Upload component test diagnostics
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: component-diagnostics-${{ matrix.test }}-${{ github.run_attempt }}
+          path: diagnostics/
+          retention-days: 7

pkg/containerprofilemanager/v1/event_reporting.go

@@ -198,12 +198,20 @@ func (cpm *ContainerProfileManager) ReportProcfsExec(containerID string, path st
 // ReportFileOpen reports a file open event for a container
 func (cpm *ContainerProfileManager) ReportFileOpen(containerID string, event utils.OpenEvent) {
 	err := cpm.withContainer(containerID, func(data *containerData) (int, error) {
+		path := event.GetPath()
+		if path == "" {
+			// No file identity can be learned. A successful open with a
+			// missing path is capture loss; a failed empty-name open is not.
+			if event.GetError() == 0 {
+				data.droppedEvents = true
+			}
+			return 0, nil
+		}
 		if data.opens == nil {
 			data.opens = &maps.SafeMap[string, mapset.Set[string]]{}
 		}
 
 		// Deduplicate /proc/1234/* into /proc/.../*
-		path := event.GetPath()
 		if strings.HasPrefix(path, "/proc/") {
 			path = procRegex.ReplaceAllString(path, "/proc/"+dynamicpathdetector.DynamicIdentifier)
 		}

pkg/containerprofilemanager/v1/open_path_reporting_test.go

@@ -0,0 +1,72 @@
+package containerprofilemanager
+
+import (
+	"testing"
+
+	"github.com/inspektor-gadget/inspektor-gadget/pkg/datasource"
+	"github.com/inspektor-gadget/inspektor-gadget/pkg/gadget-service/api"
+	"github.com/kubescape/node-agent/pkg/utils"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/sys/unix"
+)
+
+func TestReportFileOpenUnresolvedPath(t *testing.T) {
+	// Use the real open accessor: empty failed syscall names remain unresolved,
+	// whereas a captured root-directory path must remain a valid learned open.
+	ds, err := datasource.New(datasource.TypeSingle, "open")
+	require.NoError(t, err)
+	fname, err := ds.AddField("fname", api.Kind_String)
+	require.NoError(t, err)
+	fpath, err := ds.AddField("fpath", api.Kind_String)
+	require.NoError(t, err)
+	errno, err := ds.AddField("error_raw", api.Kind_Int32)
+	require.NoError(t, err)
+	flags, err := ds.AddField("flags_raw", api.Kind_Int32)
+	require.NoError(t, err)
+	for _, name := range []string{"proc.pid", "proc.tid", "fd"} {
+		_, err := ds.AddField(name, api.Kind_Uint32)
+		require.NoError(t, err)
+	}
+	dfd, err := ds.AddField("dfd", api.Kind_Int32)
+	require.NoError(t, err)
+
+	for _, tc := range []struct {
+		name, raw, resolved, wantPath string
+		errno                         int32
+		partial                       bool
+	}{
+		{name: "failed empty filename", errno: int32(unix.ENOENT)},
+		{name: "successful open with missing path", partial: true},
+		{name: "root directory", raw: ".", resolved: "/", wantPath: "/"},
+		{name: "ordinary absolute open", raw: "/etc/passwd", resolved: "/etc/passwd", wantPath: "/etc/passwd"},
+		{name: "failed named open", raw: "/missing-directory", errno: int32(unix.ENOENT), wantPath: "/missing-directory"},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			data, err := ds.NewPacketSingle()
+			require.NoError(t, err)
+			defer ds.Release(data)
+			require.NoError(t, fname.PutString(data, tc.raw))
+			require.NoError(t, fpath.PutString(data, tc.resolved))
+			require.NoError(t, errno.PutInt32(data, tc.errno))
+			require.NoError(t, dfd.PutInt32(data, utils.AT_FDCWD))
+			require.NoError(t, flags.PutInt32(data, unix.O_DIRECTORY|unix.O_CLOEXEC|unix.O_RDONLY))
+			event := &utils.DatasourceEvent{Datasource: ds, Data: data, EventType: utils.OpenEventType, FullPathTracing: true}
+			require.Equal(t, tc.wantPath, event.GetPath())
+			require.Equal(t, int64(tc.errno), event.GetError(), "open error_raw carries positive errno, not a negative syscall return")
+
+			cpm, entry := newTestManager(t, "container")
+			cpm.ReportFileOpen("container", event)
+			opens := entry.data.getOpens()
+			if tc.wantPath == "" {
+				require.Empty(t, opens, "an unresolved name must not become a profile entry that storage cleans to '.'")
+				require.Zero(t, entry.data.size.Load())
+			} else {
+				require.Len(t, opens, 1)
+				require.Equal(t, tc.wantPath, opens[0].Path)
+				require.ElementsMatch(t, event.GetFlags(), opens[0].Flags)
+			}
+			require.Equal(t, tc.partial, entry.data.droppedEvents, "only a successful open with lost identity makes capture partial")
+			require.Equal(t, tc.wantPath, event.GetPath(), "profile recording must not modify the event used by rule evaluation")
+		})
+	}
+}

pkg/containerwatcher/v2/container_watcher.go

@@ -194,7 +194,7 @@ func CreateContainerWatcher(
 
 	ebpfDropCounter, _ := otelsetup.Meter().Int64Counter(
 		"node_agent.ebpf.events_dropped.total",
-		metric.WithDescription("Total eBPF events dropped due to backpressure or profile drops"),
+		metric.WithDescription("Total eBPF events dropped due to backpressure, profile drops, or missing container attribution"),
 	)
 
 	return &ContainerWatcher{

pkg/containerwatcher/v2/event_handler_factory.go

@@ -7,6 +7,8 @@ import (
 	mapset "github.com/deckarep/golang-set/v2"
 	"github.com/goradd/maps"
 	containercollection "github.com/inspektor-gadget/inspektor-gadget/pkg/container-collection"
+	"github.com/kubescape/go-logger"
+	"github.com/kubescape/go-logger/helpers"
 	"github.com/kubescape/node-agent/pkg/config"
 	"github.com/kubescape/node-agent/pkg/containerprofilemanager"
 	"github.com/kubescape/node-agent/pkg/containerwatcher"
@@ -102,7 +104,7 @@ func NewEventHandlerFactory(
 ) *EventHandlerFactory {
 	ebpfDropCounter, _ := otelsetup.Meter().Int64Counter(
 		"node_agent.ebpf.events_dropped.total",
-		metric.WithDescription("Total eBPF events dropped due to backpressure or profile drops"),
+		metric.WithDescription("Total eBPF events dropped due to backpressure, profile drops, or missing container attribution"),
 	)
 
 	factory := &EventHandlerFactory{
@@ -337,12 +339,14 @@ func computeEventDedupKey(enrichedEvent *events.EnrichedEvent) (key uint64, ttl
 // ProcessEvent processes an event through all registered handlers
 func (ehf *EventHandlerFactory) ProcessEvent(enrichedEvent *events.EnrichedEvent) {
 	if enrichedEvent.ContainerID == "" {
+		ehf.reportAttributionDrop(enrichedEvent, "missing_container_id")
 		return
 	}
 
 	// Get container information to check if it should be ignored
 	container, err := ehf.getContainerInfo(enrichedEvent.ContainerID)
 	if err != nil || container == nil {
+		ehf.reportAttributionDrop(enrichedEvent, "container_not_found")
 		return
 	}
 
@@ -427,6 +431,24 @@ func (ehf *EventHandlerFactory) ProcessEvent(enrichedEvent *events.EnrichedEvent
 	}
 }
 
+func (ehf *EventHandlerFactory) reportAttributionDrop(event *events.EnrichedEvent, reason string) {
+	eventType := string(event.Event.GetEventType())
+	ehf.ebpfDropCounter.Add(context.Background(), 1,
+		metric.WithAttributes(attribute.String("event_type", eventType), attribute.String("reason", reason)))
+
+	// Keep process/container identity in logs only, not metric dimensions.
+	var mntns any
+	if raw, ok := event.Event.(interface{ GetMountNsID() uint64 }); ok {
+		mntns = raw.GetMountNsID()
+	}
+	logger.L().Debug("EventHandlerFactory - dropping event without container attribution",
+		helpers.String("reason", reason),
+		helpers.String("eventType", eventType),
+		helpers.Interface("pid", event.PID),
+		helpers.Interface("mntns", mntns),
+		helpers.String("containerID", event.ContainerID))
+}
+
 // registerHandlers registers all handlers for different event types
 func (ehf *EventHandlerFactory) registerHandlers(
 	containerProfileManager *ManagerAdapter,