feat(api): add memory breakdown metrics (system, vm, docker, zfs cache)

api/src/unraid-api/graph/resolvers/info/memory/memory-breakdown.service.spec.ts

@@ -0,0 +1,113 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+import { MemoryBreakdownService } from '@app/unraid-api/graph/resolvers/info/memory/memory-breakdown.service.js';
+
+const { readFileMock, execaMock, listContainersMock } = vi.hoisted(() => ({
+    readFileMock: vi.fn(),
+    execaMock: vi.fn(),
+    listContainersMock: vi.fn(),
+}));
+
+vi.mock('fs/promises', () => ({
+    readFile: readFileMock,
+}));
+
+vi.mock('execa', () => ({
+    execa: execaMock,
+}));
+
+vi.mock('@app/unraid-api/graph/resolvers/docker/utils/docker-client.js', () => ({
+    getDockerClient: () => ({ listContainers: listContainersMock }),
+}));
+
+describe('MemoryBreakdownService', () => {
+    let service: MemoryBreakdownService;
+
+    beforeEach(() => {
+        vi.clearAllMocks();
+        service = new MemoryBreakdownService();
+    });
+
+    describe('getZfsCache', () => {
+        it('returns the ARC size in bytes', async () => {
+            readFileMock.mockResolvedValue('name type data\nhits 4 100\nsize 4 1503238553\n');
+            await expect(service.getZfsCache()).resolves.toBe(1503238553);
+        });
+
+        it('returns null when the arcstats file is missing', async () => {
+            readFileMock.mockRejectedValue(new Error('ENOENT'));
+            await expect(service.getZfsCache()).resolves.toBeNull();
+        });
+    });
+
+    describe('getVmMemory', () => {
+        it('sums balloon.rss across domains and converts KiB to bytes', async () => {
+            execaMock.mockResolvedValue({
+                stdout: [
+                    'Domain: vm1',
+                    '  balloon.rss=4194304',
+                    'Domain: vm2',
+                    '  balloon.rss=2097152',
+                ].join('\n'),
+            });
+            await expect(service.getVmMemory()).resolves.toBe((4194304 + 2097152) * 1024);
+        });
+
+        it('returns null when virsh is unavailable', async () => {
+            execaMock.mockRejectedValue(new Error('not found'));
+            await expect(service.getVmMemory()).resolves.toBeNull();
+        });
+    });
+
+    describe('getDockerMemory', () => {
+        it('sums the relevant memory.stat fields across running containers', async () => {
+            listContainersMock.mockResolvedValue([{ Id: 'abc' }, { Id: 'def' }]);
+            readFileMock.mockResolvedValue(
+                ['anon 1000', 'file 9999', 'kernel 500', 'shmem 250', 'sock 50'].join('\n')
+            );
+            await expect(service.getDockerMemory()).resolves.toBe(1800 * 2);
+        });
+
+        it('returns 0 when no containers are running', async () => {
+            listContainersMock.mockResolvedValue([]);
+            await expect(service.getDockerMemory()).resolves.toBe(0);
+        });
+
+        it('returns null when the docker socket is unavailable', async () => {
+            listContainersMock.mockRejectedValue(new Error('ECONNREFUSED'));
+            await expect(service.getDockerMemory()).resolves.toBeNull();
+        });
+    });
+
+    describe('getSources caching', () => {
+        it('collects each source once within the cache window', async () => {
+            readFileMock.mockResolvedValue('size 4 100\n');
+            execaMock.mockResolvedValue({ stdout: 'balloon.rss=1024' });
+            listContainersMock.mockResolvedValue([]);
+
+            await service.getSources();
+            await service.getSources();
+
+            expect(execaMock).toHaveBeenCalledTimes(1);
+            expect(listContainersMock).toHaveBeenCalledTimes(1);
+        });
+
+        it('reuses a single in-flight collection for concurrent callers', async () => {
+            readFileMock.mockResolvedValue('size 4 100\n');
+            execaMock.mockResolvedValue({ stdout: '' });
+            let resolveContainers: (value: unknown[]) => void = () => {};
+            listContainersMock.mockReturnValue(
+                new Promise((resolve) => {
+                    resolveContainers = resolve;
+                })
+            );
+
+            const first = service.getSources();
+            const second = service.getSources();
+            resolveContainers([]);
+            await Promise.all([first, second]);
+
+            expect(listContainersMock).toHaveBeenCalledTimes(1);
+        });
+    });
+});

api/src/unraid-api/graph/resolvers/info/memory/memory-breakdown.service.ts

@@ -0,0 +1,135 @@
+import { Injectable, Logger } from '@nestjs/common';
+import { readFile } from 'fs/promises';
+
+import { execa } from 'execa';
+
+import { getDockerClient } from '@app/unraid-api/graph/resolvers/docker/utils/docker-client.js';
+
+/**
+ * cgroup memory.stat fields that the webGUI Docker hook sums to report
+ * per-container memory usage. Must stay in sync with
+ * webgui/emhttp/plugins/dynamix.docker.manager/system/Docker.
+ */
+const DOCKER_MEMORY_STAT_FIELDS = new Set([
+    'anon',
+    'kernel',
+    'kernel_stack',
+    'pagetables',
+    'sec_pagetables',
+    'percpu',
+    'sock',
+    'vmalloc',
+    'shmem',
+]);
+
+export interface MemoryBreakdownSources {
+    /** ZFS ARC cache size in bytes, or null when ZFS is not loaded. */
+    zfsCache: number | null;
+    /** Active VM balloon RSS in bytes, or null when libvirt is unavailable. */
+    vm: number | null;
+    /** Active Docker container memory in bytes, or null when Docker is unavailable. */
+    docker: number | null;
+}
+
+/**
+ * Collects the per-category memory usage (ZFS cache, VMs, Docker) that the
+ * Unraid dashboard breaks the RAM graph into. Mirrors the executable hook
+ * scripts under webgui .../system/*. Results are cached briefly so the 2s
+ * memory subscription can select these fields without shelling out repeatedly.
+ */
+@Injectable()
+export class MemoryBreakdownService {
+    private readonly logger = new Logger(MemoryBreakdownService.name);
+    private static readonly CACHE_TTL_MS = 1500;
+    private cache?: { expiresAt: number; sources: Promise<MemoryBreakdownSources> };
+
+    async getSources(): Promise<MemoryBreakdownSources> {
+        if (this.cache && this.cache.expiresAt > Date.now()) {
+            return this.cache.sources;
+        }
+        const sources = this.collectSources();
+        // Keep the in-flight promise cached so concurrent callers reuse it even
+        // if collection outlasts the TTL; start the TTL only once it settles.
+        const entry = { expiresAt: Number.POSITIVE_INFINITY, sources };
+        this.cache = entry;
+        sources
+            .then(() => {
+                if (this.cache === entry) {
+                    entry.expiresAt = Date.now() + MemoryBreakdownService.CACHE_TTL_MS;
+                }
+            })
+            .catch(() => {
+                if (this.cache === entry) this.cache = undefined;
+            });
+        return sources;
+    }
+
+    private async collectSources(): Promise<MemoryBreakdownSources> {
+        const [zfsCache, vm, docker] = await Promise.all([
+            this.getZfsCache(),
+            this.getVmMemory(),
+            this.getDockerMemory(),
+        ]);
+        return { zfsCache, vm, docker };
+    }
+
+    /** Reads the ARC `size` (already in bytes) from the ZFS kstat file. */
+    async getZfsCache(): Promise<number | null> {
+        try {
+            const contents = await readFile('/proc/spl/kstat/zfs/arcstats', 'utf8');
+            const line = contents.split('\n').find((l) => l.startsWith('size'));
+            if (!line) return null;
+            const size = Number.parseInt(line.trim().split(/\s+/)[2] ?? '', 10);
+            return Number.isFinite(size) ? size : null;
+        } catch {
+            return null;
+        }
+    }
+
+    /** Sums `balloon.rss` (reported in KiB) across active domains. */
+    async getVmMemory(): Promise<number | null> {
+        try {
+            const { stdout } = await execa('virsh', ['domstats', '--list-active', '--balloon']);
+            let total = 0;
+            for (const raw of stdout.split('\n')) {
+                const match = /^balloon\.rss=(\d+)$/.exec(raw.trim());
+                if (match) total += Number.parseInt(match[1], 10) * 1024;
+            }
+            return total;
+        } catch {
+            return null;
+        }
+    }
+
+    /** Sums the relevant cgroup memory.stat fields across running containers. */
+    async getDockerMemory(): Promise<number | null> {
+        try {
+            const containers = await getDockerClient().listContainers();
+            if (containers.length === 0) return 0;
+            const perContainer = await Promise.all(
+                containers.map((container) => this.getContainerMemory(container.Id))
+            );
+            return perContainer.reduce((sum, value) => sum + value, 0);
+        } catch {
+            return null;
+        }
+    }
+
+    private async getContainerMemory(id: string): Promise<number> {
+        try {
+            const stat = await readFile(`/sys/fs/cgroup/docker/${id}/memory.stat`, 'utf8');
+            let total = 0;
+            for (const line of stat.split('\n')) {
+                const [field, value] = line.split(/\s+/);
+                if (DOCKER_MEMORY_STAT_FIELDS.has(field)) {
+                    const bytes = Number.parseInt(value ?? '', 10);
+                    if (Number.isFinite(bytes)) total += bytes;
+                }
+            }
+            return total;
+        } catch {
+            this.logger.debug(`Unable to read memory.stat for container ${id}`);
+            return 0;
+        }
+    }
+}

api/src/unraid-api/graph/resolvers/info/memory/memory-utilization.resolver.spec.ts

@@ -0,0 +1,51 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+import type { MemoryBreakdownSources } from '@app/unraid-api/graph/resolvers/info/memory/memory-breakdown.service.js';
+import { MemoryBreakdownService } from '@app/unraid-api/graph/resolvers/info/memory/memory-breakdown.service.js';
+import { MemoryUtilizationResolver } from '@app/unraid-api/graph/resolvers/info/memory/memory-utilization.resolver.js';
+import { MemoryUtilization } from '@app/unraid-api/graph/resolvers/info/memory/memory.model.js';
+
+const parent = (total: number, available: number) => ({ total, available }) as MemoryUtilization;
+
+const resolverWithSources = (sources: MemoryBreakdownSources) => {
+    const breakdown = {
+        getSources: vi.fn().mockResolvedValue(sources),
+    } as unknown as MemoryBreakdownService;
+    return new MemoryUtilizationResolver(breakdown);
+};
+
+describe('MemoryUtilizationResolver', () => {
+    describe('source passthrough', () => {
+        let resolver: MemoryUtilizationResolver;
+
+        beforeEach(() => {
+            resolver = resolverWithSources({ zfsCache: 1, vm: 2, docker: 3 });
+        });
+
+        it('returns each collected source', async () => {
+            await expect(resolver.zfsCache()).resolves.toBe(1);
+            await expect(resolver.vm()).resolves.toBe(2);
+            await expect(resolver.docker()).resolves.toBe(3);
+        });
+    });
+
+    describe('system', () => {
+        it('is used (total - available) minus the categorized sources', async () => {
+            const resolver = resolverWithSources({ zfsCache: 1_000, vm: 2_000, docker: 3_000 });
+            const result = await resolver.system(parent(20_000, 4_000));
+            expect(result).toBe(10_000);
+        });
+
+        it('treats null sources as zero', async () => {
+            const resolver = resolverWithSources({ zfsCache: null, vm: null, docker: 500 });
+            const result = await resolver.system(parent(10_000, 4_000));
+            expect(result).toBe(5_500);
+        });
+
+        it('clamps to zero when categorized memory exceeds used', async () => {
+            const resolver = resolverWithSources({ zfsCache: 5_000, vm: 5_000, docker: 5_000 });
+            const result = await resolver.system(parent(10_000, 4_000));
+            expect(result).toBe(0);
+        });
+    });
+});

api/src/unraid-api/graph/resolvers/info/memory/memory-utilization.resolver.ts

@@ -0,0 +1,34 @@
+import { Parent, ResolveField, Resolver } from '@nestjs/graphql';
+
+import { GraphQLBigInt } from 'graphql-scalars';
+
+import { MemoryBreakdownService } from '@app/unraid-api/graph/resolvers/info/memory/memory-breakdown.service.js';
+import { MemoryUtilization } from '@app/unraid-api/graph/resolvers/info/memory/memory.model.js';
+
+@Resolver(() => MemoryUtilization)
+export class MemoryUtilizationResolver {
+    constructor(private readonly breakdown: MemoryBreakdownService) {}
+
+    @ResolveField(() => GraphQLBigInt, { nullable: true, name: 'zfsCache' })
+    async zfsCache(): Promise<number | null> {
+        return (await this.breakdown.getSources()).zfsCache;
+    }
+
+    @ResolveField(() => GraphQLBigInt, { nullable: true, name: 'vm' })
+    async vm(): Promise<number | null> {
+        return (await this.breakdown.getSources()).vm;
+    }
+
+    @ResolveField(() => GraphQLBigInt, { nullable: true, name: 'docker' })
+    async docker(): Promise<number | null> {
+        return (await this.breakdown.getSources()).docker;
+    }
+
+    @ResolveField(() => GraphQLBigInt, { nullable: true, name: 'system' })
+    async system(@Parent() memory: MemoryUtilization): Promise<number> {
+        const { zfsCache, vm, docker } = await this.breakdown.getSources();
+        const used = memory.total - memory.available;
+        const categorized = (vm ?? 0) + (zfsCache ?? 0) + (docker ?? 0);
+        return Math.max(0, used - categorized);
+    }
+}

api/src/unraid-api/graph/resolvers/info/memory/memory.model.ts

@@ -73,6 +73,18 @@ export class MemoryUtilization extends Node {
 
     @Field(() => Float, { description: 'Swap usage percentage' })
     percentSwapTotal!: number;
+
+    @Field(() => GraphQLBigInt, { nullable: true, description: 'ZFS ARC cache memory in bytes' })
+    zfsCache?: number;
+
+    @Field(() => GraphQLBigInt, { nullable: true, description: 'VM memory in bytes' })
+    vm?: number;
+
+    @Field(() => GraphQLBigInt, { nullable: true, description: 'Docker memory in bytes' })
+    docker?: number;
+
+    @Field(() => GraphQLBigInt, { nullable: true, description: 'System memory in bytes' })
+    system?: number;
 }
 
 @ObjectType({ implements: () => Node })

api/src/unraid-api/graph/resolvers/metrics/metrics.module.ts

@@ -1,6 +1,8 @@
 import { Module } from '@nestjs/common';
 
 import { CpuModule } from '@app/unraid-api/graph/resolvers/info/cpu/cpu.module.js';
+import { MemoryBreakdownService } from '@app/unraid-api/graph/resolvers/info/memory/memory-breakdown.service.js';
+import { MemoryUtilizationResolver } from '@app/unraid-api/graph/resolvers/info/memory/memory-utilization.resolver.js';
 import { MemoryService } from '@app/unraid-api/graph/resolvers/info/memory/memory.service.js';
 import { MetricsResolver } from '@app/unraid-api/graph/resolvers/metrics/metrics.resolver.js';
 import { NetworkMetricsService } from '@app/unraid-api/graph/resolvers/metrics/network/network.service.js';
@@ -9,7 +11,13 @@ import { ServicesModule } from '@app/unraid-api/graph/services/services.module.j
 
 @Module({
     imports: [ServicesModule, CpuModule, TemperatureModule],
-    providers: [MetricsResolver, MemoryService, NetworkMetricsService],
+    providers: [
+        MetricsResolver,
+        MemoryService,
+        MemoryBreakdownService,
+        MemoryUtilizationResolver,
+        NetworkMetricsService,
+    ],
     exports: [MetricsResolver],
 })
 export class MetricsModule {}