From 7dd36f4400886b3559345b59e3ec736143b56484 Mon Sep 17 00:00:00 2001 From: Richard <61517945+rkozyak@users.noreply.github.com> Date: Thu, 2 Jul 2026 14:38:18 -0400 Subject: [PATCH 1/2] feat(api): add memory breakdown metrics (system, vm, docker, zfs cache) Exposes per-category memory breakdown metrics for system, vm, docker and zfs cache mirroring the Unraid dashboard's ram breakdown. Add four new fields to memory metrics: - zfsCache - vm - docker - system --- .../memory/memory-breakdown.service.spec.ts | 95 ++++++++++++++ .../info/memory/memory-breakdown.service.ts | 124 ++++++++++++++++++ .../memory-utilization.resolver.spec.ts | 51 +++++++ .../memory/memory-utilization.resolver.ts | 46 +++++++ .../graph/resolvers/metrics/metrics.module.ts | 10 +- 5 files changed, 325 insertions(+), 1 deletion(-) create mode 100644 api/src/unraid-api/graph/resolvers/info/memory/memory-breakdown.service.spec.ts create mode 100644 api/src/unraid-api/graph/resolvers/info/memory/memory-breakdown.service.ts create mode 100644 api/src/unraid-api/graph/resolvers/info/memory/memory-utilization.resolver.spec.ts create mode 100644 api/src/unraid-api/graph/resolvers/info/memory/memory-utilization.resolver.ts diff --git a/api/src/unraid-api/graph/resolvers/info/memory/memory-breakdown.service.spec.ts b/api/src/unraid-api/graph/resolvers/info/memory/memory-breakdown.service.spec.ts new file mode 100644 index 0000000000..ca97b36f62 --- /dev/null +++ b/api/src/unraid-api/graph/resolvers/info/memory/memory-breakdown.service.spec.ts @@ -0,0 +1,95 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { MemoryBreakdownService } from '@app/unraid-api/graph/resolvers/info/memory/memory-breakdown.service.js'; + +const { readFileMock, execaMock, listContainersMock } = vi.hoisted(() => ({ + readFileMock: vi.fn(), + execaMock: vi.fn(), + listContainersMock: vi.fn(), +})); + +vi.mock('fs/promises', () => ({ + readFile: readFileMock, +})); + +vi.mock('execa', () => ({ + execa: execaMock, +})); + +vi.mock('@app/unraid-api/graph/resolvers/docker/utils/docker-client.js', () => ({ + getDockerClient: () => ({ listContainers: listContainersMock }), +})); + +describe('MemoryBreakdownService', () => { + let service: MemoryBreakdownService; + + beforeEach(() => { + vi.clearAllMocks(); + service = new MemoryBreakdownService(); + }); + + describe('getZfsCache', () => { + it('returns the ARC size in bytes', async () => { + readFileMock.mockResolvedValue('name type data\nhits 4 100\nsize 4 1503238553\n'); + await expect(service.getZfsCache()).resolves.toBe(1503238553); + }); + + it('returns null when the arcstats file is missing', async () => { + readFileMock.mockRejectedValue(new Error('ENOENT')); + await expect(service.getZfsCache()).resolves.toBeNull(); + }); + }); + + describe('getVmMemory', () => { + it('sums balloon.rss across domains and converts KiB to bytes', async () => { + execaMock.mockResolvedValue({ + stdout: [ + 'Domain: vm1', + ' balloon.rss=4194304', + 'Domain: vm2', + ' balloon.rss=2097152', + ].join('\n'), + }); + await expect(service.getVmMemory()).resolves.toBe((4194304 + 2097152) * 1024); + }); + + it('returns null when virsh is unavailable', async () => { + execaMock.mockRejectedValue(new Error('not found')); + await expect(service.getVmMemory()).resolves.toBeNull(); + }); + }); + + describe('getDockerMemory', () => { + it('sums the relevant memory.stat fields across running containers', async () => { + listContainersMock.mockResolvedValue([{ Id: 'abc' }, { Id: 'def' }]); + readFileMock.mockResolvedValue( + ['anon 1000', 'file 9999', 'kernel 500', 'shmem 250', 'sock 50'].join('\n') + ); + await expect(service.getDockerMemory()).resolves.toBe(1800 * 2); + }); + + it('returns 0 when no containers are running', async () => { + listContainersMock.mockResolvedValue([]); + await expect(service.getDockerMemory()).resolves.toBe(0); + }); + + it('returns null when the docker socket is unavailable', async () => { + listContainersMock.mockRejectedValue(new Error('ECONNREFUSED')); + await expect(service.getDockerMemory()).resolves.toBeNull(); + }); + }); + + describe('getSources caching', () => { + it('collects each source once within the cache window', async () => { + readFileMock.mockResolvedValue('size 4 100\n'); + execaMock.mockResolvedValue({ stdout: 'balloon.rss=1024' }); + listContainersMock.mockResolvedValue([]); + + await service.getSources(); + await service.getSources(); + + expect(execaMock).toHaveBeenCalledTimes(1); + expect(listContainersMock).toHaveBeenCalledTimes(1); + }); + }); +}); diff --git a/api/src/unraid-api/graph/resolvers/info/memory/memory-breakdown.service.ts b/api/src/unraid-api/graph/resolvers/info/memory/memory-breakdown.service.ts new file mode 100644 index 0000000000..332b24bebe --- /dev/null +++ b/api/src/unraid-api/graph/resolvers/info/memory/memory-breakdown.service.ts @@ -0,0 +1,124 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { readFile } from 'fs/promises'; + +import { execa } from 'execa'; + +import { getDockerClient } from '@app/unraid-api/graph/resolvers/docker/utils/docker-client.js'; + +/** + * cgroup memory.stat fields that the webGUI Docker hook sums to report + * per-container memory usage. Must stay in sync with + * webgui/emhttp/plugins/dynamix.docker.manager/system/Docker. + */ +const DOCKER_MEMORY_STAT_FIELDS = new Set([ + 'anon', + 'kernel', + 'kernel_stack', + 'pagetables', + 'sec_pagetables', + 'percpu', + 'sock', + 'vmalloc', + 'shmem', +]); + +export interface MemoryBreakdownSources { + /** ZFS ARC cache size in bytes, or null when ZFS is not loaded. */ + zfsCache: number | null; + /** Active VM balloon RSS in bytes, or null when libvirt is unavailable. */ + vm: number | null; + /** Active Docker container memory in bytes, or null when Docker is unavailable. */ + docker: number | null; +} + +/** + * Collects the per-category memory usage (ZFS cache, VMs, Docker) that the + * Unraid dashboard breaks the RAM graph into. Mirrors the executable hook + * scripts under webgui .../system/*. Results are cached briefly so the 2s + * memory subscription can select these fields without shelling out repeatedly. + */ +@Injectable() +export class MemoryBreakdownService { + private readonly logger = new Logger(MemoryBreakdownService.name); + private static readonly CACHE_TTL_MS = 1500; + private cache?: { expiresAt: number; sources: Promise }; + + async getSources(): Promise { + const now = Date.now(); + if (this.cache && this.cache.expiresAt > now) { + return this.cache.sources; + } + const sources = this.collectSources(); + this.cache = { expiresAt: now + MemoryBreakdownService.CACHE_TTL_MS, sources }; + return sources; + } + + private async collectSources(): Promise { + const [zfsCache, vm, docker] = await Promise.all([ + this.getZfsCache(), + this.getVmMemory(), + this.getDockerMemory(), + ]); + return { zfsCache, vm, docker }; + } + + /** Reads the ARC `size` (already in bytes) from the ZFS kstat file. */ + async getZfsCache(): Promise { + try { + const contents = await readFile('/proc/spl/kstat/zfs/arcstats', 'utf8'); + const line = contents.split('\n').find((l) => l.startsWith('size')); + if (!line) return null; + const size = Number.parseInt(line.trim().split(/\s+/)[2] ?? '', 10); + return Number.isFinite(size) ? size : null; + } catch { + return null; + } + } + + /** Sums `balloon.rss` (reported in KiB) across active domains. */ + async getVmMemory(): Promise { + try { + const { stdout } = await execa('virsh', ['domstats', '--list-active', '--balloon']); + let total = 0; + for (const raw of stdout.split('\n')) { + const match = /^balloon\.rss=(\d+)$/.exec(raw.trim()); + if (match) total += Number.parseInt(match[1], 10) * 1024; + } + return total; + } catch { + return null; + } + } + + /** Sums the relevant cgroup memory.stat fields across running containers. */ + async getDockerMemory(): Promise { + try { + const containers = await getDockerClient().listContainers(); + if (containers.length === 0) return 0; + const perContainer = await Promise.all( + containers.map((container) => this.getContainerMemory(container.Id)) + ); + return perContainer.reduce((sum, value) => sum + value, 0); + } catch { + return null; + } + } + + private async getContainerMemory(id: string): Promise { + try { + const stat = await readFile(`/sys/fs/cgroup/docker/${id}/memory.stat`, 'utf8'); + let total = 0; + for (const line of stat.split('\n')) { + const [field, value] = line.split(/\s+/); + if (DOCKER_MEMORY_STAT_FIELDS.has(field)) { + const bytes = Number.parseInt(value ?? '', 10); + if (Number.isFinite(bytes)) total += bytes; + } + } + return total; + } catch { + this.logger.debug(`Unable to read memory.stat for container ${id}`); + return 0; + } + } +} diff --git a/api/src/unraid-api/graph/resolvers/info/memory/memory-utilization.resolver.spec.ts b/api/src/unraid-api/graph/resolvers/info/memory/memory-utilization.resolver.spec.ts new file mode 100644 index 0000000000..c7586b1907 --- /dev/null +++ b/api/src/unraid-api/graph/resolvers/info/memory/memory-utilization.resolver.spec.ts @@ -0,0 +1,51 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import type { MemoryBreakdownSources } from '@app/unraid-api/graph/resolvers/info/memory/memory-breakdown.service.js'; +import { MemoryBreakdownService } from '@app/unraid-api/graph/resolvers/info/memory/memory-breakdown.service.js'; +import { MemoryUtilizationResolver } from '@app/unraid-api/graph/resolvers/info/memory/memory-utilization.resolver.js'; +import { MemoryUtilization } from '@app/unraid-api/graph/resolvers/info/memory/memory.model.js'; + +const parent = (total: number, available: number) => ({ total, available }) as MemoryUtilization; + +const resolverWithSources = (sources: MemoryBreakdownSources) => { + const breakdown = { + getSources: vi.fn().mockResolvedValue(sources), + } as unknown as MemoryBreakdownService; + return new MemoryUtilizationResolver(breakdown); +}; + +describe('MemoryUtilizationResolver', () => { + describe('source passthrough', () => { + let resolver: MemoryUtilizationResolver; + + beforeEach(() => { + resolver = resolverWithSources({ zfsCache: 1, vm: 2, docker: 3 }); + }); + + it('returns each collected source', async () => { + await expect(resolver.zfsCache()).resolves.toBe(1); + await expect(resolver.vm()).resolves.toBe(2); + await expect(resolver.docker()).resolves.toBe(3); + }); + }); + + describe('system', () => { + it('is used (total - available) minus the categorized sources', async () => { + const resolver = resolverWithSources({ zfsCache: 1_000, vm: 2_000, docker: 3_000 }); + const result = await resolver.system(parent(20_000, 4_000)); + expect(result).toBe(10_000); + }); + + it('treats null sources as zero', async () => { + const resolver = resolverWithSources({ zfsCache: null, vm: null, docker: 500 }); + const result = await resolver.system(parent(10_000, 4_000)); + expect(result).toBe(5_500); + }); + + it('clamps to zero when categorized memory exceeds used', async () => { + const resolver = resolverWithSources({ zfsCache: 5_000, vm: 5_000, docker: 5_000 }); + const result = await resolver.system(parent(10_000, 4_000)); + expect(result).toBe(0); + }); + }); +}); diff --git a/api/src/unraid-api/graph/resolvers/info/memory/memory-utilization.resolver.ts b/api/src/unraid-api/graph/resolvers/info/memory/memory-utilization.resolver.ts new file mode 100644 index 0000000000..5fc6b684da --- /dev/null +++ b/api/src/unraid-api/graph/resolvers/info/memory/memory-utilization.resolver.ts @@ -0,0 +1,46 @@ +import { Parent, ResolveField, Resolver } from '@nestjs/graphql'; + +import { GraphQLBigInt } from 'graphql-scalars'; + +import { MemoryBreakdownService } from '@app/unraid-api/graph/resolvers/info/memory/memory-breakdown.service.js'; +import { MemoryUtilization } from '@app/unraid-api/graph/resolvers/info/memory/memory.model.js'; + +@Resolver(() => MemoryUtilization) +export class MemoryUtilizationResolver { + constructor(private readonly breakdown: MemoryBreakdownService) {} + + @ResolveField(() => GraphQLBigInt, { + nullable: true, + description: 'ZFS ARC cache memory in bytes', + }) + async zfsCache(): Promise { + return (await this.breakdown.getSources()).zfsCache; + } + + @ResolveField(() => GraphQLBigInt, { + nullable: true, + description: 'VM memory in bytes', + }) + async vm(): Promise { + return (await this.breakdown.getSources()).vm; + } + + @ResolveField(() => GraphQLBigInt, { + nullable: true, + description: 'Docker memory in bytes', + }) + async docker(): Promise { + return (await this.breakdown.getSources()).docker; + } + + @ResolveField(() => GraphQLBigInt, { + nullable: true, + description: 'System memory in bytes', + }) + async system(@Parent() memory: MemoryUtilization): Promise { + const { zfsCache, vm, docker } = await this.breakdown.getSources(); + const used = memory.total - memory.available; + const categorized = (vm ?? 0) + (zfsCache ?? 0) + (docker ?? 0); + return Math.max(0, used - categorized); + } +} diff --git a/api/src/unraid-api/graph/resolvers/metrics/metrics.module.ts b/api/src/unraid-api/graph/resolvers/metrics/metrics.module.ts index 2934cda407..90fdf46319 100644 --- a/api/src/unraid-api/graph/resolvers/metrics/metrics.module.ts +++ b/api/src/unraid-api/graph/resolvers/metrics/metrics.module.ts @@ -1,6 +1,8 @@ import { Module } from '@nestjs/common'; import { CpuModule } from '@app/unraid-api/graph/resolvers/info/cpu/cpu.module.js'; +import { MemoryBreakdownService } from '@app/unraid-api/graph/resolvers/info/memory/memory-breakdown.service.js'; +import { MemoryUtilizationResolver } from '@app/unraid-api/graph/resolvers/info/memory/memory-utilization.resolver.js'; import { MemoryService } from '@app/unraid-api/graph/resolvers/info/memory/memory.service.js'; import { MetricsResolver } from '@app/unraid-api/graph/resolvers/metrics/metrics.resolver.js'; import { NetworkMetricsService } from '@app/unraid-api/graph/resolvers/metrics/network/network.service.js'; @@ -9,7 +11,13 @@ import { ServicesModule } from '@app/unraid-api/graph/services/services.module.j @Module({ imports: [ServicesModule, CpuModule, TemperatureModule], - providers: [MetricsResolver, MemoryService, NetworkMetricsService], + providers: [ + MetricsResolver, + MemoryService, + MemoryBreakdownService, + MemoryUtilizationResolver, + NetworkMetricsService, + ], exports: [MetricsResolver], }) export class MetricsModule {} From a324f670ab1bdd4b78a1d31a4092dc6f5765c12a Mon Sep 17 00:00:00 2001 From: Richard <61517945+rkozyak@users.noreply.github.com> Date: Thu, 2 Jul 2026 15:13:20 -0400 Subject: [PATCH 2/2] refactor(api): address review feedback on memory breakdown metrics - Prevents duplicate in flight breakdown collecitons - Declare as @Fields for zfs/vm/docker/system memory --- .../memory/memory-breakdown.service.spec.ts | 18 +++++++++++++++++ .../info/memory/memory-breakdown.service.ts | 17 +++++++++++++--- .../memory/memory-utilization.resolver.ts | 20 ++++--------------- .../resolvers/info/memory/memory.model.ts | 12 +++++++++++ 4 files changed, 48 insertions(+), 19 deletions(-) diff --git a/api/src/unraid-api/graph/resolvers/info/memory/memory-breakdown.service.spec.ts b/api/src/unraid-api/graph/resolvers/info/memory/memory-breakdown.service.spec.ts index ca97b36f62..5987aa5c5b 100644 --- a/api/src/unraid-api/graph/resolvers/info/memory/memory-breakdown.service.spec.ts +++ b/api/src/unraid-api/graph/resolvers/info/memory/memory-breakdown.service.spec.ts @@ -91,5 +91,23 @@ describe('MemoryBreakdownService', () => { expect(execaMock).toHaveBeenCalledTimes(1); expect(listContainersMock).toHaveBeenCalledTimes(1); }); + + it('reuses a single in-flight collection for concurrent callers', async () => { + readFileMock.mockResolvedValue('size 4 100\n'); + execaMock.mockResolvedValue({ stdout: '' }); + let resolveContainers: (value: unknown[]) => void = () => {}; + listContainersMock.mockReturnValue( + new Promise((resolve) => { + resolveContainers = resolve; + }) + ); + + const first = service.getSources(); + const second = service.getSources(); + resolveContainers([]); + await Promise.all([first, second]); + + expect(listContainersMock).toHaveBeenCalledTimes(1); + }); }); }); diff --git a/api/src/unraid-api/graph/resolvers/info/memory/memory-breakdown.service.ts b/api/src/unraid-api/graph/resolvers/info/memory/memory-breakdown.service.ts index 332b24bebe..19ae3efaee 100644 --- a/api/src/unraid-api/graph/resolvers/info/memory/memory-breakdown.service.ts +++ b/api/src/unraid-api/graph/resolvers/info/memory/memory-breakdown.service.ts @@ -44,12 +44,23 @@ export class MemoryBreakdownService { private cache?: { expiresAt: number; sources: Promise }; async getSources(): Promise { - const now = Date.now(); - if (this.cache && this.cache.expiresAt > now) { + if (this.cache && this.cache.expiresAt > Date.now()) { return this.cache.sources; } const sources = this.collectSources(); - this.cache = { expiresAt: now + MemoryBreakdownService.CACHE_TTL_MS, sources }; + // Keep the in-flight promise cached so concurrent callers reuse it even + // if collection outlasts the TTL; start the TTL only once it settles. + const entry = { expiresAt: Number.POSITIVE_INFINITY, sources }; + this.cache = entry; + sources + .then(() => { + if (this.cache === entry) { + entry.expiresAt = Date.now() + MemoryBreakdownService.CACHE_TTL_MS; + } + }) + .catch(() => { + if (this.cache === entry) this.cache = undefined; + }); return sources; } diff --git a/api/src/unraid-api/graph/resolvers/info/memory/memory-utilization.resolver.ts b/api/src/unraid-api/graph/resolvers/info/memory/memory-utilization.resolver.ts index 5fc6b684da..db432697d9 100644 --- a/api/src/unraid-api/graph/resolvers/info/memory/memory-utilization.resolver.ts +++ b/api/src/unraid-api/graph/resolvers/info/memory/memory-utilization.resolver.ts @@ -9,34 +9,22 @@ import { MemoryUtilization } from '@app/unraid-api/graph/resolvers/info/memory/m export class MemoryUtilizationResolver { constructor(private readonly breakdown: MemoryBreakdownService) {} - @ResolveField(() => GraphQLBigInt, { - nullable: true, - description: 'ZFS ARC cache memory in bytes', - }) + @ResolveField(() => GraphQLBigInt, { nullable: true, name: 'zfsCache' }) async zfsCache(): Promise { return (await this.breakdown.getSources()).zfsCache; } - @ResolveField(() => GraphQLBigInt, { - nullable: true, - description: 'VM memory in bytes', - }) + @ResolveField(() => GraphQLBigInt, { nullable: true, name: 'vm' }) async vm(): Promise { return (await this.breakdown.getSources()).vm; } - @ResolveField(() => GraphQLBigInt, { - nullable: true, - description: 'Docker memory in bytes', - }) + @ResolveField(() => GraphQLBigInt, { nullable: true, name: 'docker' }) async docker(): Promise { return (await this.breakdown.getSources()).docker; } - @ResolveField(() => GraphQLBigInt, { - nullable: true, - description: 'System memory in bytes', - }) + @ResolveField(() => GraphQLBigInt, { nullable: true, name: 'system' }) async system(@Parent() memory: MemoryUtilization): Promise { const { zfsCache, vm, docker } = await this.breakdown.getSources(); const used = memory.total - memory.available; diff --git a/api/src/unraid-api/graph/resolvers/info/memory/memory.model.ts b/api/src/unraid-api/graph/resolvers/info/memory/memory.model.ts index 4e2aa46b3a..ac0507d178 100644 --- a/api/src/unraid-api/graph/resolvers/info/memory/memory.model.ts +++ b/api/src/unraid-api/graph/resolvers/info/memory/memory.model.ts @@ -73,6 +73,18 @@ export class MemoryUtilization extends Node { @Field(() => Float, { description: 'Swap usage percentage' }) percentSwapTotal!: number; + + @Field(() => GraphQLBigInt, { nullable: true, description: 'ZFS ARC cache memory in bytes' }) + zfsCache?: number; + + @Field(() => GraphQLBigInt, { nullable: true, description: 'VM memory in bytes' }) + vm?: number; + + @Field(() => GraphQLBigInt, { nullable: true, description: 'Docker memory in bytes' }) + docker?: number; + + @Field(() => GraphQLBigInt, { nullable: true, description: 'System memory in bytes' }) + system?: number; } @ObjectType({ implements: () => Node })