From b4f73d31ce7603ce61093f36fc3f159149911168 Mon Sep 17 00:00:00 2001 From: TenSt Date: Fri, 12 Jun 2026 12:34:09 +0200 Subject: [PATCH 1/7] RHINENG-26548: drop system_inventory.workspaces column --- ..._drop_system_inventory_workspaces.down.sql | 52 +++++++++++++++++++ ...60_drop_system_inventory_workspaces.up.sql | 45 ++++++++++++++++ database_admin/schema/create_schema.sql | 11 ++-- 3 files changed, 101 insertions(+), 7 deletions(-) create mode 100644 database_admin/migrations/160_drop_system_inventory_workspaces.down.sql create mode 100644 database_admin/migrations/160_drop_system_inventory_workspaces.up.sql diff --git a/database_admin/migrations/160_drop_system_inventory_workspaces.down.sql b/database_admin/migrations/160_drop_system_inventory_workspaces.down.sql new file mode 100644 index 000000000..ee224fa3b --- /dev/null +++ b/database_admin/migrations/160_drop_system_inventory_workspaces.down.sql @@ -0,0 +1,52 @@ +ALTER TABLE system_inventory ADD COLUMN IF NOT EXISTS workspaces JSONB; + +UPDATE system_inventory + SET workspaces = JSONB_BUILD_ARRAY(JSONB_BUILD_OBJECT('id', workspace_id, 'name', workspace_name)) + WHERE workspace_id IS NOT NULL + AND workspace_name IS NOT NULL + AND workspaces IS NULL; + +CREATE INDEX IF NOT EXISTS system_inventory_workspaces_index ON system_inventory USING GIN (workspaces); + +CREATE OR REPLACE FUNCTION refresh_account_advisory_caches_multi(advisory_ids_in INTEGER[] DEFAULT NULL, + rh_account_id_in INTEGER DEFAULT NULL) + RETURNS VOID AS +$refresh_account_advisory$ +BEGIN + PERFORM aa.rh_account_id, aa.workspace_id, aa.advisory_id + FROM account_advisory aa + WHERE (aa.advisory_id = ANY (advisory_ids_in) OR advisory_ids_in IS NULL) + AND (aa.rh_account_id = rh_account_id_in OR rh_account_id_in IS NULL) + FOR UPDATE OF aa; + + WITH current_counts AS ( + SELECT sa.advisory_id, sa.rh_account_id, (ws->>'id')::UUID AS workspace_id, + count(sa.*) FILTER (WHERE sa.status_id = 0) AS systems_installable, + count(sa.*) AS systems_applicable + FROM system_advisories sa + JOIN system_inventory si + ON sa.rh_account_id = si.rh_account_id AND sa.system_id = si.id + JOIN system_patch sp + ON si.id = sp.system_id AND sp.rh_account_id = si.rh_account_id + CROSS JOIN LATERAL jsonb_array_elements(si.workspaces) AS ws + WHERE sp.last_evaluation IS NOT NULL + AND si.stale = FALSE + AND si.workspaces IS NOT NULL + AND (sa.advisory_id = ANY (advisory_ids_in) OR advisory_ids_in IS NULL) + AND (si.rh_account_id = rh_account_id_in OR rh_account_id_in IS NULL) + GROUP BY sa.advisory_id, sa.rh_account_id, (ws->>'id')::UUID + ), + upserted AS ( + INSERT INTO account_advisory (advisory_id, rh_account_id, workspace_id, systems_installable, systems_applicable) + SELECT advisory_id, rh_account_id, workspace_id, systems_installable, systems_applicable + FROM current_counts + ON CONFLICT (rh_account_id, workspace_id, advisory_id) DO UPDATE SET + systems_installable = EXCLUDED.systems_installable, + systems_applicable = EXCLUDED.systems_applicable + ) + DELETE FROM account_advisory + WHERE (advisory_id, rh_account_id, workspace_id) NOT IN (SELECT advisory_id, rh_account_id, workspace_id FROM current_counts) + AND (advisory_id = ANY (advisory_ids_in) OR advisory_ids_in IS NULL) + AND (rh_account_id = rh_account_id_in OR rh_account_id_in IS NULL); +END; +$refresh_account_advisory$ LANGUAGE plpgsql; diff --git a/database_admin/migrations/160_drop_system_inventory_workspaces.up.sql b/database_admin/migrations/160_drop_system_inventory_workspaces.up.sql new file mode 100644 index 000000000..b98f5adc1 --- /dev/null +++ b/database_admin/migrations/160_drop_system_inventory_workspaces.up.sql @@ -0,0 +1,45 @@ +CREATE OR REPLACE FUNCTION refresh_account_advisory_caches_multi(advisory_ids_in INTEGER[] DEFAULT NULL, + rh_account_id_in INTEGER DEFAULT NULL) + RETURNS VOID AS +$refresh_account_advisory$ +BEGIN + PERFORM aa.rh_account_id, aa.workspace_id, aa.advisory_id + FROM account_advisory aa + WHERE (aa.advisory_id = ANY (advisory_ids_in) OR advisory_ids_in IS NULL) + AND (aa.rh_account_id = rh_account_id_in OR rh_account_id_in IS NULL) + FOR UPDATE OF aa; + + WITH current_counts AS ( + SELECT sa.advisory_id, sa.rh_account_id, si.workspace_id, + count(sa.*) FILTER (WHERE sa.status_id = 0) AS systems_installable, + count(sa.*) AS systems_applicable + FROM system_advisories sa + JOIN system_inventory si + ON sa.rh_account_id = si.rh_account_id AND sa.system_id = si.id + JOIN system_patch sp + ON si.id = sp.system_id AND sp.rh_account_id = si.rh_account_id + WHERE sp.last_evaluation IS NOT NULL + AND si.stale = FALSE + AND si.workspace_id IS NOT NULL + AND (sa.advisory_id = ANY (advisory_ids_in) OR advisory_ids_in IS NULL) + AND (si.rh_account_id = rh_account_id_in OR rh_account_id_in IS NULL) + GROUP BY sa.advisory_id, sa.rh_account_id, si.workspace_id + ), + upserted AS ( + INSERT INTO account_advisory (advisory_id, rh_account_id, workspace_id, systems_installable, systems_applicable) + SELECT advisory_id, rh_account_id, workspace_id, systems_installable, systems_applicable + FROM current_counts + ON CONFLICT (rh_account_id, workspace_id, advisory_id) DO UPDATE SET + systems_installable = EXCLUDED.systems_installable, + systems_applicable = EXCLUDED.systems_applicable + ) + DELETE FROM account_advisory + WHERE (advisory_id, rh_account_id, workspace_id) NOT IN (SELECT advisory_id, rh_account_id, workspace_id FROM current_counts) + AND (advisory_id = ANY (advisory_ids_in) OR advisory_ids_in IS NULL) + AND (rh_account_id = rh_account_id_in OR rh_account_id_in IS NULL); +END; +$refresh_account_advisory$ LANGUAGE plpgsql; + +DROP INDEX IF EXISTS system_inventory_workspaces_index; + +ALTER TABLE system_inventory DROP COLUMN IF EXISTS workspaces; diff --git a/database_admin/schema/create_schema.sql b/database_admin/schema/create_schema.sql index 1019113c5..d90a868d5 100644 --- a/database_admin/schema/create_schema.sql +++ b/database_admin/schema/create_schema.sql @@ -7,7 +7,7 @@ CREATE TABLE IF NOT EXISTS schema_migrations INSERT INTO schema_migrations -VALUES (159, false); +VALUES (160, false); -- --------------------------------------------------------------------------- -- Functions @@ -182,7 +182,7 @@ BEGIN FOR UPDATE OF aa; WITH current_counts AS ( - SELECT sa.advisory_id, sa.rh_account_id, (ws->>'id')::UUID AS workspace_id, + SELECT sa.advisory_id, sa.rh_account_id, si.workspace_id, count(sa.*) FILTER (WHERE sa.status_id = 0) AS systems_installable, count(sa.*) AS systems_applicable FROM system_advisories sa @@ -190,13 +190,12 @@ BEGIN ON sa.rh_account_id = si.rh_account_id AND sa.system_id = si.id JOIN system_patch sp ON si.id = sp.system_id AND sp.rh_account_id = si.rh_account_id - CROSS JOIN LATERAL jsonb_array_elements(si.workspaces) AS ws WHERE sp.last_evaluation IS NOT NULL AND si.stale = FALSE - AND si.workspaces IS NOT NULL + AND si.workspace_id IS NOT NULL AND (sa.advisory_id = ANY (advisory_ids_in) OR advisory_ids_in IS NULL) AND (si.rh_account_id = rh_account_id_in OR rh_account_id_in IS NULL) - GROUP BY sa.advisory_id, sa.rh_account_id, (ws->>'id')::UUID + GROUP BY sa.advisory_id, sa.rh_account_id, si.workspace_id ), upserted AS ( INSERT INTO account_advisory (advisory_id, rh_account_id, workspace_id, systems_installable, systems_applicable) @@ -655,7 +654,6 @@ CREATE TABLE IF NOT EXISTS system_inventory ansible_workload_controller_version TEXT CHECK (NOT empty(ansible_workload_controller_version)), mssql_workload BOOLEAN NOT NULL DEFAULT false, mssql_workload_version TEXT CHECK (NOT empty(mssql_workload_version)), - workspaces JSONB, workspace_id UUID, workspace_name TEXT CHECK (NOT empty(workspace_name)), PRIMARY KEY (rh_account_id, id), @@ -689,7 +687,6 @@ SELECT create_table_partition_triggers('system_inventory_on_update', CREATE INDEX IF NOT EXISTS system_inventory_inventory_id_idx ON system_inventory (inventory_id); CREATE INDEX IF NOT EXISTS system_inventory_tags_index ON system_inventory USING GIN (tags JSONB_PATH_OPS); CREATE INDEX IF NOT EXISTS system_inventory_stale_timestamp_index ON system_inventory (stale_timestamp); -CREATE INDEX IF NOT EXISTS system_inventory_workspaces_index ON system_inventory USING GIN (workspaces); CREATE INDEX IF NOT EXISTS system_inventory_workspace_id_index ON system_inventory (workspace_id); CREATE TABLE IF NOT EXISTS deleted_system From 79730dfa97a41e4f51ab409e07007632b5755c02 Mon Sep 17 00:00:00 2001 From: TenSt Date: Fri, 12 Jun 2026 12:34:20 +0200 Subject: [PATCH 2/7] RHINENG-26548: remove workspace_backfill job --- main.go | 3 - tasks/config.go | 6 - tasks/workspace_backfill/metrics.go | 36 --- .../workspace_backfill/workspace_backfill.go | 207 ------------------ .../workspace_backfill_test.go | 95 -------- 5 files changed, 347 deletions(-) delete mode 100644 tasks/workspace_backfill/metrics.go delete mode 100644 tasks/workspace_backfill/workspace_backfill.go delete mode 100644 tasks/workspace_backfill/workspace_backfill_test.go diff --git a/main.go b/main.go index edeb78280..90f4828aa 100644 --- a/main.go +++ b/main.go @@ -13,7 +13,6 @@ import ( "app/tasks/repack" "app/tasks/system_culling" "app/tasks/vmaas_sync" - "app/tasks/workspace_backfill" "app/turnpike" "log" "os" @@ -75,7 +74,5 @@ func runJob(name string) { repack.RunRepack() case "clean_advisory_account_data": cleaning.RunCleanAdvisoryAccountData() - case "workspace_backfill": - workspace_backfill.RunWorkspaceBackfill() } } diff --git a/tasks/config.go b/tasks/config.go index ca921a3c9..d059dc58c 100644 --- a/tasks/config.go +++ b/tasks/config.go @@ -42,10 +42,4 @@ var ( MaxChangedPackages = utils.PodConfig.GetInt("max_changed_packages", 30000) // prune deleted_system table records older than threshold DeletedSystemsThreshold = time.Hour * time.Duration(utils.PodConfig.GetInt("system_delete_hrs", 4)) - // Max system_inventory rows updated per workspace backfill job run - WorkspaceBackfillMaxRowsPerRun = utils.PodConfig.GetInt("workspace_backfill_max_rows_per_run", 50000) - // Batch size per account in workspace backfill job - WorkspaceBackfillBatchSize = utils.PodConfig.GetInt("workspace_backfill_batch_size", 1000) - // Sleep between workspace backfill batches in milliseconds - WorkspaceBackfillBatchSleepMs = utils.PodConfig.GetInt("workspace_backfill_batch_sleep_ms", 0) ) diff --git a/tasks/workspace_backfill/metrics.go b/tasks/workspace_backfill/metrics.go deleted file mode 100644 index 09d483541..000000000 --- a/tasks/workspace_backfill/metrics.go +++ /dev/null @@ -1,36 +0,0 @@ -package workspace_backfill - -import ( - "app/base/utils" - - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/push" -) - -var ( - backfillRowsCnt = prometheus.NewCounter(prometheus.CounterOpts{ - Help: "How many system_inventory rows were backfilled with workspace_id and workspace_name", - Namespace: "patchman_engine", - Subsystem: "workspace_backfill", - Name: "rows_updated", - }) - backfillBatchesCnt = prometheus.NewCounter(prometheus.CounterOpts{ - Help: "How many workspace backfill batches completed successfully", - Namespace: "patchman_engine", - Subsystem: "workspace_backfill", - Name: "batches", - }) - backfillErrorsCnt = prometheus.NewCounter(prometheus.CounterOpts{ - Help: "How many workspace backfill batches failed", - Namespace: "patchman_engine", - Subsystem: "workspace_backfill", - Name: "batch_errors", - }) -) - -// Metrics returns a pushgateway pusher for workspace backfill counters. -func Metrics() *push.Pusher { - registry := prometheus.NewRegistry() - registry.MustRegister(backfillRowsCnt, backfillBatchesCnt, backfillErrorsCnt) - return push.New(utils.CoreCfg.PrometheusPushGateway, "workspace_backfill").Gatherer(registry) -} diff --git a/tasks/workspace_backfill/workspace_backfill.go b/tasks/workspace_backfill/workspace_backfill.go deleted file mode 100644 index c48a7fc1f..000000000 --- a/tasks/workspace_backfill/workspace_backfill.go +++ /dev/null @@ -1,207 +0,0 @@ -package workspace_backfill - -import ( - "app/base/core" - "app/base/utils" - "app/tasks" - "time" - - "gorm.io/gorm" -) - -const setReplicaRoleSQL = `SET LOCAL session_replication_role = replica` - -const workspacePendingPredicate = ` -workspace_id IS NULL - AND workspaces IS NOT NULL - AND jsonb_typeof(workspaces) = 'array' - AND jsonb_array_length(workspaces) > 0 - AND workspaces->0->>'id' IS NOT NULL - AND workspaces->0->>'name' IS NOT NULL - AND NOT empty(workspaces->0->>'name')` - -const validWorkspacesJSONPredicate = ` -jsonb_typeof(workspaces) = 'array' - AND jsonb_array_length(workspaces) > 0 - AND workspaces->0->>'id' IS NOT NULL - AND workspaces->0->>'name' IS NOT NULL - AND NOT empty(workspaces->0->>'name')` - -const backfillUpdateSQL = ` -UPDATE system_inventory si -SET workspace_id = (si.workspaces->0->>'id')::uuid, - workspace_name = si.workspaces->0->>'name' -FROM ( - SELECT rh_account_id, id - FROM system_inventory - WHERE rh_account_id = ? - AND ` + workspacePendingPredicate + ` - ORDER BY id - LIMIT ? -) batch -WHERE si.rh_account_id = batch.rh_account_id - AND si.id = batch.id` - -const pendingAccountsSQL = ` -SELECT rh_account_id -FROM system_inventory -WHERE ` + workspacePendingPredicate + ` -GROUP BY rh_account_id -ORDER BY hash_partition_id(rh_account_id, 128), rh_account_id` - -const pendingRowsSQL = workspacePendingPredicate - -const invalidPendingRowsSQL = ` -workspace_id IS NULL - AND workspaces IS NOT NULL - AND NOT (` + validWorkspacesJSONPredicate + `)` - -func configure() { - // Admin DB user (same as migrations): UPDATE on partitions and session_replication_role = replica. - core.ConfigureAdminApp() -} - -// RunWorkspaceBackfill runs the batched workspace_id / workspace_name backfill job. -func RunWorkspaceBackfill() { - tasks.HandleContextCancel(tasks.WaitAndExit) - configure() - - nUpdated, complete, err := runWorkspaceBackfill() - if err != nil { - utils.LogError("err", err.Error(), "Workspace backfill") - return - } - if err := Metrics().Add(); err != nil { - utils.LogInfo("err", err, "Could not push to pushgateway") - } - if complete { - utils.LogInfo("nUpdated", nUpdated, "Workspace backfill complete") - } else { - utils.LogInfo("nUpdated", nUpdated, "Workspace backfill paused (per-run limit); more rows remain") - } -} - -func runWorkspaceBackfill() (nUpdated int64, complete bool, err error) { - if err := logPendingStats(); err != nil { - return 0, false, err - } - - accounts, err := loadPendingAccounts() - if err != nil { - return 0, false, err - } - if len(accounts) == 0 { - return 0, true, nil - } - - utils.LogInfo("accounts", len(accounts), "Starting workspace backfill") - - maxRows := int64(tasks.WorkspaceBackfillMaxRowsPerRun) - var total int64 - - for i, rhAccountID := range accounts { - rows, hitLimit, batchErr := processAccountBatches(i, rhAccountID, maxRows, total) - total += rows - if batchErr != nil { - continue - } - if hitLimit { - return total, false, nil - } - } - - pending, err := countPending() - if err != nil { - return total, false, err - } - return total, pending == 0, nil -} - -//nolint:lll -func processAccountBatches(idx, rhAccountID int, maxRows, totalSoFar int64) (rowsUpdated int64, hitLimit bool, err error) { - for totalSoFar+rowsUpdated < maxRows { - remaining := maxRows - (totalSoFar + rowsUpdated) - batchLimit := tasks.WorkspaceBackfillBatchSize - if int64(batchLimit) > remaining { - batchLimit = int(remaining) - } - - rows, batchErr := backfillBatch(rhAccountID, batchLimit) - if batchErr != nil { - utils.LogWarn("rhAccountID", rhAccountID, "err", batchErr.Error(), "Workspace backfill batch failed") - backfillErrorsCnt.Inc() - return rowsUpdated, false, batchErr - } - if rows == 0 { - return rowsUpdated, false, nil - } - - rowsUpdated += rows - backfillRowsCnt.Add(float64(rows)) - backfillBatchesCnt.Inc() - utils.LogInfo("i", idx, "rhAccountID", rhAccountID, "nRows", rows, "total", totalSoFar+rowsUpdated, - "Workspace backfill batch") - - if tasks.WorkspaceBackfillBatchSleepMs > 0 { - time.Sleep(time.Duration(tasks.WorkspaceBackfillBatchSleepMs) * time.Millisecond) - } - } - - return rowsUpdated, true, nil -} - -func logPendingStats() error { - pending, err := countPending() - if err != nil { - return err - } - invalid, err := countInvalidPending() - if err != nil { - return err - } - utils.LogInfo("pending", pending, "invalidPending", invalid, "Workspace backfill pending rows") - if invalid > 0 { - utils.LogWarn("invalidPending", invalid, "Rows with workspace_id NULL and invalid workspaces are skipped") - } - return nil -} - -func loadPendingAccounts() ([]int, error) { - var accounts []int - err := tasks.WithReadReplicaTx(func(tx *gorm.DB) error { - return tx.Raw(pendingAccountsSQL).Scan(&accounts).Error - }) - return accounts, err -} - -func countPending() (int64, error) { - var cnt int64 - err := tasks.WithReadReplicaTx(func(tx *gorm.DB) error { - return tx.Table("system_inventory").Where(pendingRowsSQL).Count(&cnt).Error - }) - return cnt, err -} - -func countInvalidPending() (int64, error) { - var cnt int64 - err := tasks.WithReadReplicaTx(func(tx *gorm.DB) error { - return tx.Table("system_inventory").Where(invalidPendingRowsSQL).Count(&cnt).Error - }) - return cnt, err -} - -func backfillBatch(rhAccountID, batchLimit int) (int64, error) { - var rows int64 - err := tasks.CancelableDB().Transaction(func(tx *gorm.DB) error { - if err := tx.Exec(setReplicaRoleSQL).Error; err != nil { - return err - } - res := tx.Exec(backfillUpdateSQL, rhAccountID, batchLimit) - if res.Error != nil { - return res.Error - } - rows = res.RowsAffected - return nil - }) - return rows, err -} diff --git a/tasks/workspace_backfill/workspace_backfill_test.go b/tasks/workspace_backfill/workspace_backfill_test.go deleted file mode 100644 index bee8179d2..000000000 --- a/tasks/workspace_backfill/workspace_backfill_test.go +++ /dev/null @@ -1,95 +0,0 @@ -package workspace_backfill - -import ( - "app/base/core" - "app/base/database" - "app/base/models" - "app/base/utils" - "fmt" - "testing" - "time" - - "github.com/google/uuid" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "gorm.io/gorm" -) - -type workspaceRow struct { - WorkspaceID *string `gorm:"column:workspace_id"` - WorkspaceName *string `gorm:"column:workspace_name"` - LastUpdated *time.Time `gorm:"column:last_updated"` -} - -//nolint:funlen -func TestBackfillBatch(t *testing.T) { - utils.SkipWithoutDB(t) - core.SetupTestEnvironment() - - lastUpdated := time.Date(2020, 1, 15, 12, 0, 0, 0, time.UTC) - var systemIDs []int64 - - for i := 0; i < 3; i++ { - invID := fmt.Sprintf("00000000-0000-4000-8000-00000000%04x", i) - inv := models.SystemInventory{ - InventoryID: uuid.MustParse(invID), - RhAccountID: 1, - DisplayName: invID, - Tags: []byte("[]"), - Workspaces: database.TestWorkspacesGroup1(), - } - require.NoError(t, database.DB.Create(&inv).Error) - systemIDs = append(systemIDs, inv.ID) - - require.NoError(t, database.DB.Transaction(func(tx *gorm.DB) error { - if err := tx.Exec(setReplicaRoleSQL).Error; err != nil { - return err - } - return tx.Exec( - `UPDATE system_inventory - SET workspace_id = NULL, workspace_name = NULL, last_updated = ? - WHERE rh_account_id = ? AND id = ?`, - lastUpdated, 1, inv.ID, - ).Error - })) - } - - for _, systemID := range systemIDs { - var row workspaceRow - require.NoError(t, database.DB.Table("system_inventory"). - Select("workspace_id, workspace_name, last_updated"). - Where("rh_account_id = ? AND id = ?", 1, systemID). - Scan(&row).Error) - assert.Nil(t, row.WorkspaceID) - assert.Nil(t, row.WorkspaceName) - require.NotNil(t, row.LastUpdated) - assert.True(t, row.LastUpdated.Equal(lastUpdated)) - } - - rows, err := backfillBatch(1, 1000) - require.NoError(t, err) - assert.Equal(t, int64(3), rows) - - for _, systemID := range systemIDs { - var row workspaceRow - require.NoError(t, database.DB.Table("system_inventory"). - Select("workspace_id, workspace_name, last_updated"). - Where("rh_account_id = ? AND id = ?", 1, systemID). - Scan(&row).Error) - require.NotNil(t, row.WorkspaceID) - assert.Equal(t, database.TestWorkspace1ID, *row.WorkspaceID) - require.NotNil(t, row.WorkspaceName) - assert.Equal(t, "group1", *row.WorkspaceName) - require.NotNil(t, row.LastUpdated) - assert.True(t, row.LastUpdated.Equal(lastUpdated), "last_updated must be preserved") - } - - rows, err = backfillBatch(1, 1000) - require.NoError(t, err) - assert.Equal(t, int64(0), rows) - - for i := range systemIDs { - invID := fmt.Sprintf("00000000-0000-4000-8000-00000000%04x", i) - require.NoError(t, database.DB.Exec("SELECT delete_system(?::uuid)", invID).Error) - } -} From fc7b061b8b4c2777cb5723aa8c462baa9000610b Mon Sep 17 00:00:00 2001 From: TenSt Date: Fri, 12 Jun 2026 12:34:33 +0200 Subject: [PATCH 3/7] RHINENG-26548: remove workspace backfill CronJob and dashboard --- ...hts-patchman-engine-general.configmap.yaml | 128 ------------------ deploy/clowdapp.yaml | 29 ---- 2 files changed, 157 deletions(-) diff --git a/dashboards/app-sre/grafana-dashboard-insights-patchman-engine-general.configmap.yaml b/dashboards/app-sre/grafana-dashboard-insights-patchman-engine-general.configmap.yaml index 34dc5f66a..e45fddaee 100644 --- a/dashboards/app-sre/grafana-dashboard-insights-patchman-engine-general.configmap.yaml +++ b/dashboards/app-sre/grafana-dashboard-insights-patchman-engine-general.configmap.yaml @@ -5500,134 +5500,6 @@ data: "title": "System culling", "type": "timeseries" }, - { - "datasource": { - "uid": "$datasource" - }, - "description": "Workspace backfill CronJob: rows updated per interval, batches completed, and batch errors (Pushgateway job workspace_backfill).", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Count", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [ - - ], - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 103 - }, - "id": 77, - "options": { - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.6.3", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "sum(increase(patchman_engine_workspace_backfill_rows_updated_total{}[$interval]))", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "rows updated", - "refId": "A" - }, - { - "datasource": { - "uid": "$datasource" - }, - "expr": "sum(increase(patchman_engine_workspace_backfill_batches_total{}[$interval]))", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "batches", - "refId": "B" - }, - { - "datasource": { - "uid": "$datasource" - }, - "expr": "sum(increase(patchman_engine_workspace_backfill_batch_errors_total{}[$interval]))", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "batch errors", - "refId": "C" - } - ], - "title": "Workspace backfill", - "type": "timeseries" - }, { "collapsed": false, "gridPos": { diff --git a/deploy/clowdapp.yaml b/deploy/clowdapp.yaml index 7d87a2d3c..ebf2e41d9 100644 --- a/deploy/clowdapp.yaml +++ b/deploy/clowdapp.yaml @@ -519,31 +519,6 @@ objects: key: vmaas-sync-database-password}}} - {name: POD_CONFIG, value: '${JOBS_CONFIG}'} - - name: workspace-backfill - activeDeadlineSeconds: ${{JOBS_TIMEOUT}} - schedule: ${WORKSPACE_BACKFILL_SCHEDULE} - suspend: ${{WORKSPACE_BACKFILL_SUSPEND}} - concurrencyPolicy: Forbid - podSpec: - image: ${IMAGE}:${IMAGE_TAG} - initContainers: - - name: check-for-db - image: ${IMAGE}:${IMAGE_TAG} - command: - - ./database_admin/check-upgraded.sh - env: - - {name: POD_CONFIG, value: '${DATABASE_ADMIN_CONFIG}'} - command: - - ./scripts/entrypoint.sh - - job - - workspace_backfill - env: - - {name: LOG_LEVEL, value: '${LOG_LEVEL_JOBS}'} - - {name: GIN_MODE, value: '${GIN_MODE}'} - - {name: DB_DEBUG, value: '${DB_DEBUG_JOBS}'} - - {name: PROMETHEUS_PUSHGATEWAY,value: '${PROMETHEUS_PUSHGATEWAY}'} - - {name: POD_CONFIG, value: '${WORKSPACE_BACKFILL_CONFIG}'} - database: name: patchman version: 16 @@ -790,10 +765,6 @@ parameters: # Clean advisory_account_data - {name: CLEAN_AAD_SCHEDULE, value: '0 12 * * *'} # Cronjob schedule definition - {name: CLEAN_AAD_SUSPEND, value: 'false'} # Disable cronjob execution -# Workspace backfill (one-time data migration; keep suspended until ready to run) -- {name: WORKSPACE_BACKFILL_SCHEDULE, value: '*/10 * * * *'} # Cronjob schedule definition -- {name: WORKSPACE_BACKFILL_SUSPEND, value: 'true'} # Disable cronjob execution -- {name: WORKSPACE_BACKFILL_CONFIG, value: 'workspace_backfill_batch_size=1000;workspace_backfill_max_rows_per_run=50000;workspace_backfill_batch_sleep_ms=0'} # Database admin - {name: LOG_LEVEL_DATABASE_ADMIN, value: debug} From a936b0f8df23c1bebe0325c9b9119b7554d98bdc Mon Sep 17 00:00:00 2001 From: TenSt Date: Fri, 12 Jun 2026 12:34:37 +0200 Subject: [PATCH 4/7] RHINENG-26548: remove local workspace backfill tooling --- conf/workspace_backfill.env | 4 - .../prepare_workspace_backfill_test.sql | 20 --- .../test_generate_system_inventory.sql | 126 ----------------- .../verify_workspace_backfill.sql | 21 --- dev/workspace_backfill/workspace_backfill.md | 127 ------------------ docker-compose.workspace-backfill.yml | 41 ------ scripts/workspace_backfill_e2e.sh | 51 ------- 7 files changed, 390 deletions(-) delete mode 100644 conf/workspace_backfill.env delete mode 100644 dev/workspace_backfill/prepare_workspace_backfill_test.sql delete mode 100644 dev/workspace_backfill/test_generate_system_inventory.sql delete mode 100644 dev/workspace_backfill/verify_workspace_backfill.sql delete mode 100644 dev/workspace_backfill/workspace_backfill.md delete mode 100644 docker-compose.workspace-backfill.yml delete mode 100755 scripts/workspace_backfill_e2e.sh diff --git a/conf/workspace_backfill.env b/conf/workspace_backfill.env deleted file mode 100644 index 0c1fc5e2a..000000000 --- a/conf/workspace_backfill.env +++ /dev/null @@ -1,4 +0,0 @@ -LOG_LEVEL=info - -# Local Docker only: 1000 rows per job run (re-run job until complete). Production uses 50000 via WORKSPACE_BACKFILL_CONFIG. -POD_CONFIG=update_users;update_db_config;wait_for_db=empty;use_testing_db;workspace_backfill_batch_size=1000;workspace_backfill_max_rows_per_run=10000 diff --git a/dev/workspace_backfill/prepare_workspace_backfill_test.sql b/dev/workspace_backfill/prepare_workspace_backfill_test.sql deleted file mode 100644 index dbc59392b..000000000 --- a/dev/workspace_backfill/prepare_workspace_backfill_test.sql +++ /dev/null @@ -1,20 +0,0 @@ --- Clear denormalized workspace columns without firing row triggers (for backfill e2e). -\set ON_ERROR_STOP on - -BEGIN; -SET LOCAL session_replication_role = replica; - -UPDATE system_inventory -SET workspace_id = NULL, - workspace_name = NULL -WHERE workspace_id IS NOT NULL - OR workspace_name IS NOT NULL; - -COMMIT; - -SELECT count(*) AS pending_backfill -FROM system_inventory -WHERE workspace_id IS NULL - AND workspaces IS NOT NULL - AND jsonb_typeof(workspaces) = 'array' - AND jsonb_array_length(workspaces) > 0; diff --git a/dev/workspace_backfill/test_generate_system_inventory.sql b/dev/workspace_backfill/test_generate_system_inventory.sql deleted file mode 100644 index 0b2e92976..000000000 --- a/dev/workspace_backfill/test_generate_system_inventory.sql +++ /dev/null @@ -1,126 +0,0 @@ --- Load-test generator for system_inventory (+ required rh_account, system_patch). --- Faster subset of dev/test_generate_data.sql (no advisories, repos, packages, etc.). -\timing on -\set ON_ERROR_STOP on - -CREATE TABLE IF NOT EXISTS _const ( - key TEXT PRIMARY KEY, - val INT -); - -TRUNCATE _const; -INSERT INTO _const VALUES - ('accounts', 50), -- rh_account rows - ('systems', 7500), -- system_inventory + system_patch rows - ('progress_pct', 10); -- progress NOTICE every N% - --- Minimal vmaas_json samples (same as test_generate_data.sql) -CREATE TABLE IF NOT EXISTS _json ( - id INT PRIMARY KEY, - data TEXT, - hash TEXT -); -INSERT INTO _json VALUES - (1, '{ "package_list": [ "kernel-2.6.32-696.20.1.el6.x86_64" ]}'), - (2, '{ "package_list": [ "libsmbclient-4.6.2-12.el7_4.x86_64", "dconf-0.26.0-2.el7.x86_64"]}'), - (3, '{ "repository_list": [ "rhel-7-server-rpms" ], "releasever": "7Server", "basearch": "x86_64", "package_list": [ "libsmbclient-4.6.2-12.el7_4.x86_64"]}') -ON CONFLICT DO NOTHING; -UPDATE _json SET hash = encode(sha256(data::bytea), 'hex'); - --- Wipes accounts and all dependent host data (inventory, patch, advisories links, packages, …) -TRUNCATE rh_account CASCADE; - -ALTER SEQUENCE rh_account_id_seq RESTART WITH 1; -DO $$ -DECLARE - cnt INT := 0; - wanted INT; - id INT; -BEGIN - SELECT val INTO wanted FROM _const WHERE key = 'accounts'; - WHILE cnt < wanted LOOP - id := nextval('rh_account_id_seq'); - INSERT INTO rh_account (id, org_id) VALUES (id, 'RHACCOUNT-' || id); - cnt := cnt + 1; - END LOOP; - RAISE NOTICE 'created % rh_accounts', wanted; -END; -$$; - -CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; -ALTER SEQUENCE system_inventory_id_seq RESTART WITH 1; - -DO $$ -DECLARE - cnt INT := 0; - wanted INT; - progress INT; - gen_uuid UUID; - rh_accounts INT; - rnd FLOAT; - json_data TEXT[]; - json_hash TEXT[]; - rnd_date1 TIMESTAMPTZ; - rnd_date2 TIMESTAMPTZ; - acc_id INT; - new_id BIGINT; - ji INT; - workspace_ids UUID[]; -BEGIN - SELECT val INTO wanted FROM _const WHERE key = 'systems'; - SELECT val INTO progress FROM _const WHERE key = 'progress_pct'; - SELECT count(*) INTO rh_accounts FROM rh_account; - json_data := array(SELECT data FROM _json ORDER BY id); - json_hash := array(SELECT hash FROM _json ORDER BY id); - workspace_ids := array(SELECT uuid_generate_v4() FROM generate_series(1, 3)); - - WHILE cnt < wanted LOOP - gen_uuid := uuid_generate_v4(); - rnd := random(); - rnd_date1 := now() - make_interval(days => (rnd * 30)::INT); - rnd_date2 := rnd_date1 + make_interval(days => (rnd * 10)::INT); - acc_id := trunc(rnd * rh_accounts) + 1; - ji := trunc(rnd * 3) + 1; - - INSERT INTO system_inventory - (inventory_id, display_name, rh_account_id, vmaas_json, json_checksum, - last_updated, last_upload, arch, tags, created, os_name, os_major, rhsm_version, - workspaces, workspace_id, workspace_name) - VALUES - (gen_uuid, gen_uuid::text, acc_id, json_data[ji], json_hash[ji], - rnd_date2, rnd_date2, 'x86_64', '[]'::jsonb, rnd_date1, 'RHEL', 8, '8.0', - jsonb_build_array(jsonb_build_object( - 'id', workspace_ids[cnt % 3 + 1]::text, - 'name', workspace_ids[cnt % 3 + 1]::text)), - workspace_ids[cnt % 3 + 1], workspace_ids[cnt % 3 + 1]::text) - RETURNING id INTO new_id; - - INSERT INTO system_patch - (rh_account_id, system_id, last_evaluation, - packages_installed, packages_installable, packages_applicable) - VALUES - (acc_id, new_id, rnd_date2, trunc(rnd * 1000), trunc(rnd * 50), trunc(rnd * 50)); - - IF mod(cnt, greatest(1, ceil((wanted::numeric * progress) / 100.0)::int)) = 0 THEN - RAISE NOTICE 'created % systems (inventory + patch)', cnt; - END IF; - cnt := cnt + 1; - END LOOP; - RAISE NOTICE 'created % systems (inventory + patch)', wanted; -END; -$$; - -SELECT 'rh_account' AS tbl, count(*) FROM rh_account -UNION ALL -SELECT 'system_inventory', count(*) FROM system_inventory -UNION ALL -SELECT 'system_patch', count(*) FROM system_patch; - -SELECT parent.relname AS parent, - child.relname AS child, - pg_size_pretty(pg_relation_size(child.oid)) AS size -FROM pg_inherits - JOIN pg_class parent ON pg_inherits.inhparent = parent.oid - JOIN pg_class child ON pg_inherits.inhrelid = child.oid -WHERE parent.relname IN ('system_inventory', 'system_patch') -ORDER BY 1, 2; diff --git a/dev/workspace_backfill/verify_workspace_backfill.sql b/dev/workspace_backfill/verify_workspace_backfill.sql deleted file mode 100644 index c2ca2149f..000000000 --- a/dev/workspace_backfill/verify_workspace_backfill.sql +++ /dev/null @@ -1,21 +0,0 @@ --- Pending predicate must match workspacePendingPredicate in tasks/workspace_backfill/workspace_backfill.go -\set ON_ERROR_STOP on - -SELECT count(*) AS pending -FROM system_inventory -WHERE workspace_id IS NULL - AND workspaces IS NOT NULL - AND jsonb_typeof(workspaces) = 'array' - AND jsonb_array_length(workspaces) > 0 - AND workspaces->0->>'id' IS NOT NULL - AND workspaces->0->>'name' IS NOT NULL - AND NOT empty(workspaces->0->>'name'); - -SELECT count(*) AS mismatched -FROM system_inventory -WHERE workspace_id IS NOT NULL - AND workspaces IS NOT NULL - AND ( - workspace_id::text IS DISTINCT FROM workspaces->0->>'id' - OR workspace_name IS DISTINCT FROM workspaces->0->>'name' - ); diff --git a/dev/workspace_backfill/workspace_backfill.md b/dev/workspace_backfill/workspace_backfill.md deleted file mode 100644 index c9bae42d2..000000000 --- a/dev/workspace_backfill/workspace_backfill.md +++ /dev/null @@ -1,127 +0,0 @@ -# Workspace backfill — local testing - -Backfill copies `workspaces` JSON into `workspace_id` and `workspace_name` on `system_inventory` via the `workspace_backfill` job. - -**Do not run** `docker-compose.test.yml` and `docker-compose.workspace-backfill.yml` at the same time (both use host port **5433**). - -## Files - -| File | Purpose | -|------|---------| -| [`docker-compose.workspace-backfill.yml`](../../docker-compose.workspace-backfill.yml) | `db` + optional one-shot e2e runner | -| [`conf/workspace_backfill.env`](../../conf/workspace_backfill.env) | Local Docker `POD_CONFIG` (**1000** rows per job run) | -| [`test_generate_system_inventory.sql`](test_generate_system_inventory.sql) | Fast load: accounts + inventory + patch only | -| [`prepare_workspace_backfill_test.sql`](prepare_workspace_backfill_test.sql) | Clear `workspace_id` / `workspace_name` without triggers | -| [`verify_workspace_backfill.sql`](verify_workspace_backfill.sql) | Check pending / mismatched rows | -| [`scripts/workspace_backfill_e2e.sh`](../../scripts/workspace_backfill_e2e.sh) | Automated pipeline: setup + **one** job run + verify | - -## Configuration (`POD_CONFIG`) - -| Key | Local Docker ([`conf/workspace_backfill.env`](../../conf/workspace_backfill.env)) | Code / production default | -|-----|----------------------------------------------------------------------------------|---------------------------| -| `workspace_backfill_batch_size` | `1000` | `1000` | -| `workspace_backfill_max_rows_per_run` | **`1000`** | **`50000`** ([`tasks/config.go`](../../tasks/config.go), [`deploy/clowdapp.yaml`](../../deploy/clowdapp.yaml)) | -| `workspace_backfill_batch_sleep_ms` | `0` | `0` | - -Local compose loads `workspace_backfill.env` so each job invocation updates at most **1000** rows. Re-run the job manually until logs say `Workspace backfill complete`. - -Override per run: `docker compose run --rm -e 'POD_CONFIG=...' workspace-backfill ...` - -## Manual workflow (recommended for batched local runs) - -### 1. Run DB - -```bash -docker compose -f docker-compose.workspace-backfill.yml up -d db -``` - -Wait for Postgres: - -```bash -until PGPASSWORD=passwd psql -h localhost -p 5433 -U admin -d patchman -c 'SELECT 1' >/dev/null 2>&1; do sleep 1; done -``` - -### 2. Setup once - -Migrates, loads test data, clears workspace columns. **Destructive** (`TRUNCATE rh_account CASCADE` in the generator). - -```bash -docker compose -f docker-compose.workspace-backfill.yml run --rm workspace-backfill bash -c ' -set -e -o pipefail -cd /go/src/app -export WAIT_FOR_EMPTY_DB=1 -./dev/scripts/wait-for-services.sh true -go run main.go migrate file://./database_admin/migrations -unset WAIT_FOR_EMPTY_DB -export WAIT_FOR_FULL_DB=1 -./dev/scripts/wait-for-services.sh true -unset WAIT_FOR_FULL_DB -PGPASSWORD=passwd psql -h db -p 5432 -U admin -d patchman -f dev/workspace_backfill/test_generate_system_inventory.sql -PGPASSWORD=passwd psql -h db -p 5432 -U admin -d patchman -f dev/workspace_backfill/prepare_workspace_backfill_test.sql -' -``` - -Default generator creates **7500** systems. Lower `_const` in `test_generate_system_inventory.sql` first if you want a smaller dataset (e.g. `500` systems). - -### 3. Run job (repeat until complete) - -Each run processes up to **1000** rows (local env): - -```bash -docker compose -f docker-compose.workspace-backfill.yml run --rm workspace-backfill \ - ./scripts/entrypoint.sh job workspace_backfill -``` - -Logs: - -- `Workspace backfill paused (per-run limit); more rows remain` — run step 3 again -- `Workspace backfill complete` — done - -Check pending count: - -```bash -PGPASSWORD=passwd psql -h localhost -p 5433 -U admin -d patchman -c \ - "SELECT count(*) AS pending FROM system_inventory WHERE workspace_id IS NULL AND workspaces IS NOT NULL;" -``` - -Verify when `pending` is 0: - -```bash -PGPASSWORD=passwd psql -h localhost -p 5433 -U admin -d patchman -f dev/workspace_backfill/verify_workspace_backfill.sql -``` - -Expect `pending = 0` and `mismatched = 0`. - -### Teardown - -```bash -docker compose -f docker-compose.workspace-backfill.yml down -``` - -## One-shot e2e (automated script) - -Runs setup, **one** job invocation (local limit 1000 rows), then verification: - -```bash -docker compose -f docker-compose.workspace-backfill.yml up --build --abort-on-container-exit -``` - -With the default generator (**7500** systems), verification fails after a single job run because only 1000 rows are backfilled. Use either: - -- **≤1000** systems in `_const` before e2e, or -- The **manual workflow** above (setup once, job repeated), or -- A one-off higher limit for a full single-run test only: - -```bash -docker compose -f docker-compose.workspace-backfill.yml run --rm \ - -e 'POD_CONFIG=update_users;update_db_config;wait_for_db=empty;use_testing_db;workspace_backfill_batch_size=1000;workspace_backfill_max_rows_per_run=10000000' \ - workspace-backfill ./scripts/workspace_backfill_e2e.sh -``` - -## Database user - -The job connects as the **admin** user (`core.ConfigureAdminApp()`), same credentials as migrations (`database.adminUsername` / `adminPassword` from Clowder). That is required for `SET LOCAL session_replication_role = replica` and matches local testing via `cdappconfig.json`. - -## Production - -CronJob `workspace-backfill` in [`deploy/clowdapp.yaml`](../../deploy/clowdapp.yaml) uses `WORKSPACE_BACKFILL_CONFIG` with `workspace_backfill_max_rows_per_run=50000` (suspended by default). Run on a schedule until pending rows are gone. Admin DB credentials are provided by the platform (Clowder), not component secrets. diff --git a/docker-compose.workspace-backfill.yml b/docker-compose.workspace-backfill.yml deleted file mode 100644 index 76c6c8ee7..000000000 --- a/docker-compose.workspace-backfill.yml +++ /dev/null @@ -1,41 +0,0 @@ -# E2E workspace backfill: DB + one-shot runner (no Kafka/platform). -# -# docker compose -f docker-compose.workspace-backfill.yml up --build --abort-on-container-exit -# -# Uses host port 5433 (same as docker-compose.test.yml). Do not run both stacks at once. - -services: - db: - container_name: patchman-wb-db - build: - context: . - dockerfile: dev/database/Dockerfile - image: patchman-engine-db - ports: - - 5433:5432 - env_file: - - ./conf/database.env - - workspace-backfill: - container_name: patchman-wb-runner - build: - context: . - dockerfile: Dockerfile - args: - - INSTALL_TOOLS=yes - target: buildimg - image: patchman-engine-app - env_file: - - ./conf/common.env - - ./conf/database.env - - ./conf/database_admin.env - - ./conf/gorun.env - - ./conf/workspace_backfill.env - depends_on: - - db - user: root - command: ./scripts/workspace_backfill_e2e.sh - volumes: - - ./:/go/src/app/ - security_opt: - - label=disable diff --git a/scripts/workspace_backfill_e2e.sh b/scripts/workspace_backfill_e2e.sh deleted file mode 100755 index a440cac3b..000000000 --- a/scripts/workspace_backfill_e2e.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env bash -# End-to-end local run: migrate DB, load system_inventory test data, clear workspace -# columns, run workspace_backfill job once, verify. -set -e -o pipefail - -cd "$(dirname "$0")/.." - -psql_admin() { - PGPASSWORD="${DB_ADMIN_PASSWD:-passwd}" psql \ - -h "${DB_HOST:-db}" \ - -p "${DB_PORT:-5432}" \ - -U "${DB_ADMIN_USER:-admin}" \ - -d "${DB_NAME:-patchman}" \ - "$@" -} - -echo "==> Waiting for PostgreSQL" -export WAIT_FOR_EMPTY_DB=1 -./dev/scripts/wait-for-services.sh true - -echo "==> Running migrations" -go run main.go migrate file://./database_admin/migrations - -echo "==> Waiting for migrations to finish" -unset WAIT_FOR_EMPTY_DB -export WAIT_FOR_FULL_DB=1 -./dev/scripts/wait-for-services.sh true -unset WAIT_FOR_FULL_DB - -echo "==> Loading system_inventory test data" -psql_admin -f dev/workspace_backfill/test_generate_system_inventory.sql - -echo "==> Clearing workspace_id / workspace_name (triggers disabled per transaction)" -psql_admin -f dev/workspace_backfill/prepare_workspace_backfill_test.sql - -echo "==> Running workspace_backfill job" -./scripts/entrypoint.sh job workspace_backfill - -echo "==> Verifying backfill" -mapfile -t _wb_counts < <(psql_admin -t -A -f dev/workspace_backfill/verify_workspace_backfill.sql) -pending="${_wb_counts[0]}" -mismatched="${_wb_counts[1]}" - -echo "pending=${pending} mismatched=${mismatched}" - -if [[ "${pending}" != "0" || "${mismatched}" != "0" ]]; then - echo "Workspace backfill e2e verification failed" >&2 - exit 1 -fi - -echo "Workspace backfill e2e finished successfully" From d5d6ff58d18a556bd589c2fcaae6ef6c0325127d Mon Sep 17 00:00:00 2001 From: TenSt Date: Fri, 12 Jun 2026 12:34:43 +0200 Subject: [PATCH 5/7] RHINENG-26548: remove workspaces field from SystemInventory model --- base/database/testing.go | 10 +--------- base/inventory/inventory.go | 32 -------------------------------- base/models/models.go | 10 ++++------ 3 files changed, 5 insertions(+), 47 deletions(-) diff --git a/base/database/testing.go b/base/database/testing.go index 8012571ce..2e8e4d49d 100644 --- a/base/database/testing.go +++ b/base/database/testing.go @@ -2,7 +2,6 @@ package database import ( "app/base" - "app/base/inventory" "app/base/models" "app/base/utils" "fmt" @@ -16,16 +15,9 @@ import ( ) const ( - TestWorkspace1ID = "aaaaaaaa-0000-0000-0000-000000000001" - TestWorkspace2ID = "aaaaaaaa-0000-0000-0000-000000000002" - TestWorkspaceOtherID = "bbbbbbbb-0000-0000-0000-000000000003" + TestWorkspace1ID = "aaaaaaaa-0000-0000-0000-000000000001" ) -func TestWorkspacesGroup1() *inventory.Groups { - g := inventory.Groups{{ID: TestWorkspace1ID, Name: "group1"}} - return &g -} - func TestWorkspace1IDPtr() *uuid.UUID { id := uuid.MustParse(TestWorkspace1ID) return &id diff --git a/base/inventory/inventory.go b/base/inventory/inventory.go index db615358f..e2cfcdc31 100644 --- a/base/inventory/inventory.go +++ b/base/inventory/inventory.go @@ -2,9 +2,6 @@ package inventory import ( "app/base/types" - "database/sql/driver" - "encoding/json" - "errors" "github.com/google/uuid" ) @@ -86,35 +83,6 @@ type Group struct { Name string `json:"name"` } -// Groups is a slice of Group that implements driver.Valuer and sql.Scanner -// for storing/loading as JSONB in the database (e.g. system_inventory.workspaces). -type Groups []Group - -// Value implements driver.Valuer for GORM: marshal to JSON for DB write. -func (g *Groups) Value() (driver.Value, error) { - if g == nil { - return nil, nil - } - return json.Marshal(g) -} - -// Scan implements sql.Scanner for GORM: unmarshal from JSON on DB read. -func (g *Groups) Scan(value interface{}) error { - if value == nil { - *g = nil - return nil - } - b, ok := value.([]byte) - if !ok { - return errors.New("inventory.Groups: type assertion to []byte failed") - } - if len(b) == 0 { - *g = nil - return nil - } - return json.Unmarshal(b, g) -} - type Workloads struct { Sap SapWorkload `json:"sap,omitempty"` Ansible AnsibleWorkload `json:"ansible,omitempty"` diff --git a/base/models/models.go b/base/models/models.go index 0c2c0887e..d9f41658c 100644 --- a/base/models/models.go +++ b/base/models/models.go @@ -1,7 +1,6 @@ package models import ( - "app/base/inventory" "time" "github.com/google/uuid" @@ -71,11 +70,10 @@ type SystemInventory struct { BuiltPkgcache bool `gorm:"column:built_pkgcache"` Arch *string Bootc bool - Tags []byte `gorm:"column:tags"` - Created time.Time // set by trigger system_platform_insert_trigger - Workspaces *inventory.Groups `gorm:"column:workspaces"` - WorkspaceID *uuid.UUID `gorm:"column:workspace_id"` - WorkspaceName *string `gorm:"column:workspace_name"` + Tags []byte `gorm:"column:tags"` + Created time.Time // set by trigger system_platform_insert_trigger + WorkspaceID *uuid.UUID `gorm:"column:workspace_id"` + WorkspaceName *string `gorm:"column:workspace_name"` StaleTimestamp *time.Time StaleWarningTimestamp *time.Time CulledTimestamp *time.Time From 990f623c18be70eb45cccfb0e80aafae89b0b77a Mon Sep 17 00:00:00 2001 From: TenSt Date: Fri, 12 Jun 2026 12:34:51 +0200 Subject: [PATCH 6/7] RHINENG-26548: update test data generator for workspace columns --- dev/test_generate_data.sql | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/dev/test_generate_data.sql b/dev/test_generate_data.sql index 4eebb6156..7dff77364 100644 --- a/dev/test_generate_data.sql +++ b/dev/test_generate_data.sql @@ -114,13 +114,10 @@ do $$ insert into system_inventory (inventory_id, display_name, rh_account_id, vmaas_json, json_checksum, last_updated, last_upload, arch, tags, created, os_name, os_major, rhsm_version, - workspaces, workspace_id, workspace_name) + workspace_id, workspace_name) values (gen_uuid, gen_uuid::text, acc_id, json_data[ji], json_hash[ji], rnd_date2, rnd_date2, 'x86_64', '[]'::jsonb, rnd_date1, 'RHEL', 8, '8.0', - jsonb_build_array(jsonb_build_object( - 'id', workspace_ids[cnt % 3 + 1]::text, - 'name', workspace_ids[cnt % 3 + 1]::text)), workspace_ids[cnt % 3 + 1], workspace_ids[cnt % 3 + 1]::text) returning id into new_id; insert into system_patch From 8f57ec8b46423cb8fabe921c8f9465d6ba8783aa Mon Sep 17 00:00:00 2001 From: TenSt Date: Fri, 12 Jun 2026 12:34:55 +0200 Subject: [PATCH 7/7] RHINENG-26548: update system_inventory database docs --- docs/md/database.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/md/database.md b/docs/md/database.md index fa191a700..254a90f40 100644 --- a/docs/md/database.md +++ b/docs/md/database.md @@ -2,7 +2,7 @@ ## Tables Main database tables description: -- **system_inventory** — Partitioned table for the registered host / inventory profile: internal `id`, Insights `inventory_id`, `rh_account_id`, `vmaas_json` (packages, repos, modules for VMaaS), `yum_updates` and related checksums, staleness and culling timestamps, `display_name`, OS fields, tags/workspaces, and workload flags. **`system_repo`** (and similar link tables) use this internal `id` as the system key. The **listener** upserts rows here and relies on **system_inventory** for upload locks and unchanged detection; the **evaluator** reads it via a join to **system_patch**. +- **system_inventory** — Partitioned table for the registered host / inventory profile: internal `id`, Insights `inventory_id`, `rh_account_id`, `vmaas_json` (packages, repos, modules for VMaaS), `yum_updates` and related checksums, staleness and culling timestamps, `display_name`, OS fields, tags, workspace fields, and workload flags. **`system_repo`** (and similar link tables) use this internal `id` as the system key. The **listener** upserts rows here and relies on **system_inventory** for upload locks and unchanged detection; the **evaluator** reads it via a join to **system_patch**. - **system_patch** — Partitioned evaluation output for each system, keyed by `rh_account_id` and `system_id` where `system_id` equals **system_inventory.id** on the same account. Holds advisory and package count caches, `last_evaluation`, `third_party`, `template_id`, and related aggregates. Rows are created or updated by the **listener** together with **system_inventory**; the **evaluator** persists evaluation results here (not into a single legacy table). - **advisory_metadata** - stores info about advisories (`description`, `summary`, `solution` etc.). It's synced and stored on trigger by `vmaas_sync` component. It allows to display detail information about the advisory. - **system_advisories** - stores info about advisories evaluated for particular systems (system - advisory M-N mapping table). `system_id` references **system_inventory.id**. Contains info when system advisory was firstly reported and patched (if so). Records are created and updated by `evaluator` component. It allows to display list of advisories related to a system.