",
- "
high-verified
",
+ "
verified
",
"
high and above
",
"
medium and above
",
"
low and above
",
@@ -359,9 +361,9 @@ var pageTemplate = strings.Join([]string{
" const normalize = function(value) { return (value || '').toLowerCase(); };",
" function updateNotificationStatus() { if (!('Notification' in window)) { enableNotificationsButton.hidden = true; notificationStatus.textContent = 'Browser notifications are not supported.'; return; } if (Notification.permission === 'granted') { enableNotificationsButton.hidden = true; notificationStatus.textContent = notificationThreshold.value === 'none' ? 'Notifications are off.' : 'Browser notifications enabled.'; return; } if (Notification.permission === 'denied') { enableNotificationsButton.hidden = true; notificationStatus.textContent = 'Notifications are blocked in browser settings.'; return; } enableNotificationsButton.hidden = notificationThreshold.value === 'none'; notificationStatus.textContent = notificationThreshold.value === 'none' ? 'Notifications are off.' : 'Allow notifications for matching findings.'; }",
" function requestNotificationPermission() { if (!('Notification' in window)) { updateNotificationStatus(); return; } try { Notification.requestPermission().then(updateNotificationStatus).catch(updateNotificationStatus); } catch (_) { updateNotificationStatus(); } }",
- " function meetsNotificationThreshold(confidence) { var threshold = notificationThreshold.value; if (threshold === 'none') return false; if (threshold === 'any') return true; var levels = ['high-verified', 'high', 'medium', 'low']; var findingLevel = levels.indexOf(normalize(confidence)); return findingLevel >= 0 && findingLevel <= levels.indexOf(threshold); }",
+ " function meetsNotificationThreshold(confidence) { var threshold = notificationThreshold.value; if (threshold === 'none') return false; if (threshold === 'any') return true; var levels = ['verified', 'high', 'medium', 'low']; var findingLevel = levels.indexOf(normalize(confidence)); return findingLevel >= 0 && findingLevel <= levels.indexOf(threshold); }",
" function closeActiveNotifications() { activeNotifications.forEach(function(notification) { notification.close(); }); activeNotifications = []; }",
- " function notifyFinding(item) { var findingTime = Number(item.timestampMillis); if (!state.notificationsReady || (findingTime > 0 && findingTime < notificationsSince) || !('Notification' in window) || Notification.permission !== 'granted' || !meetsNotificationThreshold(item.confidence)) return; try { var title = item.confidence === 'high-verified' ? 'Verified credential found' : 'Credential finding detected'; var notification = new Notification(title, { body: 'Confidence: ' + (item.confidence || 'unknown') + '. Open Pipeleek to review.' }); activeNotifications.push(notification); notification.onclose = function() { activeNotifications = activeNotifications.filter(function(openNotification) { return openNotification !== notification; }); }; notification.onclick = function() { window.focus(); notification.close(); }; } catch (_) {} }",
+ " function notifyFinding(item) { var findingTime = Number(item.timestampMillis); if (!state.notificationsReady || (findingTime > 0 && findingTime < notificationsSince) || !('Notification' in window) || Notification.permission !== 'granted' || !meetsNotificationThreshold(item.confidence)) return; try { var title = item.confidence === 'verified' ? 'Verified credential found' : 'Credential finding detected'; var notification = new Notification(title, { body: 'Confidence: ' + (item.confidence || 'unknown') + '. Open Pipeleek to review.' }); activeNotifications.push(notification); notification.onclose = function() { activeNotifications = activeNotifications.filter(function(openNotification) { return openNotification !== notification; }); }; notification.onclick = function() { window.focus(); notification.close(); }; } catch (_) {} }",
" function severityClass(value) { return String(value || 'unknown').toLowerCase().replace(/\\s+/g, '-'); }",
" function typeClass(value) { return String(value || 'log').toLowerCase().replace(/\\s+/g, '-'); }",
" function showToast(message) { toast.textContent = message; toast.classList.add('visible'); clearTimeout(showToast.timeout); showToast.timeout = setTimeout(function() { toast.classList.remove('visible'); }, 1200); }",
@@ -372,10 +374,10 @@ var pageTemplate = strings.Join([]string{
" return date.toLocaleString(undefined, { year: 'numeric', month: 'short', day: 'numeric', hour: '2-digit', minute: '2-digit', second: '2-digit', hour12: false });",
" }",
" function renderSummary() {",
- " var counts = { low: 0, medium: 0, high: 0, 'high-verified': 0 };",
+ " var counts = { low: 0, medium: 0, high: 0, verified: 0, trufflehog: 0 };",
" state.all.forEach(function(item) { var key = normalize(item.confidence || 'unknown'); if (key in counts) counts[key] += 1; });",
" summaryCards.replaceChildren();",
- " ['high-verified', 'high', 'medium', 'low'].forEach(function(key) { var metric = document.createElement('div'); metric.className = 'metric'; var label = document.createElement('label'); label.textContent = key.charAt(0).toUpperCase() + key.slice(1); var count = document.createElement('strong'); count.textContent = String(counts[key] || 0); metric.append(label, count); summaryCards.appendChild(metric); });",
+ " ['verified', 'high', 'medium', 'low', 'trufflehog'].forEach(function(key) { var metric = document.createElement('div'); metric.className = 'metric'; var label = document.createElement('label'); label.textContent = key.charAt(0).toUpperCase() + key.slice(1); if (key === 'trufflehog') label.title = 'TruffleHog detection without successful verification'; var count = document.createElement('strong'); count.textContent = String(counts[key] || 0); metric.append(label, count); summaryCards.appendChild(metric); });",
" }",
" function filteredFindings() {",
" var query = normalize(searchFilter.value);",
diff --git a/pkg/webui/server_test.go b/pkg/webui/server_test.go
index 5d6e6491..030ae35f 100644
--- a/pkg/webui/server_test.go
+++ b/pkg/webui/server_test.go
@@ -51,6 +51,30 @@ func TestHandleHitAssignsUniqueIDs(t *testing.T) {
}
}
+func TestHandleHitPreservesDetectionLevels(t *testing.T) {
+ for _, confidence := range []string{"verified", "trufflehog"} {
+ s := &Server{clients: make(map[chan string]struct{})}
+ s.handleHit(logging.HitRecord{
+ Time: time.Now(), Confidence: confidence, Value: "fixture",
+ Fields: map[string]interface{}{"engine": "trufflehog"},
+ })
+ if len(s.findings) != 1 || s.findings[0].Confidence != confidence || s.findings[0].Details["engine"] != "trufflehog" {
+ t.Fatal("expected the live finding to preserve its detection level and engine")
+ }
+ data, err := json.Marshal(s.findings[0])
+ if err != nil {
+ t.Fatal(err)
+ }
+ var decoded Finding
+ if err := json.Unmarshal(data, &decoded); err != nil {
+ t.Fatal(err)
+ }
+ if decoded.Confidence != confidence || decoded.Details["engine"] != "trufflehog" {
+ t.Fatal("expected SSE JSON to preserve detection metadata")
+ }
+ }
+}
+
func TestCompletedEventsReplayFindingsAndDone(t *testing.T) {
startedAt := time.Date(2026, time.October, 1, 12, 0, 0, 0, time.UTC)
s := &Server{token: "abc", clients: make(map[chan string]struct{}), startedAt: startedAt}
@@ -133,14 +157,20 @@ func TestHandleRootAllowsInlineScript(t *testing.T) {
if !strings.Contains(body, "Live secret findings") {
t.Fatalf("expected rendered page body, got %q", body)
}
+ if !strings.Contains(body, `data-value="trufflehog"`) || !strings.Contains(body, ".badge.trufflehog") || !strings.Contains(body, "trufflehog: 0") || !strings.Contains(body, "['verified', 'high', 'medium', 'low', 'trufflehog']") {
+ t.Fatal("expected a neutral TruffleHog candidate filter, badge and summary count")
+ }
+ if !strings.Contains(body, "var levels = ['verified', 'high', 'medium', 'low']") || strings.Contains(body, "high-verified") || strings.Contains(body, "trufflehog-unverified") {
+ t.Fatal("expected migrated notification ranking without ranking TruffleHog candidates as verified")
+ }
if !strings.Contains(body, "No findings yet. Results will appear here as the scan runs.") || !strings.Contains(body, "No findings were detected.") || !strings.Contains(body, "No findings match the current filters.") {
t.Fatal("expected distinct empty states for an active scan, a completed scan, and filtered results")
}
if !strings.Contains(body, "id=\"elapsed-time\"") || !strings.Contains(body, "var elapsedBase = ") || !strings.Contains(body, "window.setInterval(updateElapsedTime, 1000)") {
t.Fatal("expected elapsed timer element, server-provided baseline, and periodic updates")
}
- if !strings.Contains(body, `id="notification-threshold-picker"`) || !strings.Contains(body, `aria-multiselectable="false"`) || !strings.Contains(body, `data-value="high-verified" aria-selected="true"`) || !strings.Contains(body, `data-value="none" aria-selected="false"`) || !strings.Contains(body, `class="badge high-verified">high-verified`) || !strings.Contains(body, `id="enable-notifications"`) {
- t.Fatal("expected single-select badge picker with high-verified default, None option, and permission fallback control")
+ if !strings.Contains(body, `id="notification-threshold-picker"`) || !strings.Contains(body, `aria-multiselectable="false"`) || !strings.Contains(body, `data-value="verified" aria-selected="true"`) || !strings.Contains(body, `data-value="none" aria-selected="false"`) || !strings.Contains(body, `class="badge verified">verified`) || !strings.Contains(body, `id="enable-notifications"`) {
+ t.Fatal("expected single-select badge picker with verified default, None option, and permission fallback control")
}
if strings.Index(body, `id="notification-status"`) < strings.Index(body, `id="notification-threshold-picker"`) {
t.Fatal("expected notification status to appear after its threshold picker")
@@ -151,8 +181,8 @@ func TestHandleRootAllowsInlineScript(t *testing.T) {
if !strings.Contains(body, "notificationsSince = Date.now(); closeActiveNotifications()") || !strings.Contains(body, "Number(item.timestampMillis)") || !strings.Contains(body, "findingTime < notificationsSince") || !strings.Contains(body, "function closeActiveNotifications()") {
t.Fatal("expected threshold changes to close and suppress already-queued notifications")
}
- if !strings.Contains(body, `id="notification-threshold" type="hidden" value="high-verified"`) || strings.Contains(body, "document.cookie") || strings.Contains(body, "localStorage") {
- t.Fatal("expected the notification threshold to reset to high-verified for each new WebUI page")
+ if !strings.Contains(body, `id="notification-threshold" type="hidden" value="verified"`) || strings.Contains(body, "document.cookie") || strings.Contains(body, "localStorage") {
+ t.Fatal("expected the notification threshold to reset to verified for each new WebUI page")
}
if strings.Contains(body, "tableBody.innerHTML") || !strings.Contains(body, "function appendDetailValue") || !strings.Contains(body, "parsed.protocol === 'http:' || parsed.protocol === 'https:'") {
t.Fatal("expected finding values to render via DOM text APIs with HTTP(S)-only links")
@@ -169,7 +199,7 @@ func TestHandleRootAllowsInlineScript(t *testing.T) {
if !strings.Contains(body, ".filters { display: flex; flex-wrap: wrap; gap: .75rem; align-items: flex-start; }") || !strings.Contains(body, ".filters > .control-btn { align-self: flex-start; margin-top: 1.5rem; }") {
t.Fatal("expected filter labels to align at the top while action buttons align with the controls")
}
- if strings.Count(body, `aria-multiselectable="true"`) != 2 || !strings.Contains(body, `class="badge high-verified">high-verified`) {
+ if strings.Count(body, `aria-multiselectable="true"`) != 2 || !strings.Contains(body, `class="badge verified">verified`) {
t.Fatal("expected severity and type filters to render badge-style multi-select options")
}
if !strings.Contains(body, `id="toggle-full-width"`) || !strings.Contains(body, "main.main-wide") {
@@ -305,7 +335,7 @@ func TestExportCSVPreservesSpecialValues(t *testing.T) {
{
Time: "2026-10-01T12:34:56Z",
Type: "log",
- Confidence: "high",
+ Confidence: "trufflehog",
RuleName: `rule, with "quotes"`,
Value: "first line,\r\nsecond line \"quoted\" 雪",
Details: map[string]string{
@@ -316,7 +346,7 @@ func TestExportCSVPreservesSpecialValues(t *testing.T) {
{
Time: "2026-10-01T12:35:00Z",
Type: "dotenv",
- Confidence: "low",
+ Confidence: "verified",
RuleName: "formula value",
Value: "\t=1+1",
},
@@ -345,6 +375,9 @@ func TestExportCSVPreservesSpecialValues(t *testing.T) {
if len(records) != 3 {
t.Fatalf("expected header and two findings, got %d records", len(records))
}
+ if records[1][0] != "trufflehog" || records[2][0] != "verified" {
+ t.Fatal("expected CSV export to preserve the detection levels")
+ }
if got, want := records[1][1], findings[0].RuleName; got != want {
t.Errorf("rule name mismatch: got %q, want %q", got, want)
}
diff --git a/tests/e2e/github/scan/local_rules_test.go b/tests/e2e/github/scan/local_rules_test.go
new file mode 100644
index 00000000..8b60bb7f
--- /dev/null
+++ b/tests/e2e/github/scan/local_rules_test.go
@@ -0,0 +1,143 @@
+package e2e
+
+import (
+ "archive/zip"
+ "bytes"
+ "encoding/json"
+ "net/http"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/CompassSecurity/pipeleek/tests/e2e/internal/testutil"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestGitHubScan_LocalRulesAndLevels(t *testing.T) {
+ require.True(t, filepath.IsAbs(os.Getenv("PIPELEEK_BINARY")), "Use an absolute PIPELEEK_BINARY for isolated working-directory tests")
+ dir := t.TempDir()
+ t.Chdir(dir)
+ archive := func(name, content string) []byte {
+ var buffer bytes.Buffer
+ writer := zip.NewWriter(&buffer)
+ file, err := writer.Create(name)
+ require.NoError(t, err)
+ _, err = file.Write([]byte(content))
+ require.NoError(t, err)
+ require.NoError(t, writer.Close())
+ return buffer.Bytes()
+ }
+ logs := archive("job.log", "GITLAB_USER_ID=19357\nGITLAB_TOKEN=glpat-abcdefghijklmnopqrst\nPIPELEEK_LOG_FIXTURE\n")
+ artifact := archive("settings.txt", "KUBECONFIG=/tmp/local-config\nPIPELEEK_ARTIFACT_FIXTURE\n")
+ server, requests, cleanup := testutil.StartMockServerWithRecording(t, func(w http.ResponseWriter, r *http.Request) {
+ switch r.URL.Path {
+ case "/api/v3/user/repos":
+ testutil.JSON(w, http.StatusOK, []map[string]interface{}{{
+ "id": 1, "name": "test-repo", "full_name": "user/test-repo",
+ "html_url": "https://example.test/user/test-repo",
+ "owner": map[string]interface{}{"login": "user"},
+ }})
+ case "/api/v3/repos/user/test-repo/actions/runs":
+ testutil.JSON(w, http.StatusOK, map[string]interface{}{
+ "total_count": 1,
+ "workflow_runs": []map[string]interface{}{{
+ "id": 100, "name": "Local rules", "status": "completed", "display_title": "Local rules",
+ "html_url": "https://example.test/runs/100",
+ "repository": map[string]interface{}{"name": "test-repo", "owner": map[string]interface{}{"login": "user"}},
+ }},
+ })
+ case "/api/v3/repos/user/test-repo/actions/runs/100/jobs":
+ testutil.JSON(w, http.StatusOK, map[string]interface{}{"jobs": []map[string]interface{}{{"id": 1000, "name": "fixture-job"}}, "total_count": 1})
+ case "/api/v3/repos/user/test-repo/actions/runs/100/logs":
+ w.Header().Set("Location", "http://"+r.Host+"/download/logs.zip")
+ w.WriteHeader(http.StatusFound)
+ case "/download/logs.zip":
+ w.Header().Set("Content-Type", "application/zip")
+ _, _ = w.Write(logs)
+ case "/api/v3/repos/user/test-repo/actions/runs/100/artifacts":
+ testutil.JSON(w, http.StatusOK, map[string]interface{}{
+ "artifacts": []map[string]interface{}{{
+ "id": 1, "name": "fixture-output",
+ "archive_download_url": "http://" + r.Host + "/api/v3/repos/user/test-repo/actions/artifacts/1/zip",
+ }}, "total_count": 1,
+ })
+ case "/api/v3/repos/user/test-repo/actions/artifacts/1/zip":
+ w.Header().Set("Location", "http://"+r.Host+"/download/artifact.zip")
+ w.WriteHeader(http.StatusFound)
+ case "/download/artifact.zip":
+ w.Header().Set("Content-Type", "application/zip")
+ _, _ = w.Write(artifact)
+ default:
+ testutil.JSON(w, http.StatusOK, map[string]interface{}{})
+ }
+ })
+ defer cleanup()
+
+ run := func(confidence string) []map[string]interface{} {
+ t.Helper()
+ args := []string{"gh", "scan", "--url", server.URL, "--token", "fixture-token", "--owned", "--artifacts", "--secrets-verification=false", "--json", "--verbose"}
+ if confidence != "" {
+ args = append(args, "--confidence="+confidence)
+ }
+ stdout, stderr, err := testutil.RunCLI(t, args, nil, time.Minute)
+ require.NoError(t, err, stdout+stderr)
+ var findings []map[string]interface{}
+ for _, line := range strings.Split(stdout+"\n"+stderr, "\n") {
+ var record map[string]interface{}
+ if json.Unmarshal([]byte(line), &record) == nil && record["message"] == "SECRET" {
+ findings = append(findings, record)
+ }
+ }
+ return findings
+ }
+ has := func(findings []map[string]interface{}, confidence, value string) bool {
+ for _, finding := range findings {
+ text, ok := finding["value"].(string)
+ if ok && finding["confidence"] == confidence && strings.Contains(text, value) {
+ return true
+ }
+ }
+ return false
+ }
+ findings := run("")
+ assert.True(t, has(findings, "medium", "GITLAB_USER_ID=19357"), "Retain the identifier rule without a local rules file")
+ assert.True(t, has(findings, "medium", "KUBECONFIG=/tmp/local-config"), "Retain path matching in artifacts")
+ assert.True(t, has(findings, "trufflehog", "glpat-abcdefghijklmnopqrst"), "Offline TruffleHog candidates use the new level")
+ _, err := os.Stat(filepath.Join(dir, "rules.yml"))
+ assert.True(t, os.IsNotExist(err), "Scanning must not create or download rules")
+ assert.Empty(t, run("verified"), "Offline fixtures must not become verified")
+ candidates := run("trufflehog")
+ require.NotEmpty(t, candidates)
+ for _, finding := range candidates {
+ assert.Equal(t, "trufflehog", finding["confidence"])
+ assert.Equal(t, "trufflehog", finding["engine"])
+ }
+
+ custom := []byte(`patterns:
+ - pattern:
+ name: Gitlab - Predefined Environment Variable
+ regex: PIPELEEK_(LOG|ARTIFACT)_FIXTURE
+ confidence: custom-confidence
+`)
+ require.NoError(t, os.WriteFile(filepath.Join(dir, "rules.yml"), custom, 0600))
+ findings = run("custom-confidence,medium")
+ assert.True(t, has(findings, "custom-confidence", "PIPELEEK_LOG_FIXTURE"))
+ assert.True(t, has(findings, "custom-confidence", "PIPELEEK_ARTIFACT_FIXTURE"))
+ for _, finding := range findings {
+ assert.Equal(t, "custom-confidence", finding["confidence"], "The overridden variable rule must not emit medium findings")
+ }
+ unchanged, err := os.ReadFile(filepath.Join(dir, "rules.yml"))
+ require.NoError(t, err)
+ assert.Equal(t, custom, unchanged)
+ assert.NotEmpty(t, requests())
+
+ require.NoError(t, os.WriteFile(filepath.Join(dir, "rules.yml"), []byte("patterns:\n- pattern: {name: InvalidFixture, regex: '[', confidence: high}\n"), 0600))
+ stdout, stderr, err := testutil.RunCLI(t, []string{"gh", "scan", "--url", server.URL, "--token", "fixture-token", "--secrets-verification=false"}, nil, time.Minute)
+ require.Error(t, err)
+ assert.Contains(t, stdout+stderr, "InvalidFixture")
+ assert.Contains(t, stdout+stderr, "invalid regex")
+ assert.Contains(t, stdout+stderr, "Failed loading rules")
+}
diff --git a/tests/e2e/rules/download_test.go b/tests/e2e/rules/download_test.go
new file mode 100644
index 00000000..0131a1ae
--- /dev/null
+++ b/tests/e2e/rules/download_test.go
@@ -0,0 +1,98 @@
+package rules
+
+import (
+ "io"
+ "net"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "testing"
+ "time"
+
+ "github.com/CompassSecurity/pipeleek/tests/e2e/internal/testutil"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+ "gopkg.in/yaml.v3"
+)
+
+func TestRulesDownload(t *testing.T) {
+ require.True(t, filepath.IsAbs(os.Getenv("PIPELEEK_BINARY")))
+ t.Chdir(t.TempDir())
+ const fixture = "patterns:\n - pattern:\n name: Fixture\n regex: FIXTURE_[A-Z]+\n confidence: high\n"
+ backend := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ assert.Equal(t, "/mazen160/secrets-patterns-db/master/db/rules-stable.yml", r.URL.Path)
+ _, _ = w.Write([]byte(fixture))
+ }))
+ defer backend.Close()
+ proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if !assert.Equal(t, http.MethodConnect, r.Method) || !assert.Equal(t, "raw.githubusercontent.com:443", r.Host) {
+ w.WriteHeader(http.StatusBadRequest)
+ return
+ }
+ upstream, err := net.DialTimeout("tcp", backend.Listener.Addr().String(), 5*time.Second)
+ if !assert.NoError(t, err) {
+ w.WriteHeader(http.StatusBadGateway)
+ return
+ }
+ defer upstream.Close()
+ conn, buffer, err := w.(http.Hijacker).Hijack()
+ if !assert.NoError(t, err) {
+ return
+ }
+ defer conn.Close()
+ _, err = conn.Write([]byte("HTTP/1.1 200 Connection Established\r\n\r\n"))
+ if !assert.NoError(t, err) {
+ return
+ }
+ go func() { _, _ = io.Copy(upstream, buffer) }()
+ _, _ = io.Copy(conn, upstream)
+ }))
+ defer proxy.Close()
+ run := func(extra ...string) (string, error) {
+ t.Helper()
+ args := append([]string{"rules", "download", "--proxy", proxy.URL, "--http-timeout", "10s"}, extra...)
+ stdout, stderr, err := testutil.RunCLI(t, args, nil, 30*time.Second)
+ return stdout + stderr, err
+ }
+ output, err := run()
+ require.NoError(t, err, output)
+ assert.Contains(t, output, "Downloaded optional rules")
+ data, err := os.ReadFile("rules.yml")
+ require.NoError(t, err)
+ require.Equal(t, fixture, string(data))
+
+ require.NoError(t, os.WriteFile("rules.yml", []byte("original"), 0600))
+ output, err = run()
+ require.Error(t, err, output)
+ assert.Contains(t, output, "--force")
+ data, err = os.ReadFile("rules.yml")
+ require.NoError(t, err)
+ require.Equal(t, "original", string(data))
+
+ output, err = run("--force")
+ require.NoError(t, err, output)
+ data, err = os.ReadFile("rules.yml")
+ require.NoError(t, err)
+ require.Equal(t, fixture, string(data))
+
+ output, err = run("--output", "custom.yml")
+ require.NoError(t, err, output)
+ data, err = os.ReadFile("custom.yml")
+ require.NoError(t, err)
+ require.Equal(t, fixture, string(data))
+
+ stdout, stderr, err := testutil.RunCLI(t, []string{"config", "gen"}, nil, 30*time.Second)
+ require.NoError(t, err, stderr)
+ var generated struct {
+ Rules struct {
+ Download struct {
+ Output string
+ Force bool
+ }
+ }
+ }
+ require.NoError(t, yaml.Unmarshal([]byte(stdout), &generated))
+ require.Equal(t, "rules.yml", generated.Rules.Download.Output)
+ require.False(t, generated.Rules.Download.Force)
+}
From 93093ffaae08b4672eb176676b50781e8cff02f8 Mon Sep 17 00:00:00 2001
From: frjcomp <107982661+frjcomp@users.noreply.github.com>
Date: Mon, 19 Oct 2026 08:27:23 +0000
Subject: [PATCH 2/5] fix: identify CI YAML findings and streamline scanner
docs
Report merged GitLab CI configuration findings as cicd-yaml and link to the CI YAML file. Add the source type to the web UI and regression tests. Consolidate rules documentation and update ELK verified-confidence queries.
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
docs/guides/scanning.md | 4 +-
docs/introduction/configuration.md | 15 +---
docs/introduction/secrets_verification.md | 95 ++++++++---------------
examples/compose-elk/README.md | 2 +-
examples/compose-elk/setup-kibana.sh | 14 ++--
pkg/gitlab/scan/pipeline.go | 2 +-
pkg/gitlab/scan/pipeline_test.go | 6 +-
pkg/gitlab/scan/queue.go | 1 +
pkg/logging/hit.go | 3 +-
pkg/webui/server.go | 2 +
pkg/webui/server_test.go | 3 +
tests/e2e/gitlab/cicd/scan/scan_test.go | 12 ++-
12 files changed, 70 insertions(+), 89 deletions(-)
diff --git a/docs/guides/scanning.md b/docs/guides/scanning.md
index 1b22dae8..c4ec8240 100644
--- a/docs/guides/scanning.md
+++ b/docs/guides/scanning.md
@@ -29,7 +29,9 @@ pipeleek gl scan -u https://gitlab.com -t glpat-[redacted] --cookie [redacted] -
2025-09-30T09:53:37Z hit SECRET confidence=high type=artifact file=an_artifact.txt jobName=artifact-job ruleName="Generic - 1719" url=gitlab.com/testgroup/project/-/jobs/11484162833 value="datadog_api_key=secret_artifact_value "
```
-As shown, Pipeleek can detect secrets in job logs and build artifacts using its embedded/custom YAML patterns, Betterleaks rules, and TruffleHog detectors. The example above uses custom patterns; the external pattern database is no longer downloaded. Security findings are logged at the custom `hit` level to distinguish them from regular warnings. Manually review the hits to verify if they're valid credentials. If you see `confidence=verified`, TruffleHog, Betterleaks, or the GitLab detector confirmed the credential with its provider; enabling `--secrets-verification` can make outbound validation requests. With verification enabled, TruffleHog reports only successfully verified findings. With `--secrets-verification=false`, returned non-verified TruffleHog candidates are reported as `confidence=trufflehog`. Use `--confidence=verified` to show only successfully verified credentials across all engines.
+Findings use the `hit` log level. The sample output above uses additional YAML rules; your findings depend on the rules and credentials present. Review non-verified hits manually, or use `--confidence=verified` to show only successfully verified credentials. See [detection, verification, and rules](../introduction/secrets_verification.md) for reporting behavior and optional rule downloads.
+
+Job-log (`type=log`) and artifact (`type=archive`) findings link to the job. Findings labeled `.gitlab-ci.yml` use `type=cicd-yaml` and link to that file on the default branch. The scanned content is GitLab's merged CI configuration, so a match may originate from an included file.
### Live Findings Web UI
diff --git a/docs/introduction/configuration.md b/docs/introduction/configuration.md
index c5157d94..3874dae8 100644
--- a/docs/introduction/configuration.md
+++ b/docs/introduction/configuration.md
@@ -79,16 +79,11 @@ common:
`secrets_verification` replaces the deprecated `trufflehog_verification` key. The old key (and `PIPELEEK_COMMON_TRUFFLEHOG_VERIFICATION`) is still read, with a warning, when the new key is not set.
-With verification enabled, TruffleHog reports only successfully verified findings as `verified`; non-verified candidates, including verification errors, are suppressed. With verification disabled, returned non-verified candidates use `trufflehog`. Confidence filtering applies to every engine. Migrate old filters from `high-verified` to `verified` and `trufflehog-unverified` to `trufflehog`; these old names are not aliases.
-
-The 15 Pipeleek built-in rules are embedded locally, including identifier/path rules. Optional working-directory `rules.yml` adds custom rules or overrides built-ins by exact name in the YAML engine. No rules are downloaded or written automatically; existing files remain unchanged. See [secret verification and custom rules](secrets_verification.md) for details.
+See [verification and confidence filtering](secrets_verification.md#verification) for engine-specific reporting behavior and migration notes.
### Optional Rules Download
-`pipeleek rules download` explicitly downloads the Secrets Patterns Database stable
-ruleset using Pipeleek's HTTP client (including global proxy, TLS, and timeout settings).
-These additional regex rules may increase false positives and do not verify credentials.
-Scans never download rules automatically.
+Settings for [`pipeleek rules download`](secrets_verification.md#optional-external-rules-download):
```yaml
rules:
@@ -97,14 +92,8 @@ rules:
force: false # --force; PIPELEEK_RULES_DOWNLOAD_FORCE
```
-By default, existing files are not overwritten. Use `--force` to replace one.
-`--output` changes the download destination only; scans still load `rules.yml`
-from their working directory.
-
### Per-command Overrides
-Override per-command:
-
```yaml
gitlab:
scan:
diff --git a/docs/introduction/secrets_verification.md b/docs/introduction/secrets_verification.md
index 6b1a856b..882034bf 100644
--- a/docs/introduction/secrets_verification.md
+++ b/docs/introduction/secrets_verification.md
@@ -1,6 +1,6 @@
---
-title: Secret Verification with TruffleHog
-description: Learn how Pipeleek uses TruffleHog to automatically verify detected secrets, understand confidence levels, and how to disable verification for operational security.
+title: Secret Detection, Verification, and Rules
+description: Understand Pipeleek's detection engines, confidence levels, custom rules, and optional rule downloads.
keywords:
- secret verification
- TruffleHog
@@ -12,50 +12,44 @@ keywords:
- opsec
---
-Pipeleek uses [TruffleHog](https://github.com/trufflesecurity/trufflehog) and [Betterleaks](https://github.com/betterleaks/betterleaks) to detect secrets in CI/CD logs and artifacts. When verification is enabled, both engines can check supported credentials against their providers.
+# Secret Detection, Verification, and Rules
-### How It Works
+Pipeleek scans CI/CD logs and artifacts using:
-When Pipeleek scans logs or artifacts, it uses three detection sources:
+- **YAML rules:** 15 embedded Pipeleek rules plus optional working-directory `rules.yml`.
+- **[Betterleaks](https://github.com/betterleaks/betterleaks):** embedded rules with false-positive filtering.
+- **[TruffleHog](https://github.com/trufflesecurity/trufflehog):** specialized credential detectors.
+- **GitLab URL-aware detector:** checks supported tokens against your configured GitLab instance.
-1. **Pattern-based detection**: 15 embedded Pipeleek YAML rules plus optional custom rules from local `rules.yml`. Scans never download rules automatically.
-2. **Betterleaks rules**: Embedded default rules with keyword, path, capture, and false-positive filtering
-3. **TruffleHog detectors**: Specialized detectors with active verification
+## Verification
-The `--secrets-verification` option controls provider verification for both TruffleHog and Betterleaks. With verification enabled, either engine may make outbound requests to credential providers for supported credentials. Disable verification to scan without those provider checks; findings are still reported, but their confidence depends on the detection source:
+Verification is enabled by default. `--secrets-verification` controls provider checks for TruffleHog, Betterleaks, and the GitLab detector. These checks can send detected credentials to provider APIs.
-- With verification enabled, TruffleHog reports only successfully verified hits as `verified`. Other candidates, including those with verification errors, are suppressed. With verification disabled, returned non-verified candidates are reported as `trufflehog`.
-- Betterleaks reports a validation result of `valid` as `verified`. Without verification, all Betterleaks detections are reported at their rule confidence (`high`, `medium`, or `low`); Betterleaks has no separate unverified confidence level.
-- The GitLab URL-aware detector uses `verified` for successful verification against your configured instance and `high` for retained non-verified detections. Existing GitLab rejection behavior and Betterleaks invalid/revoked suppression are unchanged.
+| Source | Verification enabled | Verification disabled |
+| ------ | -------------------- | --------------------- |
+| TruffleHog | Only successfully verified findings (`verified`) | Returned candidates (`trufflehog`) |
+| Betterleaks | Valid credentials use `verified`; invalid/revoked results are suppressed; other retained findings use rule confidence | Rule confidence (`high`, `medium`, or `low`) |
+| GitLab detector | Verified credentials use `verified`; retained non-verified findings use `high` | Retained findings use `high` |
+| YAML rules | Rule confidence; no provider checks | Unchanged |
-Disabling verification exposes more TruffleHog candidates and may mean more false positives; `trufflehog` indicates a detection source, not a confidence ranking or a working credential. A failed verification request does not prove that a credential is invalid, but the candidate is suppressed when verification is enabled.
+A verification error does not prove a credential is invalid. TruffleHog findings with verification errors are suppressed when verification is enabled. Disabling verification can expose more false positives.
-### Confidence Levels
-
-Pipeleek assigns confidence levels to all detected secrets:
-
-| Level | Source | Description | Verified |
-| ----- | ------ | ----------- | -------- |
-| **verified** | TruffleHog, Betterleaks, GitLab detector | Successfully verified against a provider | Yes |
-| **trufflehog** | TruffleHog | Non-verified detection reported when verification is disabled | No |
-| **high** | Embedded/custom YAML, Betterleaks, GitLab detector | High confidence pattern match | No |
-| **medium** | Embedded/custom YAML, Betterleaks | Medium confidence pattern match | No |
-| **low** | Custom YAML, Betterleaks | Low confidence pattern match | No |
-| **custom** | Custom YAML | User-defined confidence label | No |
-
-### Disabling Verification
-
-For operational security (OpSec) or privacy, disable verification to prevent TruffleHog and Betterleaks from sending credential checks to provider APIs.
-
-Use the `--secrets-verification=false` flag:
+To skip provider checks:
```bash
pipeleek gl scan -u https://gitlab.com -t glpat-[redacted] --secrets-verification=false
```
-### Confidence Filtering
+## Confidence and Filtering
-Results from every engine can be filtered using the `--confidence` flag. An empty filter reports all levels. Filtering does not disable provider verification.
+| Level | Meaning |
+| ----- | ------- |
+| `verified` | Successfully verified credential |
+| `trufflehog` | Non-verified TruffleHog candidate; not a confidence ranking |
+| `high`, `medium`, `low` | Pattern confidence, not proof that a credential works |
+| User-defined label | Confidence assigned by a custom YAML rule |
+
+`--confidence` filters results from every engine. An empty filter includes all levels reported under the verification policy above. Filtering does not disable provider checks.
```bash
# Higher-confidence findings, including verified credentials
@@ -72,8 +66,7 @@ pipeleek gl scan -u https://gitlab.com -t glpat-[redacted] --secrets-verificatio
### Optional external rules download
-To explicitly install the Secrets Patterns Database stable ruleset from
-`mazen160/secrets-patterns-db`, run:
+To install the [Secrets Patterns Database](https://github.com/mazen160/secrets-patterns-db) stable ruleset:
```bash
pipeleek rules download
@@ -98,44 +91,24 @@ files unchanged.
### Local customization
-The built-in definitions live in `pkg/scanner/rules/rules.yaml` and are embedded in the binary. They include GitLab token patterns and the predefined-variable rule, which intentionally also matches identifiers and paths such as `GITLAB_USER_ID` and `KUBECONFIG`. These matches are indicators to review, not proof of a leaked credential.
+Built-in rules include GitLab token patterns and predefined variables such as `GITLAB_USER_ID` and `KUBECONFIG`. Identifier/path matches are indicators to review, not proof of a leaked credential.
-To add a rule, create `rules.yml` in your working directory. Pipeleek neither downloads nor creates this file. A rule with the same exact name as a built-in overrides that built-in in the YAML detection engine; other names add rules. Overrides do not alter the separate GitLab verifier or other engines.
+Create `rules.yml` in your working directory to add rules. An exact-name match overrides a built-in in the YAML engine; other names add rules. Overrides do not affect the separate GitLab detector or other engines. Without this file, scans use the embedded defaults.
-For example:
-
-```yaml
-patterns:
- - pattern:
- name: AWS API Key
- regex: AKIA[0-9A-Z]{16}
- confidence: high
-```
-
-Each rule requires a nonempty name, valid Go regex, and nonempty confidence label. Duplicate names within one file, invalid YAML/regexes, and unreadable files fail initialization with an error. A missing file is normal and uses embedded defaults. Regexes are compiled during initialization rather than for each scanned item.
-
-> **💡Tip:** Test your regexes at [regex101.com](https://regex101.com/) (select Golang flavor).
-
-A simple example that detects strings that follow the Regex pattern `PIPELEEK_.*` and that are logged with a custom confidence:
+For example, to report `PIPELEEK_` markers with a custom confidence:
```yaml
patterns:
- pattern:
name: Pipeleek Custom Rule
- regex: PIPELEEK_.*
+ regex: PIPELEEK_[A-Z0-9_]+
confidence: custom-confidence
```
-When you run Pipeleek, you'll see results for your custom rule and any built-in rules:
-
-```bash
-pipeleek gl scan -u https://gitlab.com -t glpat-[redacted] --secrets-verification=false --verbose
-2025-09-30T11:39:08Z hit SECRET confidence=custom-confidence type=log jobName=build-job-hidden ruleName="Pipeleek Custom Rule" url=gitlab.com/testgroup/project/-/jobs/11547853360 value="PIPELEEK_HIT=secret"
-```
+Each rule needs a nonempty name, valid Go regex, and nonempty confidence label. Duplicate names, invalid YAML/regexes, and unreadable files fail initialization.
## Migration
- Update scripts and confidence filters: `high-verified` is now `verified`; `trufflehog-unverified` is now `trufflehog`. Legacy filter names are not aliases. Custom YAML confidence labels remain user-defined.
- Existing local `rules.yml` files are preserved and loaded as custom rules. A previously downloaded Secrets Patterns Database will therefore still contribute its patterns. Back up and remove or curate that file if you want only the 15 embedded defaults.
-- Automatic rule downloads and the old Go helpers (`rules.DownloadRules` and `scanner.DownloadRules`) have been removed. Use `pipeleek rules download` for an explicit external rules download. Go callers should initialize the embedded/local rules with `InitRules`.
-- The live UI includes a `trufflehog` filter and count. These candidates are outside the verified/high severity ranking; the existing “any” notification threshold includes them.
+- Scans no longer download rules automatically. Use the optional download command above.
diff --git a/examples/compose-elk/README.md b/examples/compose-elk/README.md
index a4be3455..db3e746d 100644
--- a/examples/compose-elk/README.md
+++ b/examples/compose-elk/README.md
@@ -80,7 +80,7 @@ Once you've run `bash start-elk.sh`, the Kibana data view is automatically confi
3. You will see:
- `Pipeleek - High Hits` (count of `scan.confidence: high` hits)
-- `Pipeleek - High-Verified Hits` (count of `scan.confidence: high-verified` hits)
+- `Pipeleek - Verified Hits` (count of `scan.confidence: verified` hits)
- `Pipeleek - Log Intake Over Time` (bar chart over the last 4 hours with fixed 10-minute buckets)
- `Pipeleek - Hits by Asset Type` (pie chart, for example hit logs vs hit artifacts)
- `Pipeleek - Hits by Confidence` (pie chart of confidence ratings)
diff --git a/examples/compose-elk/setup-kibana.sh b/examples/compose-elk/setup-kibana.sh
index 130c8d51..b9129edb 100755
--- a/examples/compose-elk/setup-kibana.sh
+++ b/examples/compose-elk/setup-kibana.sh
@@ -210,12 +210,12 @@ high_metric_payload='{
}'
upsert_saved_object "visualization" "pipeleek-vis-high-count" "$high_metric_payload"
-echo "Creating metric visualization for high-verified hits..."
+echo "Creating metric visualization for verified hits..."
high_verified_metric_payload='{
"attributes": {
- "title": "Pipeleek - High-Verified Hits",
- "description": "Count of hit events with high-verified confidence",
- "visState": "{\"title\":\"Pipeleek - High-Verified Hits\",\"type\":\"metrics\",\"params\":{\"id\":\"pipeleek-high-verified-panel\",\"type\":\"metric\",\"series\":[{\"id\":\"pipeleek-high-verified-series\",\"split_mode\":\"everything\",\"split_color_mode\":\"value\",\"metrics\":[{\"id\":\"pipeleek-high-verified-count\",\"type\":\"count\"}],\"label\":\"High-Verified Hits\",\"color\":\"#D4821E\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":\"1\",\"point_size\":\"1\",\"fill\":\"0.5\",\"value_template\":\"{{value}}\",\"color_rules\":[{\"id\":\"pipeleek-high-verified-color-rule\",\"operator\":\"gte\",\"value\":0,\"color\":\"#D4821E\"}],\"background_color_rules\":[{\"id\":\"pipeleek-high-verified-bg-rule\",\"operator\":\"gte\",\"value\":0,\"color\":\"#D4821E\"}],\"gauge_color_rules\":[{\"id\":\"pipeleek-high-verified-gauge-rule\",\"operator\":\"gte\",\"value\":0,\"color\":\"#D4821E\"}]}],\"time_field\":\"@timestamp\",\"index_pattern\":\"pipeleek-logs-*\",\"default_index_pattern\":\"pipeleek-logs-*\",\"default_timefield\":\"@timestamp\",\"filter\":{\"language\":\"kuery\",\"query\":\"level:hit and confidence:\\\"high-verified\\\"\"},\"interval\":\"auto\",\"time_range_mode\":\"entire_time_range\",\"isModelInvalid\":false}}",
+ "title": "Pipeleek - Verified Hits",
+ "description": "Count of hit events with verified confidence",
+ "visState": "{\"title\":\"Pipeleek - Verified Hits\",\"type\":\"metrics\",\"params\":{\"id\":\"pipeleek-high-verified-panel\",\"type\":\"metric\",\"series\":[{\"id\":\"pipeleek-high-verified-series\",\"split_mode\":\"everything\",\"split_color_mode\":\"value\",\"metrics\":[{\"id\":\"pipeleek-high-verified-count\",\"type\":\"count\"}],\"label\":\"Verified Hits\",\"color\":\"#D4821E\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":\"1\",\"point_size\":\"1\",\"fill\":\"0.5\",\"value_template\":\"{{value}}\",\"color_rules\":[{\"id\":\"pipeleek-high-verified-color-rule\",\"operator\":\"gte\",\"value\":0,\"color\":\"#D4821E\"}],\"background_color_rules\":[{\"id\":\"pipeleek-high-verified-bg-rule\",\"operator\":\"gte\",\"value\":0,\"color\":\"#D4821E\"}],\"gauge_color_rules\":[{\"id\":\"pipeleek-high-verified-gauge-rule\",\"operator\":\"gte\",\"value\":0,\"color\":\"#D4821E\"}]}],\"time_field\":\"@timestamp\",\"index_pattern\":\"pipeleek-logs-*\",\"default_index_pattern\":\"pipeleek-logs-*\",\"default_timefield\":\"@timestamp\",\"filter\":{\"language\":\"kuery\",\"query\":\"level:hit and confidence:\\\"verified\\\"\"},\"interval\":\"auto\",\"time_range_mode\":\"entire_time_range\",\"isModelInvalid\":false}}",
"uiStateJSON": "{}",
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[]}"
@@ -272,7 +272,7 @@ ratings_pie_payload='{
"title": "Pipeleek - Hits by Confidence",
"description": "Hit count grouped by confidence rating",
"visState": "{\"title\":\"Pipeleek - Hits by Confidence\",\"type\":\"pie\",\"params\":{\"addLegend\":true,\"addTooltip\":true,\"isDonut\":true,\"legendPosition\":\"right\",\"labels\":{\"show\":true,\"values\":true}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"confidence\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"missingBucket\":true,\"missingBucketLabel\":\"unknown\"}}]}",
- "uiStateJSON": "{\"vis\":{\"colors\":{\"high-verified\":\"#D4821E\",\"high\":\"#2A4A25\",\"medium\":\"#B9CF7E\",\"low\":\"#8B6BA8\",\"warning\":\"#C4A035\",\"informational\":\"#B9CF7E\",\"unknown\":\"#4A8FA8\"}}}",
+ "uiStateJSON": "{\"vis\":{\"colors\":{\"verified\":\"#D4821E\",\"high\":\"#2A4A25\",\"medium\":\"#B9CF7E\",\"low\":\"#8B6BA8\",\"warning\":\"#C4A035\",\"informational\":\"#B9CF7E\",\"unknown\":\"#4A8FA8\"}}}",
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"query\":{\"language\":\"kuery\",\"query\":\"level:hit\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"
}
@@ -293,7 +293,7 @@ confidence_timeseries_payload='{
"title": "Pipeleek - Hit Confidence Over Time",
"description": "Hit count over time split by confidence",
"visState": "{\"title\":\"Pipeleek - Hit Confidence Over Time\",\"type\":\"histogram\",\"params\":{\"addLegend\":true,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true},\"title\":{\"text\":\"Time\"}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true},\"title\":{\"text\":\"Hit Count\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"line\",\"mode\":\"normal\",\"data\":{\"id\":\"1\",\"label\":\"Hit Count\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true,\"lineWidth\":2}],\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"times\":[],\"addTimeMarker\":false,\"dimensions\":{\"x\":{\"accessor\":\"2\",\"format\":\"date\",\"params\":{}},\"y\":[{\"accessor\":\"1\",\"format\":{\"id\":\"number\"}}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"useNormalizedEsInterval\":false,\"min_doc_count\":1,\"extended_bounds\":{},\"interval\":\"10m\",\"drop_partials\":false,\"time_zone\":\"UTC\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"confidence\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"missingBucket\":true,\"missingBucketLabel\":\"unknown\"}}]}",
- "uiStateJSON": "{\"vis\":{\"colors\":{\"high-verified\":\"#D4821E\",\"high\":\"#2A4A25\",\"medium\":\"#B9CF7E\",\"low\":\"#8B6BA8\",\"warning\":\"#C4A035\",\"informational\":\"#B9CF7E\",\"unknown\":\"#4A8FA8\"}}}",
+ "uiStateJSON": "{\"vis\":{\"colors\":{\"verified\":\"#D4821E\",\"high\":\"#2A4A25\",\"medium\":\"#B9CF7E\",\"low\":\"#8B6BA8\",\"warning\":\"#C4A035\",\"informational\":\"#B9CF7E\",\"unknown\":\"#4A8FA8\"}}}",
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"query\":{\"language\":\"kuery\",\"query\":\"level:hit\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"
}
@@ -312,7 +312,7 @@ echo "Creating prebuilt dashboard..."
dashboard_payload='{
"attributes": {
"title": "Pipeleek - Hit Monitoring",
- "description": "High/high-verified counters, log intake over time, confidence trend over time, pie charts for hit distributions, latest 100 scan entries, and latest 100 raw log entries. Use dashboard filters for scan.platform and scan.confidence.",
+ "description": "High/verified counters, log intake over time, confidence trend over time, pie charts for hit distributions, latest 100 scan entries, and latest 100 raw log entries. Use dashboard filters for scan.platform and scan.confidence.",
"hits": 0,
"optionsJSON": "{\"useMargins\":true,\"hidePanelTitles\":false}",
"panelsJSON": "[{\"type\":\"visualization\",\"panelIndex\":\"1\",\"gridData\":{\"x\":0,\"y\":0,\"w\":12,\"h\":8,\"i\":\"1\"},\"embeddableConfig\":{},\"panelRefName\":\"panel_1\",\"version\":\"8.13.4\"},{\"type\":\"visualization\",\"panelIndex\":\"2\",\"gridData\":{\"x\":12,\"y\":0,\"w\":12,\"h\":8,\"i\":\"2\"},\"embeddableConfig\":{},\"panelRefName\":\"panel_2\",\"version\":\"8.13.4\"},{\"type\":\"visualization\",\"panelIndex\":\"3\",\"gridData\":{\"x\":24,\"y\":0,\"w\":24,\"h\":8,\"i\":\"3\"},\"embeddableConfig\":{\"timeRange\":{\"from\":\"now-4h\",\"to\":\"now\"}},\"panelRefName\":\"panel_3\",\"version\":\"8.13.4\"},{\"type\":\"visualization\",\"panelIndex\":\"4\",\"gridData\":{\"x\":0,\"y\":8,\"w\":24,\"h\":12,\"i\":\"4\"},\"embeddableConfig\":{},\"panelRefName\":\"panel_4\",\"version\":\"8.13.4\"},{\"type\":\"visualization\",\"panelIndex\":\"5\",\"gridData\":{\"x\":24,\"y\":8,\"w\":24,\"h\":12,\"i\":\"5\"},\"embeddableConfig\":{},\"panelRefName\":\"panel_5\",\"version\":\"8.13.4\"},{\"type\":\"visualization\",\"panelIndex\":\"8\",\"gridData\":{\"x\":0,\"y\":20,\"w\":48,\"h\":10,\"i\":\"8\"},\"embeddableConfig\":{},\"panelRefName\":\"panel_8\",\"version\":\"8.13.4\"},{\"type\":\"search\",\"panelIndex\":\"6\",\"gridData\":{\"x\":0,\"y\":30,\"w\":48,\"h\":12,\"i\":\"6\"},\"embeddableConfig\":{},\"panelRefName\":\"panel_6\",\"version\":\"8.13.4\"},{\"type\":\"search\",\"panelIndex\":\"7\",\"gridData\":{\"x\":0,\"y\":42,\"w\":48,\"h\":12,\"i\":\"7\"},\"embeddableConfig\":{},\"panelRefName\":\"panel_7\",\"version\":\"8.13.4\"}]",
diff --git a/pkg/gitlab/scan/pipeline.go b/pkg/gitlab/scan/pipeline.go
index cdc9b1ba..103f40de 100644
--- a/pkg/gitlab/scan/pipeline.go
+++ b/pkg/gitlab/scan/pipeline.go
@@ -244,7 +244,7 @@ func getAllJobs(git *gitlab.Client, project *gitlab.Project, options *ScanOption
enqueueItem(globQueue, QueueItemCICDYaml, QueueMeta{
ProjectId: int(project.ID),
- JobWebUrl: project.WebURL,
+ JobWebUrl: strings.TrimRight(project.WebURL, "/") + "/-/blob/HEAD/.gitlab-ci.yml",
JobName: ".gitlab-ci.yml",
ProjectPathWithNamespace: project.PathWithNamespace,
}, waitGroup)
diff --git a/pkg/gitlab/scan/pipeline_test.go b/pkg/gitlab/scan/pipeline_test.go
index dfb93001..503cb0b8 100644
--- a/pkg/gitlab/scan/pipeline_test.go
+++ b/pkg/gitlab/scan/pipeline_test.go
@@ -78,7 +78,10 @@ func TestGetAllJobs_StatusScope(t *testing.T) {
globQueue, waitGroup = oldQueue, oldWG
}()
- getAllJobs(client, &gitlab.Project{ID: 1, PathWithNamespace: "group/project"}, opts)
+ getAllJobs(client, &gitlab.Project{
+ ID: 1, PathWithNamespace: "group/project",
+ WebURL: "https://gitlab.example/group/project/",
+ }, opts)
assert.Equal(t, int32(2), jobPages.Load(), "job-limit should stop pagination after two matching jobs")
wantItems := 4
if authenticated {
@@ -98,6 +101,7 @@ func TestGetAllJobs_StatusScope(t *testing.T) {
artifacts = append(artifacts, item.Meta.JobId)
case QueueItemCICDYaml:
yamlItems++
+ assert.Equal(t, "https://gitlab.example/group/project/-/blob/HEAD/.gitlab-ci.yml", item.Meta.JobWebUrl)
}
waitGroup.Done()
case <-time.After(2 * time.Second):
diff --git a/pkg/gitlab/scan/queue.go b/pkg/gitlab/scan/queue.go
index 14ac3112..d294de68 100644
--- a/pkg/gitlab/scan/queue.go
+++ b/pkg/gitlab/scan/queue.go
@@ -240,6 +240,7 @@ func analyzeCICDYaml(git *gitlab.Client, item QueueItem, options *ScanOptions) {
result.ReportFindings(logResult.Findings, result.ReportOptions{
LocationURL: item.Meta.JobWebUrl,
JobName: item.Meta.JobName,
+ Type: logging.SecretTypeCICDYaml,
})
}
diff --git a/pkg/logging/hit.go b/pkg/logging/hit.go
index 5cb26538..029135fb 100644
--- a/pkg/logging/hit.go
+++ b/pkg/logging/hit.go
@@ -22,7 +22,8 @@ const (
// NestedArchive indicates a secret found in a nested archive.
SecretTypeNestedArchive SecretType = "nested-archive"
// SecretTypeDotenv indicates a secret found in a dotenv file.
- SecretTypeDotenv SecretType = "dotenv"
+ SecretTypeDotenv SecretType = "dotenv"
+ SecretTypeCICDYaml SecretType = "cicd-yaml"
)
// HitLevel defines a custom log level for security finding hits.
diff --git a/pkg/webui/server.go b/pkg/webui/server.go
index e2777c28..1d4909a5 100644
--- a/pkg/webui/server.go
+++ b/pkg/webui/server.go
@@ -222,6 +222,7 @@ var pageTemplate = strings.Join([]string{
" .badge.type-archive { background: rgba(64,107,54,.12); color: var(--accent-dark); }",
" .badge.type-nested-archive { background: rgba(64,107,54,.18); color: var(--accent-dark); }",
" .badge.type-dotenv { background: rgba(123,136,180,.14); color: #405089; }",
+ " .badge.type-cicd-yaml { background: rgba(123,136,180,.14); color: #405089; }",
" .secret { font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; word-break: break-all; white-space: pre-wrap; background: #f7f7f7; border: 1px solid var(--line); border-radius: 6px; padding: .45rem .5rem; display: block; width: 100%; min-width: 0; color: var(--ink); cursor: pointer; transition: border-color .15s ease, background-color .15s ease; }",
" .secret:hover { border-color: rgba(64,107,54,.5); background: #f1f6eb; }",
" .secret:active { transform: translateY(1px); }",
@@ -301,6 +302,7 @@ var pageTemplate = strings.Join([]string{
"
archive
",
"
nested-archive
",
"
dotenv
",
+ "
cicd-yaml
",
"