diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 8b9d986..01c1875 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -33,7 +33,7 @@ jobs: echo "DOCKER_CLI_EXPERIMENTAL=enabled" >> $GITHUB_ENV - name: Set up Golang - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 id: go with: go-version: ${{ env.GOVERSION }} diff --git a/.golangci.yaml b/.golangci.yaml index ef9b35b..6c57039 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -2,7 +2,7 @@ version: "2" run: timeout: 5m - go: '1.23' + go: '1.25' linters: enable: diff --git a/Dockerfile b/Dockerfile index 2c76f95..7c27fd6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build stage -FROM golang:1.23.10 AS builder +FROM golang:1.25.9 AS builder # Set non-root user for build RUN useradd -u 10001 -m builder diff --git a/README.md b/README.md index 50d6faa..99ec82c 100644 --- a/README.md +++ b/README.md @@ -1,31 +1,49 @@ # Github Prometheus Exporter (promgithub) -The `promgithub` service is a lightweight service designed to receive and process GitHub webhook events (commits, pull requests, workflow jobs and workflow runs). The webhook events are converted to prometheus metrics, allowing monitoring and insights into GitHub activities. +`promgithub` is a service that receives GitHub webhook events and exposes Prometheus metrics for repository activity, workflow runs, workflow jobs, commits, and pull requests. -## Metrics Exported by the Service +It is designed to be simple to deploy and can run either: +- as a single instance +- as multiple instances with Redis for shared deduplication and state -The `promgithub` service exports the following Prometheus metrics: +## Metrics exported -| Name | Type | Labels | Description | -|------------------------------------|-----------|---------------------------------------------------------------|-------------------------------------------| -| `promgithub_workflow_status` | Counter | `repository`, `branch`, `workflow_name`, `workflow_status`, `conclusion`| Total number of workflow runs with status | -| `promgithub_workflow_duration` | Histogram | `repository`, `branch`, `workflow_name`, `workflow_status`, `conclusion`| Duration of workflow runs | -| `promgithub_workflow_queued` | Gauge | `repository`, `branch`, `workflow_name` | Number of workflow runs queued | -| `promgithub_workflow_in_progress` | Gauge | `repository`, `branch`, `workflow_name` | Number of workflow runs in progress | -| `promgithub_workflow_completed` | Gauge | `repository`, `branch`, `workflow_conclusion`,`workflow_name` | Number of workflow runs completed | +`promgithub` exports the following metrics: + +| Name | Type | Labels | Description | +|------------------------------------|-----------|-------------------------------------------------------------------------|-------------------------------------------| +| `promgithub_workflow_status` | Counter | `repository`, `branch`, `workflow_name`, `workflow_status`, `conclusion` | Total number of workflow runs with status | +| `promgithub_workflow_duration` | Histogram | `repository`, `branch`, `workflow_name`, `workflow_status`, `conclusion` | Duration of workflow runs | +| `promgithub_workflow_queued` | Gauge | `repository`, `branch`, `workflow_name` | Number of workflow runs queued | +| `promgithub_workflow_in_progress` | Gauge | `repository`, `branch`, `workflow_name` | Number of workflow runs in progress | +| `promgithub_workflow_completed` | Gauge | `repository`, `branch`, `workflow_conclusion`, `workflow_name` | Number of workflow runs completed | | `promgithub_job_status` | Counter | `repository`, `branch`, `workflow_name`, `job_status`, `job_conclusion` | Total number of jobs with status | | `promgithub_job_duration` | Histogram | `repository`, `branch`, `workflow_name`, `job_status`, `job_conclusion` | Duration of jobs runs in seconds | -| `promgithub_job_queued` | Gauge | `repository`, `branch`, `workflow_name` | Number of jobs queued | -| `promgithub_job_in_progress` | Gauge | `repository`, `branch`, `workflow_name` | Number of jobs in progress | -| `promgithub_job_completed` | Gauge | `repository`, `branch`, `job_conclusion`, `workflow_name` | Number of jobs completed | -| `promgithub_commit_pushed` | Counter | `repository` | Total number of commits pushed | -| `promgithub_pull_request` | Counter | `repository`, `base_branch`, `pull_request_status` | Total number of pull requests | +| `promgithub_job_queued` | Gauge | `repository`, `branch`, `workflow_name` | Number of jobs queued | +| `promgithub_job_in_progress` | Gauge | `repository`, `branch`, `workflow_name` | Number of jobs in progress | +| `promgithub_job_completed` | Gauge | `repository`, `branch`, `job_conclusion`, `workflow_name` | Number of jobs completed | +| `promgithub_commit_pushed` | Counter | `repository` | Total number of commits pushed | +| `promgithub_pull_request` | Counter | `repository`, `base_branch`, `pull_request_status` | Total number of pull requests | + +## Metric model + +The exporter focuses on repository and workflow health signals while avoiding noisy per-entity labels such as runner names, job names, commit author identities, and pull request authors. + +This keeps the default metric set compact and practical for Prometheus while still preserving the `branch` label for branch-specific workflow and job visibility. + +## Redis-backed multi-instance mode + +When Redis is configured, `promgithub` uses it for: +- webhook delivery deduplication using `X-GitHub-Delivery` +- shared workflow run state storage +- shared workflow job state storage +This allows multiple `promgithub` instances to share delivery and run state through a common backend. -## Using `promgithub` service +## Using promgithub -For usage information see [Usage documentation](./docs/usage.md) +See [Usage documentation](./docs/usage.md) for deployment and configuration examples. -## Contributing to `promgithub` service +## Contributing -For contributing guidelines see [Contributing documentation](./docs/contributing.md) +See [Contributing documentation](./docs/contributing.md). diff --git a/docs/usage.md b/docs/usage.md index 8c158fe..f82e426 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -1,134 +1,179 @@ # Using `promgithub` service -## Deploying the service +## Overview -The service can be deployed in your choice of infrastructure. To allow webhooks to be pushed to `promgithub` make sure your service deployment is accessible from your Github instance. +`promgithub` receives GitHub webhook events and exposes Prometheus metrics over HTTP. -### Service Parameters +It can be deployed: +- as a single instance with only the webhook secret configured +- as a multi-instance deployment with Redis configured for shared deduplication and state -The service expects the following parameters to be set: +## Configuration -- **Environment Variables**: - - `PROMGITHUB_WEBHOOK_SECRET`: The secret used to validate incoming GitHub webhook requests. - - `PROMGITHUB_SERVICE_PORT` (optional): Service API port (default is `8080`). +### Environment variables -## Deploying in kubernetes +The service supports the following environment variables: -To deploy the service in a kubernetes cluster you can use the provided helm chart from the [promgithub chart repository](https://github.com/darthfork/promgithub/pkgs/container/promgithub-charts%2Fpromgithub) +- `PROMGITHUB_WEBHOOK_SECRET`: Secret used to validate incoming GitHub webhook requests. +- `PROMGITHUB_SERVICE_PORT` (optional): HTTP port for the service, default `8080`. +- `PROMGITHUB_REDIS_ADDR` (optional): Redis address in `host:port` form. +- `PROMGITHUB_REDIS_PASSWORD` (optional): Redis password. +- `PROMGITHUB_REDIS_DB` (optional): Redis database number, default `0`. +- `PROMGITHUB_REDIS_KEY_PREFIX` (optional): Prefix used for Redis keys, default `promgithub`. +- `PROMGITHUB_REDIS_DELIVERY_TTL` (optional): TTL for webhook delivery dedupe keys, default `24h`. -When deploying with kubernetes add the following resources and configurations to your `promgithub` deployment +If Redis is configured, the service stores delivery and run state in Redis. -### Chart.yaml +## Running the service -Add the helm repository as a dependency to your chart deployment: +### Run the binary -```yaml -apiVersion: v2 -name: promgithub -description: Deployment of promgithub -type: application -version: - -dependencies: - - name: promgithub - version: "" - repository: "oci://ghcr.io/darthfork/promgithub-charts" +```bash +PROMGITHUB_WEBHOOK_SECRET="" \ +PROMGITHUB_SERVICE_PORT="8080" \ +/path/to/binary/promgithub ``` -### Ingress +### Run the binary with Redis -Add an Ingress configuration allowing your github instance to access `promgithub` deployment. More details can be found [here](https://kubernetes.io/docs/concepts/services-networking/ingress/) +```bash +PROMGITHUB_WEBHOOK_SECRET="" \ +PROMGITHUB_REDIS_ADDR="" \ +PROMGITHUB_REDIS_PASSWORD="" \ +PROMGITHUB_REDIS_DB="0" \ +PROMGITHUB_REDIS_KEY_PREFIX="promgithub" \ +PROMGITHUB_REDIS_DELIVERY_TTL="24h" \ +PROMGITHUB_SERVICE_PORT="8080" \ +/path/to/binary/promgithub +``` +### Docker -### Values +```bash +docker run \ + -e PROMGITHUB_WEBHOOK_SECRET= \ + -e PROMGITHUB_SERVICE_PORT=8080 \ + -p 8080:8080 \ + ghcr.io/darthfork/promgithub: +``` -Create a values file with the webhook secret and (optional) service port values for your chart +### Docker with Redis -```yaml -promgithub: - secrets: - github_webhook_secret: # Mounted as PROMGITHUB_WEBHOOK_SECRET in the deployment - service: - port: # optional (default is 8080) +```bash +docker run \ + -e PROMGITHUB_WEBHOOK_SECRET= \ + -e PROMGITHUB_REDIS_ADDR= \ + -e PROMGITHUB_REDIS_PASSWORD= \ + -e PROMGITHUB_REDIS_DB=0 \ + -e PROMGITHUB_REDIS_KEY_PREFIX=promgithub \ + -e PROMGITHUB_REDIS_DELIVERY_TTL=24h \ + -e PROMGITHUB_SERVICE_PORT=8080 \ + -p 8080:8080 \ + ghcr.io/darthfork/promgithub: ``` -**Default values**: The default values file for promgithub can be found [here](https://github.com/darthfork/promgithub/blob/main/helm/promgithub/values.yaml) +### Docker Compose with Redis -### Metrics Scraping +```yaml +services: + redis: + image: redis:7 + command: ["redis-server", "--appendonly", "yes"] + ports: + - "6379:6379" -Create prometheus configuration resource with the chart for scraping metrics from the `/metrics` endpoint from promgithub service. For more details see the [Prometheus scraping configuration](#prometheus-scraping-configuration) below. + promgithub: + image: ghcr.io/darthfork/promgithub: + environment: + PROMGITHUB_WEBHOOK_SECRET: + PROMGITHUB_REDIS_ADDR: redis:6379 + PROMGITHUB_REDIS_PASSWORD: + PROMGITHUB_REDIS_DB: 0 + PROMGITHUB_REDIS_KEY_PREFIX: promgithub + PROMGITHUB_REDIS_DELIVERY_TTL: 24h + PROMGITHUB_SERVICE_PORT: 8080 + ports: + - "8080:8080" + depends_on: + - redis +``` -## Deploying service in a container +## Deploying with Kubernetes -To deploy the service in a container using a container management environment like fargate/docker-compose, you can use the `promgithub` container from the [GHCR container repository](https://github.com/darthfork/promgithub/pkgs/container/promgithub) +`promgithub` includes a Helm chart. -### Docker CLI +### Add the chart dependency -Run the service using docker cli as follows: +```yaml +apiVersion: v2 +name: promgithub +description: Deployment of promgithub +type: application +version: -```bash -docker run\ - -e PROMGITHUB_WEBHOOK_SECRET=\ - -e PROMGITHUB_SERVICE_PORT=\ - -p :\ - ghcr.io/darthfork/promgithub: +dependencies: + - name: promgithub + version: "" + repository: "oci://ghcr.io/darthfork/promgithub-charts" ``` -### Docker Compose - -To run the service in docker compose, create a compose file as below: +### Values for an external Redis instance ```yaml -# promgithub-compose.yaml -services: - promgithub: - image: ghcr.io/darthfork/promgithub: - hostname: promgithub - stdin_open: false - tty: false - environment: - - PROMGITHUB_WEBHOOK_SECRET= - - PROMGITHUB_SERVICE_PORT= - ports: - - : +promgithub: + secrets: + github_webhook_secret: + redis_password: + redisConfig: + addr: redis.example.internal:6379 + db: 0 + keyPrefix: promgithub + deliveryTTL: 24h ``` -To start the `promgithub` container with compose run: +### Values for a bundled Redis deployment -```bash -docker-compose -f promgithub-compose.yaml run --rm promgithub +```yaml +promgithub: + secrets: + github_webhook_secret: + redis: + enabled: true + auth: + enabled: true + password: + redisConfig: + db: 0 + keyPrefix: promgithub + deliveryTTL: 24h ``` -## Deploying service binary +When `redis.enabled=true`, the chart deploys Redis as a dependency and configures `promgithub` to connect to it automatically. -The service binaries are also available under [github releases](https://github.com/darthfork/promgithub/releases) which can be deployed as the user wishes. +### Ingress -```bash -PROMGITHUB_WEBHOOK_SECRET="" PROMGITHUB_SERVICE_PORT="" /path/to/binary/promgithub -``` +Expose the `/webhook` endpoint to GitHub using your preferred Kubernetes ingress setup. -## Setting up the Webhook in GitHub (Repository/Organization) +## Setting up the GitHub webhook 1. Navigate to your GitHub repository or organization settings. -2. Under **Settings**, find **Webhooks** and click **Add webhook**. -3. Enter the payload URL pointing to your `promgithub` service, e.g., `http:///webhook`. -4. Set the **Content type** to `application/json`. -5. Add the **Secret**: Use the value of `PROMGITHUB_WEBHOOK_SECRET`. -6. Select the following events to trigger the webhook: +2. Under **Settings**, open **Webhooks** and click **Add webhook**. +3. Set the payload URL to your `promgithub` webhook endpoint, for example `https:///webhook`. +4. Set **Content type** to `application/json`. +5. Set the **Secret** to the value used for `PROMGITHUB_WEBHOOK_SECRET`. +6. Subscribe to these events: - **push** - **pull request** - **workflow job** - **workflow runs** -7. Click **Add webhook** to save. +7. Save the webhook. -## Prometheus scraping configuration +## Scraping metrics -Configure prometheus to scrape `promgithub`'s `/metrics` endpoint to extract metrics. +`promgithub` exposes Prometheus metrics on `/metrics`. ### Prometheus configuration -To allow prometheus to scrape `promgithub`'s `/metrics` endpoint, add the following configuration to your prometheus setup: - ```yaml scrape_configs: - job_name: 'promgithub' @@ -142,8 +187,6 @@ scrape_configs: ### VictoriaMetrics configuration -If you use victoria-metrics as your metrics provider, add a `vmservicescrape` configuration to your `promgithub` chart deployment - ```yaml apiVersion: operator.victoriametrics.com/v1beta1 kind: VMServiceScrape diff --git a/go.mod b/go.mod index f3303a4..7528ca2 100644 --- a/go.mod +++ b/go.mod @@ -1,10 +1,11 @@ module promgithub -go 1.23.10 +go 1.25.9 require ( github.com/gorilla/mux v1.8.1 github.com/prometheus/client_golang v1.20.5 + github.com/redis/go-redis/v9 v9.7.3 github.com/stretchr/testify v1.9.0 go.uber.org/zap v1.27.0 ) @@ -13,6 +14,7 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/klauspost/compress v1.17.9 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect diff --git a/go.sum b/go.sum index 527de5b..2bcf35a 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,15 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= @@ -28,6 +34,8 @@ github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/redis/go-redis/v9 v9.7.3 h1:YpPyAayJV+XErNsatSElgRZZVCwXX9QzkKYNvO7x0wM= +github.com/redis/go-redis/v9 v9.7.3/go.mod h1:bGUrSggJ9X9GUmZpZNEOQKaANxSGgOEBRltRTZHSvrA= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= diff --git a/helm/promgithub/Chart.yaml b/helm/promgithub/Chart.yaml index cc3944c..ec058af 100644 --- a/helm/promgithub/Chart.yaml +++ b/helm/promgithub/Chart.yaml @@ -10,3 +10,9 @@ version: 0.0.7 # This is the version number of the application being deployed. This version number co-relates to the version # of promgithub deployed with the chart. Update this version to deploy a new version of the application. appVersion: "0.0.7" + +dependencies: + - name: redis + version: 0.0.7 + repository: "https://charts.bitnami.com/bitnami" + condition: redis.enabled diff --git a/helm/promgithub/templates/deployment.yaml b/helm/promgithub/templates/deployment.yaml index 24e1f78..d9e8f00 100644 --- a/helm/promgithub/templates/deployment.yaml +++ b/helm/promgithub/templates/deployment.yaml @@ -39,6 +39,21 @@ spec: env: - name: ENVIRONMENT value: "{{ .Values.environment | default "production" }}" + {{- if .Values.redis.enabled }} + - name: PROMGITHUB_REDIS_ADDR + value: "{{ .Release.Name }}-redis-master:6379" + {{- else if .Values.redisConfig.addr }} + - name: PROMGITHUB_REDIS_ADDR + value: "{{ .Values.redisConfig.addr }}" + {{- end }} + {{- if or .Values.redis.enabled .Values.redisConfig.addr }} + - name: PROMGITHUB_REDIS_DB + value: "{{ .Values.redisConfig.db }}" + - name: PROMGITHUB_REDIS_KEY_PREFIX + value: "{{ .Values.redisConfig.keyPrefix | default "promgithub" }}" + - name: PROMGITHUB_REDIS_DELIVERY_TTL + value: "{{ .Values.redisConfig.deliveryTTL | default "24h" }}" + {{- end }} envFrom: - secretRef: name: "{{ include "promgithub.fullname" . }}" diff --git a/helm/promgithub/templates/secret.yaml b/helm/promgithub/templates/secret.yaml index 9440ea2..b84c33e 100644 --- a/helm/promgithub/templates/secret.yaml +++ b/helm/promgithub/templates/secret.yaml @@ -6,3 +6,6 @@ metadata: type: Opaque stringData: PROMGITHUB_WEBHOOK_SECRET: "{{ .Values.secrets.github_webhook_secret }}" + {{- if or .Values.redis.enabled .Values.redisConfig.addr }} + PROMGITHUB_REDIS_PASSWORD: "{{ default .Values.secrets.redis_password .Values.redis.auth.password }}" + {{- end }} diff --git a/helm/promgithub/values.yaml b/helm/promgithub/values.yaml index d5646ab..f69931a 100644 --- a/helm/promgithub/values.yaml +++ b/helm/promgithub/values.yaml @@ -8,6 +8,7 @@ revisionHistoryLimit: 10 secrets: github_webhook_secret: "" + redis_password: "" environment: production # Set the environment to development to enable debug logging and profiling endpoints @@ -32,6 +33,19 @@ podAnnotations: {} # This is for setting Kubernetes Labels to a Pod. podLabels: {} +redis: + enabled: false + architecture: standalone + auth: + enabled: true + password: "" + +redisConfig: + addr: "" + db: 0 + keyPrefix: promgithub + deliveryTTL: 24h + # This is for setting up the promgithub service service: # This sets the service type diff --git a/src/env.go b/src/env.go new file mode 100644 index 0000000..da89f63 --- /dev/null +++ b/src/env.go @@ -0,0 +1,45 @@ +package main + +import ( + "fmt" + "os" + "strconv" + "time" +) + +func getEnvAny(keys ...string) string { + for _, key := range keys { + if value := os.Getenv(key); value != "" { + return value + } + } + return "" +} + +func parseEnvInt(key string, defaultValue int) (int, error) { + value := os.Getenv(key) + if value == "" { + return defaultValue, nil + } + + parsed, err := strconv.Atoi(value) + if err != nil { + return 0, fmt.Errorf("parse %s: %w", key, err) + } + + return parsed, nil +} + +func parseEnvDuration(key string, defaultValue time.Duration) (time.Duration, error) { + value := os.Getenv(key) + if value == "" { + return defaultValue, nil + } + + parsed, err := time.ParseDuration(value) + if err != nil { + return 0, fmt.Errorf("parse %s: %w", key, err) + } + + return parsed, nil +} diff --git a/src/github.go b/src/github.go index dedcb11..028f17e 100644 --- a/src/github.go +++ b/src/github.go @@ -2,6 +2,7 @@ package main import ( + "context" "crypto/hmac" "crypto/sha256" "encoding/hex" @@ -92,6 +93,8 @@ type runMetricDetails struct { endedAt string } +var stateStore StateStore + func validateHMAC(body []byte, signature string, secret []byte) bool { h := hmac.New(sha256.New, secret) h.Write(body) @@ -114,12 +117,28 @@ func githubEventsHandler(w http.ResponseWriter, r *http.Request) { return } + ctx := r.Context() + deliveryID := strings.TrimSpace(r.Header.Get("X-GitHub-Delivery")) + if stateStore != nil && deliveryID != "" { + processed, storeErr := stateStore.MarkDeliveryProcessed(ctx, deliveryID) + if storeErr != nil { + http.Error(w, "Unable to record webhook delivery", http.StatusInternalServerError) + logger.Error("Unable to record webhook delivery", zap.String("deliveryID", deliveryID), zap.Error(storeErr)) + return + } + if !processed { + logger.Info("Skipping duplicate GitHub delivery", zap.String("deliveryID", deliveryID)) + w.WriteHeader(http.StatusOK) + return + } + } + eventType := r.Header.Get("X-GitHub-Event") switch eventType { case "workflow_run": - updateWorkflowMetrics(body) + updateWorkflowMetrics(ctx, body) case "workflow_job": - updateJobMetrics(body) + updateJobMetrics(ctx, body) case "push": updateCommitMetrics(body) case "pull_request": @@ -193,7 +212,25 @@ func observeRunMetrics( } } -func updateWorkflowMetrics(body []byte) { +func updateRunState(ctx context.Context, id int, details runMetricDetails, updateFn func(context.Context, int, RunState) error, entityName string) { + if stateStore == nil { + return + } + + if err := updateFn(ctx, id, RunState{ + Repository: details.repository, + Branch: details.branch, + Name: details.name, + Status: details.status, + Conclusion: details.conclusion, + StartedAt: details.startedAt, + EndedAt: details.endedAt, + }); err != nil { + logger.Error("Failed to update run state in redis", zap.String("entity", entityName), zap.Int("id", id), zap.Error(err)) + } +} + +func updateWorkflowMetrics(ctx context.Context, body []byte) { var payload GithubWorkflow if err := json.Unmarshal(body, &payload); err != nil { @@ -201,16 +238,22 @@ func updateWorkflowMetrics(body []byte) { return } + details := runMetricDetails{ + repository: payload.Workflow.Repository.FullName, + branch: payload.Workflow.Branch, + name: payload.Workflow.Name, + status: payload.Workflow.Status, + conclusion: payload.Workflow.Conclusion, + startedAt: payload.Workflow.CreatedAt, + endedAt: payload.Workflow.UpdatedAt, + } + + if stateStore != nil { + updateRunState(ctx, payload.Workflow.RunID, details, stateStore.UpdateWorkflowRun, "workflow_run") + } + observeRunMetrics( - runMetricDetails{ - repository: payload.Workflow.Repository.FullName, - branch: payload.Workflow.Branch, - name: payload.Workflow.Name, - status: payload.Workflow.Status, - conclusion: payload.Workflow.Conclusion, - startedAt: payload.Workflow.CreatedAt, - endedAt: payload.Workflow.UpdatedAt, - }, + details, workflowStatusCounter, workflowQueuedGauge, workflowInProgressGauge, @@ -219,7 +262,7 @@ func updateWorkflowMetrics(body []byte) { ) } -func updateJobMetrics(body []byte) { +func updateJobMetrics(ctx context.Context, body []byte) { var payload GithubJob if err := json.Unmarshal(body, &payload); err != nil { @@ -227,16 +270,22 @@ func updateJobMetrics(body []byte) { return } + details := runMetricDetails{ + repository: payload.Job.Repository.FullName, + branch: payload.Job.Branch, + name: payload.Job.WorkflowName, + status: payload.Job.Status, + conclusion: payload.Job.Conclusion, + startedAt: payload.Job.StartedAt, + endedAt: payload.Job.CompletedAt, + } + + if stateStore != nil { + updateRunState(ctx, payload.Job.ID, details, stateStore.UpdateWorkflowJob, "workflow_job") + } + observeRunMetrics( - runMetricDetails{ - repository: payload.Job.Repository.FullName, - branch: payload.Job.Branch, - name: payload.Job.WorkflowName, - status: payload.Job.Status, - conclusion: payload.Job.Conclusion, - startedAt: payload.Job.StartedAt, - endedAt: payload.Job.CompletedAt, - }, + details, jobStatusCounter, jobQueuedGauge, jobInProgressGauge, diff --git a/src/github_test.go b/src/github_test.go index 4b1ab8f..76c8659 100644 --- a/src/github_test.go +++ b/src/github_test.go @@ -25,6 +25,7 @@ func sendTestRequest(payload []byte, eventType string) *httptest.ResponseRecorde req := httptest.NewRequest(http.MethodPost, "/webhook", bytes.NewBuffer(payload)) req.Header.Set("X-Hub-Signature-256", signature) req.Header.Set("X-GitHub-Event", eventType) + req.Header.Set("X-GitHub-Delivery", "delivery-1") recorder := httptest.NewRecorder() handler := http.HandlerFunc(githubEventsHandler) @@ -99,3 +100,19 @@ func TestUnknownEvent(t *testing.T) { recorder := sendTestRequest(body, "unknown_event") assert.Equal(t, http.StatusOK, recorder.Code) } + +func TestDuplicateDeliveryIsIgnored(t *testing.T) { + stateStore = newInMemoryStateStore() + defer func() { stateStore = nil }() + + body, err := os.ReadFile("../test_data/workflow_run.json") + if err != nil { + t.Fatalf("Failed to read test data file: %v", err) + } + + recorder := sendTestRequest(body, "workflow_run") + assert.Equal(t, http.StatusOK, recorder.Code) + + recorder = sendTestRequest(body, "workflow_run") + assert.Equal(t, http.StatusOK, recorder.Code) +} diff --git a/src/main.go b/src/main.go index d30ad91..43fe516 100644 --- a/src/main.go +++ b/src/main.go @@ -153,6 +153,23 @@ func main() { } githubWebhookSecret = []byte(ghWebhookSecretEnv) + redisConfig, redisEnabled, err := loadRedisConfigFromEnv() + if err != nil { + logger.Fatal("Invalid Redis configuration", zap.Error(err)) + } + if redisEnabled { + stateStore, err = NewRedisStateStore(redisConfig) + if err != nil { + logger.Fatal("Unable to initialize Redis state store", zap.Error(err)) + } + defer func() { + if closeErr := stateStore.Close(); closeErr != nil { + logger.Warn("Failed to close Redis state store", zap.Error(closeErr)) + } + }() + } + logRedisMode(logger, redisEnabled, redisConfig.Addr) + r := setupRouter(logger) server := &http.Server{ diff --git a/src/metrics_test.go b/src/metrics_test.go index ac9dbec..06ad2ad 100644 --- a/src/metrics_test.go +++ b/src/metrics_test.go @@ -1,6 +1,7 @@ package main import ( + "context" "encoding/json" "os" "strings" @@ -22,7 +23,7 @@ func TestWorkflowStatusCounter(t *testing.T) { if err != nil { t.Fatalf("Failed to read test data file: %v", err) } - updateWorkflowMetrics(body) + updateWorkflowMetrics(context.Background(), body) // Test counter if err := testutil.CollectAndCompare(workflowStatusCounter, strings.NewReader(` @@ -41,7 +42,7 @@ func TestJobStatusCounter(t *testing.T) { if err != nil { t.Fatalf("Failed to read test data file: %v", err) } - updateJobMetrics(body) + updateJobMetrics(context.Background(), body) // Test counter if err := testutil.CollectAndCompare(jobStatusCounter, strings.NewReader(` @@ -98,7 +99,7 @@ func TestWorkflowDurationHistogram(t *testing.T) { if err != nil { t.Fatalf("Failed to read test data file: %v", err) } - updateWorkflowMetrics(body) + updateWorkflowMetrics(context.Background(), body) // Test histogram if err := testutil.CollectAndCompare(workflowDurationHistogram, strings.NewReader(` @@ -130,7 +131,7 @@ func TestJobDurationHistogram(t *testing.T) { if err != nil { t.Fatalf("Failed to read test data file: %v", err) } - updateJobMetrics(body) + updateJobMetrics(context.Background(), body) // Test histogram if err := testutil.CollectAndCompare(jobDurationHistogram, strings.NewReader(` @@ -179,7 +180,7 @@ func TestWorkflowQueuedGauge(t *testing.T) { t.Fatalf("Failed to marshal modified JSON data: %v", err) } - updateWorkflowMetrics(modifiedBody) + updateWorkflowMetrics(context.Background(), modifiedBody) // Test gauge if err := testutil.CollectAndCompare(workflowQueuedGauge, strings.NewReader(` @@ -215,7 +216,7 @@ func TestWorkflowInProgressGauge(t *testing.T) { t.Fatalf("Failed to marshal modified JSON data: %v", err) } - updateWorkflowMetrics(modifiedBody) + updateWorkflowMetrics(context.Background(), modifiedBody) // Test gauge if err := testutil.CollectAndCompare(workflowInProgressGauge, strings.NewReader(` @@ -234,7 +235,7 @@ func TestWorkflowCompletedGauge(t *testing.T) { if err != nil { t.Fatalf("Failed to read test data file: %v", err) } - updateWorkflowMetrics(body) + updateWorkflowMetrics(context.Background(), body) // Test gauge if err := testutil.CollectAndCompare(workflowCompletedGauge, strings.NewReader(` @@ -270,7 +271,7 @@ func TestJobQueuedGauge(t *testing.T) { t.Fatalf("Failed to marshal modified JSON data: %v", err) } - updateJobMetrics(modifiedBody) + updateJobMetrics(context.Background(), modifiedBody) // Test gauge if err := testutil.CollectAndCompare(jobQueuedGauge, strings.NewReader(` @@ -306,7 +307,7 @@ func TestJobInProgressGauge(t *testing.T) { t.Fatalf("Failed to marshal modified JSON data: %v", err) } - updateJobMetrics(modifiedBody) + updateJobMetrics(context.Background(), modifiedBody) // Test gauge if err := testutil.CollectAndCompare(jobInProgressGauge, strings.NewReader(` @@ -343,7 +344,7 @@ func TestJobCompletedGauge(t *testing.T) { t.Fatalf("Failed to marshal modified JSON data: %v", err) } - updateJobMetrics(modifiedBody) + updateJobMetrics(context.Background(), modifiedBody) // Test gauge if err := testutil.CollectAndCompare(jobCompletedGauge, strings.NewReader(` diff --git a/src/redis.go b/src/redis.go new file mode 100644 index 0000000..624bfe8 --- /dev/null +++ b/src/redis.go @@ -0,0 +1,164 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "strings" + "time" + + "github.com/redis/go-redis/v9" + "go.uber.org/zap" +) + +const ( + defaultRedisDeliveryTTL = 24 * time.Hour + redisKeyPrefix = "promgithub" +) + +type RedisConfig struct { + Addr string + Password string + DB int + KeyPrefix string + DeliveryTTL time.Duration +} + +type RunState struct { + Repository string `json:"repository"` + Branch string `json:"branch"` + Name string `json:"name"` + Status string `json:"status"` + Conclusion string `json:"conclusion"` + StartedAt string `json:"started_at"` + EndedAt string `json:"ended_at"` +} + +type RedisStateStore struct { + client *redis.Client + keyPrefix string + deliveryTTL time.Duration +} + +func NewRedisStateStore(cfg RedisConfig) (*RedisStateStore, error) { + if strings.TrimSpace(cfg.Addr) == "" { + return nil, errors.New("redis address is required") + } + + keyPrefix := strings.TrimSpace(cfg.KeyPrefix) + if keyPrefix == "" { + keyPrefix = redisKeyPrefix + } + + deliveryTTL := cfg.DeliveryTTL + if deliveryTTL <= 0 { + deliveryTTL = defaultRedisDeliveryTTL + } + + client := redis.NewClient(&redis.Options{ + Addr: cfg.Addr, + Password: cfg.Password, + DB: cfg.DB, + }) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + if err := client.Ping(ctx).Err(); err != nil { + return nil, fmt.Errorf("ping redis: %w", err) + } + + return &RedisStateStore{ + client: client, + keyPrefix: keyPrefix, + deliveryTTL: deliveryTTL, + }, nil +} + +func (s *RedisStateStore) MarkDeliveryProcessed(ctx context.Context, deliveryID string) (bool, error) { + if strings.TrimSpace(deliveryID) == "" { + return false, errors.New("delivery id is required") + } + + key := s.key("delivery", deliveryID) + created, err := s.client.SetNX(ctx, key, "1", s.deliveryTTL).Result() + if err != nil { + return false, err + } + + return created, nil +} + +func (s *RedisStateStore) UpdateWorkflowRun(ctx context.Context, runID int, state RunState) error { + if runID == 0 { + return errors.New("workflow run id is required") + } + + return s.writeState(ctx, s.key("workflow_run", fmt.Sprintf("%d", runID)), state) +} + +func (s *RedisStateStore) UpdateWorkflowJob(ctx context.Context, jobID int, state RunState) error { + if jobID == 0 { + return errors.New("workflow job id is required") + } + + return s.writeState(ctx, s.key("workflow_job", fmt.Sprintf("%d", jobID)), state) +} + +func (s *RedisStateStore) Close() error { + if s == nil || s.client == nil { + return nil + } + return s.client.Close() +} + +func (s *RedisStateStore) writeState(ctx context.Context, key string, state RunState) error { + payload, err := json.Marshal(state) + if err != nil { + return err + } + + return s.client.Set(ctx, key, payload, 0).Err() +} + +func (s *RedisStateStore) key(parts ...string) string { + return strings.Join(append([]string{s.keyPrefix}, parts...), ":") +} + +func loadRedisConfigFromEnv() (RedisConfig, bool, error) { + addr := strings.TrimSpace(getEnvAny("PROMGITHUB_REDIS_ADDR", "PROMGITHUB_REDIS_ADDRESS")) + if addr == "" { + return RedisConfig{}, false, nil + } + + db, err := parseEnvInt("PROMGITHUB_REDIS_DB", 0) + if err != nil { + return RedisConfig{}, false, err + } + + ttl, err := parseEnvDuration("PROMGITHUB_REDIS_DELIVERY_TTL", defaultRedisDeliveryTTL) + if err != nil { + return RedisConfig{}, false, err + } + + cfg := RedisConfig{ + Addr: addr, + Password: strings.TrimSpace(os.Getenv("PROMGITHUB_REDIS_PASSWORD")), + DB: db, + KeyPrefix: strings.TrimSpace(os.Getenv("PROMGITHUB_REDIS_KEY_PREFIX")), + DeliveryTTL: ttl, + } + + return cfg, true, nil +} + +func logRedisMode(logger *zap.Logger, enabled bool, addr string) { + if enabled { + logger.Info("Redis-backed multi-instance mode enabled", zap.String("redisAddr", addr)) + return + } + + logger.Info("Redis-backed multi-instance mode disabled, running without shared state") +} diff --git a/src/redis_test.go b/src/redis_test.go new file mode 100644 index 0000000..6c3a5f1 --- /dev/null +++ b/src/redis_test.go @@ -0,0 +1,89 @@ +package main + +import ( + "context" + "os" + "testing" + "time" +) + +type inMemoryStateStore struct { + deliveries map[string]struct{} + workflow map[int]RunState + jobs map[int]RunState +} + +func newInMemoryStateStore() *inMemoryStateStore { + return &inMemoryStateStore{ + deliveries: map[string]struct{}{}, + workflow: map[int]RunState{}, + jobs: map[int]RunState{}, + } +} + +func (s *inMemoryStateStore) MarkDeliveryProcessed(_ context.Context, deliveryID string) (bool, error) { + if _, ok := s.deliveries[deliveryID]; ok { + return false, nil + } + s.deliveries[deliveryID] = struct{}{} + return true, nil +} + +func (s *inMemoryStateStore) UpdateWorkflowRun(_ context.Context, runID int, state RunState) error { + s.workflow[runID] = state + return nil +} + +func (s *inMemoryStateStore) UpdateWorkflowJob(_ context.Context, jobID int, state RunState) error { + s.jobs[jobID] = state + return nil +} + +func (s *inMemoryStateStore) Close() error { + return nil +} + +func TestLoadRedisConfigFromEnv(t *testing.T) { + for _, key := range []string{ + "PROMGITHUB_REDIS_ADDR", + "PROMGITHUB_REDIS_ADDRESS", + "PROMGITHUB_REDIS_DB", + "PROMGITHUB_REDIS_PASSWORD", + "PROMGITHUB_REDIS_KEY_PREFIX", + "PROMGITHUB_REDIS_DELIVERY_TTL", + } { + _ = os.Unsetenv(key) + } + + _, enabled, err := loadRedisConfigFromEnv() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if enabled { + t.Fatalf("expected redis to be disabled when address is not configured") + } + + _ = os.Setenv("PROMGITHUB_REDIS_ADDR", "localhost:6379") + _ = os.Setenv("PROMGITHUB_REDIS_DB", "2") + _ = os.Setenv("PROMGITHUB_REDIS_PASSWORD", "secret") + _ = os.Setenv("PROMGITHUB_REDIS_KEY_PREFIX", "custom") + _ = os.Setenv("PROMGITHUB_REDIS_DELIVERY_TTL", "2h") + defer func() { + _ = os.Unsetenv("PROMGITHUB_REDIS_ADDR") + _ = os.Unsetenv("PROMGITHUB_REDIS_DB") + _ = os.Unsetenv("PROMGITHUB_REDIS_PASSWORD") + _ = os.Unsetenv("PROMGITHUB_REDIS_KEY_PREFIX") + _ = os.Unsetenv("PROMGITHUB_REDIS_DELIVERY_TTL") + }() + + cfg, enabled, err := loadRedisConfigFromEnv() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !enabled { + t.Fatalf("expected redis to be enabled") + } + if cfg.Addr != "localhost:6379" || cfg.DB != 2 || cfg.Password != "secret" || cfg.KeyPrefix != "custom" || cfg.DeliveryTTL != 2*time.Hour { + t.Fatalf("unexpected config: %+v", cfg) + } +} diff --git a/src/state_store.go b/src/state_store.go new file mode 100644 index 0000000..208b9e3 --- /dev/null +++ b/src/state_store.go @@ -0,0 +1,10 @@ +package main + +import "context" + +type StateStore interface { + MarkDeliveryProcessed(ctx context.Context, deliveryID string) (bool, error) + UpdateWorkflowRun(ctx context.Context, runID int, state RunState) error + UpdateWorkflowJob(ctx context.Context, jobID int, state RunState) error + Close() error +} diff --git a/utils/install_tools.sh b/utils/install_tools.sh index 98e40d2..5f0ba13 100755 --- a/utils/install_tools.sh +++ b/utils/install_tools.sh @@ -2,7 +2,7 @@ # Tool versions GOSEC_VERSION="v2.22.7" -GOLANGCILINT_VERSION="v2.3.1" +GOLANGCILINT_VERSION="v2.11.4" TRIVY_VERSION="latest" GOVULNCHECK_VERSION="latest"