From 511975328a7bd7dd14c5d68ad2fb4020ad8f6d1a Mon Sep 17 00:00:00 2001 From: Chose Carreras Date: Sun, 26 Jul 2026 18:07:13 +0200 Subject: [PATCH] Add incident.io alert provider Add the incidentio Provider type for incident.io. Events are posted to an HTTP alert source endpoint as alert events, authenticated with a bearer token. Error events are sent with status firing, all other severities with status resolved, using the involved object UID as the deduplication key (same as the PagerDuty provider) so that recovery events automatically resolve the alert fired for the same object, and objects with the same kind/namespace/name on different clusters never share a key. Progressing events are skipped, matching the PagerDuty provider behavior. Assisted-by: Claude Code/claude-fable-5 Signed-off-by: Chose Carreras --- api/v1beta3/provider_types.go | 3 +- ...ification.toolkit.fluxcd.io_providers.yaml | 1 + docs/spec/v1beta3/providers.md | 60 +++++++++ internal/notifier/factory.go | 5 + internal/notifier/incidentio.go | 127 ++++++++++++++++++ internal/notifier/incidentio_fuzz_test.go | 61 +++++++++ internal/notifier/incidentio_test.go | 116 ++++++++++++++++ 7 files changed, 372 insertions(+), 1 deletion(-) create mode 100644 internal/notifier/incidentio.go create mode 100644 internal/notifier/incidentio_fuzz_test.go create mode 100644 internal/notifier/incidentio_test.go diff --git a/api/v1beta3/provider_types.go b/api/v1beta3/provider_types.go index be4bd36d3..677c48ce6 100644 --- a/api/v1beta3/provider_types.go +++ b/api/v1beta3/provider_types.go @@ -58,13 +58,14 @@ const ( ZulipProvider string = "zulip" OTELProvider string = "otel" ZoomProvider string = "zoom" + IncidentioProvider string = "incidentio" ) // ProviderSpec defines the desired state of the Provider. // +kubebuilder:validation:XValidation:rule="self.type == 'github' || self.type == 'gitlab' || self.type == 'gitea' || self.type == 'bitbucketserver' || self.type == 'bitbucket' || self.type == 'azuredevops' || !has(self.commitStatusExpr)", message="spec.commitStatusExpr is only supported for the 'github', 'gitlab', 'gitea', 'bitbucketserver', 'bitbucket', 'azuredevops' provider types" type ProviderSpec struct { // Type specifies which Provider implementation to use. - // +kubebuilder:validation:Enum=slack;discord;msteams;rocket;generic;generic-hmac;github;gitlab;gitea;giteapullrequestcomment;bitbucketserver;bitbucket;azuredevops;googlechat;googlepubsub;webex;sentry;azureeventhub;telegram;lark;matrix;opsgenie;alertmanager;grafana;githubdispatch;githubpullrequestcomment;gitlabmergerequestcomment;pagerduty;datadog;nats;zulip;otel;zoom + // +kubebuilder:validation:Enum=slack;discord;msteams;rocket;generic;generic-hmac;github;gitlab;gitea;giteapullrequestcomment;bitbucketserver;bitbucket;azuredevops;googlechat;googlepubsub;webex;sentry;azureeventhub;telegram;lark;matrix;opsgenie;alertmanager;grafana;githubdispatch;githubpullrequestcomment;gitlabmergerequestcomment;pagerduty;datadog;nats;zulip;otel;zoom;incidentio // +required Type string `json:"type"` diff --git a/config/crd/bases/notification.toolkit.fluxcd.io_providers.yaml b/config/crd/bases/notification.toolkit.fluxcd.io_providers.yaml index 42b9a3e90..83e7cd037 100644 --- a/config/crd/bases/notification.toolkit.fluxcd.io_providers.yaml +++ b/config/crd/bases/notification.toolkit.fluxcd.io_providers.yaml @@ -187,6 +187,7 @@ spec: - zulip - otel - zoom + - incidentio type: string username: description: Username specifies the name under which events are posted. diff --git a/docs/spec/v1beta3/providers.md b/docs/spec/v1beta3/providers.md index a2480f378..c33f83464 100644 --- a/docs/spec/v1beta3/providers.md +++ b/docs/spec/v1beta3/providers.md @@ -96,6 +96,7 @@ The supported alerting providers are: | [Google Chat](#google-chat) | `googlechat` | | [Google Pub/Sub](#google-pubsub) | `googlepubsub` | | [Grafana](#grafana) | `grafana` | +| [incident.io](#incidentio) | `incidentio` | | [Lark](#lark) | `lark` | | [Matrix](#matrix) | `matrix` | | [Microsoft Teams](#microsoft-teams) | `msteams` | @@ -1359,6 +1360,65 @@ stringData: token: ``` +##### incident.io + +When `.spec.type` is set to `incidentio`, the controller will send a payload for +an [Event](events.md#event-structure) to the provided incident.io +[HTTP alert source](https://docs.incident.io/alerts/custom-http-sources) [Address](#address). + +The Event is mapped to an +[alert event](https://docs.incident.io/api-reference/alert-events-v2) as follows: +events with `error` severity are sent with status `firing`, all other events +with status `resolved`. The UID of the involved object is used as the +deduplication key, so a recovery event automatically resolves the alert +previously fired for the same object, and objects with the same name on +different clusters never share a key. Events with reason +`Progressing` are skipped. The event metadata, severity and reason are included +in the alert event metadata. + +Note that for automatic alert resolution to work, the referencing +[Alert](alerts.md) must allow `info` events to flow, i.e. `.spec.eventSeverity` +must be set to `info` (the default). With `.spec.eventSeverity: error`, alerts +fire but are never resolved by the controller. + +The API token generated by the incident.io alert source must be provided in the +`token` key of the referenced Secret, it is sent as a bearer token in the +`Authorization` header of the POST request. + +This Provider type does support the configuration of a [proxy URL](#https-proxy) +and [certificate secret reference](#certificate-secret-reference). + +###### incident.io example + +To configure a Provider for incident.io, create an +[HTTP alert source](https://docs.incident.io/alerts/custom-http-sources) in the +incident.io dashboard to obtain the endpoint URL and the API token, then create +a Secret with [the `address`](#address-example) set to the endpoint URL, +[the `token`](#token-example) set to the API token, and an `incidentio` +Provider with a [Secret reference](#secret-reference). + +```yaml +--- +apiVersion: notification.toolkit.fluxcd.io/v1beta3 +kind: Provider +metadata: + name: incidentio + namespace: default +spec: + type: incidentio + secretRef: + name: incidentio-alert-source +--- +apiVersion: v1 +kind: Secret +metadata: + name: incidentio-alert-source + namespace: default +stringData: + address: https://api.incident.io/v2/alert_events/http/ + token: +``` + ### Address diff --git a/internal/notifier/factory.go b/internal/notifier/factory.go index 69a392c8d..11bf73e9b 100644 --- a/internal/notifier/factory.go +++ b/internal/notifier/factory.go @@ -66,6 +66,7 @@ var ( apiv1.ZulipProvider: zulipNotifierFunc, apiv1.OTELProvider: otelNotifierFunc, apiv1.ZoomProvider: zoomNotifierFunc, + apiv1.IncidentioProvider: incidentioNotifierFunc, } ) @@ -390,3 +391,7 @@ func otelNotifierFunc(opts notifierOptions) (Interface, error) { func zoomNotifierFunc(opts notifierOptions) (Interface, error) { return NewZoom(opts.URL, opts.ProxyURL, opts.TLSConfig, opts.Token) } + +func incidentioNotifierFunc(opts notifierOptions) (Interface, error) { + return NewIncidentio(opts.URL, opts.ProxyURL, opts.TLSConfig, opts.Token) +} diff --git a/internal/notifier/incidentio.go b/internal/notifier/incidentio.go new file mode 100644 index 000000000..09cb71ed6 --- /dev/null +++ b/internal/notifier/incidentio.go @@ -0,0 +1,127 @@ +/* +Copyright 2026 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package notifier + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "net/url" + "strings" + + eventv1 "github.com/fluxcd/pkg/apis/event/v1beta1" + "github.com/fluxcd/pkg/apis/meta" + "github.com/hashicorp/go-retryablehttp" +) + +// Incidentio holds the endpoint URL of an incident.io HTTP alert source +// and its API token. +type Incidentio struct { + URL string + ProxyURL string + Token string + TLSConfig *tls.Config +} + +// IncidentioPayload is the alert event format accepted by the incident.io +// HTTP alert source endpoint (POST /v2/alert_events/http/{alert_source_config_id}). +type IncidentioPayload struct { + Title string `json:"title"` + Status string `json:"status"` + DeduplicationKey string `json:"deduplication_key"` + Description string `json:"description,omitempty"` + SourceURL string `json:"source_url,omitempty"` + Metadata map[string]string `json:"metadata,omitempty"` +} + +// NewIncidentio validates the alert source endpoint URL and token +// and returns an Incidentio object. +func NewIncidentio(endpointURL string, proxyURL string, tlsConfig *tls.Config, token string) (*Incidentio, error) { + _, err := url.ParseRequestURI(endpointURL) + if err != nil { + return nil, fmt.Errorf("invalid incident.io alert source URL %s: '%w'", endpointURL, err) + } + + if token == "" { + return nil, errors.New("empty incident.io API token") + } + + return &Incidentio{ + URL: endpointURL, + ProxyURL: proxyURL, + Token: token, + TLSConfig: tlsConfig, + }, nil +} + +// Post event to the incident.io HTTP alert source. Error events fire an +// alert, any other severity resolves it. The involved object UID is used +// as the deduplication key so that a recovery event resolves the alert +// previously fired for the same object, and objects with the same +// kind/namespace/name on different clusters never share a key. +func (i *Incidentio) Post(ctx context.Context, event eventv1.Event) error { + // Skip progressing events, we want either a terminal failure or a recovery. + if event.HasReason(meta.ProgressingReason) { + return nil + } + + obj := event.InvolvedObject + kind := strings.ToLower(obj.Kind) + objName := fmt.Sprintf("%s/%s.%s", kind, obj.Name, obj.Namespace) + + status := "resolved" + if event.Severity == eventv1.EventSeverityError { + status = "firing" + } + + metadata := make(map[string]string, len(event.Metadata)+3) + for k, v := range event.Metadata { + metadata[k] = v + } + metadata["severity"] = event.Severity + metadata["reason"] = event.Reason + if event.ReportingController != "" { + metadata["reportingController"] = event.ReportingController + } + + payload := IncidentioPayload{ + Title: fmt.Sprintf("%s: %s", objName, event.Reason), + Status: status, + DeduplicationKey: string(obj.UID), + Description: event.Message, + Metadata: metadata, + } + + opts := []postOption{ + withRequestModifier(func(req *retryablehttp.Request) { + req.Header.Set("Authorization", "Bearer "+i.Token) + }), + } + if i.ProxyURL != "" { + opts = append(opts, withProxy(i.ProxyURL)) + } + if i.TLSConfig != nil { + opts = append(opts, withTLSConfig(i.TLSConfig)) + } + + if err := postMessage(ctx, i.URL, payload, opts...); err != nil { + return fmt.Errorf("postMessage failed: %w", err) + } + + return nil +} diff --git a/internal/notifier/incidentio_fuzz_test.go b/internal/notifier/incidentio_fuzz_test.go new file mode 100644 index 000000000..af48e3e09 --- /dev/null +++ b/internal/notifier/incidentio_fuzz_test.go @@ -0,0 +1,61 @@ +/* +Copyright 2026 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package notifier + +import ( + "context" + "crypto/tls" + "fmt" + "io" + "net/http" + "net/http/httptest" + "testing" + + fuzz "github.com/AdaLogics/go-fuzz-headers" + eventv1 "github.com/fluxcd/pkg/apis/event/v1beta1" +) + +func Fuzz_Incidentio(f *testing.F) { + f.Add("token", "", "error", "", []byte{}, []byte{}) + f.Add("token", "", "info", "", []byte{}, []byte{}) + + f.Fuzz(func(t *testing.T, + token, urlSuffix, severity, message string, seed, response []byte) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write(response) + io.Copy(io.Discard, r.Body) + r.Body.Close() + })) + defer ts.Close() + + var tlsConfig tls.Config + _ = fuzz.NewConsumer(seed).GenerateStruct(&tlsConfig) + + incidentio, err := NewIncidentio(fmt.Sprintf("%s/%s", ts.URL, urlSuffix), "", &tlsConfig, token) + if err != nil { + return + } + + event := eventv1.Event{} + _ = fuzz.NewConsumer(seed).GenerateStruct(&event) + + event.Severity = severity + event.Message = message + + _ = incidentio.Post(context.TODO(), event) + }) +} diff --git a/internal/notifier/incidentio_test.go b/internal/notifier/incidentio_test.go new file mode 100644 index 000000000..bf3f121a1 --- /dev/null +++ b/internal/notifier/incidentio_test.go @@ -0,0 +1,116 @@ +/* +Copyright 2026 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package notifier + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "testing" + + eventv1 "github.com/fluxcd/pkg/apis/event/v1beta1" + "github.com/fluxcd/pkg/apis/meta" + . "github.com/onsi/gomega" +) + +func TestIncidentio_Post(t *testing.T) { + g := NewWithT(t) + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + g.Expect(r.Header.Get("Authorization")).To(Equal("Bearer token")) + g.Expect(r.Header.Get("Content-Type")).To(Equal("application/json")) + + b, err := io.ReadAll(r.Body) + g.Expect(err).ToNot(HaveOccurred()) + var payload IncidentioPayload + g.Expect(json.Unmarshal(b, &payload)).To(Succeed()) + + g.Expect(payload.Title).To(Equal("gitrepository/webapp.gitops-system: reason")) + g.Expect(payload.Status).To(Equal("resolved")) + g.Expect(payload.DeduplicationKey).To(Equal("2f5c9d21-b62c-4a35-9b17-1b0f4c8e2d3a")) + g.Expect(payload.Description).To(Equal("message")) + g.Expect(payload.Metadata).To(HaveKeyWithValue("test", "metadata")) + g.Expect(payload.Metadata).To(HaveKeyWithValue("severity", "info")) + g.Expect(payload.Metadata).To(HaveKeyWithValue("reason", "reason")) + g.Expect(payload.Metadata).To(HaveKeyWithValue("reportingController", "source-controller")) + + // incident.io responds with 202 Accepted + w.WriteHeader(http.StatusAccepted) + })) + defer ts.Close() + + incidentio, err := NewIncidentio(ts.URL, "", nil, "token") + g.Expect(err).ToNot(HaveOccurred()) + + event := testEvent() + event.InvolvedObject.UID = "2f5c9d21-b62c-4a35-9b17-1b0f4c8e2d3a" + err = incidentio.Post(context.TODO(), event) + g.Expect(err).ToNot(HaveOccurred()) +} + +func TestIncidentio_PostErrorSeverity(t *testing.T) { + g := NewWithT(t) + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + b, err := io.ReadAll(r.Body) + g.Expect(err).ToNot(HaveOccurred()) + var payload IncidentioPayload + g.Expect(json.Unmarshal(b, &payload)).To(Succeed()) + + g.Expect(payload.Status).To(Equal("firing")) + g.Expect(payload.Metadata).To(HaveKeyWithValue("severity", "error")) + + w.WriteHeader(http.StatusAccepted) + })) + defer ts.Close() + + incidentio, err := NewIncidentio(ts.URL, "", nil, "token") + g.Expect(err).ToNot(HaveOccurred()) + + event := testEvent() + event.Severity = eventv1.EventSeverityError + err = incidentio.Post(context.TODO(), event) + g.Expect(err).ToNot(HaveOccurred()) +} + +func TestIncidentio_PostSkipsProgressing(t *testing.T) { + g := NewWithT(t) + requested := false + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requested = true + })) + defer ts.Close() + + incidentio, err := NewIncidentio(ts.URL, "", nil, "token") + g.Expect(err).ToNot(HaveOccurred()) + + event := testEvent() + event.Reason = meta.ProgressingReason + err = incidentio.Post(context.TODO(), event) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(requested).To(BeFalse()) +} + +func TestNewIncidentio(t *testing.T) { + g := NewWithT(t) + + _, err := NewIncidentio("invalid-url", "", nil, "token") + g.Expect(err).To(MatchError(ContainSubstring("invalid incident.io alert source URL"))) + + _, err = NewIncidentio("https://api.incident.io/v2/alert_events/http/some-id", "", nil, "") + g.Expect(err).To(MatchError(ContainSubstring("empty incident.io API token"))) +}