diff --git a/changelog/fragments/1772577135-Fix-checkin-compression-support.yaml b/changelog/fragments/1772577135-Fix-checkin-compression-support.yaml new file mode 100644 index 000000000..cff7a912a --- /dev/null +++ b/changelog/fragments/1772577135-Fix-checkin-compression-support.yaml @@ -0,0 +1,33 @@ +# Kind can be one of: +# - breaking-change: a change to previously-documented behavior +# - deprecation: functionality that is being removed in a later release +# - bug-fix: fixes a problem in a previous version +# - enhancement: extends functionality but does not break or fix existing behavior +# - feature: new functionality +# - known-issue: problems that we are aware of in a given version +# - security: impacts on the security of a product or a user’s deployment. +# - upgrade: important information for someone upgrading from a prior version +# - other: does not fit into any of the other categories +kind: bug-fix + +# Change summary; a 80ish characters long description of the change. +summary: Fix checkin endpoint compression support + +# Long description; in case the summary is not enough to describe the change +# this field accommodate a description without length limits. +# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment. +description: | + Adds support for gzip compressed requests to the checkin endpoint. + +# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc. +component: fleet-server + +# PR URL; optional; the PR number that added the changeset. +# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added. +# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number. +# Please provide it if you are adding a fragment for a different PR. +pr: https://github.com/elastic/fleet-server/pull/6491 + +# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of). +# If not present is automatically filled by the tooling with the issue linked to the PR number. +#issue: https://github.com/owner/repo/1234 diff --git a/internal/pkg/api/handleCheckin.go b/internal/pkg/api/handleCheckin.go index 3dc793415..40d07201b 100644 --- a/internal/pkg/api/handleCheckin.go +++ b/internal/pkg/api/handleCheckin.go @@ -187,9 +187,20 @@ func (ct *CheckinT) validateRequest(zlog zerolog.Logger, w http.ResponseWriter, } readCounter := datacounter.NewReaderCounter(body) + // Decompress the body when the client signals Content-Encoding: gzip. + var bodyReader io.Reader = readCounter + if r.Header.Get("Content-Encoding") == kEncodingGzip { + gr, err := gzip.NewReader(readCounter) + if err != nil { + return validatedCheckin{}, &BadRequestErr{msg: "unable to create gzip reader for request body", nextErr: err} + } + defer gr.Close() + bodyReader = gr + } + var val validatedCheckin var req CheckinRequest - decoder := json.NewDecoder(readCounter) + decoder := json.NewDecoder(bodyReader) if err := decoder.Decode(&req); err != nil { return val, &BadRequestErr{msg: "unable to decode checkin request", nextErr: err} } @@ -666,6 +677,8 @@ func (ct *CheckinT) writeResponse(zlog zerolog.Logger, w http.ResponseWriter, r return err } +// acceptsEncoding reports whether the request includes the passed encoding. +// Only an exact match is checked as that is all the agent will send. func acceptsEncoding(r *http.Request, encoding string) bool { for _, v := range r.Header.Values("Accept-Encoding") { if v == encoding { diff --git a/internal/pkg/api/handleCheckin_test.go b/internal/pkg/api/handleCheckin_test.go index d076aaa66..f13ae1c6c 100644 --- a/internal/pkg/api/handleCheckin_test.go +++ b/internal/pkg/api/handleCheckin_test.go @@ -7,7 +7,9 @@ package api import ( + "bytes" "compress/flate" + "compress/gzip" "context" "encoding/json" "errors" @@ -1168,6 +1170,44 @@ func TestValidateCheckinRequest(t *testing.T) { rawMeta: []byte(`{"elastic": {"agent": {"id": "testid", "fips": true}}}`), }, }, + { + name: "gzip-compressed request body is decompressed before JSON decoding", + req: func() *http.Request { + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + _, _ = gz.Write([]byte(`{"status": "online", "message": "test message"}`)) + _ = gz.Close() + return &http.Request{ + Header: http.Header{"Content-Encoding": []string{"gzip"}}, + Body: io.NopCloser(&buf), + } + }(), + cfg: &config.Server{ + Limits: config.ServerLimits{ + CheckinLimit: config.Limit{ + MaxBody: 0, + }, + }, + }, + expErr: nil, + expValid: validatedCheckin{}, + }, + { + name: "invalid gzip request body returns bad request error", + req: &http.Request{ + Header: http.Header{"Content-Encoding": []string{"gzip"}}, + Body: io.NopCloser(strings.NewReader(`not gzip data`)), + }, + cfg: &config.Server{ + Limits: config.ServerLimits{ + CheckinLimit: config.Limit{ + MaxBody: 0, + }, + }, + }, + expErr: &BadRequestErr{msg: "unable to create gzip reader for request body", nextErr: gzip.ErrHeader}, + expValid: validatedCheckin{}, + }, } for _, tc := range tests { diff --git a/internal/pkg/api/openapi.gen.go b/internal/pkg/api/openapi.gen.go index 0bc7b6d9d..247910c0d 100644 --- a/internal/pkg/api/openapi.gen.go +++ b/internal/pkg/api/openapi.gen.go @@ -137,6 +137,11 @@ const ( Endpoint UploadBeginRequestSrc = "endpoint" ) +// Defines values for AgentCheckinParamsContentEncoding. +const ( + Gzip AgentCheckinParamsContentEncoding = "gzip" +) + // AckRequest The request an elastic-agent sends to fleet-serve to acknowledge the execution of one or more actions. type AckRequest struct { Events []AckRequest_Events_Item `json:"events"` @@ -999,11 +1004,13 @@ type AuditUnenrollParams struct { // AgentCheckinParams defines parameters for AgentCheckin. type AgentCheckinParams struct { - // AcceptEncoding If the agent is able to accept encoded responses. - // Used to indicate if GZIP compression may be used by the server. - // The elastic-agent does not use the accept-encoding header. + // AcceptEncoding Indicates encodings the agent can accept. The only encoding supported by the server currently is gzip. + // Comma-separated directive lists and RFC 7231 quality values (e.g. "gzip;q=1.0, deflate;q=0.5") are both accepted. AcceptEncoding *string `json:"Accept-Encoding,omitempty"` + // ContentEncoding Signals that the request body has been compressed. Server currently only supports gzip compression. + ContentEncoding *AgentCheckinParamsContentEncoding `json:"Content-Encoding,omitempty"` + // UserAgent The user-agent header that is sent. // Must have the format "elastic agent X.Y.Z" where "X.Y.Z" indicates the agent version. // The agent version must not be greater than the version of the fleet-server. @@ -1016,6 +1023,9 @@ type AgentCheckinParams struct { ElasticApiVersion *ApiVersion `json:"elastic-api-version,omitempty"` } +// AgentCheckinParamsContentEncoding defines parameters for AgentCheckin. +type AgentCheckinParamsContentEncoding string + // ArtifactParams defines parameters for Artifact. type ArtifactParams struct { // XRequestId The request tracking ID for APM. @@ -2267,6 +2277,25 @@ func (siw *ServerInterfaceWrapper) AgentCheckin(w http.ResponseWriter, r *http.R } + // ------------- Optional header parameter "Content-Encoding" ------------- + if valueList, found := headers[http.CanonicalHeaderKey("Content-Encoding")]; found { + var ContentEncoding AgentCheckinParamsContentEncoding + n := len(valueList) + if n != 1 { + siw.ErrorHandlerFunc(w, r, &TooManyValuesForParamError{ParamName: "Content-Encoding", Count: n}) + return + } + + err = runtime.BindStyledParameterWithOptions("simple", "Content-Encoding", valueList[0], &ContentEncoding, runtime.BindStyledParameterOptions{ParamLocation: runtime.ParamLocationHeader, Explode: false, Required: false}) + if err != nil { + siw.ErrorHandlerFunc(w, r, &InvalidParamFormatError{ParamName: "Content-Encoding", Err: err}) + return + } + + params.ContentEncoding = &ContentEncoding + + } + // ------------- Required header parameter "User-Agent" ------------- if valueList, found := headers[http.CanonicalHeaderKey("User-Agent")]; found { var UserAgent UserAgent diff --git a/internal/pkg/server/fleet_integration_test.go b/internal/pkg/server/fleet_integration_test.go index 7bac90de5..858a51571 100644 --- a/internal/pkg/server/fleet_integration_test.go +++ b/internal/pkg/server/fleet_integration_test.go @@ -52,7 +52,7 @@ const ( serverVersion = "8.0.0" localhost = "localhost" - testWaitServerUp = 3 * time.Second + testWaitServerUp = 10 * time.Second enrollBody = `{ "type": "PERMANENT", @@ -244,6 +244,7 @@ func startTestServer(t *testing.T, ctx context.Context, policyD model.PolicyData } func (s *tserver) waitServerUp(ctx context.Context, dur time.Duration) error { + zlog := zerolog.Ctx(ctx) ctx, cancel := context.WithTimeout(ctx, dur) defer cancel() @@ -270,6 +271,7 @@ func (s *tserver) waitServerUp(ctx context.Context, dur time.Duration) error { return false, err } + zlog.Info().Msgf("test wait for fleet-server up status: %s", status.Status) return status.Status == "HEALTHY", nil } diff --git a/model/openapi.yml b/model/openapi.yml index 857cc9eb0..8b72396c7 100644 --- a/model/openapi.yml +++ b/model/openapi.yml @@ -1489,11 +1489,18 @@ paths: - name: Accept-Encoding in: header description: | - If the agent is able to accept encoded responses. - Used to indicate if GZIP compression may be used by the server. - The elastic-agent does not use the accept-encoding header. + Indicates encodings the agent can accept. The only encoding supported by the server currently is gzip. + Comma-separated directive lists and RFC 7231 quality values (e.g. "gzip;q=1.0, deflate;q=0.5") are both accepted. schema: type: string + - name: Content-Encoding + in: header + description: | + Signals that the request body has been compressed. Server currently only supports gzip compression. + schema: + type: string + enum: + - gzip - $ref: "#/components/parameters/userAgent" - $ref: "#/components/parameters/requestId" - $ref: "#/components/parameters/apiVersion" @@ -1529,12 +1536,19 @@ paths: description: Agent checkin successful. May include actions. headers: Content-Encoding: - description: Responses may be compressed if the accept encoding indicates it. Currently not used by the agent. + description: | + Present when fleet-server has gzip-compressed the response body. + Compression is applied when all three conditions are met: the serialised + response exceeds the configured compression threshold, the server compression + level is not NoCompression, and the request's Accept-Encoding header + advertises gzip support. schema: type: string + enum: + - gzip examples: gzip: - description: Response is gzip encoded as the request headers allowed it. + description: Response body is gzip-compressed. value: gzip Elastic-Api-Version: $ref: "#/components/headers/apiVersion" @@ -2020,4 +2034,4 @@ paths: "500": $ref: "#/components/responses/internalServerError" "503": - $ref: "#/components/responses/unavailable" \ No newline at end of file + $ref: "#/components/responses/unavailable" diff --git a/pkg/api/client.gen.go b/pkg/api/client.gen.go index d0c44e036..d8498ef55 100644 --- a/pkg/api/client.gen.go +++ b/pkg/api/client.gen.go @@ -695,35 +695,46 @@ func NewAgentCheckinRequestWithBody(server string, id string, params *AgentCheck req.Header.Set("Accept-Encoding", headerParam0) } - var headerParam1 string + if params.ContentEncoding != nil { + var headerParam1 string + + headerParam1, err = runtime.StyleParamWithLocation("simple", false, "Content-Encoding", runtime.ParamLocationHeader, *params.ContentEncoding) + if err != nil { + return nil, err + } - headerParam1, err = runtime.StyleParamWithLocation("simple", false, "User-Agent", runtime.ParamLocationHeader, params.UserAgent) + req.Header.Set("Content-Encoding", headerParam1) + } + + var headerParam2 string + + headerParam2, err = runtime.StyleParamWithLocation("simple", false, "User-Agent", runtime.ParamLocationHeader, params.UserAgent) if err != nil { return nil, err } - req.Header.Set("User-Agent", headerParam1) + req.Header.Set("User-Agent", headerParam2) if params.XRequestId != nil { - var headerParam2 string + var headerParam3 string - headerParam2, err = runtime.StyleParamWithLocation("simple", false, "X-Request-Id", runtime.ParamLocationHeader, *params.XRequestId) + headerParam3, err = runtime.StyleParamWithLocation("simple", false, "X-Request-Id", runtime.ParamLocationHeader, *params.XRequestId) if err != nil { return nil, err } - req.Header.Set("X-Request-Id", headerParam2) + req.Header.Set("X-Request-Id", headerParam3) } if params.ElasticApiVersion != nil { - var headerParam3 string + var headerParam4 string - headerParam3, err = runtime.StyleParamWithLocation("simple", false, "elastic-api-version", runtime.ParamLocationHeader, *params.ElasticApiVersion) + headerParam4, err = runtime.StyleParamWithLocation("simple", false, "elastic-api-version", runtime.ParamLocationHeader, *params.ElasticApiVersion) if err != nil { return nil, err } - req.Header.Set("elastic-api-version", headerParam3) + req.Header.Set("elastic-api-version", headerParam4) } } diff --git a/pkg/api/types.gen.go b/pkg/api/types.gen.go index 9f9f78e24..285d85f86 100644 --- a/pkg/api/types.gen.go +++ b/pkg/api/types.gen.go @@ -134,6 +134,11 @@ const ( Endpoint UploadBeginRequestSrc = "endpoint" ) +// Defines values for AgentCheckinParamsContentEncoding. +const ( + Gzip AgentCheckinParamsContentEncoding = "gzip" +) + // AckRequest The request an elastic-agent sends to fleet-serve to acknowledge the execution of one or more actions. type AckRequest struct { Events []AckRequest_Events_Item `json:"events"` @@ -996,11 +1001,13 @@ type AuditUnenrollParams struct { // AgentCheckinParams defines parameters for AgentCheckin. type AgentCheckinParams struct { - // AcceptEncoding If the agent is able to accept encoded responses. - // Used to indicate if GZIP compression may be used by the server. - // The elastic-agent does not use the accept-encoding header. + // AcceptEncoding Indicates encodings the agent can accept. The only encoding supported by the server currently is gzip. + // Comma-separated directive lists and RFC 7231 quality values (e.g. "gzip;q=1.0, deflate;q=0.5") are both accepted. AcceptEncoding *string `json:"Accept-Encoding,omitempty"` + // ContentEncoding Signals that the request body has been compressed. Server currently only supports gzip compression. + ContentEncoding *AgentCheckinParamsContentEncoding `json:"Content-Encoding,omitempty"` + // UserAgent The user-agent header that is sent. // Must have the format "elastic agent X.Y.Z" where "X.Y.Z" indicates the agent version. // The agent version must not be greater than the version of the fleet-server. @@ -1013,6 +1020,9 @@ type AgentCheckinParams struct { ElasticApiVersion *ApiVersion `json:"elastic-api-version,omitempty"` } +// AgentCheckinParamsContentEncoding defines parameters for AgentCheckin. +type AgentCheckinParamsContentEncoding string + // ArtifactParams defines parameters for Artifact. type ArtifactParams struct { // XRequestId The request tracking ID for APM.