Set correct Content-Type header in query response (#4828)

* Set correct `Content-Type` header in query response

Because the internal query request handlers did not set the response
content type explicitly to `application/json`, and the external request
handler takes the response content type from the internal response to
set it for the resulting HTTP response, that response was also
incorrectly returning `Content-Type: text/plain`.

Setting the content type in the internal response correctly results in
the expected `Content-Type` header `application/json; charset=UTF-8`.

Fixes #4791

Signed-off-by: Christian Haudum <christian.haudum@gmail.com>

* fixup! Set correct `Content-Type` header in query response

Signed-off-by: Christian Haudum <christian.haudum@gmail.com>

* fixup! fixup! Set correct `Content-Type` header in query response

Signed-off-by: Christian Haudum <christian.haudum@gmail.com>

CHANGELOG.md

@@ -1,5 +1,6 @@
 ## Main
 
+* [4828](https://github.com/grafana/loki/pull/4282) **chaudum**: Set correct `Content-Type` header in query response
 * [4794](https://github.com/grafana/loki/pull/4794) **taisho6339**: Aggregate inotify watcher to file target manager
 * [4663](https://github.com/grafana/loki/pull/4663) **taisho6339**: Add SASL&mTLS authentication support for Kafka in Promtail
 * [4745](https://github.com/grafana/loki/pull/4745) **taisho6339**: Expose Kafka message key in labels

pkg/querier/worker_service.go

@@ -52,15 +52,11 @@ func InitWorkerService(
 	authMiddleware middleware.Interface,
 ) (serve services.Service, err error) {
 
-	// Create a couple Middlewares used to handle panics, perform auth, and parse Form's in http request
-	internalMiddleware := middleware.Merge(
+	// Create a couple Middlewares used to handle panics, perform auth, parse forms in http request, and set content type in response
+	handlerMiddleware := middleware.Merge(
 		serverutil.RecoveryHTTPMiddleware,
 		authMiddleware,
 		serverutil.NewPrepopulateMiddleware(),
-	)
-	// External middleware also needs to set JSON content type headers
-	externalMiddleware := middleware.Merge(
-		internalMiddleware,
 		serverutil.ResponseJSONMiddleware(),
 	)
 
@@ -72,7 +68,7 @@ func InitWorkerService(
 	// There are some routes which are always registered on the external router, add them now and
 	// wrap them with the externalMiddleware
 	for route, handler := range alwaysExternalRoutesToHandlers {
-		externalRouter.Path(route).Methods("GET", "POST").Handler(externalMiddleware.Wrap(handler))
+		externalRouter.Path(route).Methods("GET", "POST").Handler(handlerMiddleware.Wrap(handler))
 	}
 
 	// If the querier is running standalone without the query-frontend or query-scheduler, we must register the internal
@@ -91,7 +87,7 @@ func InitWorkerService(
 
 		// Register routes externally
 		for _, route := range routes {
-			externalRouter.Path(route).Methods("GET", "POST").Handler(externalMiddleware.Wrap(internalRouter))
+			externalRouter.Path(route).Methods("GET", "POST").Handler(handlerMiddleware.Wrap(internalRouter))
 		}
 
 		//If no frontend or scheduler address has been configured, then there is no place for the
@@ -129,7 +125,7 @@ func InitWorkerService(
 			return "internalQuerier"
 		}))
 
-	internalHandler = internalMiddleware.Wrap(internalHandler)
+	internalHandler = handlerMiddleware.Wrap(internalHandler)
 
 	//Return a querier worker pointed to the internal querier HTTP handler so there is not a conflict in routes between the querier
 	//and the query frontend

pkg/querier/worker_service_test.go

@@ -97,6 +97,10 @@ func Test_InitQuerierService(t *testing.T) {
 		})
 
 		t.Run("wrap external handler with response json middleware", func(t *testing.T) {
+			// note: this test only assures that the content type of the response is
+			// set if the handler function does not override it, which happens in the
+			// actual implementation, see
+			// https://github.com/grafana/loki/blob/34a012adcfade43291de3a7670f53679ea06aefe/pkg/lokifrontend/frontend/transport/handler.go#L136-L139
 			config := WorkerServiceConfig{
 				QueryFrontendEnabled:  false,
 				QuerySchedulerEnabled: false,