Skip to content

Commit

Permalink
http fault: implement header controlled faults (#6318)
Browse files Browse the repository at this point in the history
Part of #5942

Signed-off-by: Matt Klein <mklein@lyft.com>
  • Loading branch information
mattklein123 authored Mar 26, 2019
1 parent aab0545 commit 805683f
Show file tree
Hide file tree
Showing 22 changed files with 579 additions and 150 deletions.
4 changes: 4 additions & 0 deletions DEPRECATED.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,10 @@ A logged warning is expected for each deprecated item that is in deprecation win
* Use of `enabled` in `CorsPolicy`, found in
[route.proto](https://github.com/envoyproxy/envoy/blob/master/api/envoy/api/v2/route/route.proto).
Set the `filter_enabled` field instead.
* Use of the `type` field in the `FaultDelay` message (found in
[fault.proto](https://github.com/envoyproxy/envoy/blob/master/api/envoy/config/filter/fault/v2/fault.proto))
has been deprecated. It was never used and setting it has no effect. It will be removed in the
following release.
* Use of `cluster`, found in [redis-proxy.proto](https://github.com/envoyproxy/envoy/blob/master/api/envoy/config/filter/network/redis_proxy/v2/redis_proxy.proto) is deprecated. Set a `PrefixRoutes.catch_all_cluster` instead.

## Version 1.9.0 (Dec 20, 2018)
Expand Down
26 changes: 22 additions & 4 deletions api/envoy/config/filter/fault/v2/fault.proto
Original file line number Diff line number Diff line change
Expand Up @@ -19,19 +19,25 @@ import "gogoproto/gogo.proto";
// Delay specification is used to inject latency into the
// HTTP/gRPC/Mongo/Redis operation or delay proxying of TCP connections.
message FaultDelay {
// Fault delays are controlled via an HTTP header (if applicable). See the
// :ref:`http fault filter <config_http_filters_fault_injection_http_header>` documentation for
// more information.
message HeaderDelay {
}

enum FaultDelayType {
// Fixed delay (step function).
// Unused and deprecated.
FIXED = 0;
}

// Delay type to use (fixed|exponential|..). Currently, only fixed delay (step function) is
// supported.
FaultDelayType type = 1 [(validate.rules).enum.defined_only = true];
// Unused and deprecated. Will be removed in the next release.
FaultDelayType type = 1 [deprecated = true];

reserved 2;

oneof fault_delay_secifier {
option (validate.required) = true;

// Add a fixed delay before forwarding the operation upstream. See
// https://developers.google.com/protocol-buffers/docs/proto3#json for
// the JSON/YAML Duration mapping. For HTTP/Mongo/Redis, the specified
Expand All @@ -40,6 +46,9 @@ message FaultDelay {
// for the specified period. This is required if type is FIXED.
google.protobuf.Duration fixed_delay = 3
[(validate.rules).duration.gt = {}, (gogoproto.stdduration) = true];

// Fault delays are controlled via an HTTP header (if applicable).
HeaderDelay header_delay = 5;
}

// The percentage of operations/connections/requests on which the delay will be injected.
Expand All @@ -54,11 +63,20 @@ message FaultRateLimit {
uint64 limit_kbps = 1 [(validate.rules).uint64.gte = 1];
}

// Rate limits are controlled via an HTTP header (if applicable). See the
// :ref:`http fault filter <config_http_filters_fault_injection_http_header>` documentation for
// more information.
message HeaderLimit {
}

oneof limit_type {
option (validate.required) = true;

// A fixed rate limit.
FixedLimit fixed_limit = 1;

// Rate limits are controlled via an HTTP header (if applicable).
HeaderLimit header_limit = 3;
}

// The percentage of operations/connections/requests on which the rate limit will be injected.
Expand Down
4 changes: 3 additions & 1 deletion api/envoy/config/filter/http/fault/v2/fault.proto
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,9 @@ message HTTPFault {
// amount due to the implementation details.
google.protobuf.UInt32Value max_active_faults = 6;

// The response rate limit to be applied to the response body of the stream.
// The response rate limit to be applied to the response body of the stream. When configured,
// the percentage can be overridden by the :ref:`fault.http.rate_limit.response_percent
// <config_http_filters_fault_injection_runtime>` runtime key.
//
// .. attention::
// This is a per-stream limit versus a connection level limit. This means that concurrent streams
Expand Down
75 changes: 58 additions & 17 deletions docs/root/configuration/http_filters/fault_filter.rst
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,6 @@ The scope of failures is restricted to those that are observable by an
application communicating over the network. CPU and disk failures on the
local host cannot be emulated.

Currently, the fault injection filter has the following limitations:

* Abort codes are restricted to HTTP status codes only
* Delays are restricted to fixed duration.

Future versions will include support for restricting faults to specific
routes, injecting *gRPC* and *HTTP/2* specific error codes and delay
durations based on distributions.

Configuration
-------------

Expand All @@ -36,33 +27,83 @@ Configuration
* :ref:`v2 API reference <envoy_api_msg_config.filter.http.fault.v2.HTTPFault>`
* This filter should be configured with the name *envoy.fault*.

.. _config_http_filters_fault_injection_http_header:

Controlling fault injection via HTTP headers
--------------------------------------------

The fault filter has the capability to allow fault configuration to be specified by the caller.
This is useful in certain scenarios in which it is desired to allow the client to specify its own
fault configuration. The currently supported header controls are:

* Request delay configuration via the *x-envoy-fault-delay-request* header. The header value
should be an integer that specifies the number of milliseconds to throttle the latency for.
* Response rate limit configuration via the *x-envoy-fault-throughput-response* header. The
header value should be an integer that specified the limit in KiB/s and must be > 0.

.. attention::

Allowing header control is inherently dangerous if exposed to untrusted clients. In this case,
it is suggested to use the :ref:`max_active_faults
<envoy_api_field_config.filter.http.fault.v2.HTTPFault.max_active_faults>` setting to limit the
maximum concurrent faults that can be active at any given time.

The following is an example configuration that enables header control for both of the above
options:

.. code-block:: yaml
name: envoy.fault
config:
max_active_faults: 100
delay:
header_delay: {}
percentage:
numerator: 100
response_rate_limit:
header_limit: {}
percentage:
numerator: 100
.. _config_http_filters_fault_injection_runtime:

Runtime
-------

The HTTP fault injection filter supports the following global runtime settings:

.. attention::

Some of the following runtime keys require the filter to be configured for the specific fault
type and some do not. Please consult the documentation for each key for more information.

fault.http.abort.abort_percent
% of requests that will be aborted if the headers match. Defaults to the
*abort_percent* specified in config. If the config does not contain an
*abort* block, then *abort_percent* defaults to 0.
*abort* block, then *abort_percent* defaults to 0. For historic reasons, this runtime key is
available regardless of whether the filter is :ref:`configured for abort
<envoy_api_field_config.filter.http.fault.v2.HTTPFault.abort>`.

fault.http.abort.http_status
HTTP status code that will be used as the of requests that will be
aborted if the headers match. Defaults to the HTTP status code specified
in the config. If the config does not contain an *abort* block, then
*http_status* defaults to 0.
*http_status* defaults to 0. For historic reasons, this runtime key is
available regardless of whether the filter is :ref:`configured for abort
<envoy_api_field_config.filter.http.fault.v2.HTTPFault.abort>`.

fault.http.delay.fixed_delay_percent
% of requests that will be delayed if the headers match. Defaults to the
*delay_percent* specified in the config or 0 otherwise.
*delay_percent* specified in the config or 0 otherwise. This runtime key is only available when
the filter is :ref:`configured for delay
<envoy_api_field_config.filter.http.fault.v2.HTTPFault.delay>`.

fault.http.delay.fixed_duration_ms
The delay duration in milliseconds. If not specified, the
*fixed_duration_ms* specified in the config will be used. If this field
is missing from both the runtime and the config, no delays will be
injected.
injected. This runtime key is only available when the filter is :ref:`configured for delay
<envoy_api_field_config.filter.http.fault.v2.HTTPFault.delay>`.

fault.http.max_active_faults
The maximum number of active faults (of all types) that Envoy will will inject via the fault
Expand All @@ -72,10 +113,10 @@ fault.http.max_active_faults
<envoy_api_field_config.filter.http.fault.v2.HTTPFault.max_active_faults>` setting will be used.

fault.http.rate_limit.response_percent
% of requests which will have a response rate limit fault injected, if the filter is
:ref:`configured <envoy_api_field_config.filter.http.fault.v2.HTTPFault.response_rate_limit>` to
do so. Defaults to the value set in the :ref:`percentage
<envoy_api_field_config.filter.fault.v2.FaultRateLimit.percentage>` field.
% of requests which will have a response rate limit fault injected. Defaults to the value set in
the :ref:`percentage <envoy_api_field_config.filter.fault.v2.FaultRateLimit.percentage>` field.
This runtime key is only available when the filter is :ref:`configured for response rate limiting
<envoy_api_field_config.filter.http.fault.v2.HTTPFault.response_rate_limit>`.

*Note*, fault filter runtime settings for the specific downstream cluster
override the default ones if present. The following are downstream specific
Expand Down
10 changes: 6 additions & 4 deletions docs/root/intro/version_history.rst
Original file line number Diff line number Diff line change
Expand Up @@ -36,18 +36,20 @@ Version history
<envoy_api_field_config.filter.http.fault.v2.HTTPFault.max_active_faults>` setting, as well as
:ref:`statistics <config_http_filters_fault_injection_stats>` for the number of active faults
and the number of faults the overflowed.
* fault: add :ref:`response rate limit
* fault: added :ref:`response rate limit
<envoy_api_field_config.filter.http.fault.v2.HTTPFault.response_rate_limit>` fault injection.
* fault: added :ref:`HTTP header fault configuration
<config_http_filters_fault_injection_http_header>` to the HTTP fault filter.
* governance: extending Envoy deprecation policy from 1 release (0-3 months) to 2 releases (3-6 months).
* health check: expected response codes in http health checks are now :ref:`configurable <envoy_api_msg_core.HealthCheck.HttpHealthCheck>`.
* http: added new grpc_http1_reverse_bridge filter for converting gRPC requests into HTTP/1.1 requests.
* http: fixed a bug where Content-Length:0 was added to HTTP/1 204 responses.
* outlier_detection: added support for :ref:`outlier detection event protobuf-based logging <arch_overview_outlier_detection_logging>`.
* mysql: added a MySQL proxy filter that is capable of parsing SQL queries over MySQL wire protocol. Refer to ::ref:`MySQL proxy<config_network_filters_mysql_proxy>` for more details.
* http: added :ref:`max request headers size <envoy_api_field_config.filter.network.http_connection_manager.v2.HttpConnectionManager.max_request_headers_kb>`. The default behaviour is unchanged.
* http: added modifyDecodingBuffer/modifyEncodingBuffer to allow modifying the buffered request/response data.
* performance: new buffer implementation (disabled by default; to test it, add "--use-libevent-buffers 0" to the command-line arguments when starting Envoy).
* http: added encodeComplete/decodeComplete. These are invoked at the end of the stream, after all data has been encoded/decoded respectively. Default implementation is a no-op.
* outlier_detection: added support for :ref:`outlier detection event protobuf-based logging <arch_overview_outlier_detection_logging>`.
* mysql: added a MySQL proxy filter that is capable of parsing SQL queries over MySQL wire protocol. Refer to :ref:`MySQL proxy<config_network_filters_mysql_proxy>` for more details.
* performance: new buffer implementation (disabled by default; to test it, add "--use-libevent-buffers 0" to the command-line arguments when starting Envoy).
* ratelimit: removed deprecated rate limit configuration from bootstrap.
* redis: added :ref:`hashtagging <envoy_api_field_config.filter.network.redis_proxy.v2.RedisProxy.ConnPoolSettings.enable_hashtagging>` to guarantee a given key's upstream.
* redis: added :ref:`latency stats <config_network_filters_redis_proxy_per_command_stats>` for commands.
Expand Down
2 changes: 0 additions & 2 deletions source/common/config/filter_json.cc
Original file line number Diff line number Diff line change
Expand Up @@ -252,7 +252,6 @@ void FilterJson::translateMongoProxy(
auto* delay = proto_config.mutable_delay();
auto* percentage = delay->mutable_percentage();

delay->set_type(envoy::config::filter::fault::v2::FaultDelay::FIXED);
percentage->set_numerator(static_cast<uint32_t>(json_fault->getInteger("percent")));
percentage->set_denominator(envoy::type::FractionalPercent::HUNDRED);
JSON_UTIL_SET_DURATION_FROM_FIELD(*json_fault, *delay, fixed_delay, duration);
Expand Down Expand Up @@ -282,7 +281,6 @@ void FilterJson::translateFaultFilter(
if (!json_config_delay->empty()) {
auto* delay = proto_config.mutable_delay();
auto* percentage = delay->mutable_percentage();
delay->set_type(envoy::config::filter::fault::v2::FaultDelay::FIXED);
percentage->set_numerator(
static_cast<uint32_t>(json_config_delay->getInteger("fixed_delay_percent")));
percentage->set_denominator(envoy::type::FractionalPercent::HUNDRED);
Expand Down
20 changes: 20 additions & 0 deletions source/extensions/filters/common/fault/BUILD
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
licenses(["notice"]) # Apache 2

load(
"//bazel:envoy_build_system.bzl",
"envoy_cc_library",
"envoy_package",
)

envoy_package()

envoy_cc_library(
name = "fault_config_lib",
srcs = ["fault_config.cc"],
hdrs = ["fault_config.h"],
deps = [
"//include/envoy/http:header_map_interface",
"//source/common/protobuf:utility_lib",
"@envoy_api//envoy/config/filter/fault/v2:fault_cc",
],
)
78 changes: 78 additions & 0 deletions source/extensions/filters/common/fault/fault_config.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
#include "extensions/filters/common/fault/fault_config.h"

#include "common/protobuf/utility.h"

namespace Envoy {
namespace Extensions {
namespace Filters {
namespace Common {
namespace Fault {

FaultDelayConfig::FaultDelayConfig(const envoy::config::filter::fault::v2::FaultDelay& delay_config)
: percentage_(delay_config.percentage()) {
switch (delay_config.fault_delay_secifier_case()) {
case envoy::config::filter::fault::v2::FaultDelay::kFixedDelay:
provider_ = std::make_unique<FixedDelayProvider>(
std::chrono::milliseconds(PROTOBUF_GET_MS_REQUIRED(delay_config, fixed_delay)));
break;
case envoy::config::filter::fault::v2::FaultDelay::kHeaderDelay:
provider_ = std::make_unique<HeaderDelayProvider>();
break;
case envoy::config::filter::fault::v2::FaultDelay::FAULT_DELAY_SECIFIER_NOT_SET:
NOT_REACHED_GCOVR_EXCL_LINE;
}
}

absl::optional<std::chrono::milliseconds>
FaultDelayConfig::HeaderDelayProvider::duration(const Http::HeaderEntry* header) const {
if (header == nullptr) {
return absl::nullopt;
}

uint64_t value;
if (!StringUtil::atoull(header->value().getStringView().data(), value)) {
return absl::nullopt;
}

return std::chrono::milliseconds(value);
}

FaultRateLimitConfig::FaultRateLimitConfig(
const envoy::config::filter::fault::v2::FaultRateLimit& rate_limit_config)
: percentage_(rate_limit_config.percentage()) {
switch (rate_limit_config.limit_type_case()) {
case envoy::config::filter::fault::v2::FaultRateLimit::kFixedLimit:
provider_ =
std::make_unique<FixedRateLimitProvider>(rate_limit_config.fixed_limit().limit_kbps());
break;
case envoy::config::filter::fault::v2::FaultRateLimit::kHeaderLimit:
provider_ = std::make_unique<HeaderRateLimitProvider>();
break;
case envoy::config::filter::fault::v2::FaultRateLimit::LIMIT_TYPE_NOT_SET:
NOT_REACHED_GCOVR_EXCL_LINE;
}
}

absl::optional<uint64_t>
FaultRateLimitConfig::HeaderRateLimitProvider::rateKbps(const Http::HeaderEntry* header) const {
if (header == nullptr) {
return absl::nullopt;
}

uint64_t value;
if (!StringUtil::atoull(header->value().getStringView().data(), value)) {
return absl::nullopt;
}

if (value == 0) {
return absl::nullopt;
}

return value;
}

} // namespace Fault
} // namespace Common
} // namespace Filters
} // namespace Extensions
} // namespace Envoy
Loading

0 comments on commit 805683f

Please sign in to comment.