Merge remote-tracking branch 'upstream/master' into feature/use-ubunt…

…u-20

* upstream/master: (21 commits)
  Prefer W3C traceparent over elastic-apm-traceparent (elastic#1302)
  fix spacing and cross references in docs (elastic#1328)
  Update README (elastic#1325)
  Mark MicrosoftAzureBlobStorageTracer internal (elastic#1326)
  Update docs (elastic#1327)
  Update context.destination.address (elastic#1324)
  synchronize json schema specs (elastic#1320)
  Don't package Elastic.Apm.Specification (elastic#1316)
  Update setup.asciidoc (elastic#1318)
  Prepare release v.1.10.0 (elastic#1314)
  Fix nullref in Elastic.Apm.Extensions.Logging (elastic#1311)
  Capture errors with startup hook auto instrumentation (elastic#1298)
  Use Logger to log exception in AgentComponents initialization (elastic#1305)
  fix: use .NET native SDK for build and test (elastic#1301)
  Skip running Elasticsearch docker test when docker not available (elastic#1312)
  Use TraceLogger as default logger in ASP.NET Full Framework (elastic#1288)
  Create receive messaging span when inside transaction (elastic#1308)
  Fix SanitizeFieldNamesTests (elastic#1299)
  Do not capture HTTP child spans for Elasticsearch (elastic#1306)
  Use storage account in destination.service.resource (elastic#1284)
  ...

CHANGELOG.asciidoc

@@ -23,6 +23,31 @@ endif::[]
 [[release-notes-1.x]]
 === .NET Agent version 1.x
 
+[[release-notes-1.10.0]]
+==== 1.10.0
+
+[float]
+===== Features
+- {pull}1225[#1225] Add instrumentation for Azure Service Bus (issue: {issue}1157[#1157])
+- {pull}1247[#1247] Add Azure storage integration (issues: {issue}1156[#1156] and {issue}1155[#1155])
+- {pull}1241[#1241] Internalize `Newtonsoft.Json` - no more dependency on `Newtonsoft.Json`
+- {pull}1275[#1275] Internalize `Ben.Demystifier` - no more dependency on `Ben.Demystifier` (issue: {issue}1232[#1232])
+- {pull}1215[#1215] Add MongoDb support (issue: {issue}1158[#1158])
+- {pull}1277[#1277] Capture inner exceptions (issue: {issue}1267[#1267])
+- {pull}1290[#1290] Add configured hostname (issue: {issue}1289[#1289])
+- {pull}1288[#1288] Use TraceLogger as default logger in ASP.NET Full Framework (issue: {issue}1263[#1263])
+
+[float]
+===== Bug fixes
+- {pull}1252[#1252] Fix issue around setting `Recording` to `false` (issue: {issue}1250[#1250])
+- {pull}1259[#1259] ASP.NET: Move error capturing to Error event handler
+- {pull}1305[#1305] Use Logger to log exception in AgentComponents initialization (issue: {issue}1254[#1254])
+- {pull}1311[#1311] Fix `NullReferenceException` in Elastic.Apm.Extensions.Logging(issue: {issue}1309[#1309])
+
+[float]
+===== Breaking changes
+- {pull}1306[#1306] Do not capture HTTP child spans for Elasticsearch (issue: {issue}1276[#1276])
+
 [[release-notes-1.9.0]]
 ==== 1.9.0
 

Jenkinsfile

@@ -18,7 +18,7 @@ pipeline {
     SLACK_CHANNEL = '#apm-agent-dotnet'
   }
   options {
-    timeout(time: 2, unit: 'HOURS')
+    timeout(time: 4, unit: 'HOURS')
     buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20', daysToKeepStr: '30'))
     timestamps()
     ansiColor('xterm')
@@ -28,7 +28,7 @@ pipeline {
     quietPeriod(10)
   }
   triggers {
-    issueCommentTrigger('(?i).*(?:jenkins\\W+)?run\\W+(?:the\\W+)?(?:benchmark\\W+)?tests(?:\\W+please)?.*')
+    issueCommentTrigger('(?i)(.*(?:jenkins\\W+)?run\\W+(?:the\\W+)?(?:benchmark\\W+)?tests(?:\\W+please)?.*|/test)')
   }
   parameters {
     booleanParam(name: 'Run_As_Master_Branch', defaultValue: false, description: 'Allow to run any steps on a PR, some steps normally only run on master branch.')
@@ -72,20 +72,20 @@ pipeline {
               Make sure there are no code style violation in the repo.
               */
               stages{
-                // Disable until https://github.com/elastic/apm-agent-dotnet/issues/563
-                // stage('CodeStyleCheck') {
-                //   steps {
-                //     withGithubNotify(context: 'CodeStyle check') {
-                //       deleteDir()
-                //       unstash 'source'
-                //       dir("${BASE_DIR}"){
-                //         dotnet(){
-                //           sh label: 'Install and run dotnet/format', script: '.ci/linux/codestyle.sh'
-                //         }
-                //       }
-                //     }
-                //   }
-                // }
+              //   Disable until https://github.com/elastic/apm-agent-dotnet/issues/563
+              //   stage('CodeStyleCheck') {
+              //     steps {
+              //       withGithubNotify(context: 'CodeStyle check') {
+              //         deleteDir()
+              //         unstash 'source'
+              //         dir("${BASE_DIR}"){
+              //           dotnet(){
+              //             sh label: 'Install and run dotnet/format', script: '.ci/linux/codestyle.sh'
+              //           }
+              //         }
+              //       }
+              //     }
+              //   }
                 /**
                 Build the project from code..
                 */
@@ -124,9 +124,11 @@ pipeline {
                     withGithubNotify(context: 'Test - Linux', tab: 'tests') {
                       deleteDir()
                       unstash 'source'
-                      dir("${BASE_DIR}"){
-                        dotnet(){
-                          sh label: 'Test & coverage', script: '.ci/linux/test.sh'
+                      filebeat(output: "docker.log"){
+                        dir("${BASE_DIR}"){
+                          dotnet(){
+                            sh label: 'Test & coverage', script: '.ci/linux/test.sh'
+                          }
                         }
                       }
                     }
@@ -521,16 +523,48 @@ def cleanDir(path){
 }
 
 def dotnet(Closure body){
-  def dockerTagName = 'docker.elastic.co/observability-ci/apm-agent-dotnet-sdk-linux:latest'
-  sh label: 'Docker build', script: "docker build --tag ${dockerTagName} .ci/docker/sdk-linux"
+
   def homePath = "${env.WORKSPACE}/${env.BASE_DIR}"
-  docker.image("${dockerTagName}").inside("-e HOME='${homePath}' -v /var/run/docker.sock:/var/run/docker.sock"){
+  withEnv([
+    "HOME=${homePath}",
+    "DOTNET_ROOT=${homePath}/.dotnet",
+    "PATH+DOTNET=${homePath}/.dotnet/tools:${homePath}/.dotnet"
+    ]){
+    sh(label: 'Install dotnet SDK', script: """
+    mkdir -p \${DOTNET_ROOT}
+    # Download .Net SDK installer script
+    curl -s -O -L https://dotnet.microsoft.com/download/dotnet-core/scripts/v1/dotnet-install.sh
+    chmod ugo+rx dotnet-install.sh
+
+    # Install .Net SDKs
+    ./dotnet-install.sh --install-dir "\${DOTNET_ROOT}" -version '2.1.505'
+    ./dotnet-install.sh --install-dir "\${DOTNET_ROOT}" -version '3.0.103'
+    ./dotnet-install.sh --install-dir "\${DOTNET_ROOT}" -version '3.1.100'
+    ./dotnet-install.sh --install-dir "\${DOTNET_ROOT}" -version '5.0.203'
+    """)
     withAzureCredentials(path: "${homePath}", credentialsFile: '.credentials.json') {
-      body()
+      withTerraform(){
+        body()
+      }
     }
   }
 }
 
+def withTerraform(Closure body){
+  def binDir = "${HOME}/bin"
+  withEnv([
+    "PATH+TERRAFORM=${binDir}"
+    ]){
+      sh(label:'Install Terraform', script: """
+        mkdir -p ${binDir}
+        cd ${binDir}
+        curl -sSL -o terraform.zip https://releases.hashicorp.com/terraform/0.15.3/terraform_0.15.3_linux_amd64.zip
+        unzip terraform.zip
+      """)
+      body()
+  }
+}
+
 def release(Map args = [:]){
   def secret = args.secret
   def withSuffix = args.get('withSuffix', false)

README.md

@@ -15,12 +15,15 @@ Official NuGet packages can be referenced from [NuGet.org](https://www.nuget.org
 | `Elastic.Apm`           |  The core of the Agent, Public Agent API, Auto instrumentation for libraries that are part of .NET Standard 2.0.  | [![NuGet Release][ElasticApm-image]][ElasticApm-nuget-url]  |
 | `Elastic.Apm.AspNetCore` | ASP.NET Core auto instrumentation. | [![NuGet Release][ElasticApmAspNetCore-image]][ElasticApmAspNetCore-nuget-url] |
 | `Elastic.Apm.EntityFrameworkCore` | Entity Framework Core auto instrumentation. | [![NuGet Release][Elastic.Apm.EntityFrameworkCore-image]][Elastic.Apm.EntityFrameworkCore-nuget-url] |
-| `Elastic.Apm.NetCoreAll` | References every .NET Core related elastic APM package. It can be used to simply turn on the agent with a single line and activate all auto instrumentation. | [![NuGet Release][Elastic.Apm.NetCoreAll-image]][Elastic.Apm.NetCoreAll-nuget-url] |
+| `Elastic.Apm.NetCoreAll` | References every .NET Core related Elastic APM package. It can be used to simply turn on the agent and activate all auto instrumentation. | [![NuGet Release][Elastic.Apm.NetCoreAll-image]][Elastic.Apm.NetCoreAll-nuget-url] |
 | `Elastic.Apm.AspNetFullFramework` | ASP.NET (classic) auto instrumentation with an IIS Module. | [![NuGet Release][Elastic.Apm.AspNetFullFramework-image]][Elastic.Apm.AspNetFullFramework-nuget-url] |
 | `Elastic.Apm.EntityFramework6` | Entity Framework 6 auto instrumentation. | [![NuGet Release][Elastic.Apm.EntityFramework6-image]][Elastic.Apm.EntityFramework6-nuget-url] |
 | `Elastic.Apm.SqlClient` | `System.Data.SqlClient` and `Microsoft.Data.SqlClient` auto instrumentation. [More details](/src/Elastic.Apm.SqlClient/README.md) | [![NuGet Release][Elastic.Apm.SqlClient-image]][Elastic.Apm.SqlClient-nuget-url] |
 | `Elastic.Apm.Elasticsearch` | Integration with the .NET clients for Elasticsearch. | [![NuGet Release][Elastic.Apm.Elasticsearch-image]][Elastic.Apm.Elasticsearch-nuget-url] |
 | `Elastic.Apm.StackExchange.Redis` | Integration with the StackExchange.Redis client for Redis. | [![NuGet Release][Elastic.Apm.StackExchange.Redis-image]][Elastic.Apm.StackExchange.Redis-nuget-url] |
+| `Elastic.Apm.MongoDb` | Integration with the MongoDb.Driver driver for MongoDb. | [![NuGet Release][Elastic.Apm.MongoDb-image]][Elastic.Apm.MongoDb-nuget-url] |
+| `Elastic.Apm.Azure.ServiceBus` | Integration with Azure ServiceBus | [![NuGet Release][Elastic.Apm.Azure.ServiceBus-image]][Elastic.Apm.Azure.ServiceBus-nuget-url] |
+| `Elastic.Apm.Azure.Storage` | Integration with Azure Storage | [![NuGet Release][Elastic.Apm.Azure.Storage-image]][Elastic.Apm.Azure.Storage-nuget-url] |
 
 ## Documentation
 
@@ -56,6 +59,9 @@ These are the main folders within the repository:
   * `Elastic.Apm.SqlClient`: Auto-instrumentation for `System.Data.SqlClient` and `Microsoft.Data.SqlClient`.
   * `Elastic.Apm.Elasticsearch`: Auto-instrumentation for the official .NET clients for Elasticsearch.
   * `Elastic.Apm.StackExchange.Redis`: Auto-instrumentation for the StackExchange.Redis client for Redis.
+  * `Elastic.Apm.MongoDb`: Instrumentation for the MongoDb.Driver driver for MongoDb.
+  * `Elastic.Apm.Azure.ServiceBus`: Instrumentation for Azure ServiceBus.
+  * `Elastic.Apm.Azure.Storage`: Instrumentation for Azure Storage.
 * `test`: This folder contains test projects. Typically each project from the `src` folder has a corresponding test project.
   * `Elastic.Apm.Tests`: Tests the `Elastic.Apm` project.
   * `Elastic.Apm.AspNetCore.Tests`: Tests the `Elastic.Apm.AspNetCore` project.
@@ -106,3 +112,15 @@ https://img.shields.io/nuget/v/Elastic.Apm.Elasticsearch.svg
 [Elastic.Apm.StackExchange.Redis-nuget-url]:https://www.nuget.org/packages/Elastic.Apm.StackExchange.Redis/
 [Elastic.Apm.StackExchange.Redis-image]:
 https://img.shields.io/nuget/v/Elastic.Apm.StackExchange.Redis.svg
+
+[Elastic.Apm.MongoDb-nuget-url]:https://www.nuget.org/packages/Elastic.Apm.MongoDb/
+[Elastic.Apm.MongoDb-image]:
+https://img.shields.io/nuget/v/Elastic.Apm.MongoDb.svg
+
+[Elastic.Apm.Azure.ServiceBus-nuget-url]:https://www.nuget.org/packages/Elastic.Apm.Azure.ServiceBus/
+[Elastic.Apm.Azure.ServiceBus-image]:
+https://img.shields.io/nuget/v/Elastic.Apm.Azure.ServiceBus.svg
+
+[Elastic.Apm.Azure.Storage-nuget-url]:https://www.nuget.org/packages/Elastic.Apm.Azure.Storage/
+[Elastic.Apm.Azure.Storage-image]:
+https://img.shields.io/nuget/v/Elastic.Apm.Azure.Storage.svg

docs/configuration.asciidoc

@@ -33,7 +33,6 @@ using Elastic.Apm.AspNetCore;
 
 public class Startup
 {
-
     private readonly IConfiguration _configuration;
 
     public Startup(IConfiguration configuration)
@@ -44,7 +43,7 @@ public class Startup
     public void Configure(IApplicationBuilder app, IHostingEnvironment env)
     {
         //Registers the agent with an IConfiguration instance:
-        app.UseElasticApm(Configuration);
+        app.UseElasticApm(_configuration);
 
         //Rest of the Configure() method...
     }
@@ -879,7 +878,7 @@ When this setting is `true`, the agent also adds the header `elasticapm-tracepar
 [options="header"]
 |============
 | Environment variable name                    | IConfiguration or Web.config key
-| `ELASTIC_APM_USE_ELASTIC_TRACEPARENT_HEADER` | `ElasticApm:UseElasticTraceparentHeder`
+| `ELASTIC_APM_USE_ELASTIC_TRACEPARENT_HEADER` | `ElasticApm:UseElasticTraceparentHeader`
 |============
 
 [options="header"]

docs/setup.asciidoc

@@ -168,7 +168,7 @@ The following example only turns on outgoing HTTP monitoring (so, for instance,
 [[zero-code-change-setup]]
 ==== Zero code change setup on .NET Core (added[1.7])
 
-If you can't or don't want to reference NuGet packages in your application, you can use the startup hook feature to inject the agent during startup, if your application runs on .NET Core. This feature is supported on .NET Core 2.2 and newer versions.
+If you can't or don't want to reference NuGet packages in your application, you can use the startup hook feature to inject the agent during startup, if your application runs on .NET Core. This feature is supported on .NET Core 3.0 and newer versions.
 
 Steps:
 
@@ -473,7 +473,7 @@ A prerequisite for auto instrumentation with [`MongoDb.Driver`] is to configure
 ----
 var settings = MongoClientSettings.FromConnectionString(mongoConnectionString);
 
-settings.ClusterConfigurator = builder => builder.Subscribe(new MongoEventSubscriber());
+settings.ClusterConfigurator = builder => builder.Subscribe(new MongoDbEventSubscriber());
 var mongoClient = new MongoClient(settings);
 ----
 

docs/troubleshooting.asciidoc

@@ -46,15 +46,55 @@ This means the Agent will pick up the configured logging provider and log as any
 [[collect-logs-classic]]
 ==== ASP.NET Classic
 
-Unlike ASP.NET Core, ASP.NET (classic) does not have a predefined logging system.
-However, if you have a logging system in place, like NLog, Serilog, or similar, you can direct the agent logs into your
-logging system by creating a bridge between the agent's internal logger and your logging system.
+ASP.NET (classic) does not have a predefined logging system. By default, the agent is configured to
+emit log messages to a 
+https://docs.microsoft.com/en-us/dotnet/api/system.diagnostics.tracesource[`System.Diagnostics.TraceSource`] 
+with the source name `"Elastic.Apm"`. The TraceSource adheres to the log levels defined in the
+APM agent configuration.
+
+[IMPORTANT]
+--
+System.Diagnostics.TraceSource requires the https://docs.microsoft.com/en-us/dotnet/framework/debug-trace-profile/how-to-compile-conditionally-with-trace-and-debug[`TRACE` compiler directive to be specified], which is specified
+by default for both Debug and Release build configurations.
+--
+
+https://docs.microsoft.com/en-us/dotnet/api/system.diagnostics.tracelistener[TraceListeners]
+can be configured to monitor log messages for the trace source, using the https://docs.microsoft.com/en-us/dotnet/framework/configure-apps/file-schema/trace-debug/system-diagnostics-element[`<system.diagnostics>`] section of
+web.config. For example, the following web.config section writes Elastic.Apm log messages to a file
+named my_log_file.log:
+
+[source,xml]
+----
+<configuration>
+  <!-- other sections .... -->
+<system.diagnostics>
+  <sources>
+    <source name="Elastic.Apm"> <1>
+      <listeners>
+        <add name="file" 
+          type="System.Diagnostics.TextWriterTraceListener" 
+          initializeData="my_log_file.log" />
+      </listeners>
+    </source>
+  </sources>
+</system.diagnostics>
+</configuration>  
+----
+<1> Define listeners under a source with name `"Elastic.Apm"` to capture agent logs
+
+[float]
+[[collect-logs-class-other-logging-systems]]
+===== Other logging systems
+
+If you have a logging system in place such as https://nlog-project.org/[NLog], https://serilog.net/[Serilog], 
+or similar, you can direct the agent logs into your logging system by creating an adapter between 
+the agent's internal logger and your logging system.
 
 First implement the `IApmLogger` interface from the `Elastic.Apm.Logging` namespace:
 
 [source,csharp]
 ----
-internal class ApmLoggerBridge : IApmLogger
+internal class ApmLoggerAdapter : IApmLogger
 {
 	private readonly Lazy<Logger> _logger;
 	public bool IsEnabled(ApmLogLevel level)
@@ -71,31 +111,44 @@ internal class ApmLoggerBridge : IApmLogger
 }
 ----
 
-An example implementation for NLog can be seen https://github.com/elastic/apm-agent-dotnet/blob/master/sample/AspNetFullFrameworkSampleApp/App_Start/ApmLoggerToNLog.cs[in our GitHub repository].
-
-Then tell the agent to use the `ApmLoggerBridge`.
+An example implementation for NLog can be seen https://github.com/elastic/apm-agent-dotnet/blob/f6a33a185675b7b918af59d3333d94b32329a84a/sample/AspNetFullFrameworkSampleApp/App_Start/ApmLoggerToNLog.cs[in our GitHub repository].
 
-For this in ASP.NET (classic) you need to place the following code into the `Application_Start` method in the `HttpApplication` implementation of your app which is typically in the `Global.asx.cs` file:
+Then tell the agent to use the `ApmLoggerAdapter`. For ASP.NET (classic), place the following code into the `Application_Start` 
+method in the `HttpApplication` implementation of your app which is typically in the `Global.asax.cs` file:
 
 [source,csharp]
 ----
-AgentDependencies.Logger = new ApmLoggerBridge();
+using Elastic.Apm.AspNetFullFramework;
+
+namespace MyApp
+{
+	public class MyApplication : HttpApplication
+	{
+		protected void Application_Start()
+		{
+			AgentDependencies.Logger = new ApmLoggerAdapter();
+
+			// other application setup...
+		}
+	}
+}
 ----
 
-The `AgentDependencies` class lives in the `Elastic.Apm.AspNetFullFramework` namespace.
-During initialization, the agent checks if an additional logger was configured--the agent only does this once, so it's important to set it as early in the process as possible (typically in the `Application_Start` method).
+During initialization, the agent checks if an additional logger was configured-- the agent only does this once, so it's important 
+to set it as early in the process as possible, typically in the `Application_Start` method.
 
 [float]
 [[collect-logs-general]]
 ==== General .NET applications
 
-If none of the above cases apply to your application, you can still use a bridge and redirect agent logs into a .NET logging system (like NLog, Serilog, or similar).
+If none of the above cases apply to your application, you can still use a logger adapter and redirect agent logs into a .NET 
+logging system like NLog, Serilog, or similar.
 
 For this you'll need an `IApmLogger` implementation (see above) which you need to pass to the `Setup` method during agent setup:
 
 [source,csharp]
 ----
-Agent.Setup(new AgentComponents(logger: new ApmLoggerBridge()));
+Agent.Setup(new AgentComponents(logger: new ApmLoggerAdapter()));
 ----
 
 [float]
@@ -133,17 +186,17 @@ set ELASTIC_APM_STARTUP_HOOKS_LOGGING=1
 ----
 
 and then running the application in a context where the environment variable will be visible. In setting this value,
-an `ElasticApmAgentStartupHook.log` file is written to in the directory containing the startup hook assembly, in addition to
+an `ElasticApmAgentStartupHook.log` file is written to the directory containing the startup hook assembly, in addition to
 writing to standard output.
 
 [float]
 [[agent-overhead]]
 === The agent causes too much overhead
 
-A good place to start is [config-all-options-summary]. There are multiple settings with the `Performance` keyword which can help you tweak the agent for your needs.
+A good place to start is <<config-all-options-summary>>. There are multiple settings with the `Performance` keyword which can help you tweak the agent for your needs.
 
 The most expensive operation in the agent is typically stack trace capturing. The agent, by default, only captures stack traces for spans with a duration of 5ms or more, and with a limit of 50 stack frames.
 If this is too much in your environment, consider disabling stack trace capturing either partially or entirely:
 
-- To disable stack trace capturing for spans, but continue to capture stack traces for errors, set the [config-span-frames-min-duration] to `0` and leave the [config-stack-trace-limit] on its default.
-- To disable stack trace capturing entirely –which in most applications reduces the agent overhead dramatically– set [config-stack-trace-limit] to `0`.
+- To disable stack trace capturing for spans, but continue to capture stack traces for errors, set the <<config-span-frames-min-duration>> to `0` and leave the <<config-stack-trace-limit>> on its default.
+- To disable stack trace capturing entirely –which in most applications reduces the agent overhead dramatically– set <<config-stack-trace-limit>> to `0`.