Update logs sections (#10289)

* Update logs sections

* Update exception list

* Fix more validations

* Add couchbase

* Address review

.gitlab/validate-logs-intgs/validate_log_intgs.py

@@ -19,6 +19,7 @@
 
 EXCEPTIONS = {
     'amazon_eks': [ERR_UNEXPECTED_LOG_COLLECTION_CAT], # eks is just a tile
+    'aspdotnet': [ERR_MISSING_LOG_DOC], # Use iis pipeline
     'azure_active_directory': [
         ERR_MISSING_LOG_DOC,  # This is a tile only integration, the source is populated by azure directly.
         ERR_NOT_DEFINED_WEB_UI,  # The integration does not have any metrics.
@@ -27,14 +28,28 @@
         ERR_UNEXPECTED_LOG_COLLECTION_CAT,  # cilium does not need a pipeline to automatically parse the logs
         ERR_UNEXPECTED_LOG_DOC  # The documentation says to use 'source: cilium'
     ],
+    'consul_connect': [ERR_MISSING_LOG_DOC], # Use envoy pipeline
+    'docker_daemon': [ERR_UNEXPECTED_LOG_COLLECTION_CAT], # Tile only integration
+    'ecs_fargate': [
+        ERR_UNEXPECTED_LOG_COLLECTION_CAT, # Log collection but not from the agent
+        ERR_UNEXPECTED_LOG_DOC, # Not collecting logs directly, but has example in its readme
+    ],
     'eks_fargate': [ERR_UNEXPECTED_LOG_COLLECTION_CAT], # Log collection but not from the agent
     'fluentd': [ERR_UNEXPECTED_LOG_COLLECTION_CAT],  # Fluentd is about log collection but we don't collect fluentd logs
+    'jmeter': [ERR_MISSING_LOG_DOC], # Tile only in integrations-core, logs collected in DataDog/jmeter-datadog-backend-listener
+    'journald': [
+        ERR_UNEXPECTED_LOG_DOC, # Journald is a type of logs, and has its own tile
+        ERR_UNEXPECTED_LOG_COLLECTION_CAT,
+    ],
     'kubernetes': [ERR_UNEXPECTED_LOG_COLLECTION_CAT],  # The agent collects logs from kubernetes environment but there is no pipeline per se
     'mesos_master': [ERR_UNEXPECTED_LOG_COLLECTION_CAT], # We do support log collection for mesos environments
     'linkerd': [
         ERR_UNEXPECTED_LOG_COLLECTION_CAT,  # linkerd does not need a pipeline to automatically parse the logs
         ERR_UNEXPECTED_LOG_DOC
     ],
+    'openshift': [ERR_UNEXPECTED_LOG_COLLECTION_CAT],  # The agent collects logs from openshift environment but there is no pipeline
+    'pan_firewall': [ERR_NOT_DEFINED_WEB_UI], # The integration doesn't emit metric
+    'pivotal_pks': [ERR_UNEXPECTED_LOG_COLLECTION_CAT], # Using kubernetes pipeline
     'win32_event_log': [ERR_UNEXPECTED_LOG_COLLECTION_CAT],  # win32_event_log is about log collection but we don't collect win32_event_log logs
 }
 
@@ -82,7 +97,7 @@ def get_log_sources_in_readme(self) -> List[str]:
             content = f.read()
 
         code_sections: List[str] = re.findall(r'(```.*?```|`.*?`)', content, re.DOTALL)
-        sources = set(re.findall(r'(?:"source"|source): "?(\w+)"?', "\n".join(code_sections), re.MULTILINE))
+        sources = set(re.findall(r'(?:"source"|source|\\"source\\"): \\?"?(\w+)\\?"?', "\n".join(code_sections), re.MULTILINE))
 
         return list(sources)
 

activemq_xml/manifest.json

@@ -31,7 +31,9 @@
     "monitors": {},
     "dashboards": {},
     "service_checks": "assets/service_checks.json",
-    "logs": {},
+    "logs": {
+      "source": "activemq"
+    },
     "metrics_metadata": "metadata.csv"
   }
 }

aspdotnet/manifest.json

@@ -30,7 +30,9 @@
       "ASP.NET - Overview": "assets/dashboards/overview.json"
     },
     "service_checks": "assets/service_checks.json",
-    "logs": {},
+    "logs": {
+      "source": "iis"
+    },
     "metrics_metadata": "metadata.csv"
   }
 }

consul_connect/manifest.json

@@ -28,7 +28,9 @@
     "saved_views": {},
     "monitors": {},
     "service_checks": "assets/service_checks.json",
-    "logs": {},
+    "logs": {
+      "source": "envoy"
+    },
     "metrics_metadata": "metadata.csv"
   }
 }

couchbase/assets/configuration/spec.yaml

@@ -33,7 +33,7 @@ files:
   - template: logs
     example:
     - type: file
-      path: /opt/couchbase/var/lib/couchbase/logs/*.log
+      path: /opt/couchbase/var/lib/couchbase/logs/couchdb.log
       source: couchbase
 
 - name: auto_conf.yaml

couchbase/datadog_checks/couchbase/data/conf.yaml.example

@@ -396,5 +396,5 @@ instances:
 #
 # logs:
 #   - type: file
-#     path: /opt/couchbase/var/lib/couchbase/logs/*.log
+#     path: /opt/couchbase/var/lib/couchbase/logs/couchdb.log
 #     source: couchbase

couchbase/manifest.json

@@ -35,7 +35,9 @@
       "couchbase": "assets/dashboards/couchbase_dashboard.json"
     },
     "service_checks": "assets/service_checks.json",
-    "logs": {},
+    "logs": {
+      "source": "couchdb"
+    },
     "metrics_metadata": "metadata.csv",
     "saved_views": {
       "couchbase_processes": "assets/saved_views/couchbase_processes.json"

ecs_fargate/manifest.json

@@ -2,6 +2,7 @@
   "categories": [
     "aws",
     "containers",
+    "log collection",
     "orchestration"
   ],
   "creates_events": false,

teamcity/manifest.json

@@ -33,7 +33,9 @@
     "monitors": {},
     "dashboards": {},
     "service_checks": "assets/service_checks.json",
-    "logs": {},
+    "logs": {
+      "source": "teamcity"
+    },
     "metrics_metadata": "metadata.csv",
     "saved_views": {
       "teamcity_processes": "assets/saved_views/teamcity_processes.json"