feat: add opt_out_useragent_filter and $browser_type

src/__tests__/posthog-core.js

@@ -6,6 +6,7 @@ import { autocapture } from '../autocapture'
 import { truth } from './helpers/truth'
 import { _info } from '../utils/event-utils'
 import { document, window } from '../utils/globals'
+import * as globals from '../utils/globals'
 
 jest.mock('../gdpr-utils', () => ({
     ...jest.requireActual('../gdpr-utils'),
@@ -127,6 +128,44 @@ describe('posthog core', () => {
             expect(console.error).toHaveBeenCalledWith('[PostHog.js]', 'No event name provided to posthog.capture')
         })
 
+        it('respects opt_out_useragent_filter (default: false)', () => {
+            const originalUseragent = globals.userAgent
+            // eslint-disable-next-line no-import-assign
+            globals['userAgent'] =
+                'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/W.X.Y.Z Safari/537.36'
+
+            const hook = jest.fn()
+            given.lib._addCaptureHook(hook)
+            given.subject()
+            expect(hook).not.toHaveBeenCalledWith('$event')
+
+            // eslint-disable-next-line no-import-assign
+            globals['userAgent'] = originalUseragent
+        })
+
+        it('respects opt_out_useragent_filter', () => {
+            const originalUseragent = globals.userAgent
+
+            given('config', () => ({
+                opt_out_useragent_filter: true,
+                property_blacklist: [],
+                _onCapture: jest.fn(),
+            }))
+
+            // eslint-disable-next-line no-import-assign
+            globals['userAgent'] =
+                'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/W.X.Y.Z Safari/537.36'
+
+            const hook = jest.fn()
+            given.lib._addCaptureHook(hook)
+            const event = given.subject()
+            expect(hook).toHaveBeenCalledWith('$event')
+            expect(event.properties['$browser_type']).toEqual('bot')
+
+            // eslint-disable-next-line no-import-assign
+            globals['userAgent'] = originalUseragent
+        })
+
         it('truncates long properties', () => {
             given('config', () => ({
                 properties_string_max_length: 1000,

src/posthog-core.ts

@@ -130,6 +130,7 @@ export const defaultConfig = (): PostHogConfig => ({
     ip: true,
     opt_out_capturing_by_default: false,
     opt_out_persistence_by_default: false,
+    opt_out_useragent_filter: false,
     opt_out_capturing_persistence_type: 'localStorage',
     opt_out_capturing_cookie_prefix: null,
     opt_in_site_apps: false,
@@ -866,7 +867,11 @@ export class PostHog {
             return
         }
 
-        if (userAgent && _isBlockedUA(userAgent, this.config.custom_blocked_useragents)) {
+        if (
+            userAgent &&
+            !this.config.opt_out_useragent_filter &&
+            _isBlockedUA(userAgent, this.config.custom_blocked_useragents)
+        ) {
             return
         }
 
@@ -989,6 +994,12 @@ export class PostHog {
             properties['$duration'] = parseFloat((duration_in_ms / 1000).toFixed(3))
         }
 
+        if (userAgent && this.config.opt_out_useragent_filter) {
+            properties['$browser_type'] = _isBlockedUA(userAgent, this.config.custom_blocked_useragents)
+                ? 'bot'
+                : 'browser'
+        }
+
         // note: extend writes to the first object, so lets make sure we
         // don't write to the persistence properties object and info
         // properties object by passing in a new object
@@ -1631,6 +1642,9 @@ export class PostHog {
      *       // opt users out of browser data storage by this PostHog instance by default
      *       opt_out_persistence_by_default: false
      *
+     *       // opt out of user agent filtering such as googlebot or other bots
+     *       opt_out_useragent_filter: false
+     *
      *       // persistence mechanism used by opt-in/opt-out methods - cookie
      *       // or localStorage - falls back to cookie if localStorage is unavailable
      *       opt_out_capturing_persistence_type: 'localStorage'

src/types.ts

@@ -90,6 +90,8 @@ export interface PostHogConfig {
     ip: boolean
     opt_out_capturing_by_default: boolean
     opt_out_persistence_by_default: boolean
+    /** Opt out of user agent filtering such as googlebot or other bots. Defaults to `false` */
+    opt_out_useragent_filter: boolean
     opt_out_capturing_persistence_type: 'localStorage' | 'cookie'
     opt_out_capturing_cookie_prefix: string | null
     opt_in_site_apps: boolean

src/utils/blocked-uas.ts

@@ -48,9 +48,9 @@ export const DEFAULT_BLOCKED_UA_STRS = [
     'storebot-google',
 ]
 
-// _.isBlockedUA()
-// This is to block various web spiders from executing our JS and
-// sending false capturing data
+/**
+ * Block various web spiders from executing our JS and sending false capturing data
+ */
 export const _isBlockedUA = function (ua: string, customBlockedUserAgents: string[]): boolean {
     if (!ua) {
         return false