[Podman] Supports SQLite containers database back-end to support newer versions (4.8+)

LICENSE-3rdparty.csv

@@ -1220,6 +1220,7 @@ core,github.com/mattn/go-colorable,MIT,Copyright (c) 2016 Yasuhiro Matsumoto
 core,github.com/mattn/go-isatty,MIT,Copyright (c) Yasuhiro MATSUMOTO <mattn.jp@gmail.com>
 core,github.com/mattn/go-runewidth,MIT,Copyright (c) 2016 Yasuhiro Matsumoto
 core,github.com/mattn/go-shellwords,MIT,Copyright (c) 2017 Yasuhiro Matsumoto
+core,github.com/mattn/go-sqlite3,MIT,Copyright (c) 2014 Yasuhiro Matsumoto
 core,github.com/mdlayher/netlink,MIT,Copyright (C) 2016-2022 Matt Layher
 core,github.com/mdlayher/netlink/nlenc,MIT,Copyright (C) 2016-2022 Matt Layher
 core,github.com/mdlayher/socket,MIT,Copyright (C) 2021 Matt Layher

comp/core/workloadmeta/collectors/internal/podman/podman.go

@@ -63,7 +63,16 @@ func (c *collector) Start(_ context.Context, store workloadmeta.Component) error
 		return dderrors.NewDisabled(componentName, "Podman not detected")
 	}
 
-	c.client = podman.NewDBClient(config.Datadog.GetString("podman_db_path"))
+	dbPath := config.Datadog.GetString("podman_db_path")
+
+	// As the containers database file is hard-coded in Podman (non-user customizable), the client to use is determined thanks to the file extension.
+	// If `podman_db_path` references a `db.sql` file, the SQLite client is used. Defaults to BoltDB client otherwise (`bolt_state.db`).
+	if strings.Contains(dbPath, "db.sql") {
+		log.Debugf("Using SQLite client for Podman DB as provided path contains db.sql")
+		c.client = podman.NewSQLDBClient(config.Datadog.GetString(dbPath))
+	} else {
+		c.client = podman.NewDBClient(config.Datadog.GetString(dbPath))
+	}
 	c.store = store
 
 	return nil

go.mod

@@ -674,6 +674,7 @@ require (
 	github.com/godror/godror v0.37.0
 	github.com/jmoiron/sqlx v1.3.5
 	github.com/kr/pretty v0.3.1
+	github.com/mattn/go-sqlite3 v1.14.16
 	github.com/planetscale/vtprotobuf v0.6.0
 	github.com/prometheus-community/pro-bing v0.3.0
 	github.com/rickar/props v1.0.0

pkg/config/config_template.yaml

@@ -2863,7 +2863,9 @@ api_key:
   # listen_address: /var/vcap/data/garden/garden.sock
 
 ## @param podman_db_path - string - optional - default: /var/lib/containers/storage/libpod/bolt_state.db
+## @env DD_PODMAN_DB_PATH - string - optional - default: /var/lib/containers/storage/libpod/bolt_state.db
 ## Settings for Podman DB that Datadog Agent collects container metrics.
+## If using Podman 4.8+ with SQLite back-end, this parameter should be set to /var/lib/containers/storage/db.sql (rootfull containers).
 #
 # podman_db_path: /var/lib/containers/storage/libpod/bolt_state.db
 

pkg/util/podman/sqlite_db_client.go

@@ -0,0 +1,114 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2016-present Datadog, Inc.
+
+//go:build podman
+
+package podman
+
+import (
+	"database/sql"
+	"encoding/json"
+	"fmt"
+	"path/filepath"
+
+	// SQLite backend for database/sql
+	_ "github.com/mattn/go-sqlite3"
+
+	"github.com/DataDog/datadog-agent/pkg/util/log"
+)
+
+// Same strategy as for BoltDB : we do not need the full podman go package.
+// This reduces the number of dependencies and the size of the ultimately shipped binary.
+//
+// The functions in this file have been copied from
+// https://github.com/containers/podman/blob/v5.0.0/libpod/sqlite_state.go
+// The code has been adapted a bit to our needs. The only functions of that file
+// that we need are AllContainers() and NewSqliteState().
+//
+// This code could break in future versions of Podman. This has been tried with
+// v4.9.2 and v5.0.0.
+
+// SQLDBClient is a client for the podman's state database in the SQLite format.
+type SQLDBClient struct {
+	DBPath string
+}
+
+const (
+	// Deal with timezone automatically.
+	sqliteOptionLocation = "_loc=auto"
+	// Read-only mode (https://www.sqlite.org/pragma.html#pragma_query_only)
+	sqliteOptionQueryOnly = "&_query_only=true"
+	// Make sure busy timeout is set to high value to keep retrying when the db is locked.
+	// Timeout is in ms, so set it to 100s to have enough time to retry the operations.
+	sqliteOptionBusyTimeout = "&_busy_timeout=100000"
+
+	// Assembled sqlite options used when opening the database.
+	sqliteOptions = "?" + sqliteOptionLocation + sqliteOptionQueryOnly + sqliteOptionBusyTimeout
+)
+
+// NewSQLDBClient returns a DB client that uses the DB stored in dbPath.
+func NewSQLDBClient(dbPath string) *SQLDBClient {
+	return &SQLDBClient{
+		DBPath: dbPath,
+	}
+}
+
+// getDBCon opens a connection to the SQLite-backed state database.
+// Note: original function comes from https://github.com/containers/podman/blob/e71ec6f1d94d2d97fb3afe08aae0d8adaf8bddf0/libpod/sqlite_state.go#L57-L96
+// It was adapted as we don't need to write any information to the DB.
+func (client *SQLDBClient) getDBCon() (*sql.DB, error) {
+	conn, err := sql.Open("sqlite3", filepath.Join(client.DBPath, sqliteOptions))
+	if err != nil {
+		return nil, fmt.Errorf("opening sqlite database: %w", err)
+	}
+	return conn, nil
+}
+
+// GetAllContainers retrieves all the containers in the database.
+// We retrieve the state always.
+func (client *SQLDBClient) GetAllContainers() ([]Container, error) {
+	var res []Container
+
+	conn, err := client.getDBCon()
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		if errClose := conn.Close(); errClose != nil {
+			log.Warnf("failed to close libpod db: %q", err)
+		}
+	}()
+
+	rows, err := conn.Query("SELECT ContainerConfig.JSON, ContainerState.JSON AS StateJSON FROM ContainerConfig INNER JOIN ContainerState ON ContainerConfig.ID = ContainerState.ID;")
+	if err != nil {
+		return nil, fmt.Errorf("retrieving all containers from database: %w", err)
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var configJSON, stateJSON string
+		if err := rows.Scan(&configJSON, &stateJSON); err != nil {
+			return nil, fmt.Errorf("scanning container from database: %w", err)
+		}
+
+		ctr := new(Container)
+		ctr.Config = new(ContainerConfig)
+		ctr.State = new(ContainerState)
+
+		if err := json.Unmarshal([]byte(configJSON), ctr.Config); err != nil {
+			return nil, fmt.Errorf("unmarshalling container config: %w", err)
+		}
+		if err := json.Unmarshal([]byte(stateJSON), ctr.State); err != nil {
+			return nil, fmt.Errorf("unmarshalling container %s state: %w", ctr.Config.ID, err)
+		}
+
+		res = append(res, *ctr)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+
+	return res, nil
+}

releasenotes/notes/podman-sqlite-backend-support-8437c6d5254b39ef.yaml

@@ -0,0 +1,4 @@
+---
+enhancements:
+  - |
+    Supports Podman newer versions (4.8+) using SQLite instead of BoltDB for the containers database backend. Setting ``podman_db_path`` to the path with the ``db.sql`` file (e.g. ``/var/lib/containers/storage/db.sql``) will make the Datadog Agent use the SQLite format.