feat(redpanda): add option to enable topic auto-creation

The option to enable automatic topic creation is particularly useful for testing
environments, allowing users to produce data into topics without the need to
create them beforehand.

Signed-off-by: Ladislav Macoun <ladislavmacoun@gmail.com>

modules/redpanda/mounts/bootstrap.yaml.tpl

@@ -13,4 +13,8 @@ superusers:
 
 {{- if .KafkaAPIEnableAuthorization }}
 kafka_enable_authorization: true
-{{- end }}
+{{- end }}
+
+{{- if .AutoCreateTopicsEnabled }}
+auto_create_topics_enabled: true
+{{- end }}

modules/redpanda/mounts/redpanda.yaml.tpl

@@ -37,4 +37,6 @@ schema_registry:
 schema_registry_client:
   brokers:
     - address: localhost
-      port: 9093
+      port: 9093
+
+auto_create_topics_enabled: {{ .AutoCreateTopicsEnabled }}

modules/redpanda/options.go

@@ -23,6 +23,9 @@ type options struct {
 	// You must use SCRAM-SHA-256 as algorithm when authenticating on the
 	// Kafka API.
 	ServiceAccounts map[string]string
+
+	// AutoCreateTopicsEnabled is a flag to allow topic auto creation.
+	AutoCreateTopicsEnabled bool
 }
 
 func defaultOptions() options {
@@ -32,6 +35,7 @@ func defaultOptions() options {
 		KafkaAuthenticationMethod:          "none",
 		SchemaRegistryAuthenticationMethod: "none",
 		ServiceAccounts:                    make(map[string]string, 0),
+		AutoCreateTopicsEnabled:            false,
 	}
 }
 
@@ -82,3 +86,10 @@ func WithEnableSchemaRegistryHTTPBasicAuth() Option {
 		o.SchemaRegistryAuthenticationMethod = "http_basic"
 	}
 }
+
+// WithAutoCreateTopics enables topic auto creation.
+func WithAutoCreateTopics() Option {
+	return func(o *options) {
+		o.AutoCreateTopicsEnabled = true
+	}
+}

modules/redpanda/redpanda.go

@@ -229,6 +229,7 @@ func createBootstrapConfigFile(settings options) (*os.File, error) {
 	bootstrapTplParams := redpandaBootstrapConfigTplParams{
 		Superusers:                  settings.Superusers,
 		KafkaAPIEnableAuthorization: settings.KafkaEnableAuthorization,
+		AutoCreateTopicsEnabled:     settings.AutoCreateTopicsEnabled,
 	}
 
 	tpl, err := template.New("bootstrap.yaml").Parse(bootstrapConfigTpl)
@@ -257,6 +258,7 @@ func createBootstrapConfigFile(settings options) (*os.File, error) {
 // byte array.
 func renderNodeConfig(settings options, hostIP string, advertisedKafkaPort int) ([]byte, error) {
 	tplParams := redpandaConfigTplParams{
+		AutoCreateTopicsEnabled: settings.AutoCreateTopicsEnabled,
 		KafkaAPI: redpandaConfigTplParamsKafkaAPI{
 			AdvertisedHost:       hostIP,
 			AdvertisedPort:       advertisedKafkaPort,
@@ -284,11 +286,13 @@ func renderNodeConfig(settings options, hostIP string, advertisedKafkaPort int)
 type redpandaBootstrapConfigTplParams struct {
 	Superusers                  []string
 	KafkaAPIEnableAuthorization bool
+	AutoCreateTopicsEnabled     bool
 }
 
 type redpandaConfigTplParams struct {
-	KafkaAPI       redpandaConfigTplParamsKafkaAPI
-	SchemaRegistry redpandaConfigTplParamsSchemaRegistry
+	KafkaAPI                redpandaConfigTplParamsKafkaAPI
+	SchemaRegistry          redpandaConfigTplParamsSchemaRegistry
+	AutoCreateTopicsEnabled bool
 }
 
 type redpandaConfigTplParamsKafkaAPI struct {

modules/redpanda/redpanda_test.go

@@ -10,6 +10,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/twmb/franz-go/pkg/kadm"
+	"github.com/twmb/franz-go/pkg/kerr"
 	"github.com/twmb/franz-go/pkg/kgo"
 	"github.com/twmb/franz-go/pkg/sasl/scram"
 )
@@ -63,6 +64,10 @@ func TestRedpanda(t *testing.T) {
 	require.NoError(t, err)
 	defer resp.Body.Close()
 	assert.Equal(t, http.StatusOK, resp.StatusCode)
+
+	// Test produce to unknown topic
+	results := kafkaCl.ProduceSync(ctx, &kgo.Record{Topic: "test", Value: []byte("test message")})
+	require.Error(t, results.FirstErr(), kerr.UnknownTopicOrPartition)
 }
 
 func TestRedpandaWithAuthentication(t *testing.T) {
@@ -173,3 +178,29 @@ func TestRedpandaWithAuthentication(t *testing.T) {
 		resp.Body.Close()
 	}
 }
+
+func TestRedpandaProduceWithAutoCreateTopics(t *testing.T) {
+	ctx := context.Background()
+
+	container, err := RunContainer(ctx, WithAutoCreateTopics())
+	require.NoError(t, err)
+
+	t.Cleanup(func() {
+		if err := container.Terminate(ctx); err != nil {
+			t.Fatalf("failed to terminate container: %s", err)
+		}
+	})
+
+	brokers, err := container.KafkaSeedBroker(ctx)
+	require.NoError(t, err)
+
+	kafkaCl, err := kgo.NewClient(
+		kgo.SeedBrokers(brokers),
+		kgo.AllowAutoTopicCreation(),
+	)
+	require.NoError(t, err)
+	defer kafkaCl.Close()
+
+	results := kafkaCl.ProduceSync(ctx, &kgo.Record{Topic: "test", Value: []byte("test message")})
+	require.NoError(t, results.FirstErr())
+}