From 4d8599b9dea81108e27397789fa2183de0e13321 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Delbrayelle?= Date: Mon, 8 Jun 2020 21:52:55 +0200 Subject: [PATCH] feat: topic management --- README.md | 172 +++++++++-------- USE_CASES.md | 60 +++--- generators/app/files.js | 25 ++- generators/app/index.js | 11 +- generators/app/prompts.js | 174 +++++++++++++++++- .../kafka/consumer/EntityConsumer.java.ejs | 2 +- .../kafka/producer/EntityProducer.java.ejs | 2 +- .../main/resources/application-kafka.yml.ejs | 8 +- generators/app/utils.js | 8 +- generators/constants.js | 10 +- package.json | 2 +- test/app.spec.js | 140 +++++++++++++- .../.jhipster/modules/jhi-hooks.json | 2 +- .../service/kafka/consumer/FooConsumer.java | 2 +- .../service/kafka/producer/FooProducer.java | 2 +- .../src/main/resources/config/application.yml | 4 +- 16 files changed, 486 insertions(+), 138 deletions(-) diff --git a/README.md b/README.md index e5ed995..6877da5 100644 --- a/README.md +++ b/README.md @@ -14,12 +14,12 @@ - [x] Basic [Consumer/Producer API use](#consumer-api-and-producer-api) - [x] Several [prompt options](#prompt-options-tree) (`polling.timeout`, `auto.offset.reset.policy`) - [x] [AKHQ (KafkaHQ)](#akhq) support +- [x] Topic management ## 🛠 To do or doing... You can have more details about work in progress in [issues](https://github.com/fdelbrayelle/generator-jhipster-kafka/issues): -- [ ] Topic management - [ ] Producer API (ordered messages, high throughput...) - [ ] Deserialization alternatives (JacksonSerde) as a prompt option - [ ] Security (SSL protocol as a prompt option, safe mode...) @@ -112,85 +112,97 @@ Choose your own adventure module! The **END** represents the end of the prompts below, when files are written after confirmation (you can use the `--force` option with `yo jhipster-kafka` to overwrite all files). -``` -. -├── Big Bang Mode (build a configuration from scratch) (default) -│ ├── Consumer -│ │ ├── No entity (will be typed String) (default) -│ │ │ └── componentPrefix -│ │ │ └── pollingTimeoutValue (default = 10000) -│ │ │ ├── earliest (automatically reset the offset to the earliest offset) (default) -│ │ │ │ └── END -│ │ │ ├── latest (automatically reset the offset to the latest offset) -│ │ │ │ └── END -│ │ │ └── none (throw exception to the consumer if no previous offset is found for the consumer group) -│ │ │ └── END -│ │ ├── FooEntity -│ │ │ └── pollingTimeoutValue (default = 10000) -│ │ │ ├── earliest (automatically reset the offset to the earliest offset) (default) -│ │ │ │ └── END -│ │ │ ├── latest (automatically reset the offset to the latest offset) -│ │ │ │ └── END -│ │ │ └── none (throw exception to the consumer if no previous offset is found for the consumer group) -│ │ │ └── END -│ │ └── BarEntity -│ │ └── pollingTimeoutValue (default = 10000) -│ │ ├── earliest (automatically reset the offset to the earliest offset) (default) -│ │ ├── latest (automatically reset the offset to the latest offset) -│ │ └── none (throw exception to the consumer if no previous offset is found for the consumer group) -│ └── Producer -│ ├── No entity (will be typed String) -│ │ └── componentPrefix -│ │ └── END -│ ├── FooEntity -│ │ └── END -│ └── BarEntity -│ └── END -└── Incremental Mode (upgrade an existing configuration) - ├── No entity (will be typed String) (default) - │ └── componentPrefix - │ ├── Consumer - │ │ └── pollingTimeoutValue (default = 10000) - │ │ ├── earliest (automatically reset the offset to the earliest offset) (default) - │ │ │ ├── Continue adding consumers or producers (default = N) - │ │ │ └── END - │ │ ├── latest (automatically reset the offset to the latest offset) - │ │ │ ├── Continue adding consumers or producers (default = N) - │ │ │ └── END - │ │ └── none (throw exception to the consumer if no previous offset is found for the consumer group) - │ │ ├── Continue adding consumers or producers (default = N) - │ │ └── END - │ └── Producer - │ └── END - ├── FooEntity - │ ├── Consumer - │ │ └── pollingTimeoutValue (default = 10000) - │ │ ├── earliest (automatically reset the offset to the earliest offset) (default) - │ │ │ ├── Continue adding consumers or producers (default = N) - │ │ │ └── END - │ │ ├── latest (automatically reset the offset to the latest offset) - │ │ │ ├── Continue adding consumers or producers (default = N) - │ │ │ └── END - │ │ └── none (throw exception to the consumer if no previous offset is found for the consumer group) - │ │ ├── Continue adding consumers or producers (default = N) - │ │ └── END - │ └── Producer - │ └── END - └── BarEntity - ├── Consumer - │ └── pollingTimeoutValue (default = 10000) - │ ├── earliest (automatically reset the offset to the earliest offset) (default) - │ │ ├── Continue adding consumers or producers (default = N) - │ │ └── END - │ ├── latest (automatically reset the offset to the latest offset) - │ │ ├── Continue adding consumers or producers (default = N) - │ │ └── END - │ └── none (throw exception to the consumer if no previous offset is found for the consumer group) - │ ├── Continue adding consumers or producers (default = N) - │ └── END - └── Producer - └── END -``` + + ## Skip prompts diff --git a/USE_CASES.md b/USE_CASES.md index 69316b3..0cc704f 100644 --- a/USE_CASES.md +++ b/USE_CASES.md @@ -29,8 +29,10 @@ After following the first 3 steps of the [basic usage](README.md#basic-usage) ab 5. "For which entity (class name)?" - Foo (the available entities are retrieved in the `.jhipster` folder as `.json`) 6. "What is the consumer polling timeout (in ms)?" - Your answer or '10000' by default (global for all consumers) 7. "Define the auto offset reset policy?" - Your answer or 'earliest' by default (global for all consumers) -8. Overwrite all files in conflict -9. `FooConsumer` (consumes `Foo`) is available with a `FooDeserializer` +8. "Which topic for Foo?" - Any choice (choosing "Custom topic name" will add another question "What is the topic name for Foo?") +9. Loop on each entity with step 8 +10. Overwrite all files in conflict +11. `FooConsumer` (consumes `Foo`) is available with a `FooDeserializer` #### Incremental Mode @@ -39,11 +41,12 @@ After following the first 3 steps of the [basic usage](README.md#basic-usage) ab 3. "Which type of generation do you want?" - Incremental Mode (upgrade an existing configuration) 4. "For which entity (class name)?" - Foo (the available entities are retrieved in the `.jhipster` folder as `.json`) 5. "Which components would you like to generate?" - Consumer -6. "What is the consumer polling timeout (in ms)?" - Your answer or '10000' by default (global for all consumers) -7. "Define the auto offset reset policy?" - Your answer or 'earliest' by default (global for all consumers) -8. "Do you want to continue adding consumers or producers?" - Your answer or 'N' par default -9. Overwrite all files in conflict -10. `FooConsumer` (consumes `Foo`) is available with a `FooDeserializer` +6. "For which topic?" - Any choice (choosing "Custom topic name" will add another question "What is the topic name?") +7. "What is the consumer polling timeout (in ms)?" - Your answer or '10000' by default (global for all consumers) +8. "Define the auto offset reset policy?" - Your answer or 'earliest' by default (global for all consumers) +9. "Do you want to continue adding consumers or producers?" - Your answer or 'N' par default +10. Overwrite all files in conflict +11. `FooConsumer` (consumes `Foo`) is available with a `FooDeserializer` ### Create a producer linked to an entity @@ -56,8 +59,10 @@ After following the first 3 steps of the [basic usage](README.md#basic-usage) ab 3. "Which type of generation do you want?" - Big Bang Mode (build a configuration from scratch) 4. "Which components would you like to generate?" - Producer 5. "For which entity (class name)?" - Foo (the available entities are retrieved in the `.jhipster` folder as `.json`) -6. Overwrite all files in conflict -7. `FooProducer` (produces `Foo`) is available with a `FooSerializer` and a `FooKafkaResource` to [help testing](README.md#test-consumers-and-producers) +6. "Which topic for Foo?" - Any choice (choosing "Custom topic name" will add another question "What is the topic name for Foo?") +7. Loop on each entity with step 6 +8. Overwrite all files in conflict +9. `FooProducer` (produces `Foo`) is available with a `FooSerializer` and a `FooKafkaResource` to [help testing](README.md#test-consumers-and-producers) #### Incremental Mode @@ -66,9 +71,10 @@ After following the first 3 steps of the [basic usage](README.md#basic-usage) ab 3. "Which type of generation do you want?" - Incremental Mode (upgrade an existing configuration) 4. "For which entity (class name)?" - Foo (the available entities are retrieved in the `.jhipster` folder as `.json`) 5. "Which components would you like to generate?" - Producer -6. "Do you want to continue adding consumers or producers?" - Your answer or 'N' par default -7. Overwrite all files in conflict -8. `FooProducer` (produces `Foo`) is available with a `FooSerializer` and a `FooKafkaResource` to [help testing](README.md#test-consumers-and-producers) +6. "For which topic?" - Any choice (choosing "Custom topic name" will add another question "What is the topic name?") +7. "Do you want to continue adding consumers or producers?" - Your answer or 'N' par default +8. Overwrite all files in conflict +9. `FooProducer` (produces `Foo`) is available with a `FooSerializer` and a `FooKafkaResource` to [help testing](README.md#test-consumers-and-producers) ### Create a consumer NOT linked to an entity @@ -84,8 +90,10 @@ After following the first 3 steps of the [basic usage](README.md#basic-usage) ab 6. "How would you prefix your objects (no entity, for instance: [SomeEventType]Consumer|Producer...)?" - someEventType 7. "What is the consumer polling timeout (in ms)?" - Your answer or '10000' by default (global for all consumers) 8. "Define the auto offset reset policy?" - Your answer or 'earliest' by default (global for all consumers) -9. Overwrite all files in conflict -10. `SomeEventTypeConsumer` (consumes `String`) is available with a `SomeEventTypeDeserializer` +9. "Which topic for Foo?" - Any choice (choosing "Custom topic name" will add another question "What is the topic name for Foo?") +10. Loop on each entity with step 9 +11. Overwrite all files in conflict +12. `SomeEventTypeConsumer` (consumes `String`) is available with a `SomeEventTypeDeserializer` #### Incremental Mode @@ -95,11 +103,12 @@ After following the first 3 steps of the [basic usage](README.md#basic-usage) ab 4. "For which entity (class name)?" - No entity (will be typed String) 5. "How would you prefix your objects (no entity, for instance: [SomeEventType]Consumer|Producer...)?" - someEventType 6. "Which components would you like to generate?" - Consumer -7. "What is the consumer polling timeout (in ms)?" - Your answer or '10000' by default (global for all consumers) -8. "Define the auto offset reset policy?" - Your answer or 'earliest' by default (global for all consumers) -9. "Do you want to continue adding consumers or producers?" - Your answer or 'N' par default -10. Overwrite all files in conflict -11. `SomeEventTypeConsumer` (consumes `String`) is available with a `SomeEventTypeDeserializer` +7. "For which topic?" - Any choice (choosing "Custom topic name" will add another question "What is the topic name?") +8. "What is the consumer polling timeout (in ms)?" - Your answer or '10000' by default (global for all consumers) +9. "Define the auto offset reset policy?" - Your answer or 'earliest' by default (global for all consumers) +10. "Do you want to continue adding consumers or producers?" - Your answer or 'N' par default +11. Overwrite all files in conflict +12. `SomeEventTypeConsumer` (consumes `String`) is available with a `SomeEventTypeDeserializer` ### Create a producer NOT linked to an entity @@ -113,8 +122,10 @@ After following the first 3 steps of the [basic usage](README.md#basic-usage) ab 4. "Which components would you like to generate?" - Producer 5. "For which entity (class name)?" - No entity (will be typed String) 6. "How would you prefix your objects (no entity, for instance: [SomeEventType]Consumer|Producer...)?" - someEventType -7. Overwrite all files in conflict -8. `SomeEventTypeProducer` (produces `String`) is available with a `SomeEventTypeSerializer` and a `SomeEventTypeKafkaResource` to [help testing](README.md#test-consumers-and-producers) +7. "Which topic for Foo?" - Any choice (choosing "Custom topic name" will add another question "What is the topic name for Foo?") +8. Loop on each entity with step 7 +9. Overwrite all files in conflict +10. `SomeEventTypeProducer` (produces `String`) is available with a `SomeEventTypeSerializer` and a `SomeEventTypeKafkaResource` to [help testing](README.md#test-consumers-and-producers) #### Incremental Mode @@ -124,6 +135,7 @@ After following the first 3 steps of the [basic usage](README.md#basic-usage) ab 4. "For which entity (class name)?" - No entity (will be typed String) 5. "How would you prefix your objects (no entity, for instance: [SomeEventType]Consumer|Producer...)?" - someEventType 6. "Which components would you like to generate?" - Producer -7. "Do you want to continue adding consumers or producers?" - Your answer or 'N' par default -8. Overwrite all files in conflict -9. `SomeEventTypeProducer` (produces `String`) is available with a `SomeEventTypeSerializer` and a `SomeEventTypeKafkaResource` to [help testing](README.md#test-consumers-and-producers) +7. "For which topic?" - Any choice (choosing "Custom topic name" will add another question "What is the topic name?") +8. "Do you want to continue adding consumers or producers?" - Your answer or 'N' par default +9. Overwrite all files in conflict +10. `SomeEventTypeProducer` (produces `String`) is available with a `SomeEventTypeSerializer` and a `SomeEventTypeKafkaResource` to [help testing](README.md#test-consumers-and-producers) diff --git a/generators/app/files.js b/generators/app/files.js index 74e4acf..38fc9ac 100644 --- a/generators/app/files.js +++ b/generators/app/files.js @@ -12,9 +12,6 @@ module.exports = { writeFiles }; -// This is a default topic naming convention which can be updated (see also application-kafka.yml.ejs) -const topicNamingFormat = (generator, entity) => `queuing.${generator.snakeCaseBaseName}.${_.snakeCase(entity)}`; - function initVariables(generator) { const vavrVersion = '0.10.3'; generator.addMavenProperty('vavr.version', vavrVersion); @@ -47,6 +44,7 @@ function initVariables(generator) { generator.componentsPrefixes = generator.props.componentsPrefixes || []; generator.components = generator.props.components; generator.componentsByEntityConfig = generator.props.componentsByEntityConfig || []; + generator.topics = generator.props.topics; generator.pollingTimeout = generator.props.pollingTimeout; generator.autoOffsetResetPolicy = generator.props.autoOffsetResetPolicy; @@ -320,6 +318,19 @@ function writeFiles(generator) { writeProperties(kafkaPreviousConfiguration, kafkaPreviousTestConfiguration, utils.transformToJavaClassNameCase(prefix)); }); + if (!kafkaPreviousConfiguration.kafka.topic) { + kafkaPreviousConfiguration.kafka.topic = {}; + } + + if (!kafkaPreviousTestConfiguration.kafka.topic) { + kafkaPreviousTestConfiguration.kafka.topic = {}; + } + + generator.topics.forEach(topic => { + kafkaPreviousConfiguration.kafka.topic[topic.key] = topic.value; + kafkaPreviousTestConfiguration.kafka.topic[topic.key] = topic.value; + }); + const kafkaProperties = jsYaml.dump(utils.orderKafkaProperties(kafkaPreviousConfiguration), { lineWidth: -1 }); @@ -356,7 +367,6 @@ function writeFiles(generator) { function buildJsonConsumerConfiguration(generator, entity, enabled) { return { - name: topicNamingFormat(generator, entity), enabled, '[key.deserializer]': 'org.apache.kafka.common.serialization.StringDeserializer', '[value.deserializer]': `${generator.packageName}.service.kafka.deserializer.${entity}Deserializer`, @@ -367,16 +377,15 @@ function buildJsonConsumerConfiguration(generator, entity, enabled) { function buildJsonProducerConfiguration(generator, entity, enabled) { return { - name: topicNamingFormat(generator, entity), enabled, '[key.serializer]': 'org.apache.kafka.common.serialization.StringSerializer', '[value.serializer]': `${generator.packageName}.service.kafka.serializer.${entity}Serializer` }; } function sanitizeProperties(jsyamlGeneratedProperties) { - // related to https://github.com/nodeca/js-yaml/issues/470 + // Related to: https://github.com/nodeca/js-yaml/issues/470 const patternContainingSingleQuote = /^(\s.+)(:[ ]+)('((.+:)+.*)')$/gm; - // related to https://github.com/nodeca/js-yaml/issues/478 + // Related to: https://github.com/nodeca/js-yaml/issues/478 const patternNullGeneratedValue = /^(\s.+)(:)([ ]+null.*)$/gm; return jsyamlGeneratedProperties.replace(patternContainingSingleQuote, '$1$2$4').replace(patternNullGeneratedValue, '$1$2'); } @@ -407,7 +416,7 @@ function registerToEntityPostHook(generator) { 'entity', 'post', 'entity', - 'A JHipster module to generate Apache Kafka consumers and producers.' + 'A JHipster module that generates Apache Kafka consumers and producers and more!' ); } catch (e) { generator.log(`${chalk.red.bold('WARN!')} Could not register as a jhipster entity post creation hook...\n`, e); diff --git a/generators/app/index.js b/generators/app/index.js index 58a8da4..78a9e14 100644 --- a/generators/app/index.js +++ b/generators/app/index.js @@ -19,12 +19,13 @@ module.exports = class extends BaseGenerator { defaults: false }); - // init props + // props used for writing this.props = { entities: [], components: [], componentsByEntityConfig: [], - componentsPrefixes: [] + componentsPrefixes: [], + topics: [] }; this.setupClientOptions(this); @@ -54,7 +55,11 @@ module.exports = class extends BaseGenerator { this.printJHipsterLogo(); // Have Yeoman greet the user. - this.log(`\nWelcome to the ${chalk.bold.yellow('JHipster kafka')} generator! ${chalk.yellow(`v${packagejs.version}\n`)}`); + this.log( + `\nWelcome to the ${chalk.bold.yellow('Kafka')} Module for ${chalk.bold.green('J')}${chalk.bold.red( + 'Hipster' + )}! ${chalk.yellow(`v${packagejs.version}\n`)}` + ); }, checkJhipster() { const currentJhipsterVersion = this.jhipsterAppConfig.jhipsterVersion; diff --git a/generators/app/prompts.js b/generators/app/prompts.js index 5b7bc22..ba509f5 100644 --- a/generators/app/prompts.js +++ b/generators/app/prompts.js @@ -43,7 +43,7 @@ function offsetChoices() { ]; } -function componentChoices() { +function componentsChoices() { return [ { name: 'Consumer', @@ -56,6 +56,34 @@ function componentChoices() { ]; } +function topicsChoices(generator, previousConfiguration) { + const topicsChoices = []; + topicsChoices.push({ + name: 'Default topic name following this convention: message_type.application_type.entity_name', + value: constants.DEFAULT_TOPIC + }); + topicsChoices.push({ + name: 'Custom topic name', + value: constants.CUSTOM_TOPIC + }); + + if (previousConfiguration && previousConfiguration.topic) { + Object.entries(previousConfiguration.topic).forEach(topic => { + if (!topicsChoices.some(t => t.name === topic[1])) { + topicsChoices.push({ name: topic[1], value: topic[1] }); + } + }); + } + + generator.props.topics.forEach(topic => { + if (!topic.existingTopicName) { + topicsChoices.push({ name: topic.value, value: topic.value }); + } + }); + + return topicsChoices; +} + /** * Retrieve from .jhipster metadata, the list of all project entities. * @@ -120,7 +148,7 @@ function askForBigBangOperations(generator, done) { type: 'checkbox', name: 'components', message: 'Which components would you like to generate?', - choices: componentChoices(), + choices: componentsChoices(), default: [], validate: input => (_.isEmpty(input) ? 'You have to choose at least one component' : true) }, @@ -178,7 +206,77 @@ function askForBigBangOperations(generator, done) { } generator.props = _.merge(generator.props, answers); - done(); + + askForBigBangEntityOperations(generator, answers, done); + }); +} + +function askForBigBangEntityOperations(generator, answers, done, entityIndex = 0) { + let name = answers.entities[entityIndex]; + if (answers.entities[entityIndex] === constants.NO_ENTITY) { + name = answers.componentPrefix; + } + + const bigbangEntityPrompt = [ + { + when: answers.components.includes(constants.CONSUMER_COMPONENT) || answers.components.includes(constants.PRODUCER_COMPONENT), + type: 'list', + name: 'topic', + message: `Which topic for ${name}?`, + choices: topicsChoices(generator, null), + default: constants.DEFAULT_TOPIC + }, + { + when: response => response.topic === constants.CUSTOM_TOPIC, + type: 'input', + name: 'topicName', + message: `What is the topic name for ${name}?`, + validate: input => { + if (_.isEmpty(input)) return 'You have to choose a topic name'; + + const legalChars = /^[A-Za-z0-9._]+$/gm; + if (!input.match(new RegExp(legalChars))) { + return 'You can only use alphanumeric characters, dots and underscores'; + } + + if (input.length > constants.TOPIC_NAME_MAX_SIZE) { + return `Your topic name cannot exceed ${constants.TOPIC_NAME_MAX_SIZE} characters`; + } + + return true; + } + }, + { + when: entityIndex < answers.entities.length - 1, + type: 'confirm', + name: 'confirmBigBangEntityOperations', + message: 'Do you want to continue to the next entity/prefix or exit?', + default: true + } + ]; + + generator.prompt(bigbangEntityPrompt).then(subAnswers => { + if (!generator.props.topics) { + generator.props.topics = []; + } + + generator.props.currentEntity = answers.entities[entityIndex]; + + if (generator.props.currentEntity === constants.NO_ENTITY) { + generator.props.currentPrefix = answers.componentPrefix; + } + + pushTopicName(generator, subAnswers.topic, subAnswers.topicName); + + generator.props = _.merge(generator.props, subAnswers); + + if (entityIndex === answers.entities.length - 1) { + done(); + } + + if (subAnswers.confirmBigBangEntityOperations) { + askForBigBangEntityOperations(generator, answers, done, ++entityIndex); + } }); } @@ -238,7 +336,7 @@ function askForIncrementalOperations(generator, done) { if (answers.currentPrefix && !generator.props.componentsPrefixes.includes(answers.currentPrefix)) { generator.props.componentsPrefixes.push(answers.currentPrefix); } - askForUnitaryEntityOperations(generator, done); + askForIncrementalEntityOperations(generator, done); } else { done(); } @@ -247,7 +345,7 @@ function askForIncrementalOperations(generator, done) { function getAvailableComponentsWithoutEntity(generator, previousConfiguration, prefix) { const availableComponents = []; - const allComponentChoices = componentChoices(); + const allComponentChoices = componentsChoices(); const entitiesComponents = utils.extractEntitiesComponents(previousConfiguration); const prefixJavaClassName = utils.transformToJavaClassNameCase(prefix); if (generator.props.componentsByEntityConfig) { @@ -268,10 +366,10 @@ function getAvailableComponentsWithoutEntity(generator, previousConfiguration, p return availableComponents; } -function askForUnitaryEntityOperations(generator, done) { +function askForIncrementalEntityOperations(generator, done) { const getConcernedComponents = (previousConfiguration, entityName, currentPrefix) => { const availableComponents = []; - const allComponentChoices = componentChoices(); + const allComponentChoices = componentsChoices(); const entitiesComponents = utils.extractEntitiesComponents(previousConfiguration); if (entityName === constants.NO_ENTITY) { @@ -308,10 +406,40 @@ function askForUnitaryEntityOperations(generator, done) { when: generator.props.currentEntity, type: 'checkbox', name: 'currentEntityComponents', - validate: input => (_.isEmpty(input) ? 'You have to choose at least one component' : true), message: 'Which components would you like to generate?', choices: getConcernedComponents(previousConfiguration(generator), generator.props.currentEntity, generator.props.currentPrefix), - default: [] + default: [], + validate: input => (_.isEmpty(input) ? 'You have to choose at least one component' : true) + }, + { + when: response => + response.currentEntityComponents.includes(constants.CONSUMER_COMPONENT) || + response.currentEntityComponents.includes(constants.PRODUCER_COMPONENT), + type: 'list', + name: 'topic', + message: 'For which topic?', + choices: topicsChoices(generator, previousConfiguration(generator)), + default: constants.DEFAULT_TOPIC + }, + { + when: response => response.topic === constants.CUSTOM_TOPIC, + type: 'input', + name: 'topicName', + message: 'What is the topic name?', + validate: input => { + if (_.isEmpty(input)) return 'You have to choose a topic name'; + + const legalChars = /^[A-Za-z0-9._]+$/gm; + if (!input.match(new RegExp(legalChars))) { + return 'You can only use alphanumeric characters, dots and underscores'; + } + + if (input.length > constants.TOPIC_NAME_MAX_SIZE) { + return `Your topic name cannot exceed ${constants.TOPIC_NAME_MAX_SIZE} characters`; + } + + return true; + } }, { when: response => @@ -348,6 +476,13 @@ function askForUnitaryEntityOperations(generator, done) { if (!generator.props.componentsByEntityConfig) { generator.props.componentsByEntityConfig = []; } + + if (!generator.props.topics) { + generator.props.topics = []; + } + + pushTopicName(generator, answers.topic, answers.topicName); + if (answers.currentEntityComponents && answers.currentEntityComponents.length > 0) { if (generator.props.currentEntity === constants.NO_ENTITY) { pushComponentsByEntity(generator, answers, utils.transformToJavaClassNameCase(generator.props.currentPrefix)); @@ -355,12 +490,15 @@ function askForUnitaryEntityOperations(generator, done) { pushComponentsByEntity(generator, answers, generator.props.currentEntity); } } + if (answers.pollingTimeout) { generator.props.pollingTimeout = +answers.pollingTimeout; // force conversion to int } + if (answers.autoOffsetResetPolicy) { generator.props.autoOffsetResetPolicy = answers.autoOffsetResetPolicy; } + if (answers.continueAddingEntitiesComponents) { askForIncrementalOperations(generator, done); } else { @@ -380,3 +518,21 @@ function pushComponentsByEntity(generator, answers, entity) { generator.props.componentsByEntityConfig[entity] = [...answers.currentEntityComponents]; } } + +function pushTopicName(generator, topicChoice, topicName) { + const name = generator.props.currentEntity === constants.NO_ENTITY ? generator.props.currentPrefix : generator.props.currentEntity; + + if (topicChoice) { + if (topicChoice === constants.CUSTOM_TOPIC) { + generator.props.topics.push({ key: _.camelCase(name), value: topicName, existingTopicName: false }); + } else if (topicChoice === constants.DEFAULT_TOPIC) { + generator.props.topics.push({ + key: _.camelCase(name), + value: utils.topicNamingFormat(_.snakeCase(generator.jhipsterAppConfig.baseName), _.snakeCase(name)), + existingTopicName: false + }); + } else { + generator.props.topics.push({ key: _.camelCase(name), value: topicChoice, existingTopicName: true }); + } + } +} diff --git a/generators/app/templates/src/main/java/package/service/kafka/consumer/EntityConsumer.java.ejs b/generators/app/templates/src/main/java/package/service/kafka/consumer/EntityConsumer.java.ejs index affaa85..8f8b28d 100644 --- a/generators/app/templates/src/main/java/package/service/kafka/consumer/EntityConsumer.java.ejs +++ b/generators/app/templates/src/main/java/package/service/kafka/consumer/EntityConsumer.java.ejs @@ -38,7 +38,7 @@ public class <%= entityClass %>Consumer extends GenericConsumer<<%= type %>> { private final Logger log = LoggerFactory.getLogger(<%= entityClass %>Consumer.class); - public <%= entityClass %>Consumer(@Value("${kafka.consumer.<%= camelCaseEntityClass %>.name}") final String topicName, final KafkaProperties kafkaProperties) { + public <%= entityClass %>Consumer(@Value("${kafka.topic.<%= camelCaseEntityClass %>}") final String topicName, final KafkaProperties kafkaProperties) { super(topicName, kafkaProperties.getConsumer().get("<%= camelCaseEntityClass %>"), kafkaProperties.getPollingTimeout()); } diff --git a/generators/app/templates/src/main/java/package/service/kafka/producer/EntityProducer.java.ejs b/generators/app/templates/src/main/java/package/service/kafka/producer/EntityProducer.java.ejs index fbbbffa..220f22a 100644 --- a/generators/app/templates/src/main/java/package/service/kafka/producer/EntityProducer.java.ejs +++ b/generators/app/templates/src/main/java/package/service/kafka/producer/EntityProducer.java.ejs @@ -40,7 +40,7 @@ public class <%= entityClass %>Producer { private final String topicName; - public <%= entityClass %>Producer(@Value("${kafka.producer.<%= camelCaseEntityClass %>.name}") final String topicName, final KafkaProperties kafkaProperties) { + public <%= entityClass %>Producer(@Value("${kafka.topic.<%= camelCaseEntityClass %>}") final String topicName, final KafkaProperties kafkaProperties) { this.topicName = topicName; this.kafkaProducer = new KafkaProducer<>(kafkaProperties.getProducer().get("<%= camelCaseEntityClass %>")); } diff --git a/generators/app/templates/src/main/resources/application-kafka.yml.ejs b/generators/app/templates/src/main/resources/application-kafka.yml.ejs index d38a675..f9530a6 100644 --- a/generators/app/templates/src/main/resources/application-kafka.yml.ejs +++ b/generators/app/templates/src/main/resources/application-kafka.yml.ejs @@ -13,7 +13,6 @@ kafka: } _%> <%= _.camelCase(entity) _%>: - name: queuing.<%= snakeCaseBaseName %>.<%= _.snakeCase(entity) %> enabled: <%= enabled %> '[key.deserializer]': org.apache.kafka.common.serialization.StringDeserializer '[value.deserializer]': <%= packageName %>.service.kafka.deserializer.<%= _.upperFirst(_.camelCase(entity)) %>Deserializer @@ -31,10 +30,15 @@ _%> } _%> <%= _.camelCase(entity) _%>: - name: queuing.<%= snakeCaseBaseName %>.<%= _.snakeCase(entity) %> enabled: <%= enabled %> '[key.serializer]': org.apache.kafka.common.serialization.StringSerializer '[value.serializer]': <%= packageName %>.service.kafka.serializer.<%= _.upperFirst(_.camelCase(entity)) %>Serializer <%_ } _%> <%_ } _%> +<%_ if (components.includes('consumer') || components.includes('producer')) { _%> + topic: +<%_ for (topic of topics) { _%> + <%= topic.key %>: <%= topic.value %> +<%_ } _%> +<%_ } _%> diff --git a/generators/app/utils.js b/generators/app/utils.js index 05f2334..64ca2a3 100644 --- a/generators/app/utils.js +++ b/generators/app/utils.js @@ -7,9 +7,15 @@ module.exports = { getPreviousKafkaConfiguration, extractEntitiesComponents, orderKafkaProperties, - transformToJavaClassNameCase + transformToJavaClassNameCase, + topicNamingFormat }; +// This is a default topic naming convention which can be updated (see also application-kafka.yml.ejs) +function topicNamingFormat(baseName, name) { + return `queuing.${baseName}.${name}`; +} + function transformToJavaClassNameCase(entityName) { return _.upperFirst(_.camelCase(entityName)); } diff --git a/generators/constants.js b/generators/constants.js index b0a50bd..061aae9 100644 --- a/generators/constants.js +++ b/generators/constants.js @@ -31,6 +31,11 @@ const NONE_OFFSET = 'none'; const DEFAULT_POLLING_TIMEOUT = '10000'; const JSON_EXTENSION = '.json'; const EMPTY_STRING = ''; +const DEFAULT_TOPIC = 'default_topic'; +const CUSTOM_TOPIC = 'custom_topic'; + +// Related to: https://github.com/apache/kafka/commit/ad3dfc6ab25c3f80d2425e24e72ae732b850dc60 +const TOPIC_NAME_MAX_SIZE = 249; const constants = { JHIPSTER_CONFIG_DIR, @@ -46,7 +51,10 @@ const constants = { NONE_OFFSET, DEFAULT_POLLING_TIMEOUT, JSON_EXTENSION, - EMPTY_STRING + EMPTY_STRING, + DEFAULT_TOPIC, + CUSTOM_TOPIC, + TOPIC_NAME_MAX_SIZE }; module.exports = constants; diff --git a/package.json b/package.json index f835a2d..cb00369 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "generator-jhipster-kafka", "version": "0.3.0", - "description": "A JHipster module to generate Apache Kafka consumers and producers.", + "description": "A JHipster module that generates Apache Kafka consumers and producers and more!", "keywords": [ "yeoman-generator", "jhipster-module", diff --git a/test/app.spec.js b/test/app.spec.js index d0e6764..29b6066 100644 --- a/test/app.spec.js +++ b/test/app.spec.js @@ -14,6 +14,8 @@ const FOO_ENTITY = 'Foo'; const AWESOME_ENTITY = 'AwesomeEntity'; const COMPONENT_PREFIX = 'ComponentsWithoutEntity'; const COMPONENTS_CHOSEN = Object.freeze({ all: 1, consumer: 2, producer: 3 }); +const CUSTOM_TOPIC_NAME = 'custom_topic_name'; +const EXISTING_TOPIC_NAME = 'queuing.message_broker_with_entities.foo'; describe('JHipster generator kafka', () => { describe('with no message broker', () => { @@ -175,6 +177,51 @@ describe('JHipster generator kafka', () => { assert.fileContent(`${jhipsterConstants.SERVER_TEST_RES_DIR}config/application.yml`, /'\[auto.offset.reset\].: latest/); }); }); + + describe('with a consumer and a producer for a single entity with a default topic name', () => { + before(done => { + helpers + .run(path.join(__dirname, '../generators/app')) + .inTmpDir(dir => { + fse.copySync(path.join(__dirname, '../test/templates/message-broker-with-entities-1st-call'), dir); + }) + .withPrompts({ + generationType: constants.BIGBANG_MODE, + components: [constants.CONSUMER_COMPONENT, constants.PRODUCER_COMPONENT], + entities: [FOO_ENTITY], + topic: constants.DEFAULT_TOPIC + }) + .on('end', done); + }); + + it('should put a default topic name in application.yml', () => { + const { applicationYml, testApplicationYml } = loadApplicationYaml(); + assertTopicName(applicationYml, testApplicationYml, FOO_ENTITY, constants.DEFAULT_TOPIC, null); + }); + }); + + describe('with a consumer and a producer for a single entity with a custom topic name', () => { + before(done => { + helpers + .run(path.join(__dirname, '../generators/app')) + .inTmpDir(dir => { + fse.copySync(path.join(__dirname, '../test/templates/message-broker-with-entities-1st-call'), dir); + }) + .withPrompts({ + generationType: constants.BIGBANG_MODE, + components: [constants.CONSUMER_COMPONENT, constants.PRODUCER_COMPONENT], + entities: [FOO_ENTITY], + topic: constants.CUSTOM_TOPIC, + topicName: CUSTOM_TOPIC_NAME + }) + .on('end', done); + }); + + it('should put a custom topic name in application.yml', () => { + const { applicationYml, testApplicationYml } = loadApplicationYaml(); + assertTopicName(applicationYml, testApplicationYml, FOO_ENTITY, constants.CUSTOM_TOPIC, CUSTOM_TOPIC_NAME); + }); + }); }); describe('with the incremental mode', () => { @@ -422,6 +469,53 @@ describe('JHipster generator kafka', () => { itShouldUpdatesPropertiesWithGivenValue(); }); + + describe('with a default topic name', () => { + before(done => { + helpers + .run(path.join(__dirname, '../generators/app')) + .inTmpDir(dir => { + fse.copySync(path.join(__dirname, '../test/templates/message-broker-with-entities-1st-call'), dir); + }) + .withPrompts({ + generationType: constants.INCREMENTAL_MODE, + currentEntity: FOO_ENTITY, + currentEntityComponents: [constants.CONSUMER_COMPONENT, constants.PRODUCER_COMPONENT], + topic: constants.DEFAULT_TOPIC, + continueAddingEntitiesComponents: false + }) + .on('end', done); + }); + + it('should put a default topic name in application.yml', () => { + const { applicationYml, testApplicationYml } = loadApplicationYaml(); + assertTopicName(applicationYml, testApplicationYml, FOO_ENTITY, constants.DEFAULT_TOPIC, null); + }); + }); + + describe('with a custom topic name', () => { + before(done => { + helpers + .run(path.join(__dirname, '../generators/app')) + .inTmpDir(dir => { + fse.copySync(path.join(__dirname, '../test/templates/message-broker-with-entities-1st-call'), dir); + }) + .withPrompts({ + generationType: constants.INCREMENTAL_MODE, + currentEntity: FOO_ENTITY, + currentEntityComponents: [constants.CONSUMER_COMPONENT, constants.PRODUCER_COMPONENT], + topic: constants.CUSTOM_TOPIC, + topicName: CUSTOM_TOPIC_NAME, + continueAddingEntitiesComponents: false + }) + .on('end', done); + }); + + it('should put a custom topic name in application.yml', () => { + const { applicationYml, testApplicationYml } = loadApplicationYaml(); + assertTopicName(applicationYml, testApplicationYml, FOO_ENTITY, constants.CUSTOM_TOPIC, CUSTOM_TOPIC_NAME); + }); + }); }); describe('with a consumer and a producer without entity', () => { @@ -612,6 +706,29 @@ describe('JHipster generator kafka', () => { }); }); + describe('with a consumer and a producer for a single entity (using an existing topic name)', () => { + before(done => { + helpers + .run(path.join(__dirname, '../generators/app')) + .inTmpDir(dir => { + fse.copySync(path.join(__dirname, '../test/templates/message-broker-with-entities-2nd-call'), dir); + }) + .withPrompts({ + generationType: constants.INCREMENTAL_MODE, + currentEntity: AWESOME_ENTITY, + currentEntityComponents: [constants.PRODUCER_COMPONENT, constants.CONSUMER_COMPONENT], + topic: EXISTING_TOPIC_NAME, + continueAddingEntitiesComponents: false + }) + .on('end', done); + }); + + it('should put an existing topic name in application.yml', () => { + const { applicationYml, testApplicationYml } = loadApplicationYaml(); + assertTopicName(applicationYml, testApplicationYml, AWESOME_ENTITY, null, EXISTING_TOPIC_NAME); + }); + }); + describe('with only a consumer without entity', () => { before(done => { helpers @@ -846,10 +963,29 @@ function assertMinimalProperties(applicationYml, testApplicationYml, entityName, entityTestYmlBlock = testApplicationYml.kafka.producer[`${_.camelCase(entityName)}`]; } - assert.textEqual(entityYmlBlock.name, `queuing.message_broker_with_entities.${_.snakeCase(entityName)}`); assert.textEqual(entityYmlBlock.enabled.toString(), 'true'); - assert.textEqual(entityTestYmlBlock.name, `queuing.message_broker_with_entities.${_.snakeCase(entityName)}`); assert.textEqual(entityTestYmlBlock.enabled.toString(), 'false'); + + assertTopicName(applicationYml, testApplicationYml, entityName, constants.DEFAULT_TOPIC, null); +} + +function assertTopicName(applicationYml, testApplicationYml, entityName, topicChoice, topicName) { + let expectedTopicName = `queuing.message_broker_with_entities.${_.snakeCase(entityName)}`; + + if (topicChoice !== constants.DEFAULT_TOPIC) { + expectedTopicName = topicName; + } + + Object.keys(applicationYml.kafka.topic).forEach(key => { + if (key === _.camelCase(entityName)) { + assert.textEqual(applicationYml.kafka.topic[key], expectedTopicName); + } + }); + Object.keys(testApplicationYml.kafka.topic).forEach(key => { + if (key === _.camelCase(entityName)) { + assert.textEqual(testApplicationYml.kafka.topic[key], expectedTopicName); + } + }); } function assertThatKafkaPropertiesAreOrdered(applicationYml) { diff --git a/test/templates/message-broker-with-entities-2nd-call/.jhipster/modules/jhi-hooks.json b/test/templates/message-broker-with-entities-2nd-call/.jhipster/modules/jhi-hooks.json index 2632164..3ef6f3e 100644 --- a/test/templates/message-broker-with-entities-2nd-call/.jhipster/modules/jhi-hooks.json +++ b/test/templates/message-broker-with-entities-2nd-call/.jhipster/modules/jhi-hooks.json @@ -2,7 +2,7 @@ { "name": "Kafka generator", "npmPackageName": "generator-jhipster-kafka", - "description": "A JHipster module to generate Apache Kafka consumers and producers.", + "description": "A JHipster module that generates Apache Kafka consumers and producers and more!", "hookFor": "entity", "hookType": "post", "generatorCallback": "jhipster-kafka:entity" diff --git a/test/templates/message-broker-with-entities-2nd-call/src/main/java/com/mycompany/myapp/service/kafka/consumer/FooConsumer.java b/test/templates/message-broker-with-entities-2nd-call/src/main/java/com/mycompany/myapp/service/kafka/consumer/FooConsumer.java index 607c1db..0e8bc5b 100644 --- a/test/templates/message-broker-with-entities-2nd-call/src/main/java/com/mycompany/myapp/service/kafka/consumer/FooConsumer.java +++ b/test/templates/message-broker-with-entities-2nd-call/src/main/java/com/mycompany/myapp/service/kafka/consumer/FooConsumer.java @@ -18,7 +18,7 @@ public class FooConsumer extends GenericConsumer { private final Logger log = LoggerFactory.getLogger(FooConsumer.class); - public FooConsumer(@Value("${kafka.consumer.foo.name}") final String topicName, final KafkaProperties kafkaProperties) { + public FooConsumer(@Value("${kafka.topic.foo}") final String topicName, final KafkaProperties kafkaProperties) { super(topicName, kafkaProperties.getConsumer().get("foo"), kafkaProperties.getPollingTimeout()); } diff --git a/test/templates/message-broker-with-entities-2nd-call/src/main/java/com/mycompany/myapp/service/kafka/producer/FooProducer.java b/test/templates/message-broker-with-entities-2nd-call/src/main/java/com/mycompany/myapp/service/kafka/producer/FooProducer.java index 33b691e..e8e65c4 100644 --- a/test/templates/message-broker-with-entities-2nd-call/src/main/java/com/mycompany/myapp/service/kafka/producer/FooProducer.java +++ b/test/templates/message-broker-with-entities-2nd-call/src/main/java/com/mycompany/myapp/service/kafka/producer/FooProducer.java @@ -20,7 +20,7 @@ public class FooProducer { private final String topicName; - public FooProducer(@Value("${kafka.producer.foo.name}") final String topicName, final KafkaProperties kafkaProperties) { + public FooProducer(@Value("${kafka.topic.foo}") final String topicName, final KafkaProperties kafkaProperties) { this.topicName = topicName; this.kafkaProducer = new KafkaProducer<>(kafkaProperties.getProducer().get("foo")); } diff --git a/test/templates/message-broker-with-entities-2nd-call/src/main/resources/config/application.yml b/test/templates/message-broker-with-entities-2nd-call/src/main/resources/config/application.yml index 1818f36..73686c3 100644 --- a/test/templates/message-broker-with-entities-2nd-call/src/main/resources/config/application.yml +++ b/test/templates/message-broker-with-entities-2nd-call/src/main/resources/config/application.yml @@ -163,19 +163,19 @@ kafka: bootstrap.servers: ${KAFKA_BOOTSTRAP_SERVERS:localhost:9092} producer: foo: - name: queuing.message_broker_with_entities.foo enabled: true '[key.serializer]': org.apache.kafka.common.serialization.StringSerializer '[value.serializer]': com.mycompany.myapp.service.kafka.serializer.FooSerializer polling.timeout: 10000 consumer: foo: - name: queuing.message_broker_with_entities.foo enabled: true '[key.deserializer]': org.apache.kafka.common.serialization.StringDeserializer '[value.deserializer]': com.mycompany.myapp.service.kafka.deserializer.FooDeserializer '[group.id]': message-broker-with-entities '[auto.offset.reset]': earliest + topic: + foo: queuing.message_broker_with_entities.foo # =================================================================== # Application specific properties # Add your own application properties here, see the ApplicationProperties class