From 1949651bae93a5360877d9958478534710a76146 Mon Sep 17 00:00:00 2001 From: Jeff Garratt Date: Wed, 10 May 2017 12:40:16 -0400 Subject: [PATCH] [FAB-3751] Add support to configure kafka in BDD Added concept of composition to step ordererBootstrapAdmin creating the genesis block. This makes mor explicit the requirement that the orderer boostrap requires network configuration information. Now the addition of the orderer addresses and kafka specific genesis block configuration information depend on the composition information. Increased the topology graph view. Change-Id: Icf1fc49265a24898c6c924c56c8778dc3b8e74d6 Signed-off-by: Jeff Garratt --- bddtests/features/bootstrap.feature | 6 +++--- bddtests/steps/bootstrap_impl.py | 15 ++++++++++----- bddtests/steps/bootstrap_util.py | 23 +++++++++++------------ bddtests/steps/compose.py | 13 +++++++------ bddtests/templates/html/graph.html | 2 +- 5 files changed, 32 insertions(+), 27 deletions(-) diff --git a/bddtests/features/bootstrap.feature b/bddtests/features/bootstrap.feature index a62972134ae..6130f428cae 100644 --- a/bddtests/features/bootstrap.feature +++ b/bddtests/features/bootstrap.feature @@ -68,9 +68,9 @@ Feature: Bootstrap # Order info includes orderer admin/orderer information and address (host:port) from previous steps # Only the peer organizations can vary. - And the ordererBootstrapAdmin using cert alias "bootstrapCertAlias" creates the genesis block "ordererGenesisBlock" for chain "OrdererSystemChainId" for network config policy "" and consensus "" using consortiums: - | Consortium | - | consortium1 | + And the ordererBootstrapAdmin using cert alias "bootstrapCertAlias" creates the genesis block "ordererGenesisBlock" for chain "OrdererSystemChainId" for composition "" and consensus "" using consortiums: + | Consortium | + | consortium1 | And the orderer admins inspect and approve the genesis block for chain "OrdererSystemChainId" diff --git a/bddtests/steps/bootstrap_impl.py b/bddtests/steps/bootstrap_impl.py index bf45cbb1df8..7f03a8765a4 100644 --- a/bddtests/steps/bootstrap_impl.py +++ b/bddtests/steps/bootstrap_impl.py @@ -62,8 +62,8 @@ def step_impl(context): # Simply create the user bootstrap_util.getOrdererBootstrapAdmin(context, shouldCreate=True) -@given(u'the ordererBootstrapAdmin using cert alias "{certAlias}" creates the genesis block "{ordererGenesisBlockName}" for chain "{ordererSystemChainIdName}" for network config policy "{networkConfigPolicy}" and consensus "{consensusType}" using consortiums') -def step_impl(context, certAlias, ordererGenesisBlockName, ordererSystemChainIdName, networkConfigPolicy, consensusType): +@given(u'the ordererBootstrapAdmin using cert alias "{certAlias}" creates the genesis block "{ordererGenesisBlockName}" for chain "{ordererSystemChainIdName}" for composition "{composeFile}" and consensus "{consensusType}" using consortiums') +def step_impl(context, certAlias, ordererGenesisBlockName, ordererSystemChainIdName, composeFile, consensusType): directory = bootstrap_util.getDirectory(context=context) ordererBootstrapAdmin = bootstrap_util.getOrdererBootstrapAdmin(context) ordererSystemChainIdGUUID = ordererBootstrapAdmin.tags[ordererSystemChainIdName] @@ -74,12 +74,17 @@ def step_impl(context, certAlias, ordererGenesisBlockName, ordererSystemChainIdN configGroups += ordererBootstrapAdmin.tags[configGroupName] # Concatenate signedConfigItems + service_names = compose.Composition(context, composeFilesYaml=composeFile, register_and_up=False).getServiceNames() + # Construct block nodeAdminTuple = ordererBootstrapAdmin.tags[certAlias] bootstrapCert = directory.findCertForNodeAdminTuple(nodeAdminTuple=nodeAdminTuple) - (genesisBlock, envelope, genesis_block_channel_config) = bootstrap_util.createGenesisBlock(context, ordererSystemChainIdGUUID, consensusType, - nodeAdminTuple=nodeAdminTuple, - signedConfigItems=configGroups) + (genesisBlock, envelope, genesis_block_channel_config) = bootstrap_util.createGenesisBlock(context=context, + service_names=service_names, + chainId=ordererSystemChainIdGUUID, + consensusType=consensusType, + nodeAdminTuple=nodeAdminTuple, + signedConfigItems=configGroups) ordererBootstrapAdmin.setTagValue(ordererGenesisBlockName + "_genesis_channel_config", genesis_block_channel_config) ordererBootstrapAdmin.setTagValue(ordererGenesisBlockName, genesisBlock) ordererBootstrapAdmin.setTagValue("ConsensusType", consensusType) diff --git a/bddtests/steps/bootstrap_util.py b/bddtests/steps/bootstrap_util.py index 58bb13cd915..5aa9926dde5 100644 --- a/bddtests/steps/bootstrap_util.py +++ b/bddtests/steps/bootstrap_util.py @@ -595,7 +595,7 @@ def setDefaultPoliciesForOrgs(channel, orgs, group_name, version=0, policy_versi -def createChannelConfigGroup(directory, hashingAlgoName="SHA256", consensusType="solo", batchTimeout="1s", batchSizeMaxMessageCount=10, batchSizeAbsoluteMaxBytes=100000000, batchSizePreferredMaxBytes=512 * 1024, channel_max_count=0): +def createChannelConfigGroup(directory, service_names, hashingAlgoName="SHA256", consensusType="solo", batchTimeout="1s", batchSizeMaxMessageCount=10, batchSizeAbsoluteMaxBytes=100000000, batchSizePreferredMaxBytes=512 * 1024, channel_max_count=0): channel = common_dot_configtx_pb2.ConfigGroup() # channel.groups[ApplicationGroup] = common_dot_configtx_pb2.ConfigGroup() @@ -647,12 +647,12 @@ def createChannelConfigGroup(directory, hashingAlgoName="SHA256", consensusType= channel.groups[OrdererGroup].groups[ordererOrg.name].values[BootstrapHelper.KEY_MSP_INFO].value = toValue( getMSPConfig(org=ordererOrg, directory=directory)) channel.groups[OrdererGroup].groups[ordererOrg.name].values[BootstrapHelper.KEY_MSP_INFO].mod_policy=BootstrapHelper.KEY_POLICY_ADMINS - # #Kafka specific - # matchingNATs = [nat for nat in directory.getNamedCtxTuples() if (("orderer" in nat.user) and ("Signer" in nat.user) and ((compose_service in nat.nodeName)))] - # for broker in [org for org in directory.getOrganizations().values() if Network.Orderer in org.networks]: - # channel.groups[OrdererGroup].groups[ordererOrg.name].values[BootstrapHelper.KEY_MSP_INFO].value = toValue( - # ordererOrg.getMSPConfig()) - # channel.groups[OrdererGroup].values[BootstrapHelper.KEY_ORDERER_KAFKA_BROKERS].value = toValue(orderer_dot_configuration_pb2.KafkaBrokers(brokers=["kafka0:9092"])) + + #Kafka specific + kafka_brokers = ["{0}:9092".format(service_name) for service_name in service_names if "kafka" in service_name] + if len(kafka_brokers) > 0: + channel.groups[OrdererGroup].values[BootstrapHelper.KEY_ORDERER_KAFKA_BROKERS].value = toValue( + orderer_dot_configuration_pb2.KafkaBrokers(brokers=kafka_brokers)) for vKey, vVal in channel.groups[OrdererGroup].values.iteritems(): vVal.mod_policy=BootstrapHelper.KEY_POLICY_ADMINS @@ -666,9 +666,8 @@ def createChannelConfigGroup(directory, hashingAlgoName="SHA256", consensusType= #New OrdererAddress ordererAddress = common_dot_configuration_pb2.OrdererAddresses() - for ordererNodeTuple, cert in [(user_node_tuple, cert) for user_node_tuple, cert in directory.ordererAdminTuples.iteritems() if - "orderer" in user_node_tuple.user and "signer" in user_node_tuple.user.lower()]: - ordererAddress.addresses.append("{0}:7050".format(ordererNodeTuple.nodeName)) + for orderer_service_name in [service_name for service_name in service_names if "orderer" in service_name]: + ordererAddress.addresses.append("{0}:7050".format(orderer_service_name)) assert len(ordererAddress.addresses) > 0, "No orderer nodes were found while trying to create channel ConfigGroup" channel.values[BootstrapHelper.KEY_ORDERER_ADDRESSES].value = toValue(ordererAddress) @@ -739,13 +738,13 @@ def mergeConfigGroups(configGroupTarget, configGroupSource): configGroupTarget.values[k].CopyFrom(v) -def createGenesisBlock(context, chainId, consensusType, nodeAdminTuple, signedConfigItems=[]): +def createGenesisBlock(context, service_names, chainId, consensusType, nodeAdminTuple, signedConfigItems=[]): 'Generates the genesis block for starting the oderers and for use in the chain config transaction by peers' # assert not "bootstrapGenesisBlock" in context,"Genesis block already created:\n{0}".format(context.bootstrapGenesisBlock) directory = getDirectory(context) assert len(directory.ordererAdminTuples) > 0, "No orderer admin tuples defined!!!" - channelConfig = createChannelConfigGroup(directory=directory, consensusType=consensusType) + channelConfig = createChannelConfigGroup(directory=directory, service_names=service_names, consensusType=consensusType) for configGroup in signedConfigItems: mergeConfigGroups(channelConfig, configGroup) diff --git a/bddtests/steps/compose.py b/bddtests/steps/compose.py index 69960dc105f..bc569ca23cb 100644 --- a/bddtests/steps/compose.py +++ b/bddtests/steps/compose.py @@ -82,8 +82,8 @@ def GetCompositionCallbacksFromContext(cls, context): def GetUUID(cls): return GetDockerSafeUUID() - def __init__(self, context, composeFilesYaml, projectName = None, - force_recreate = True, components = []): + def __init__(self, context, composeFilesYaml, projectName=None, + force_recreate=True, components=[], register_and_up=True): self.contextHelper = ContextHelper.GetHelper(context=context) if not projectName: projectName = self.contextHelper.getGuuid() @@ -93,10 +93,11 @@ def __init__(self, context, composeFilesYaml, projectName = None, self.composeFilesYaml = composeFilesYaml self.serviceNames = [] self.serviceNames = self._collectServiceNames() - # Register with contextHelper (Supports docgen) - self.contextHelper.registerComposition(self) - [callback.composing(self, context) for callback in Composition.GetCompositionCallbacksFromContext(context)] - self.up(context, force_recreate, components) + if register_and_up: + # Register with contextHelper (Supports docgen) + self.contextHelper.registerComposition(self) + [callback.composing(self, context) for callback in Composition.GetCompositionCallbacksFromContext(context)] + self.up(context, force_recreate, components) def _collectServiceNames(self): 'First collect the services names.' diff --git a/bddtests/templates/html/graph.html b/bddtests/templates/html/graph.html index bf9c2e8e96a..ec6acda14c3 100644 --- a/bddtests/templates/html/graph.html +++ b/bddtests/templates/html/graph.html @@ -12,7 +12,7 @@
- +