From 2dcddce8ed061acd3a77727b52753f1b77ed51cd Mon Sep 17 00:00:00 2001 From: liweitian Date: Wed, 2 Dec 2020 20:14:13 +0800 Subject: [PATCH] fix: merge with main (#5048) * try-catch around trigger grouping expressions (#5017) Co-authored-by: Ben Yackley <61990921+beyackle@users.noreply.github.com> * chore: Updated installOneAuth script to target correct version. (#5020) * Updated installOneAuth script to target correct version. * Better refactor * feat: adaptive expression functions menu (#5015) * expressions menu * Fixing aligment of menu * PR comments Co-authored-by: Soroush * fix: Correct handling of focus for Intellisense fields (#5039) * fix * Improved handling of focus when leveraging an Intellisense suggestion * comments on wrong lines * feat: azure publish with orchestrator (#5011) * refactor the luis build in azure publish * update the orchestrator path * update the cleanup * fix typo * catch the build error * remove qna endpoint * remove console * add qnaconfig Co-authored-by: Geoff Cox (Microsoft) Co-authored-by: Ben Yackley <61990921+beyackle@users.noreply.github.com> Co-authored-by: Tony Anziano Co-authored-by: LouisEugeneMSFT <66701106+LouisEugeneMSFT@users.noreply.github.com> Co-authored-by: Soroush Co-authored-by: leileizhang --- Composer/packages/adaptive-form/package.json | 3 +- .../src/components/SchemaField.tsx | 1 - .../ExpressionSwitchWindow.tsx | 0 .../expressions/ExpressionsListMenu.tsx | 62 +++ .../utils/expressionsListMenuUtils.ts | 48 ++ .../components/fields/IntellisenseFields.tsx | 42 +- .../src/components/fields/StringField.tsx | 9 +- .../src/recoilModel/dispatchers/publisher.ts | 25 +- .../electron-server/scripts/installOneAuth.js | 3 +- .../extension-client/src/types/form.ts | 1 + .../src/components/CompletionList.tsx | 2 +- .../src/components/Intellisense.tsx | 23 +- .../lib/indexers/src/groupTriggers.ts | 13 +- .../server/src/models/bot/botProject.ts | 1 + .../packages/server/src/models/bot/builder.ts | 63 ++- .../src/builtInFunctionsGrouping.ts | 191 ++++++++ .../src/builtInFunctionsUtils.ts | 20 + .../tools/built-in-functions/src/index.ts | 2 + .../intellisense/src/resolvers/expressions.ts | 11 +- extensions/azurePublish/package.json | 2 - extensions/azurePublish/src/deploy.ts | 24 +- extensions/azurePublish/src/index.ts | 9 +- extensions/azurePublish/src/luisAndQnA.ts | 423 +++--------------- .../azurePublish/src/utils/crossTrainUtil.ts | 216 --------- extensions/azurePublish/src/utils/fileUtil.ts | 12 - extensions/azurePublish/src/utils/jsonWalk.ts | 37 -- 26 files changed, 585 insertions(+), 658 deletions(-) rename Composer/packages/adaptive-form/src/components/{ => expressions}/ExpressionSwitchWindow.tsx (100%) create mode 100644 Composer/packages/adaptive-form/src/components/expressions/ExpressionsListMenu.tsx create mode 100644 Composer/packages/adaptive-form/src/components/expressions/utils/expressionsListMenuUtils.ts create mode 100644 Composer/packages/tools/built-in-functions/src/builtInFunctionsGrouping.ts create mode 100644 Composer/packages/tools/built-in-functions/src/builtInFunctionsUtils.ts delete mode 100644 extensions/azurePublish/src/utils/crossTrainUtil.ts delete mode 100644 extensions/azurePublish/src/utils/fileUtil.ts delete mode 100644 extensions/azurePublish/src/utils/jsonWalk.ts diff --git a/Composer/packages/adaptive-form/package.json b/Composer/packages/adaptive-form/package.json index 1dcbf60460..541e9151de 100644 --- a/Composer/packages/adaptive-form/package.json +++ b/Composer/packages/adaptive-form/package.json @@ -43,6 +43,7 @@ "dependencies": { "@emotion/core": "^10.0.27", "lodash": "^4.17.19", - "react-error-boundary": "^1.2.5" + "react-error-boundary": "^1.2.5", + "@bfc/built-in-functions": "*" } } diff --git a/Composer/packages/adaptive-form/src/components/SchemaField.tsx b/Composer/packages/adaptive-form/src/components/SchemaField.tsx index 427c289626..155f887d20 100644 --- a/Composer/packages/adaptive-form/src/components/SchemaField.tsx +++ b/Composer/packages/adaptive-form/src/components/SchemaField.tsx @@ -51,7 +51,6 @@ export const SchemaField: React.FC = (props) => { typeof uiOptions?.serializer?.set === 'function' ? uiOptions.serializer.set(newValue) : newValue; onChange(serializedValue); - setFieldFocused(true); }; useEffect(() => { diff --git a/Composer/packages/adaptive-form/src/components/ExpressionSwitchWindow.tsx b/Composer/packages/adaptive-form/src/components/expressions/ExpressionSwitchWindow.tsx similarity index 100% rename from Composer/packages/adaptive-form/src/components/ExpressionSwitchWindow.tsx rename to Composer/packages/adaptive-form/src/components/expressions/ExpressionSwitchWindow.tsx diff --git a/Composer/packages/adaptive-form/src/components/expressions/ExpressionsListMenu.tsx b/Composer/packages/adaptive-form/src/components/expressions/ExpressionsListMenu.tsx new file mode 100644 index 0000000000..51efc6e68c --- /dev/null +++ b/Composer/packages/adaptive-form/src/components/expressions/ExpressionsListMenu.tsx @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { ContextualMenu, DirectionalHint } from 'office-ui-fabric-react/lib/ContextualMenu'; +import React, { useCallback, useMemo } from 'react'; +import { builtInFunctionsGrouping, getBuiltInFunctionInsertText } from '@bfc/built-in-functions'; + +import { expressionGroupingsToMenuItems } from './utils/expressionsListMenuUtils'; + +const componentMaxHeight = 400; + +type ExpressionsListMenuProps = { + onExpressionSelected: (expression: string) => void; + onMenuMount: (menuContainerElms: HTMLDivElement[]) => void; +}; +export const ExpressionsListMenu = (props: ExpressionsListMenuProps) => { + const { onExpressionSelected, onMenuMount } = props; + + const containerRef = React.createRef(); + + const onExpressionKeySelected = useCallback( + (key) => { + const insertText = getBuiltInFunctionInsertText(key); + onExpressionSelected('= ' + insertText); + }, + [onExpressionSelected] + ); + + const onLayerMounted = useCallback(() => { + const elms = document.querySelectorAll('.ms-ContextualMenu-Callout'); + onMenuMount(Array.prototype.slice.call(elms)); + }, [onMenuMount]); + + const menuItems = useMemo( + () => + expressionGroupingsToMenuItems( + builtInFunctionsGrouping, + onExpressionKeySelected, + onLayerMounted, + componentMaxHeight + ), + [onExpressionKeySelected, onLayerMounted] + ); + + return ( +
+
+ ); +}; diff --git a/Composer/packages/adaptive-form/src/components/expressions/utils/expressionsListMenuUtils.ts b/Composer/packages/adaptive-form/src/components/expressions/utils/expressionsListMenuUtils.ts new file mode 100644 index 0000000000..af1503a91d --- /dev/null +++ b/Composer/packages/adaptive-form/src/components/expressions/utils/expressionsListMenuUtils.ts @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { ContextualMenuItemType, IContextualMenuItem } from 'office-ui-fabric-react/lib/ContextualMenu'; + +type ExpressionGroupingType = { + key: string; + name: string; + children: string[]; +}; + +export const expressionGroupingsToMenuItems = ( + expressionGroupings: ExpressionGroupingType[], + menuItemSelectedHandler: (key: string) => void, + onLayerMounted: () => void, + maxHeight?: number +): IContextualMenuItem[] => { + const menuItems: IContextualMenuItem[] = + expressionGroupings?.map((grouping: ExpressionGroupingType) => { + return { + key: grouping.key, + text: grouping.name, + target: '_blank', + subMenuProps: { + calloutProps: { onLayerMounted: onLayerMounted }, + items: grouping.children.map((key: string) => { + return { + key: key, + text: key, + onClick: () => menuItemSelectedHandler(key), + }; + }), + styles: { container: { maxHeight } }, + }, + }; + }) || []; + + const header = { + key: 'header', + itemType: ContextualMenuItemType.Header, + text: 'Pre-built functions', + itemProps: { lang: 'en-us' }, + }; + + menuItems.unshift(header); + + return menuItems; +}; diff --git a/Composer/packages/adaptive-form/src/components/fields/IntellisenseFields.tsx b/Composer/packages/adaptive-form/src/components/fields/IntellisenseFields.tsx index 2173a55b13..fafbf7fbbb 100644 --- a/Composer/packages/adaptive-form/src/components/fields/IntellisenseFields.tsx +++ b/Composer/packages/adaptive-form/src/components/fields/IntellisenseFields.tsx @@ -3,10 +3,11 @@ import { FieldProps } from '@bfc/extension-client'; import { Intellisense } from '@bfc/intellisense'; -import React, { useRef } from 'react'; +import React, { useRef, useState } from 'react'; import { getIntellisenseUrl } from '../../utils/getIntellisenseUrl'; -import { ExpressionSwitchWindow } from '../ExpressionSwitchWindow'; +import { ExpressionSwitchWindow } from '../expressions/ExpressionSwitchWindow'; +import { ExpressionsListMenu } from '../expressions/ExpressionsListMenu'; import { JsonField } from './JsonField'; import { NumberField } from './NumberField'; @@ -35,9 +36,18 @@ export const IntellisenseTextField: React.FC> = (props) => { onBlur={props.onBlur} onChange={onChange} > - {({ textFieldValue, focused, onValueChanged, onKeyDownTextField, onKeyUpTextField, onClickTextField }) => ( + {({ + textFieldValue, + focused, + cursorPosition, + onValueChanged, + onKeyDownTextField, + onKeyUpTextField, + onClickTextField, + }) => ( > = (props) const scopes = ['expressions', 'user-variables']; const intellisenseServerUrlRef = useRef(getIntellisenseUrl()); + const [expressionsListContainerElements, setExpressionsListContainerElements] = useState([]); + + const completionListOverrideResolver = (value: string) => { + return value === '=' ? ( + onChange(expression)} + onMenuMount={(refs) => { + setExpressionsListContainerElements(refs); + }} + /> + ) : null; + }; + return ( > = (props) onBlur={props.onBlur} onChange={onChange} > - {({ textFieldValue, focused, onValueChanged, onKeyDownTextField, onKeyUpTextField, onClickTextField }) => ( + {({ + textFieldValue, + focused, + cursorPosition, + onValueChanged, + onKeyDownTextField, + onKeyUpTextField, + onClickTextField, + }) => ( > = function StringField(pr uiOptions, required, focused, + cursorPosition, } = props; const textFieldRef = React.createRef(); @@ -49,7 +50,13 @@ export const StringField: React.FC> = function StringField(pr if (focused && textFieldRef.current) { textFieldRef.current.focus(); } - }, [focused, textFieldRef.current]); + }, [focused, textFieldRef.current, value]); + + useEffect(() => { + if (cursorPosition !== undefined && cursorPosition > -1 && textFieldRef.current) { + textFieldRef.current.setSelectionRange(cursorPosition, cursorPosition); + } + }, [cursorPosition]); const handleFocus = (e: React.FocusEvent) => { if (typeof onFocus === 'function') { diff --git a/Composer/packages/client/src/recoilModel/dispatchers/publisher.ts b/Composer/packages/client/src/recoilModel/dispatchers/publisher.ts index 6f03f085df..8589c6f2a3 100644 --- a/Composer/packages/client/src/recoilModel/dispatchers/publisher.ts +++ b/Composer/packages/client/src/recoilModel/dispatchers/publisher.ts @@ -14,10 +14,13 @@ import { isEjectRuntimeExistState, filePersistenceState, settingsState, + luFilesState, + qnaFilesState, } from '../atoms/botState'; -import { botEndpointsState } from '../atoms'; import { openInEmulator } from '../../utils/navigation'; -import { rootBotProjectIdSelector } from '../selectors'; +import { rootBotProjectIdSelector, dialogsSelectorFamily } from '../selectors'; +import { botEndpointsState } from '../atoms'; +import * as luUtil from '../../utils/luUtil'; import { BotStatus, Text } from './../../constants'; import httpClient from './../../utils/httpUtil'; @@ -139,10 +142,24 @@ export const publisherDispatcher = () => { }); const publishToTarget = useRecoilCallback( - (callbackHelpers: CallbackInterface) => async (projectId: string, target: any, metadata, sensitiveSettings) => { + (callbackHelpers: CallbackInterface) => async ( + projectId: string, + target: any, + metadata: any, + sensitiveSettings + ) => { try { + const { snapshot } = callbackHelpers; + const dialogs = await snapshot.getPromise(dialogsSelectorFamily(projectId)); + const luFiles = await snapshot.getPromise(luFilesState(projectId)); + const qnaFiles = await snapshot.getPromise(qnaFilesState(projectId)); + const referredLuFiles = luUtil.checkLuisBuild(luFiles, dialogs); const response = await httpClient.post(`/publish/${projectId}/publish/${target.name}`, { - metadata, + metadata: { + ...metadata, + luResources: referredLuFiles.map((file) => file.id), + qnaResources: qnaFiles.map((file) => file.id), + }, sensitiveSettings, }); await publishSuccess(callbackHelpers, projectId, response.data, target); diff --git a/Composer/packages/electron-server/scripts/installOneAuth.js b/Composer/packages/electron-server/scripts/installOneAuth.js index b77be1ffe5..cbe3097348 100644 --- a/Composer/packages/electron-server/scripts/installOneAuth.js +++ b/Composer/packages/electron-server/scripts/installOneAuth.js @@ -21,6 +21,7 @@ const { log } = require('./common'); */ let packageName = null; +const packageVersion = '1.14.0'; switch (process.platform) { case 'darwin': @@ -54,7 +55,7 @@ async function downloadPackage() { log.info('Starting download.'); await ensureDir(outDir); try { - execSync(`cd ${outDir} && npm pack ${packageName}`, { encoding: 'utf-8' }); + execSync(`cd ${outDir} && npm pack ${packageName}@${packageVersion}`, { encoding: 'utf-8' }); } catch (err) { process.exit(1); return; diff --git a/Composer/packages/extension-client/src/types/form.ts b/Composer/packages/extension-client/src/types/form.ts index b7d4b5276e..477757f639 100644 --- a/Composer/packages/extension-client/src/types/form.ts +++ b/Composer/packages/extension-client/src/types/form.ts @@ -36,6 +36,7 @@ export interface FieldProps { value?: T; focused?: boolean; style?: React.CSSProperties; + cursorPosition?: number; onChange: ChangeHandler; onFocus?: (id: string, value?: T) => void; diff --git a/Composer/packages/intellisense/src/components/CompletionList.tsx b/Composer/packages/intellisense/src/components/CompletionList.tsx index a1557a48eb..e2491f2145 100644 --- a/Composer/packages/intellisense/src/components/CompletionList.tsx +++ b/Composer/packages/intellisense/src/components/CompletionList.tsx @@ -11,7 +11,7 @@ import { CompletionElement } from './CompletionElement'; const styles = { completionList: css` position: absolute; - top: 32; + top: 32px; left: 0; max-height: 300px; width: 100%; diff --git a/Composer/packages/intellisense/src/components/Intellisense.tsx b/Composer/packages/intellisense/src/components/Intellisense.tsx index 321a5ba026..48a0a5f44d 100644 --- a/Composer/packages/intellisense/src/components/Intellisense.tsx +++ b/Composer/packages/intellisense/src/components/Intellisense.tsx @@ -16,12 +16,14 @@ export const Intellisense = React.memo( id: string; value?: any; focused?: boolean; + completionListOverrideContainerElements?: HTMLDivElement[]; completionListOverrideResolver?: (value: any) => JSX.Element | null; onChange: (newValue: string) => void; onBlur?: (id: string) => void; children: (renderProps: { textFieldValue: any; focused?: boolean; + cursorPosition?: number; onValueChanged: (newValue: any) => void; onKeyDownTextField: (event: React.KeyboardEvent) => void; onKeyUpTextField: (event: React.KeyboardEvent) => void; @@ -39,6 +41,7 @@ export const Intellisense = React.memo( onChange, onBlur, children, + completionListOverrideContainerElements, } = props; const [textFieldValue, setTextFieldValue] = React.useState(value); @@ -90,6 +93,13 @@ export const Intellisense = React.memo( shouldBlur = false; } + if ( + completionListOverrideContainerElements && + completionListOverrideContainerElements.some((item) => !checkIsOutside(x, y, item)) + ) { + shouldBlur = false; + } + if (shouldBlur) { setShowCompletionList(false); setCursorPosition(-1); @@ -111,7 +121,7 @@ export const Intellisense = React.memo( document.body.removeEventListener('click', outsideClickHandler); document.body.removeEventListener('keydown', keydownHandler); }; - }, [focused, onBlur]); + }, [focused, onBlur, completionListOverrideContainerElements]); // When textField value is changed const onValueChanged = (newValue: string) => { @@ -132,6 +142,7 @@ export const Intellisense = React.memo( selectedSuggestion + textFieldValue.substr(range.end.character); onValueChanged(newValue); + setCursorPosition(range.start.character + selectedSuggestion.length); } else { onValueChanged(selectedSuggestion); } @@ -188,7 +199,15 @@ export const Intellisense = React.memo( return (
- {children({ textFieldValue, focused, onValueChanged, onKeyDownTextField, onKeyUpTextField, onClickTextField })} + {children({ + textFieldValue, + focused, + cursorPosition, + onValueChanged, + onKeyDownTextField, + onKeyUpTextField, + onClickTextField, + })} {completionListOverride || showCompletionList ? ( { // has condition : "" if (content.condition) { - const expressionParser = new ExpressionParser(); - const expression = expressionParser.parse(content.condition); - const references = expression.references().map((r) => (r.startsWith('$') ? r.substring(1) : r)); - foundProperties.push(...references); + try { + const expressionParser = new ExpressionParser(); + const expression = expressionParser.parse(content.condition); + const references = expression.references().map((r) => (r.startsWith('$') ? r.substring(1) : r)); + foundProperties.push(...references); + } catch (err) { + // eslint-disable-next-line no-console + console.error(`Could not parse condition expression ${content.condition}. ${err}`); + } } } diff --git a/Composer/packages/server/src/models/bot/botProject.ts b/Composer/packages/server/src/models/bot/botProject.ts index cfdc42d47d..8f06c67aaa 100644 --- a/Composer/packages/server/src/models/bot/botProject.ts +++ b/Composer/packages/server/src/models/bot/botProject.ts @@ -487,6 +487,7 @@ export class BotProject implements IBotProject { } }); + this.builder.rootDir = this.dir; this.builder.setBuildConfig( { ...luisConfig, subscriptionKey: qnaConfig.subscriptionKey, qnaRegion: qnaConfig.qnaRegion }, this.settings.downsampling diff --git a/Composer/packages/server/src/models/bot/builder.ts b/Composer/packages/server/src/models/bot/builder.ts index bd11751898..f8b031cab8 100644 --- a/Composer/packages/server/src/models/bot/builder.ts +++ b/Composer/packages/server/src/models/bot/builder.ts @@ -2,12 +2,13 @@ // Licensed under the MIT License. /* eslint-disable @typescript-eslint/no-var-requires */ -import { pathExists, writeFile } from 'fs-extra'; +import { pathExists, writeFile, copy } from 'fs-extra'; import { FileInfo, IConfig, SDKKinds } from '@bfc/shared'; import { ComposerReservoirSampler } from '@microsoft/bf-dispatcher/lib/mathematics/sampler/ComposerReservoirSampler'; import { ComposerBootstrapSampler } from '@microsoft/bf-dispatcher/lib/mathematics/sampler/ComposerBootstrapSampler'; import { luImportResolverGenerator, getLUFiles, getQnAFiles } from '@bfc/shared/lib/luBuildResolver'; import { Orchestrator } from '@microsoft/bf-orchestrator'; +import keys from 'lodash/keys'; import { Path } from '../../utility/path'; import { IFileStorage } from '../storage/interface'; @@ -26,6 +27,7 @@ const SETTINGS = 'settings'; const INTERRUPTION = 'interruption'; const SAMPLE_SIZE_CONFIGURATION = 2; const CrossTrainConfigName = 'cross-train.config.json'; +const MODEL = 'model'; export type SingleConfig = { rootDialog: boolean; @@ -51,6 +53,7 @@ export class Builder { public config: IConfig | null = null; public downSamplingConfig: DownSamplingConfig = { maxImbalanceRatio: 0, maxUtteranceAllowed: 0 }; private _locale: string; + private containOrchestrator = false; private luBuilder = new luBuild.Builder((message) => { log(message); @@ -67,6 +70,12 @@ export class Builder { this._locale = locale; } + public set rootDir(path: string) { + this.botDir = path; + this.generatedFolderPath = Path.join(this.botDir, GENERATEDFOLDER); + this.interruptionFolderPath = Path.join(this.generatedFolderPath, INTERRUPTION); + } + public build = async (luFiles: FileInfo[], qnaFiles: FileInfo[], allFiles: FileInfo[]) => { try { await this.createGeneratedDir(); @@ -76,7 +85,11 @@ export class Builder { const { interruptionLuFiles, interruptionQnaFiles } = await this.getInterruptionFiles(); const { luBuildFiles, orchestratorBuildFiles } = this.separateLuFiles(interruptionLuFiles, allFiles); - + if (orchestratorBuildFiles.length) { + this.containOrchestrator = true; + } else { + this.containOrchestrator = false; + } await this.runLuBuild(luBuildFiles); await this.runQnaBuild(interruptionQnaFiles); await this.runOrchestratorBuild(orchestratorBuildFiles); @@ -219,6 +232,49 @@ export class Builder { return await Orchestrator.buildAsync(modelPath, luObjects, isDialog, null, fullEmbedding); } + public async copyModelPathToBot() { + if (this.containOrchestrator) { + const nlrList = await this.runOrchestratorNlrList(); + const defaultNLR = nlrList.default; + const folderName = defaultNLR.replace('.onnx', ''); + const modelPath = Path.resolve(await this.getModelPathAsync(), folderName); + const destDir = Path.resolve(Path.join(this.botDir, MODEL), folderName); + await copy(modelPath, destDir); + await this.updateOrchestratorSetting(folderName); + } + } + + public async getQnaConfig() { + const config = this._getConfig([]); + const subscriptionKeyEndpoint = `https://${config?.qnaRegion}.api.cognitive.microsoft.com/qnamaker/v4.0`; + // Find any files that contain the name 'qnamaker.settings' in them + // These are generated by the LuBuild process and placed in the generated folder + // These contain dialog-to-luis app id mapping + const paths = await this.storage.glob('qnamaker.settings.*', this.generatedFolderPath); + if (!paths.length) return {}; + + const qnaConfigFile = await this.storage.readFile(Path.join(this.generatedFolderPath, paths[0])); + const qna: any = {}; + + const qnaConfig = await JSON.parse(qnaConfigFile); + const endpointKey = await this.qnaBuilder.getEndpointKeys(config.subscriptionKey, subscriptionKeyEndpoint); + Object.assign(qna, qnaConfig.qna, { endpointKey: endpointKey.primaryEndpointKey }); + + return qna; + } + + private async updateOrchestratorSetting(dirName: string) { + const runtimeRootPath = './ComposerDialogs'; + const settingPath = Path.join(this.generatedFolderPath, 'orchestrator.settings.json'); + const content = JSON.parse(await this.storage.readFile(settingPath)); + content.orchestrator.ModelPath = `${runtimeRootPath}/${MODEL}/${dirName}`; + keys(content.orchestrator.snapshots).forEach((key) => { + const values = content.orchestrator.snapshots[key].split('ComposerDialogs'); + content.orchestrator.snapshots[key] = `${runtimeRootPath}${values[1]}`; + }); + await this.storage.writeFile(settingPath, JSON.stringify(content, null, 2)); + } + private async createGeneratedDir() { // clear previous folder await this.deleteDir(this.generatedFolderPath); @@ -370,8 +426,7 @@ export class Builder { }); if (qnaContents) { - const subscriptionKeyEndpoint = - config.endpoint ?? `https://${config.qnaRegion}.api.cognitive.microsoft.com/qnamaker/v4.0`; + const subscriptionKeyEndpoint = `https://${config.qnaRegion}.api.cognitive.microsoft.com/qnamaker/v4.0`; const buildResult = await this.qnaBuilder.build(qnaContents, config.subscriptionKey, config.botName, { endpoint: subscriptionKeyEndpoint, diff --git a/Composer/packages/tools/built-in-functions/src/builtInFunctionsGrouping.ts b/Composer/packages/tools/built-in-functions/src/builtInFunctionsGrouping.ts new file mode 100644 index 0000000000..4a84c236f0 --- /dev/null +++ b/Composer/packages/tools/built-in-functions/src/builtInFunctionsGrouping.ts @@ -0,0 +1,191 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// This is the grouping of adaptive expression functions by type as seen on: +// https://docs.microsoft.com/en-us/azure/bot-service/adaptive-expressions/adaptive-expressions-prebuilt-functions?view=azure-bot-service-4.0 + +export const builtInFunctionsGrouping = [ + { + key: 'string', + name: 'String', + children: [ + 'length', + 'replace', + 'replaceIgnoreCase', + 'split', + 'substring', + 'toLower', + 'toUpper', + 'trim', + 'addOrdinal', + 'endsWith', + 'startsWith', + 'countWord', + 'concat', + 'newGuid', + 'indexOf', + 'lastIndexOf', + 'sentenceCase', + 'titleCase', + ], + }, + { + key: 'collection', + name: 'Collection', + children: [ + 'contains', + 'first', + 'join', + 'last', + 'count', + 'foreach', + 'union', + 'skip', + 'take', + 'intersection', + 'subArray', + 'select', + 'where', + 'sortBy', + 'sortByDescending', + 'indicesAndValues', + 'flatten', + 'unique', + ], + }, + { + key: 'logicalComparison', + name: 'Logical comparison', + children: [ + 'and', + 'equals', + 'empty', + 'greater', + 'greaterOrEquals', + 'if', + 'less', + 'lessOrEquals', + 'not', + 'or', + 'exists', + ], + }, + { + key: 'conversion', + name: 'Conversion', + children: [ + 'float', + 'int', + 'string', + 'bool', + 'createArray', + 'json', + 'base64', + 'base64ToBinary', + 'base64ToString', + 'binary', + 'dataUri', + 'dataUriToBinary', + 'dataUriToString', + 'uriComponent', + 'uriComponentToString', + 'xml', + 'formatNumber', + ], + }, + { + key: 'math', + name: 'Math', + children: [ + 'add', + 'div', + 'max', + 'min', + 'mod', + 'mul', + 'rand', + 'sub', + 'sum', + 'range', + 'exp', + 'average', + 'floor', + 'ceiling', + 'round', + ], + }, + { + key: 'dateAndTime', + name: 'Date and time', + children: [ + 'addDays', + 'addHours', + 'addMinutes', + 'addSeconds', + 'dayOfMonth', + 'dayOfWeek', + 'dayOfYear', + 'formatDateTime', + 'formatEpoch', + 'formatTicks', + 'subtractFromTime', + 'utcNow', + 'dateReadBack', + 'month', + 'date', + 'year', + 'getTimeOfDay', + 'getFutureTime', + 'getPastTime', + 'addToTime', + 'convertFromUTC', + 'convertToUTC', + 'startOfDay', + 'startOfHour', + 'startOfMonth', + 'ticks', + 'ticksToDays', + 'ticksToHours', + 'ticksToMinutes', + 'dateTimeDiff', + 'getPreviousViableDate', + 'getNextViableDate', + 'getPreviousViableTime', + 'getNextViableTime', + ], + }, + { + key: 'timex', + name: 'Timex', + children: ['isPresent', 'isDuration', 'isTime', 'isDate', 'isTimeRange', 'isDateRange', 'isDefinite'], + }, + { + key: 'uriParsing', + name: 'URI parsing', + children: ['uriHost', 'uriPath', 'uriPathAndQuery', 'uriPort', 'uriQuery', 'uriScheme'], + }, + { + key: 'objectManipulationAndConstruction', + name: 'Object manipulation and construction', + children: [ + 'addProperty', + 'removeProperty', + 'setProperty', + 'getProperty', + 'coalesce', + 'xPath', + 'jPath', + 'setPathToValue', + ], + }, + { + key: 'regularExpression', + name: 'Regular expression', + children: ['isMatch'], + }, + { + key: 'typeChecking', + name: 'Type checking', + children: ['EOL', 'isInteger', 'isFloat', 'isBoolean', 'isArray', 'isObject', 'isDateTime', 'isString'], + }, +]; diff --git a/Composer/packages/tools/built-in-functions/src/builtInFunctionsUtils.ts b/Composer/packages/tools/built-in-functions/src/builtInFunctionsUtils.ts new file mode 100644 index 0000000000..751bee58c1 --- /dev/null +++ b/Composer/packages/tools/built-in-functions/src/builtInFunctionsUtils.ts @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { buildInFunctionsMap } from './builtInFunctionsMap'; + +const removeParamFormat = (params: string): string => { + const resultArr = params.split(',').map((element) => { + return element.trim().split(':')[0]; + }); + return resultArr.join(' ,'); +}; + +export const getBuiltInFunctionInsertText = (builtInFunctionKey: string): string => { + const builtInFunction = buildInFunctionsMap.get(builtInFunctionKey); + + if (builtInFunction) { + return `${builtInFunctionKey}(${removeParamFormat(builtInFunction.Params.toString())})`; + } + return ''; +}; diff --git a/Composer/packages/tools/built-in-functions/src/index.ts b/Composer/packages/tools/built-in-functions/src/index.ts index c246a98832..d888c93f0c 100644 --- a/Composer/packages/tools/built-in-functions/src/index.ts +++ b/Composer/packages/tools/built-in-functions/src/index.ts @@ -2,3 +2,5 @@ // Licensed under the MIT License. export * from './builtInFunctionsMap'; +export * from './builtInFunctionsGrouping'; +export * from './builtInFunctionsUtils'; diff --git a/Composer/packages/tools/language-servers/intellisense/src/resolvers/expressions.ts b/Composer/packages/tools/language-servers/intellisense/src/resolvers/expressions.ts index c0308aacd5..668f9d7209 100644 --- a/Composer/packages/tools/language-servers/intellisense/src/resolvers/expressions.ts +++ b/Composer/packages/tools/language-servers/intellisense/src/resolvers/expressions.ts @@ -1,6 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import { buildInFunctionsMap } from '@bfc/built-in-functions'; +import { buildInFunctionsMap, getBuiltInFunctionInsertText } from '@bfc/built-in-functions'; import { CompletionItem, CompletionItemKind } from 'vscode-languageserver'; export const expressionsResolver = (): CompletionItem[] => { @@ -9,15 +9,8 @@ export const expressionsResolver = (): CompletionItem[] => { return { label: key, kind: CompletionItemKind.Function, - insertText: `${key}(${removeParamFormat(value.Params.toString())})`, + insertText: getBuiltInFunctionInsertText(key), documentation: value.Introduction, }; }); }; - -const removeParamFormat = (params: string): string => { - const resultArr = params.split(',').map((element) => { - return element.trim().split(':')[0]; - }); - return resultArr.join(' ,'); -}; diff --git a/extensions/azurePublish/package.json b/extensions/azurePublish/package.json index 560354c35f..7e5da1f2e1 100644 --- a/extensions/azurePublish/package.json +++ b/extensions/azurePublish/package.json @@ -20,8 +20,6 @@ "@azure/ms-rest-nodeauth": "3.0.3", "@bfc/indexers": "../../Composer/packages/lib/indexers", "@bfc/shared": "../../Composer/packages/lib/shared", - "@microsoft/bf-lu": "4.11.0-dev.20201005.7e5e1b8", - "@microsoft/bf-luis-cli": "^4.10.0-dev.20200721.8bb21ac", "adal-node": "0.2.1", "archiver": "^5.0.2", "fs-extra": "8.1.0", diff --git a/extensions/azurePublish/src/deploy.ts b/extensions/azurePublish/src/deploy.ts index 0e26915d6d..028a3c237c 100644 --- a/extensions/azurePublish/src/deploy.ts +++ b/extensions/azurePublish/src/deploy.ts @@ -8,7 +8,7 @@ import * as rp from 'request-promise'; import { BotProjectDeployConfig } from './botProjectDeployConfig'; import { BotProjectDeployLoggerType } from './botProjectLoggerType'; -import { LuisAndQnaPublish } from './luisAndQnA'; +import { build, publishLuisToPrediction } from './luisAndQnA'; import archiver = require('archiver'); export class BotProjectDeploy { @@ -58,20 +58,32 @@ export class BotProjectDeploy { if (!language) { language = 'en-us'; } - const publisher = new LuisAndQnaPublish({ logger: this.logger, projPath: this.projPath }); + + this.logger({ + status: BotProjectDeployLoggerType.DEPLOY_INFO, + message: "Building the bot's resources ...", + }); + await build(project, this.projPath, settings); + + this.logger({ + status: BotProjectDeployLoggerType.DEPLOY_INFO, + message: 'Build Success!', + }); // this function returns an object that contains the luis APP ids mapping // each dialog to its matching app. - const { luisAppIds, qnaConfig } = await publisher.publishLuisAndQna( + const luisAppIds = await publishLuisToPrediction( name, environment, this.accessToken, - language, settings.luis, - settings.qna, - luisResource + luisResource, + this.projPath, + this.logger ); + const qnaConfig = await project.builder.getQnaConfig(); + // amend luis settings with newly generated values settings.luis = { ...settings.luis, diff --git a/extensions/azurePublish/src/index.ts b/extensions/azurePublish/src/index.ts index 7c99e6cce1..3887859ab1 100644 --- a/extensions/azurePublish/src/index.ts +++ b/extensions/azurePublish/src/index.ts @@ -126,7 +126,7 @@ export default async (composer: IExtensionRegistration): Promise => { */ private init = async (project: any, srcTemplate: string, resourcekey: string, runtime: any) => { // point to the declarative assets (possibly in remote storage) - const botFiles = project.getProject().filesWithoutRecognizers; + const botFiles = project.getProject().files; const botFolder = this.getBotFolder(resourcekey, this.mode); const runtimeFolder = this.getRuntimeFolder(resourcekey); @@ -321,6 +321,8 @@ export default async (composer: IExtensionRegistration): Promise => { defaultLanguage, settings, accessToken, + luResources, + qnaResources } = config; // get the appropriate runtime template which contains methods to build and configure the runtime @@ -347,7 +349,7 @@ export default async (composer: IExtensionRegistration): Promise => { // Merge all the settings // this combines the bot-wide settings, the environment specific settings, and 2 new fields needed for deployed bots // these will be written to the appropriate settings file inside the appropriate runtime plugin. - const mergedSettings = mergeDeep(fullSettings, settings); + const mergedSettings = mergeDeep(fullSettings, settings, {luResources, qnaResources}); // Prepare parameters and then perform the actual deployment action const customizeConfiguration: CreateAndDeployResources = { @@ -392,6 +394,7 @@ export default async (composer: IExtensionRegistration): Promise => { * plugin methods *************************************************************************************************/ publish = async (config: PublishConfig, project: IBotProject, metadata, user) => { + const {luResources, qnaResources} = metadata; const { // these are provided by Composer profileName, // the name of the publishing profile "My Azure Prod Slot" @@ -436,7 +439,7 @@ export default async (composer: IExtensionRegistration): Promise => { throw new Error('Required field `settings` is missing from publishing profile.'); } - this.asyncPublish(config, project, resourcekey, jobId); + this.asyncPublish({...config, luResources, qnaResources}, project, resourcekey, jobId); } catch (err) { this.logger('%O', err); if (err instanceof Error) { diff --git a/extensions/azurePublish/src/luisAndQnA.ts b/extensions/azurePublish/src/luisAndQnA.ts index 474c763713..aa6f788661 100644 --- a/extensions/azurePublish/src/luisAndQnA.ts +++ b/extensions/azurePublish/src/luisAndQnA.ts @@ -6,182 +6,68 @@ import { promisify } from 'util'; import * as fs from 'fs-extra'; import * as rp from 'request-promise'; -import { ILuisConfig, FileInfo, IQnAConfig } from '@botframework-composer/types'; +import { ILuisConfig, FileInfo, IQnAConfig, IBotProject } from '@botframework-composer/types'; -import { ICrossTrainConfig, createCrossTrainConfig } from './utils/crossTrainUtil'; import { BotProjectDeployLoggerType } from './botProjectLoggerType'; -import { luImportResolverGenerator } from '@bfc/shared/lib/luBuildResolver'; -const crossTrainer = require('@microsoft/bf-lu/lib/parser/cross-train/crossTrainer.js'); -const luBuild = require('@microsoft/bf-lu/lib/parser/lubuild/builder.js'); -const qnaBuild = require('@microsoft/bf-lu/lib/parser/qnabuild/builder.js'); const readdir: any = promisify(fs.readdir); -export interface PublishConfig { - // Logger - logger: (string) => any; - projPath: string; - [key: string]: any; -} - -const INTERRUPTION = 'interruption'; - -export class LuisAndQnaPublish { - private logger: (string) => any; - private remoteBotPath: string; - private generatedFolder: string; - private interruptionFolderPath: string; - private crossTrainConfig: ICrossTrainConfig; - - constructor(config: PublishConfig) { - this.logger = config.logger; - // path to the ready to deploy generated folder - this.remoteBotPath = path.join(config.projPath, 'ComposerDialogs'); - this.generatedFolder = path.join(this.remoteBotPath, 'generated'); - this.interruptionFolderPath = path.join(this.generatedFolder, INTERRUPTION); - - // Cross Train config - this.crossTrainConfig = { - rootIds: [], - triggerRules: {}, - intentName: '_Interruption', - verbose: true, - botName: '', - }; - } +const botPath = (projPath: string) => path.join(projPath, 'ComposerDialogs') - /*******************************************************************************************************************************/ - /* This section has to do with publishing LU files to LUIS - /*******************************************************************************************************************************/ +type QnaConfigType = { + subscriptionKey: string; + qnaRegion: string | 'westus'; +}; - /** - * return an array of all the files in a given directory - * @param dir - */ - private async getFiles(dir: string): Promise { - const dirents = await readdir(dir, { withFileTypes: true }); - const files = await Promise.all( - dirents.map((dirent) => { - const res = path.resolve(dir, dirent.name); - return dirent.isDirectory() ? this.getFiles(res) : res; - }) - ); - return Array.prototype.concat(...files); - } - - /** - * Helper function to get the appropriate account out of a list of accounts - * @param accounts - * @param filter - */ - private getAccount(accounts: any, filter: string) { - for (const account of accounts) { - if (account.AccountName === filter) { - return account; - } - } - } +type Resources = { + luResources: string[]; + qnaResources: string[]; +} - private notEmptyModel(file: string) { - return fs.readFileSync(file).length > 0; - } +type BuildSettingType = { + luis: ILuisConfig, + qna: QnaConfigType +} & Resources - private async createGeneratedDir() { - if (!(await fs.pathExists(this.generatedFolder))) { - await fs.mkdir(this.generatedFolder); +function getAccount(accounts: any, filter: string) { + for (const account of accounts) { + if (account.AccountName === filter) { + return account; } } +} - private async setCrossTrainConfig(botName: string, dialogFiles: string[], luFiles: string[]) { - const dialogs: { [key: string]: any }[] = []; - for (const dialog of dialogFiles) { - dialogs.push({ - id: dialog.substring(dialog.lastIndexOf('\\') + 1, dialog.length), - isRoot: dialog.indexOf(path.join(this.remoteBotPath, 'dialogs')) === -1, - content: fs.readJSONSync(dialog), - }); - } - const luFileInfos: FileInfo[] = luFiles.map((luFile) => { - const fileStats = fs.statSync(luFile); - return { - name: luFile.substring(luFile.lastIndexOf('\\') + 1), - content: fs.readFileSync(luFile, 'utf-8'), - lastModified: fileStats.mtime.toString(), - path: luFile, - relativePath: luFile.substring(luFile.lastIndexOf(this.remoteBotPath) + 1), - }; - }); - this.crossTrainConfig = createCrossTrainConfig(dialogs, luFileInfos); - } - private async writeCrossTrainFiles(crossTrainResult) { - if (!(await fs.pathExists(this.interruptionFolderPath))) { - await fs.mkdir(this.interruptionFolderPath); - } +/** +* return an array of all the files in a given directory +* @param dir +*/ +async function getFiles(dir: string): Promise { + const dirents = await readdir(dir, { withFileTypes: true }); + const files = await Promise.all( + dirents.map((dirent) => { + const res = path.resolve(dir, dirent.name); + return dirent.isDirectory() ? getFiles(res) : res; + }) + ); + return Array.prototype.concat(...files); +} - await Promise.all( - [...crossTrainResult.keys()].map(async (key: string) => { - const fileName = path.basename(key); - const newFileId = path.join(this.interruptionFolderPath, fileName); - await fs.writeFile(newFileId, crossTrainResult.get(key).Content); - }) - ); - } +export async function publishLuisToPrediction( + name: string, + environment: string, + accessToken: string, + luisSettings: ILuisConfig, + luisResource: string, + path: string, + logger + ) { + let { authoringKey: luisAuthoringKey, endpoint: luisEndpoint, authoringRegion: luisAuthoringRegion } = luisSettings; - private async crossTrain(luFiles: string[], qnaFiles: string[]) { - const luContents: { [key: string]: any }[] = []; - const qnaContents: { [key: string]: any }[] = []; - for (const luFile of luFiles) { - luContents.push({ - content: fs.readFileSync(luFile, { encoding: 'utf-8' }), - name: path.basename(luFile), - id: path.basename(luFile), - path: luFile, - }); - } - for (const qnaFile of qnaFiles) { - qnaContents.push({ - content: fs.readFileSync(qnaFile, { encoding: 'utf-8' }), - name: path.basename(qnaFile), - id: path.basename(qnaFile), - path: qnaFile, - }); + if (!luisSettings.endpoint) { + luisEndpoint = `https://${luisAuthoringRegion}.api.cognitive.microsoft.com`; } - const importResolver = luImportResolverGenerator([...qnaContents, ...luContents] as FileInfo[]); - const result = await crossTrainer.crossTrain(luContents, qnaContents, this.crossTrainConfig, importResolver); - await this.writeCrossTrainFiles(result.luResult); - await this.writeCrossTrainFiles(result.qnaResult); - } - - private async cleanCrossTrain() { - fs.rmdirSync(this.interruptionFolderPath, { recursive: true }); - } - private async getInterruptionFiles() { - const files = await this.getFiles(this.interruptionFolderPath); - const interruptionLuFiles: string[] = []; - const interruptionQnaFiles: string[] = []; - files.forEach((file) => { - if (file.endsWith('qna')) { - interruptionQnaFiles.push(file); - } else if (file.endsWith('lu')) { - interruptionLuFiles.push(file); - } - }); - return { interruptionLuFiles, interruptionQnaFiles }; - } - - private async publishLuis( - name: string, - environment: string, - accessToken: string, - language: string, - luisSettings: ILuisConfig, - interruptionLuFiles: string[], - luisResource?: string - ) { - const { authoringKey: luisAuthoringKey, endpoint: luisEndpoint } = luisSettings; - - this.logger({ + logger({ status: BotProjectDeployLoggerType.DEPLOY_INFO, message: 'start publish luis', }); @@ -189,7 +75,7 @@ export class LuisAndQnaPublish { // Find any files that contain the name 'luis.settings' in them // These are generated by the LuBuild process and placed in the generated folder // These contain dialog-to-luis app id mapping - const luisConfigFiles = (await this.getFiles(this.remoteBotPath)).filter((filename) => + const luisConfigFiles = (await getFiles(botPath(path))).filter((filename) => filename.includes('luis.settings') ); const luisAppIds: any = {}; @@ -228,13 +114,13 @@ export class LuisAndQnaPublish { } // Extract the accoutn object that matches the expected resource name. // This is the name that would appear in the azure portal associated with the luis endpoint key. - const account = this.getAccount(accountList, luisResource ? luisResource : `${name}-${environment}-luis`); + const account = getAccount(accountList, luisResource ? luisResource : `${name}-${environment}-luis`); // Assign the appropriate account to each of the applicable LUIS apps for this bot. // DOCS HERE: https://westus.dev.cognitive.microsoft.com/docs/services/5890b47c39e2bb17b84a55ff/operations/5be32228e8473de116325515 for (const dialogKey in luisAppIds) { const luisAppId = luisAppIds[dialogKey].appId; - this.logger({ + logger({ status: BotProjectDeployLoggerType.DEPLOY_INFO, message: `Assigning to luis app id: ${luisAppId}`, }); @@ -249,209 +135,46 @@ export class LuisAndQnaPublish { // TODO: Add some error handling on this API call. As it is, errors will just throw by default and be caught by the catch all try/catch in the deploy method - this.logger({ + logger({ status: BotProjectDeployLoggerType.DEPLOY_INFO, message: response, }); } // The process has now completed. - this.logger({ + logger({ status: BotProjectDeployLoggerType.DEPLOY_INFO, message: 'Luis Publish Success! ...', }); // return the new settings that need to be added to the main settings file. return luisAppIds; - } - - // Run through the lubuild process - // This happens in the build folder, NOT in the original source folder - private async buildLuis( - name: string, - environment: string, - language: string, - luisSettings: ILuisConfig, - interruptionLuFiles: string[] - ) { - const { authoringKey: luisAuthoringKey, authoringRegion: luisAuthoringRegion } = luisSettings; - - // Instantiate the LuBuild object from the LU parsing library - // This object is responsible for parsing the LU files and sending them to LUIS - const builder = new luBuild.Builder((msg) => - this.logger({ - status: BotProjectDeployLoggerType.DEPLOY_INFO, - message: msg, - }) - ); - - // Pass in the list of the non-empty LU files we got above... - const loadResult = await builder.loadContents( - interruptionLuFiles, - language || 'en-us', - environment || '', - luisAuthoringRegion || '' - ); - - // set the default endpoint - if (!luisSettings.endpoint) { - luisSettings.endpoint = `https://${luisAuthoringRegion}.api.cognitive.microsoft.com`; - } - - // if not specified, set the authoring endpoint - if (!luisSettings.authoringEndpoint) { - luisSettings.authoringEndpoint = luisSettings.endpoint; - } - - // Perform the Lubuild process - // This will create new luis apps for each of the luis models represented in the LU files - const buildResult = await builder.build( - loadResult.luContents, - loadResult.recognizers, - luisAuthoringKey, - luisSettings.authoringEndpoint, - name, - environment, - language, - true, - false, - loadResult.multiRecognizers, - loadResult.settings, - loadResult.crosstrainedRecognizers, - 'crosstrained' - ); - - // Write the generated files to the generated folder - await builder.writeDialogAssets(buildResult, true, this.generatedFolder); - - this.logger({ - status: BotProjectDeployLoggerType.DEPLOY_INFO, - message: `lubuild succeed`, - }); - } - - private async buildQna( - name: string, - environment: string, - language: string, - qnaSettings: IQnAConfig, - interruptionQnaFiles: string[] - ) { - // eslint-disable-next-line prefer-const - let { subscriptionKey } = qnaSettings; - const authoringRegion = 'westus'; - // publishing luis - const builder = new qnaBuild.Builder((msg) => - this.logger({ - status: BotProjectDeployLoggerType.DEPLOY_INFO, - message: msg, - }) - ); - - const loadResult = await builder.loadContents( - interruptionQnaFiles, - name, - environment || '', - authoringRegion || '', - language || '' - ); - - const endpoint = `https://${authoringRegion}.api.cognitive.microsoft.com/qnamaker/v4.0`; - - const buildResult = await builder.build( - loadResult.qnaContents, - loadResult.recognizers, - subscriptionKey, - endpoint, - name, - environment, - language, - loadResult.multiRecognizers, - loadResult.settings, - loadResult.crosstrainedRecognizers, - 'crosstrained' - ); - await builder.writeDialogAssets(buildResult, true, this.generatedFolder); - - this.logger({ - status: BotProjectDeployLoggerType.DEPLOY_INFO, - message: `qnabuild succeed`, - }); - - // Find any files that contain the name 'qnamaker.settings' in them - // These are generated by the LuBuild process and placed in the generated folder - // These contain dialog-to-luis app id mapping - const qnaConfigFile = (await this.getFiles(this.remoteBotPath)).find((filename) => - filename.includes('qnamaker.settings') - ); - const qna: any = {}; - - // Read the qna settings - if (qnaConfigFile) { - const qnaConfig = await fs.readJson(qnaConfigFile); - const endpointKey = await builder.getEndpointKeys(subscriptionKey, endpoint); - Object.assign(qna, qnaConfig.qna, { endpointKey: endpointKey.primaryEndpointKey }); - } - return qna; - } +} - // Run through the build process - // This happens in the build folder, NOT in the original source folder - public async publishLuisAndQna( - name: string, - environment: string, - accessToken: string, - language: string, - luisSettings: ILuisConfig, - qnaSettings: IQnAConfig, - luisResource?: string - ) { - const { authoringKey, authoringRegion } = luisSettings; - const { subscriptionKey } = qnaSettings; - const botFiles = await this.getFiles(this.remoteBotPath); - const luFiles = botFiles.filter((name) => { - return name.endsWith('.lu'); - }); - const qnaFiles = botFiles.filter((name) => { - return name.endsWith('.qna'); - }); +export async function build(project: IBotProject, path: string, settings: BuildSettingType) { + const {luResources, qnaResources, luis: luisConfig, qna: qnaConfig} = settings; - // check content - const notEmptyLuFiles = luFiles.some((name) => this.notEmptyModel(name)); - const notEmptyQnaFiles = qnaFiles.some((name) => this.notEmptyModel(name)); + const {builder, files} = project; - if (notEmptyLuFiles && !(authoringKey && authoringRegion)) { - throw Error('Should have luis authoringKey and authoringRegion when lu file not empty'); + const luFiles: FileInfo[] = []; + luResources.forEach((id) => { + const fileName = `${id}.lu`; + const f = files.get(fileName); + if (f) { + luFiles.push(f); } - if (notEmptyQnaFiles && !subscriptionKey) { - throw Error('Should have qna subscriptionKey when qna file not empty'); + }); + const qnaFiles: FileInfo[] = []; + qnaResources.forEach((id) => { + const fileName = `${id}.qna`; + const f = files.get(fileName); + if (f) { + qnaFiles.push(f); } - const dialogFiles = botFiles.filter((name) => { - return name.endsWith('.dialog') && this.notEmptyModel(name); - }); - - await this.setCrossTrainConfig(name, dialogFiles, luFiles); - await this.createGeneratedDir(); - await this.crossTrain(luFiles, qnaFiles); - const { interruptionLuFiles, interruptionQnaFiles } = await this.getInterruptionFiles(); + }); - await this.buildLuis(name, environment, language, luisSettings, interruptionLuFiles); - let luisAppIds = {}; - // publish luis only when Lu files not empty - if (notEmptyLuFiles) { - luisAppIds = await this.publishLuis( - name, - environment, - accessToken, - language, - luisSettings, - interruptionLuFiles, - luisResource - ); - } - - const qnaConfig = await this.buildQna(name, environment, language, qnaSettings, interruptionQnaFiles); - await this.cleanCrossTrain(); - return { luisAppIds, qnaConfig }; - } + builder.rootDir = botPath(path); + builder.setBuildConfig( {...luisConfig, ...qnaConfig}, project.settings.downsampling ); + await builder.build(luFiles, qnaFiles, Array.from(files.values()) as FileInfo[]); + await builder.copyModelPathToBot(); } diff --git a/extensions/azurePublish/src/utils/crossTrainUtil.ts b/extensions/azurePublish/src/utils/crossTrainUtil.ts deleted file mode 100644 index 5dad90688c..0000000000 --- a/extensions/azurePublish/src/utils/crossTrainUtil.ts +++ /dev/null @@ -1,216 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -/** - * luUtil.ts is a single place use lu-parser handle lu file operation. - * it's designed have no state, input text file, output text file. - * for more usage detail, please check client/__tests__/utils/luUtil.test.ts - */ -import keys from 'lodash/keys'; -import { FieldNames } from '@bfc/shared'; -import { LuFile, DialogInfo, IIntentTrigger, SDKKinds, FileInfo } from '@botframework-composer/types'; -import { luIndexer } from '@bfc/indexers'; - -import { getBaseName, getExtension } from './fileUtil'; -import { VisitorFunc, JsonWalk } from './jsonWalk'; - -export function getReferredLuFiles(luFiles: LuFile[], dialogs: DialogInfo[], checkContent = true) { - return luFiles.filter((file) => { - const idWithOutLocale = getBaseName(file.id); - return dialogs.some( - (dialog) => dialog.luFile === idWithOutLocale && ((checkContent && !!file.content) || !checkContent) - ); - }); -} - -function ExtractAllBeginDialogs(value: any): string[] { - const dialogs: string[] = []; - - const visitor: VisitorFunc = (path: string, value: any): boolean => { - if (value?.$kind === SDKKinds.BeginDialog && value?.dialog) { - dialogs.push(value.dialog); - return true; - } - return false; - }; - - JsonWalk('$', value, visitor); - - return dialogs; -} - -// find out all properties from given dialog -function ExtractIntentTriggers(value: any): IIntentTrigger[] { - const intentTriggers: IIntentTrigger[] = []; - const triggers = value?.[FieldNames.Events]; - - if (triggers && triggers.length) { - for (const trigger of triggers) { - const dialogs = ExtractAllBeginDialogs(trigger); - - if (trigger.$kind === SDKKinds.OnIntent && trigger.intent) { - intentTriggers.push({ intent: trigger.intent, dialogs }); - } else if (trigger.$kind !== SDKKinds.OnIntent && dialogs.length) { - const emptyIntent = intentTriggers.find((e) => e.intent === ''); - if (emptyIntent) { - //remove the duplication dialogs - const all = new Set([...emptyIntent.dialogs, ...dialogs]); - emptyIntent.dialogs = Array.from(all); - } else { - intentTriggers.push({ intent: '', dialogs }); - } - } - } - } - - return intentTriggers; -} - -function createConfigId(fileId) { - return `${fileId}.lu`; -} - -function getLuFilesByDialogId(dialogId: string, luFiles: LuFile[]) { - return luFiles.filter((lu) => getBaseName(lu.id) === dialogId).map((lu) => createConfigId(lu.id)); -} - -function getFileLocale(fileName: string) { - //file name = 'a.en-us.lu' - return getExtension(getBaseName(fileName)); -} - -//replace the dialogId with luFile's name -function addLocaleToConfig(config: ICrossTrainConfig, luFiles: LuFile[]) { - const { rootIds, triggerRules } = config; - config.rootIds = rootIds.reduce((result: string[], id: string) => { - return [...result, ...getLuFilesByDialogId(id, luFiles)]; - }, []); - config.triggerRules = keys(triggerRules).reduce((result, key) => { - const fileNames = getLuFilesByDialogId(key, luFiles); - return { - ...result, - ...fileNames.reduce((result, name) => { - const locale = getFileLocale(name); - const triggers = triggerRules[key]; - keys(triggers).forEach((trigger) => { - if (!result[name]) result[name] = {}; - const ids = triggers[trigger]; - if (Array.isArray(ids)) { - result[name][trigger] = ids.map((id) => (id ? `${id}.${locale}.lu` : id)); - } else { - result[name][trigger] = ids ? `${ids}.${locale}.lu` : ids; - } - }); - return result; - }, {}), - }; - }, {}); - return config; -} - -function parse(dialog) { - const { id, content, isRoot } = dialog; - const luFile = typeof content.recognizer === 'string' ? getBaseName(id) : ''; - const qnaFile = typeof content.recognizer === 'string' ? getBaseName(id) : ''; - - return { - id: getBaseName(id), - isRoot: isRoot, - content, - luFile: luFile, - qnaFile: qnaFile, - intentTriggers: ExtractIntentTriggers(content), - }; -} -export interface ICrossTrainConfig { - rootIds: string[]; - triggerRules: { [key: string]: any }; - intentName: string; - verbose: boolean; - botName: string; -} - -//generate the cross-train config without locale -/* the config is like - { - rootIds: [ - 'main.en-us.lu', - 'main.fr-fr.lu' - ], - triggerRules: { - 'main.en-us.lu': { - 'dia1_trigger': 'dia1.en-us.lu', - 'dia2_trigger': 'dia2.en-us.lu' - }, - 'dia2.en-us.lu': { - 'dia3_trigger': 'dia3.en-us.lu', - 'dia4_trigger': 'dia4.en-us.lu' - }, - 'main.fr-fr.lu': { - 'dia1_trigger': 'dia1.fr-fr.lu' - } - }, - intentName: '_Interruption', - verbose: true - } - */ -export function createCrossTrainConfig(dialogs: any[], luFilesInfo: FileInfo[], luFeatures = {}): ICrossTrainConfig { - const triggerRules = {}; - const countMap = {}; - const wrapDialogs: { [key: string]: any }[] = []; - for (const dialog of dialogs) { - wrapDialogs.push(parse(dialog)); - } - - const luFiles = luIndexer.index(luFilesInfo, luFeatures); - - //map all referred lu files - luFiles.forEach((file) => { - countMap[getBaseName(file.id)] = 1; - }); - - let rootId = ''; - let botName = ''; - wrapDialogs.forEach((dialog) => { - if (dialog.isRoot) { - rootId = dialog.id; - botName = dialog.content.$designer.name; - } - - if (luFiles.find((luFile) => getBaseName(luFile.id) === dialog.luFile)) { - const { intentTriggers } = dialog; - const fileId = dialog.id; - //find the trigger's dialog that use a recognizer - intentTriggers.forEach((item) => { - //find all dialogs in trigger that has a luis recognizer - const used = item.dialogs.filter((dialog) => !!countMap[dialog]); - - const deduped = Array.from(new Set(used)); - - const result = {}; - if (deduped.length === 1) { - result[item.intent] = deduped[0]; - } else if (deduped.length) { - result[item.intent] = deduped; - } else { - result[item.intent] = ''; - } - - triggerRules[fileId] = { ...triggerRules[fileId], ...result }; - }); - } - }); - - const crossTrainConfig: ICrossTrainConfig = { - botName: botName, - rootIds: [], - triggerRules: {}, - intentName: '_Interruption', - verbose: true, - }; - crossTrainConfig.rootIds = keys(countMap).filter( - (key) => (countMap[key] === 0 || key === rootId) && triggerRules[key] - ); - crossTrainConfig.triggerRules = triggerRules; - return addLocaleToConfig(crossTrainConfig, luFiles); -} diff --git a/extensions/azurePublish/src/utils/fileUtil.ts b/extensions/azurePublish/src/utils/fileUtil.ts deleted file mode 100644 index 0e5ab3af4a..0000000000 --- a/extensions/azurePublish/src/utils/fileUtil.ts +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -export function getExtension(filename?: string): string | any { - if (typeof filename !== 'string') return filename; - return filename.substring(filename.lastIndexOf('.') + 1, filename.length) || filename; -} - -export function getBaseName(filename: string, sep?: string): string | any { - if (sep) return filename.substr(0, filename.lastIndexOf(sep)); - return filename.substring(0, filename.lastIndexOf('.')) || filename; -} diff --git a/extensions/azurePublish/src/utils/jsonWalk.ts b/extensions/azurePublish/src/utils/jsonWalk.ts deleted file mode 100644 index 6f4efcd4e1..0000000000 --- a/extensions/azurePublish/src/utils/jsonWalk.ts +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -/** - * visitor function used by JsonWalk - * @param path jsonPath string - * @param value current node value - * @return boolean, true to stop walk deep - */ -export interface VisitorFunc { - (path: string, value: any): boolean; -} - -/** - * - * @param path jsonPath string - * @param value current node value - * @param visitor - */ - -export const JsonWalk = (path: string, value: any, visitor: VisitorFunc) => { - const stop = visitor(path, value); - if (stop === true) return; - - // extract array - if (Array.isArray(value)) { - value.forEach((child, index) => { - JsonWalk(`${path}[${index}]`, child, visitor); - }); - - // extract object - } else if (typeof value === 'object' && value) { - Object.keys(value).forEach((key) => { - JsonWalk(`${path}.${key}`, value[key], visitor); - }); - } -};