From ad6ee2cefe1e2556f6d120fceed93ecad24aef4e Mon Sep 17 00:00:00 2001 From: Zhixiang Zhan Date: Fri, 8 Jan 2021 15:44:34 +0800 Subject: [PATCH] fix: project tree show incorrect imports (#5361) * get imports from parser * clean up * define type for lu sections * merge conflict * clean up Co-authored-by: Soroush Co-authored-by: Dong Lei --- .../dispatchers/__tests__/lg.test.tsx | 1 + .../selectors/__test__/dialogImports.test.tsx | 11 +++++- .../recoilModel/selectors/dialogImports.ts | 37 ++++++------------- .../lib/indexers/__tests__/lgIndexer.test.ts | 18 ++++----- .../lib/indexers/__tests__/luIndexer.test.ts | 19 ++++++++-- .../lib/indexers/__tests__/qnaIndexer.test.ts | 8 +++- .../lib/indexers/__tests__/qnaUtil.test.ts | 7 ++++ .../packages/lib/indexers/src/utils/lgUtil.ts | 11 +++++- .../packages/lib/indexers/src/utils/luUtil.ts | 21 ++++++++++- .../lib/indexers/src/utils/qnaUtil.ts | 17 +++++---- Composer/packages/types/src/indexers.ts | 19 +++++++++- 11 files changed, 116 insertions(+), 53 deletions(-) diff --git a/Composer/packages/client/src/recoilModel/dispatchers/__tests__/lg.test.tsx b/Composer/packages/client/src/recoilModel/dispatchers/__tests__/lg.test.tsx index b3079aeee0..f066164769 100644 --- a/Composer/packages/client/src/recoilModel/dispatchers/__tests__/lg.test.tsx +++ b/Composer/packages/client/src/recoilModel/dispatchers/__tests__/lg.test.tsx @@ -42,6 +42,7 @@ const lgFiles = [ content: `\r\n# Hello\r\n-hi`, templates: [{ name: 'Hello', body: '-hi', parameters: [] }], diagnostics: [], + imports: [], allTemplates: [{ name: 'Hello', body: '-hi', parameters: [] }], }, ] as LgFile[]; diff --git a/Composer/packages/client/src/recoilModel/selectors/__test__/dialogImports.test.tsx b/Composer/packages/client/src/recoilModel/selectors/__test__/dialogImports.test.tsx index e82a68133e..1be280b137 100644 --- a/Composer/packages/client/src/recoilModel/selectors/__test__/dialogImports.test.tsx +++ b/Composer/packages/client/src/recoilModel/selectors/__test__/dialogImports.test.tsx @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +import { lgUtil } from '@bfc/indexers'; + import { getBaseName } from '../../../utils/fileUtil'; import { getLanguageFileImports } from '../dialogImports'; @@ -37,7 +39,14 @@ const files = [ describe('dialogImports selectors', () => { it('should follow all imports and list all unique imports', () => { - const getFile = (id) => files.find((f) => getBaseName(f.id) === id) as { id: string; content: string }; + const getFile = (id) => { + const file = files.find((f) => getBaseName(f.id) === id); + if (file) { + return lgUtil.parse(file.id, file.content, []); + } else { + throw new Error(`file ${id} not found`); + } + }; const fileImports = getLanguageFileImports('name1', getFile); expect(fileImports).toEqual([ diff --git a/Composer/packages/client/src/recoilModel/selectors/dialogImports.ts b/Composer/packages/client/src/recoilModel/selectors/dialogImports.ts index 6281da42ab..ff92849130 100644 --- a/Composer/packages/client/src/recoilModel/selectors/dialogImports.ts +++ b/Composer/packages/client/src/recoilModel/selectors/dialogImports.ts @@ -1,37 +1,15 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import { LanguageFileImport, LgFile, LuFile } from '@bfc/shared'; +import { LanguageFileImport, LgFile, LuFile, QnAFile } from '@bfc/shared'; import uniqBy from 'lodash/uniqBy'; import { selectorFamily } from 'recoil'; -import { getBaseName, getFileName } from '../../utils/fileUtil'; +import { getBaseName } from '../../utils/fileUtil'; import { localeState, lgFilesState, luFilesState } from '../atoms'; -// eslint-disable-next-line security/detect-unsafe-regex -const importRegex = /\[(?.*?)]\((?.*?)(?="|\))(?".*")?\)/g; - -const getImportsHelper = (content: string): LanguageFileImport[] => { - const lines = content.split(/\r?\n/g).filter((l) => !!l) ?? []; - - return (lines - .map((l) => { - importRegex.lastIndex = 0; - return importRegex.exec(l) as RegExpExecArray; - }) - .filter(Boolean) as RegExpExecArray[]).map((regExecArr) => { - const importPath = regExecArr.groups?.importPath ?? ''; - - return { - displayName: regExecArr.groups?.id ?? '', - importPath, - id: getBaseName(getFileName(importPath)), - }; - }); -}; - // Finds all the file imports starting from a given dialog file. -export const getLanguageFileImports = ( +export const getLanguageFileImports = ( rootDialogId: string, getFile: (fileId: string) => T ): LanguageFileImport[] => { @@ -55,7 +33,14 @@ export const getLanguageFileImports = { + return { + displayName: item.description, + importPath: item.path, + id: getBaseName(item.id), + }; + }); + visitedIds.push(currentId); imports.push(...currentImports); const newIds = currentImports.map((ci) => getBaseName(ci.id)); diff --git a/Composer/packages/lib/indexers/__tests__/lgIndexer.test.ts b/Composer/packages/lib/indexers/__tests__/lgIndexer.test.ts index e0c8f0c83c..1dab46ade0 100644 --- a/Composer/packages/lib/indexers/__tests__/lgIndexer.test.ts +++ b/Composer/packages/lib/indexers/__tests__/lgIndexer.test.ts @@ -1,7 +1,8 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import { FileInfo } from '@bfc/shared'; +import { FileInfo, lgImportResolverGenerator, ResolverResource } from '@bfc/shared'; +import { LGResource } from 'botbuilder-lg'; import { lgIndexer } from '../src/lgIndexer'; import { getBaseName } from '../src/utils/help'; @@ -54,25 +55,24 @@ describe('index', () => { lastModified: '', }; - const files = { - common: { + const files = [ + { id: 'common', content: `# Greeting -What's up bro`, }, - }; + ] as ResolverResource[]; - const importResolver = (_source: string, _id: string) => { - const id = getBaseName(_id.split('/').pop() || '', '.lg'); - return files[id]; - }; + const importresolver = lgImportResolverGenerator(files, '.lg'); it('should index lg file with [import]', () => { - const { id, templates, diagnostics }: any = index([file], importResolver)[0]; + const { id, templates, diagnostics, imports }: any = index([file], importresolver)[0]; expect(id).toEqual('test'); expect(templates.length).toEqual(2); expect(diagnostics.length).toEqual(0); expect(templates[0].name).toEqual('Exit'); expect(templates[1].name).toEqual('Hi'); + expect(imports.length).toEqual(1); + expect(imports[0]).toEqual({ id: 'common.lg', path: '../common/common.lg', description: 'import' }); }); }); diff --git a/Composer/packages/lib/indexers/__tests__/luIndexer.test.ts b/Composer/packages/lib/indexers/__tests__/luIndexer.test.ts index b4d6d4f054..4d897e01ef 100644 --- a/Composer/packages/lib/indexers/__tests__/luIndexer.test.ts +++ b/Composer/packages/lib/indexers/__tests__/luIndexer.test.ts @@ -31,7 +31,7 @@ describe('parse', () => { @ simple fooName `; - const { intents, diagnostics }: any = parse(content); + const { intents, diagnostics }: any = parse(content, '', {}); expect(diagnostics.length).toEqual(0); expect(intents.length).toEqual(4); expect(intents[0].Name).toEqual('CheckTodo'); @@ -44,6 +44,19 @@ describe('parse', () => { expect(intents[0].Children[0].Entities[0]).toEqual('todoTitle'); }); + it('should parse LU file with import', () => { + const content = `[import](greating.lu) + [import](../common/aks.lu) + `; + + const result = parse(content, '', {}); + expect(result.imports.length).toEqual(2); + expect(result.imports[0]).toEqual({ id: 'greating.lu', path: 'greating.lu', description: 'import' }); + expect(result.imports[1]).toEqual({ id: 'aks.lu', path: '../common/aks.lu', description: 'import' }); + expect(result.empty).toEqual(false); + expect(result.intents.length).toEqual(0); + }); + it('should parse lu file with diagnostic', () => { const content = `# Greeting hi @@ -52,7 +65,7 @@ hi @ simple friendsName `; - const { intents, diagnostics }: any = parse(content); + const { intents, diagnostics }: any = parse(content, '', {}); expect(intents.length).toEqual(1); expect(diagnostics.length).toEqual(1); expect(diagnostics[0].range.start.line).toEqual(2); @@ -80,7 +93,7 @@ describe('index', () => { }; it('should index lu file', () => { - const { id, intents, diagnostics }: any = index([file])[0]; + const { id, intents, diagnostics }: any = index([file], {})[0]; expect(id).toEqual('test'); expect(diagnostics.length).toEqual(0); expect(intents.length).toEqual(1); diff --git a/Composer/packages/lib/indexers/__tests__/qnaIndexer.test.ts b/Composer/packages/lib/indexers/__tests__/qnaIndexer.test.ts index a08ad31952..5dfe890d73 100644 --- a/Composer/packages/lib/indexers/__tests__/qnaIndexer.test.ts +++ b/Composer/packages/lib/indexers/__tests__/qnaIndexer.test.ts @@ -53,8 +53,12 @@ describe('parse', () => { const result = parse('a.qna', content); expect(result.imports.length).toEqual(2); - expect(result.imports[0]).toEqual({ id: 'windows-guide.source.qna', path: 'windows-guide.source.qna' }); - expect(result.imports[1]).toEqual({ id: 'aks.qna', path: '../common/aks.qna' }); + expect(result.imports[0]).toEqual({ + id: 'windows-guide.source.qna', + path: 'windows-guide.source.qna', + description: 'import', + }); + expect(result.imports[1]).toEqual({ id: 'aks.qna', path: '../common/aks.qna', description: 'import' }); expect(result.empty).toEqual(false); expect(result.qnaSections.length).toEqual(0); }); diff --git a/Composer/packages/lib/indexers/__tests__/qnaUtil.test.ts b/Composer/packages/lib/indexers/__tests__/qnaUtil.test.ts index f18f8a59c3..d5c2535ad3 100644 --- a/Composer/packages/lib/indexers/__tests__/qnaUtil.test.ts +++ b/Composer/packages/lib/indexers/__tests__/qnaUtil.test.ts @@ -292,10 +292,12 @@ ${content1} expect(imports).toEqual([ { id: 'help.qna', + description: 'import', path: '../common/help.qna', }, { id: 'windows.source.qna', + description: 'import', path: 'windows.source.qna', }, ]); @@ -311,14 +313,17 @@ ${content1} { id: 'chitchat.qna', path: 'chitchat.qna', + description: 'import', }, { id: 'help.qna', path: '../common/help.qna', + description: 'import', }, { id: 'windows.source.qna', path: 'windows.source.qna', + description: 'import', }, ]); }); @@ -334,6 +339,7 @@ ${content1} { id: 'chitchat.qna', path: 'chitchat.qna', + description: 'import', }, ]); }); @@ -348,6 +354,7 @@ ${content1} { id: 'help.qna', path: '../common/help.qna', + description: 'import', }, ]); }); diff --git a/Composer/packages/lib/indexers/src/utils/lgUtil.ts b/Composer/packages/lib/indexers/src/utils/lgUtil.ts index 263e3fde92..0c84e756cc 100644 --- a/Composer/packages/lib/indexers/src/utils/lgUtil.ts +++ b/Composer/packages/lib/indexers/src/utils/lgUtil.ts @@ -15,6 +15,8 @@ import { SourceRange } from 'botbuilder-lg/lib/sourceRange'; import { lgIndexer } from '../lgIndexer'; +import { getFileName } from './help'; + // NOTE: LGDiagnostic is defined in PascalCase which should be corrected function convertLGDiagnostic(d: LGDiagnostic, source: string): Diagnostic { const result = new Diagnostic(d.message, source, d.severity); @@ -58,8 +60,15 @@ export function convertTemplatesToLgFile(id = '', content: string, parseResult: const templates = templateToLgTemplate(parseResult.toArray()); const allTemplates = templateToLgTemplate(parseResult.allTemplates); + const imports = parseResult.imports.map((item) => { + return { + id: getFileName(item.id), + path: item.id, + description: item.description, + }; + }); - return { id, content, templates, allTemplates, diagnostics, options: parseResult.options, parseResult }; + return { id, content, templates, allTemplates, diagnostics, imports, options: parseResult.options, parseResult }; } export function increaseNameUtilNotExist(templates: LgTemplate[], name: string): string { diff --git a/Composer/packages/lib/indexers/src/utils/luUtil.ts b/Composer/packages/lib/indexers/src/utils/luUtil.ts index 2676e06686..830be3a6b2 100644 --- a/Composer/packages/lib/indexers/src/utils/luUtil.ts +++ b/Composer/packages/lib/indexers/src/utils/luUtil.ts @@ -20,10 +20,12 @@ import { Range, DiagnosticSeverity, ILUFeaturesConfig, + LuParseResource, } from '@bfc/shared'; import formatMessage from 'format-message'; -import { buildNewlineText, splitNewlineText } from './help'; +import { buildNewlineText, getFileName, splitNewlineText } from './help'; +import { SectionTypes } from './qnaUtil'; const { luParser, sectionOperator } = sectionHandler; const { parseFile, validateResource } = BFLUParser; @@ -62,7 +64,11 @@ export function convertLuDiagnostic(d: any, source: string, offset = 0): Diagnos return result; } -export function convertLuParseResultToLuFile(id: string, resource, luFeatures: ILUFeaturesConfig): LuFile { +export function convertLuParseResultToLuFile( + id: string, + resource: LuParseResource, + luFeatures: ILUFeaturesConfig +): LuFile { // filter structured-object from LUParser result. const { Sections, Errors, Content } = resource; const intents: LuIntentSection[] = []; @@ -104,6 +110,16 @@ export function convertLuParseResultToLuFile(id: string, resource, luFeatures: I convertLuDiagnostic(e, id) ) as Diagnostic[]; + const imports = Sections.filter(({ SectionType }) => SectionType === SectionTypes.ImportSection).map( + ({ Path, Description }) => { + return { + id: getFileName(Path), + description: Description, + path: Path, + }; + } + ); + const diagnostics = syntaxDiagnostics.concat(semanticDiagnostics); return { id, @@ -111,6 +127,7 @@ export function convertLuParseResultToLuFile(id: string, resource, luFeatures: I empty: !Sections.length, intents, diagnostics, + imports, resource: { Sections, Errors, Content }, }; } diff --git a/Composer/packages/lib/indexers/src/utils/qnaUtil.ts b/Composer/packages/lib/indexers/src/utils/qnaUtil.ts index 351cad149f..77043decec 100644 --- a/Composer/packages/lib/indexers/src/utils/qnaUtil.ts +++ b/Composer/packages/lib/indexers/src/utils/qnaUtil.ts @@ -19,7 +19,7 @@ const { luParser, sectionOperator } = sectionHandler; const NEWLINE = '\n'; -enum SectionTypes { +export enum SectionTypes { QnASection = 'qnaSection', ImportSection = 'importSection', LUModelInfo = 'modelInfoSection', @@ -115,12 +115,15 @@ export function convertQnAParseResultToQnAFile(id = '', resource: LuParseResourc }; }); - const imports = Sections.filter(({ SectionType }) => SectionType === SectionTypes.ImportSection).map(({ Path }) => { - return { - id: getFileName(Path), - path: Path, - }; - }); + const imports = Sections.filter(({ SectionType }) => SectionType === SectionTypes.ImportSection).map( + ({ Path, Description }) => { + return { + id: getFileName(Path), + description: Description, + path: Path, + }; + } + ); const optionRegExp = new RegExp(/@source\.(\w+)\s*=\s*(.*)/); const options: { id: string; name: string; value: string }[] = []; diff --git a/Composer/packages/types/src/indexers.ts b/Composer/packages/types/src/indexers.ts index 34faefbf27..ce110e4525 100644 --- a/Composer/packages/types/src/indexers.ts +++ b/Composer/packages/types/src/indexers.ts @@ -114,11 +114,25 @@ export type LuFile = { intents: LuIntentSection[]; empty: boolean; resource: LuParseResource; + imports: { id: string; path: string; description: string }[]; published?: boolean; }; +export type LuParseResourceSection = { + Name: string; + Body: string; + SectionType: string; + Path: string; + Id: string; + Description: string; + Answer: string; + Questions: string[]; + ModelInfo: string; + [key: string]: any; +}; + export type LuParseResource = { - Sections: any[]; + Sections: LuParseResourceSection[]; Errors: any[]; Content: string; }; @@ -136,7 +150,7 @@ export type QnAFile = { content: string; diagnostics: IDiagnostic[]; qnaSections: QnASection[]; - imports: { id: string; path: string }[]; + imports: { id: string; path: string; description: string }[]; options: { id: string; name: string; value: string }[]; empty: boolean; resource: LuParseResource; @@ -177,6 +191,7 @@ export type LgFile = { diagnostics: IDiagnostic[]; templates: LgTemplate[]; allTemplates: LgTemplate[]; + imports: { id: string; path: string; description: string }[]; options?: string[]; parseResult?: any; };