refactor: Move indexer to client and add file persistence layer (#2348)

* add file persistence layer

* fix some confilct

* support lu publish

* add locale for create

* update the undo/redo

* fix unit test

* handle error

* update the resover

* add args

* remove some notes

* fix create dialog from form

* fix find wrong root dialog

* add error handler

* add locale when search the common lg file

* move file persistence from action to reducer

* fix some conflicts

* remove file persisten midleware

* update the navigate

* fix some conflicts

* change type->kind

* fix conflict

* add unit test for persistence layer

* add some unit test

* add unit tests for lu publish status

* add unit test for file reloatedreducer

* error handling

* wrap the reducer

* refine the persistence

* fix lint

* fix some spelling mistakes

Co-authored-by: Andy Brown <asbrown002@gmail.com>
Co-authored-by: zeye <2295905420@qq.com>
Co-authored-by: Chris Whitten <christopher.whitten@microsoft.com>

Composer/packages/client/__tests__/store/index.test.ts

@@ -0,0 +1,40 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+import { applyMiddleware } from './../../src/store';
+import { Store, State } from './../../src/store/types';
+import { ActionTypes } from './../../src/constants';
+
+describe('applyMiddleware', () => {
+  it('warns when dispatching during middleware setup', () => {
+    const store: Store = {
+      dispatch: jest.fn(),
+      getState: () => {
+        return {} as State;
+      },
+    };
+    const mockFunction1 = jest.fn();
+    const mockFunction2 = jest.fn();
+
+    const middleWare1 = (store: Store) => next => {
+      return () => {
+        mockFunction1();
+        return next();
+      };
+    };
+
+    const middleWare2 = (store: Store) => next => {
+      return () => {
+        mockFunction2();
+        return next();
+      };
+    };
+    const dispatch = applyMiddleware(store, middleWare1, middleWare2);
+    dispatch({
+      type: ActionTypes.UPDATE_BOTSTATUS,
+      payload: { a: 'a' },
+    });
+    expect(mockFunction1).toBeCalledTimes(1);
+    expect(mockFunction2).toBeCalledTimes(1);
+    expect(store.dispatch).toBeCalledTimes(1);
+  });
+});

Composer/packages/client/__tests__/store/middlewares/undo/history.test.js

@@ -5,10 +5,10 @@ import undoHistory from './../../../../src/store/middlewares/undo/history';
 
 describe('test undo history', () => {
   it('test undo history', async () => {
-    const mockStack1Undo = jest.fn((store, args) => args);
-    const mockStack1Redo = jest.fn((store, args) => args);
-    const mockStack2Undo = jest.fn((store, args) => args);
-    const mockStack2Redo = jest.fn((store, args) => args);
+    const mockStack1Undo = jest.fn((store, from, to) => to[0]);
+    const mockStack1Redo = jest.fn((store, from, to) => to[0]);
+    const mockStack2Undo = jest.fn((store, from, to) => to[0]);
+    const mockStack2Redo = jest.fn((store, from, to) => to[0]);
     const stack1 = undoHistory.createStack(mockStack1Undo, mockStack1Redo);
     const stack2 = undoHistory.createStack(mockStack2Undo, mockStack2Redo);
     stack1.add(['t0']);

Composer/packages/client/__tests__/store/persistence/filePersistence.test.ts

@@ -0,0 +1,170 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+import { DialogInfo, LgFile, LuFile } from '@bfc/shared';
+
+import { ActionTypes } from './../../../src/constants';
+import filePersistence from './../../../src/store/persistence/FilePersistence';
+import { State } from './../../../src/store/types';
+
+jest.mock('axios', () => {
+  return {
+    create: jest.fn(() => {
+      return {
+        put: jest.fn(() => Promise.resolve({ data: {} })),
+        post: jest.fn((url, data) => Promise.resolve({ data })),
+        delete: jest.fn(() => Promise.resolve({ data: {} })),
+      };
+    }),
+  };
+});
+
+const files = [
+  {
+    name: 'a.dialog',
+    content: '',
+    lastModified: 'Tue Mar 31 2020 23:08:15 GMT+0800 (GMT+08:00)',
+    path: 'C:/a.dialog',
+    relativePath: 'a.dialog',
+  },
+  {
+    name: 'a.en-us.lg',
+    content: '',
+    lastModified: 'Tue Mar 31 2020 23:08:15 GMT+0800 (GMT+08:00)',
+    path: 'C:/a.dialog',
+    relativePath: 'a.en-us.lg',
+  },
+  {
+    name: 'a.en-us.lu',
+    content: '',
+    lastModified: 'Tue Mar 31 2020 23:08:15 GMT+0800 (GMT+08:00)',
+    path: 'C:/a.en-us.lu',
+    relativePath: 'a.en-us.lu',
+  },
+];
+
+describe('test persistence layer', () => {
+  it('test attach file', () => {
+    expect(Object.keys(filePersistence.files).length).toBe(0);
+    expect(filePersistence.projectId).toBe('');
+    filePersistence.projectId = 'projectId';
+    files.forEach(file => filePersistence.attach(file.name, file));
+    expect(Object.keys(filePersistence.files).length).toBe(3);
+  });
+
+  it('test notify update', async () => {
+    const state1 = {
+      dialogs: [{ id: 'a', content: {} }] as DialogInfo[],
+      lgFiles: [{ id: 'a.en-us', content: '' }] as LgFile[],
+      luFiles: [{ id: 'a.en-us', content: '' }] as LuFile[],
+    } as State;
+
+    const state2 = {
+      dialogs: [{ id: 'a', content: { a: 'a' } }] as DialogInfo[],
+      lgFiles: [{ id: 'a.en-us', content: 'a' }] as LgFile[],
+      luFiles: [{ id: 'a.en-us', content: 'a' }] as LuFile[],
+    } as State;
+
+    await filePersistence.notify(state1, state2, { type: ActionTypes.UPDATE_DIALOG, payload: { id: 'a' } });
+    await filePersistence.notify(state1, state2, { type: ActionTypes.UPDATE_LG, payload: { id: 'a.en-us' } });
+    await filePersistence.notify(state1, state2, { type: ActionTypes.UPDATE_LU, payload: { id: 'a.en-us' } });
+    await new Promise(res =>
+      setTimeout(() => {
+        const dialog = filePersistence.files['a.dialog'].file;
+        const lg = filePersistence.files['a.en-us.lg'].file;
+        const lu = filePersistence.files['a.en-us.lu'].file;
+        expect(dialog).toBeDefined();
+        if (dialog) {
+          expect(JSON.parse(dialog.content).a).toBe('a');
+        }
+        expect(lg).toBeDefined();
+        if (lg) {
+          expect(lg.content).toBe('a');
+        }
+        expect(lu).toBeDefined();
+        if (lu) {
+          expect(lu.content).toBe('a');
+        }
+        res();
+      }, 601)
+    );
+  });
+
+  it('test notify create', async () => {
+    const state1 = {
+      dialogs: [{ id: 'a', content: { a: 'a' } }] as DialogInfo[],
+      lgFiles: [{ id: 'a.en-us', content: 'a' }] as LgFile[],
+      luFiles: [{ id: 'a.en-us', content: 'a' }] as LuFile[],
+    } as State;
+
+    const state2 = {
+      dialogs: [
+        { id: 'a', content: { a: 'a' } },
+        { id: 'b', content: { b: 'b' } },
+      ] as DialogInfo[],
+      lgFiles: [
+        { id: 'a.en-us', content: 'a' },
+        { id: 'b.en-us', content: 'b' },
+      ] as LgFile[],
+      luFiles: [
+        { id: 'a.en-us', content: 'a' },
+        { id: 'b.en-us', content: 'b' },
+      ] as LuFile[],
+    } as State;
+
+    await filePersistence.notify(state1, state2, { type: ActionTypes.CREATE_DIALOG, payload: { id: 'b' } });
+    const dialog = filePersistence.files['b.dialog'].file;
+    const lg = filePersistence.files['b.en-us.lg'].file;
+    const lu = filePersistence.files['b.en-us.lu'].file;
+    expect(dialog).toBeDefined();
+    if (dialog) {
+      expect(JSON.parse(dialog.content).b).toBe('b');
+    }
+    expect(lg).toBeDefined();
+    if (lg) {
+      expect(lg.content).toBe('b');
+    }
+    expect(lu).toBeDefined();
+    if (lu) {
+      expect(lu.content).toBe('b');
+    }
+  });
+
+  it('test notify remove', async () => {
+    const state1 = {
+      dialogs: [
+        { id: 'a', content: { a: 'a' } },
+        { id: 'b', content: { b: 'b' } },
+      ] as DialogInfo[],
+      lgFiles: [
+        { id: 'a.en-us', content: 'a' },
+        { id: 'b.en-us', content: 'b' },
+      ] as LgFile[],
+      luFiles: [
+        { id: 'a.en-us', content: 'a' },
+        { id: 'b.en-us', content: 'b' },
+      ] as LuFile[],
+    } as State;
+
+    const state2 = {
+      dialogs: [{ id: 'a', content: { a: 'a' } }] as DialogInfo[],
+      lgFiles: [{ id: 'a.en-us', content: 'a' }] as LgFile[],
+      luFiles: [{ id: 'a.en-us', content: 'a' }] as LuFile[],
+    } as State;
+
+    await filePersistence.notify(state1, state2, { type: ActionTypes.REMOVE_DIALOG, payload: { id: 'b' } });
+    const dialog = filePersistence.files['b.dialog'];
+    const lg = filePersistence.files['b.en-us.lg'];
+    const lu = filePersistence.files['b.en-us.lu'];
+    expect(dialog).toBeUndefined();
+    expect(lg).toBeUndefined();
+    expect(lu).toBeUndefined();
+  });
+  it('test detach', async () => {
+    filePersistence.detach('a.dialog');
+    expect(filePersistence.files['a.dialog']).toBeUndefined();
+  });
+  it('test clear', async () => {
+    filePersistence.clear();
+    expect(Object.keys(filePersistence.files).length).toBe(0);
+  });
+});

Composer/packages/client/__tests__/store/reducer/reducer.test.js

@@ -6,39 +6,12 @@ import { reducer } from '../../../src/store/reducer/index';
 
 const mockResponse = {
   data: {
-    dialogs: ['test dialogs'],
-    lgFiles: ['test lgFiles'],
-    luFiles: ['test luFiles'],
+    files: ['test files'],
     schemas: 'test schemas',
   },
 };
 
 describe('test all reducer handlers', () => {
-  it('test getProjectSuccess reducer', () => {
-    const result = reducer({}, { type: ActionTypes.GET_PROJECT_SUCCESS, payload: { response: mockResponse } });
-    expect(result.dialogs[0]).toBe('test dialogs');
-    expect(result.lgFiles[0]).toBe('test lgFiles');
-    expect(result.schemas).toBe('test schemas');
-  });
-  it('test createDialogSuccess reducer', () => {
-    const result = reducer({}, { type: ActionTypes.CREATE_DIALOG_SUCCESS, payload: { response: mockResponse } });
-    expect(result.dialogs[0]).toBe('test dialogs');
-  });
-  it('test updateLgTemplate reducer', () => {
-    const result = reducer(
-      { lgFiles: [{ id: 'common.lg', content: 'test lgFiles' }] },
-      {
-        type: ActionTypes.UPDATE_LG_SUCCESS,
-        payload: {
-          id: 'common.lg',
-          content: ` # bfdactivity-003038
-        - You said '\${turn.activity.text}'`,
-        },
-      }
-    );
-    expect(result.lgFiles[0].templates.length).toBe(1);
-  });
-
   it('test getStorageFileSuccess reducer', () => {
     const mockStorageFile = {
       data: {
@@ -63,4 +36,68 @@ describe('test all reducer handlers', () => {
     expect(result.focusedStorageFolder).toEqual(expect.objectContaining({ children: expect.any(Array) }));
     expect(result.focusedStorageFolder.children).toHaveLength(2);
   });
+
+  it('remove lg file', () => {
+    const result = reducer(
+      { lgFiles: [{ id: '1' }, { id: '2' }] },
+      { type: ActionTypes.REMOVE_LG, payload: { id: '1' } }
+    );
+    expect(result.lgFiles.length).toBe(1);
+    expect(result.lgFiles[0].id).toBe('2');
+  });
+
+  it('create lg file', () => {
+    const result = reducer(
+      { lgFiles: [{ id: '1' }, { id: '2' }], locale: 'en-us' },
+      { type: ActionTypes.CREATE_LG, payload: { id: '3', content: '' } }
+    );
+    expect(result.lgFiles.length).toBe(3);
+    expect(result.lgFiles[2].id).toBe('3.en-us');
+  });
+
+  it('update lg file', () => {
+    const result = reducer(
+      { lgFiles: [{ id: '1', content: 'old' }, { id: '2' }] },
+      { type: ActionTypes.UPDATE_LG, payload: { id: '1', content: 'new' } }
+    );
+    expect(result.lgFiles.length).toBe(2);
+    expect(result.lgFiles[0].content).toBe('new');
+  });
+
+  it('remove dialog file', () => {
+    const result = reducer(
+      { dialogs: [{ id: '1' }, { id: '2' }], lgFiles: [{ id: '1' }], luFiles: [{ id: '1' }] },
+      { type: ActionTypes.REMOVE_DIALOG, payload: { id: '1' } }
+    );
+    expect(result.dialogs.length).toBe(1);
+    expect(result.dialogs[0].id).toBe('2');
+    expect(result.luFiles.length).toBe(0);
+    expect(result.lgFiles.length).toBe(0);
+  });
+
+  it('create dialog file', () => {
+    const result = reducer(
+      {
+        dialogs: [{ id: '1' }, { id: '2' }],
+        locale: 'en-us',
+        lgFiles: [],
+        luFiles: [],
+        schemas: { sdk: { content: {} } },
+      },
+      { type: ActionTypes.CREATE_DIALOG, payload: { id: '3', content: '' } }
+    );
+    expect(result.dialogs.length).toBe(3);
+    expect(result.dialogs[2].id).toBe('3');
+    expect(result.luFiles.length).toBe(1);
+    expect(result.lgFiles.length).toBe(1);
+  });
+
+  it('update dialog file', () => {
+    const result = reducer(
+      { dialogs: [{ id: '1', content: 'old' }, { id: '2' }], schemas: { sdk: { content: {} } } },
+      { type: ActionTypes.UPDATE_DIALOG, payload: { id: '1', content: 'new' } }
+    );
+    expect(result.dialogs.length).toBe(2);
+    expect(result.dialogs[0].content).toBe('new');
+  });
 });

Composer/packages/client/__tests__/utils/luFileStatusStorage.test.ts

@@ -0,0 +1,48 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import luFileStatusStorage from '../../src/utils/luFileStatusStorage';
+
+const fileIds = ['1', '2', '3', '4'];
+const botName = 'test_status';
+
+afterAll(() => luFileStatusStorage.removeAllStatuses(botName));
+
+describe('luFileStatusStorage', () => {
+  it('check file status', () => {
+    luFileStatusStorage.checkFileStatus(botName, fileIds);
+    expect(Object.keys(luFileStatusStorage.get(botName)).length).toEqual(4);
+  });
+
+  it('the statuses after publishing', () => {
+    luFileStatusStorage.publishAll(botName);
+    const result = luFileStatusStorage.get(botName);
+    Object.keys(result).forEach(id => {
+      expect(result[id]).toBeTruthy();
+    });
+  });
+
+  it('update one luis file', () => {
+    luFileStatusStorage.updateFileStatus(botName, fileIds[0]);
+    const result = luFileStatusStorage.get(botName);
+    expect(result[fileIds[0]]).toBeFalsy();
+    expect(result[fileIds[1]]).toBeTruthy();
+    expect(result[fileIds[2]]).toBeTruthy();
+    expect(result[fileIds[3]]).toBeTruthy();
+  });
+
+  it('remove one luis file', () => {
+    luFileStatusStorage.removeFileStatus(botName, fileIds[0]);
+    const result = luFileStatusStorage.get(botName);
+    expect(result[fileIds[0]]).toBeUndefined;
+    expect(result[fileIds[1]]).toBeTruthy();
+    expect(result[fileIds[2]]).toBeTruthy();
+    expect(result[fileIds[3]]).toBeTruthy();
+  });
+
+  it('remove all statuses', () => {
+    luFileStatusStorage.removeAllStatuses(botName);
+    const result = luFileStatusStorage.get(botName);
+    expect(result).toBeUndefined;
+  });
+});