merge: merge branch 'THU/add_tests_for_dataset_manager_PROD-13333'

cypress/commons/actions/generic/DatasetManager.js

@@ -19,6 +19,8 @@ export const getNoDatasetsPlaceholderUserSubtitle = (timeout = 5) =>
 
 export const getCreateDatasetButton = () => cy.get(SELECTORS.createDatasetButton);
 export const startDatasetCreation = () => getCreateDatasetButton().click();
+export const getCreateSubdatasetButton = () => cy.get(SELECTORS.createSubdatasetButton);
+export const startSubdatasetCreation = () => getCreateSubdatasetButton().click();
 
 export const getDatasetSearchBar = () => cy.get(SELECTORS.list.searchBar);
 export const getDatasetSearchBarInput = () =>
@@ -66,6 +68,7 @@ export const getDatasetMetadataApiUrlButton = () => cy.get(SELECTORS.metadata.ap
 export const getDatasetMetadataEditDescriptionButton = () => cy.get(SELECTORS.metadata.editDescriptionButton);
 export const getDatasetMetadataDescriptionTextField = () => cy.get(SELECTORS.metadata.descriptionTextField);
 export const getDatasetMetadataDescription = () => cy.get(SELECTORS.metadata.description);
+export const getDatasetMetadataParent = () => cy.get(SELECTORS.metadata.parent);
 
 export const getDatasetMetadataNewTagTextField = () => cy.get(SELECTORS.metadata.newTagTextField);
 export const getDatasetMetadataAddTagButton = () => cy.get(SELECTORS.metadata.addTagButton);
@@ -75,7 +78,6 @@ export const getDatasetMetadataDeleteTagButton = (parent) => {
 };
 export const getDatasetMetadataTagsContainer = () => cy.get(SELECTORS.metadata.tagsContainer);
 export const getDatasetMetadataTags = () => cy.get(SELECTORS.metadata.tags);
-
 export const getDatasetMetadataTag = (index) => cy.get(SELECTORS.metadata.tagByIndex.replace('$INDEX', index));
 
 export const getDatasetNameInOverview = (timeout) =>
@@ -144,6 +146,7 @@ export const deleteDatasetTag = (tagIndex, options) => {
 };
 
 export const getDatasetCreationDialog = () => cy.get(SELECTORS.wizard.dialog);
+export const getParentNameSubtitle = () => cy.get(SELECTORS.wizard.parentNameSubtitle);
 export const getCancelDatasetCreation = () => cy.get(SELECTORS.wizard.cancelDatasetCreation);
 export const getDatasetCreationPreviousStep = () => cy.get(SELECTORS.wizard.previous);
 export const getDatasetCreationNextStep = () => cy.get(SELECTORS.wizard.next);
@@ -154,14 +157,23 @@ export const getConfirmDatasetCreation = () => cy.get(SELECTORS.wizard.confirmDa
 //    - id (optional): id of the created dataset (only used when stubbing is enabled; if undefined, a random id is used)
 //    - validateRequest (optional): validation function to run on the dataset update request
 //    - importJobOptions: options to provide to the interception of the "create dataset" query (default: undefined)
-//    - isFile: boolean defining if the created dataset is imported from a file upload (needs to be set to trigger the
-//      correct interceptions)
+//    - runnerCreationOptions: options to provide to the interception of the "create runner" query (default: undefined)
+//    - runnerUpdateOptions: options to provide to the interception of the "update runner" query (default: undefined)
+//    - isFile: boolean defining if the created dataset is imported from a file upload (when enabled, queries to refresh
+//      and status endpoints won't be run)
+//    - isETL: boolean defining if the created dataset is created from an ETL runner (when enabled, queries to create
+//      and patch the runner will be run)
 export const confirmDatasetCreation = (options = {}) => {
   options.customDatasetPatch = {
     main: true,
     ...options.customDatasetPatch,
   };
-  const aliases = [api.interceptCreateDataset(options, options.importJobOptions), api.interceptLinkDataset()];
+  const aliases = [];
+  if (options.isETL) aliases.push(api.interceptCreateRunner(options.runnerCreationOptions));
+  aliases.push(api.interceptCreateDataset(options, options.importJobOptions));
+  aliases.push(api.interceptLinkDataset());
+  if (options.isETL) aliases.push(api.interceptUpdateRunner(options.runnerUpdateOptions));
+
   if (!options.isFile) {
     aliases.push(api.interceptRefreshDataset());
     aliases.push(api.interceptGetDatasetStatus(options.importJobOptions?.expectedPollsCount));
@@ -178,6 +190,8 @@ export const getNewDatasetNameInput = () =>
 export const setNewDatasetName = (name) => getNewDatasetName().type(name + '{enter}');
 export const getNewDatasetTagsContainer = () => cy.get(SELECTORS.wizard.tagsContainer);
 export const getNewDatasetTagsInput = () => cy.get(SELECTORS.wizard.tagsInput);
+export const getNewDatasetTag = (index) => cy.get(SELECTORS.wizard.tagByIndex.replace('$INDEX', index));
+
 export const getNewDatasetDeleteTagButtons = () => getNewDatasetTagsContainer().find(SELECTORS.wizard.deleteTagButton);
 export const addNewDatasetTag = (newTag) => getNewDatasetTagsInput().type(newTag + '{enter}');
 export const deleteNewDatasetTag = (tagIndex) => getNewDatasetDeleteTagButtons().eq(tagIndex).click();
@@ -186,13 +200,24 @@ export const selectNewDatasetFromExistingData = () => getNewDatasetLocationOptio
 export const getNewDatasetLocationOptionFromScratch = () => cy.get(SELECTORS.wizard.locationOptionFromScratch);
 export const selectNewDatasetFromScratch = () => getNewDatasetLocationOptionFromScratch().click();
 export const getNewDatasetSourceTypeSelect = () => cy.get(SELECTORS.wizard.sourceTypeSelect);
+export const getNewDatasetSourceTypeOptionsMenu = () => cy.get(SELECTORS.wizard.sourceTypeOptionsMenu);
+export const getNewDatasetSourceTypeOptions = () =>
+  getNewDatasetSourceTypeOptionsMenu().find(SELECTORS.wizard.sourceTypeOptions);
+export const getNewDatasetSourceTypeOption = (runTemplateId) =>
+  getNewDatasetSourceTypeOptionsMenu().find(
+    SELECTORS.wizard.sourceTypeOption.replace('$RUN_TEMPLATE_ID', runTemplateId)
+  );
 export const getNewDatasetSourceTypeOptionAzureStorage = () => cy.get(SELECTORS.wizard.sourceTypeOptionAzureStorage);
 export const getNewDatasetSourceTypeOptionFile = () => cy.get(SELECTORS.wizard.sourceTypeOptionFile);
 export const getNewDatasetSourceTypeOptionADT = () => cy.get(SELECTORS.wizard.sourceTypeOptionADT);
 export const getNewDatasetAzureStorageAccountName = () => cy.get(SELECTORS.wizard.azureStorageAccountName);
 export const getNewDatasetAzureStorageContainerName = () => cy.get(SELECTORS.wizard.azureStorageContainerName);
 export const getNewDatasetAzureStoragePath = () => cy.get(SELECTORS.wizard.azureStoragePath);
 export const getNewDatasetADTURL = () => cy.get(SELECTORS.wizard.adtURL);
+export const selectNewDatasetSourceType = (runTemplateId) => {
+  getNewDatasetSourceTypeSelect().click();
+  getNewDatasetSourceTypeOption(runTemplateId).click();
+};
 
 export const setNewDatasetAzureStorageAccountName = (value) =>
   getNewDatasetAzureStorageAccountName().type('{selectAll}{backspace}' + value);
@@ -234,3 +259,14 @@ export const rollbackDatasetStatus = () => {
   getDatasetOverviewPlaceholderRollbackButton().click();
   api.waitAlias(alias);
 };
+
+// Parameters:
+//   - response (optional): JSON response to the twingraph query that is simulated if stubbing is enabled. Example:
+//       [{"id":"Dynamic value 1"},{"id":"Dynamic value 2"},{"id":"Dynamic value 3"}]
+//   - validateRequest (optional): a function, taking the request object as argument, that can be used to perform
+//       cypress checks on the content of the intercepted query
+// Return value: a callback function to call in your test to wait for the interception
+export const expectDatasetTwingraphQuery = (response = {}, validateRequest) => {
+  const alias = api.interceptPostDatasetTwingraphQuery(response, validateRequest);
+  return () => api.waitAlias(alias);
+};

cypress/commons/constants/generic/IdConstants.js

@@ -143,6 +143,7 @@ export const GENERIC_SELECTORS = {
   },
   datasetmanager: {
     createDatasetButton: '[data-cy=create-dataset-button]',
+    createSubdatasetButton: '[data-cy=create-subdataset-button]',
     list: {
       container: '[data-cy=datasets-list]',
       deleteButtons: '[data-cy^=dataset-delete-button-]',
@@ -171,6 +172,7 @@ export const GENERIC_SELECTORS = {
       editDescriptionButton: '[data-cy=edit-description]',
       descriptionTextField: '[data-cy=description-textfield]',
       description: '[data-cy=dataset-metadata-description]',
+      parent: '[data-cy=dataset-metadata-parent]',
       newTagTextField: '[data-cy=new-tag-textfield]',
       addTagButton: '[data-cy=add-tag]',
       deleteTagButton: '[data-testid=CancelIcon]',
@@ -185,16 +187,21 @@ export const GENERIC_SELECTORS = {
     view: '[data-cy=dataset-manager-view]',
     wizard: {
       dialog: '[data-cy=dataset-creation-dialog]',
+      parentNameSubtitle: '[data-cy=wizard-subtitle-parent-dataset-name]',
       cancelDatasetCreation: '[data-cy=cancel-dataset-creation]',
       confirmDatasetCreation: '[data-cy=confirm-dataset-creation]',
       description: '[data-cy=text-input-new-dataset-description]',
       name: '[data-cy=text-input-new-dataset-title]',
       tagsContainer: '[data-cy="new-dataset-tags-container"]',
       tagsInput: '[id=new-dataset-tags]',
+      tagByIndex: '[data-cy=new-dataset-tags-tag-$INDEX]',
       deleteTagButton: '[data-testid=CancelIcon]',
       locationOptionExistingData: '[data-cy=radio-button-existingData]',
       locationOptionFromScratch: '[data-cy=radio-button-fromScratch]',
       sourceTypeSelect: '[data-cy="enum-input-select-new-dataset-sourceType"]',
+      sourceTypeOptionsMenu: '[data-cy="enum-input-menu-new-dataset-sourceType"]',
+      sourceTypeOptions: 'li[role="option"]',
+      sourceTypeOption: '[data-cy="$RUN_TEMPLATE_ID"]',
       sourceTypeOptionAzureStorage: '[data-cy="AzureStorage"]',
       sourceTypeOptionFile: '[data-cy="File"]',
       sourceTypeOptionADT: '[data-cy="ADT"]',

cypress/commons/constants/generic/TestConstants.js

@@ -42,7 +42,10 @@ export const API_ENDPOINT = {
   DATASET_REFRESH: URL_ROOT + '/.*/datasets/((d|D)-[\\w]+)/refresh',
   DATASET_ROLLBACK: URL_ROOT + '/.*/datasets/((d|D)-[\\w]+)/refresh/rollback',
   DATASET_STATUS: URL_ROOT + '/.*/datasets/((d|D)-[\\w]+)/status',
-  RUNNER_DEFAULT_SECURITY: URL_ROOT + '/.*/runners/((d|D)-[\\w]+)/security/default',
+  DATASET_TWINGRAPH: URL_ROOT + '/.*/datasets/((d|D)-[\\w]+)/twingraph',
+  RUNNERS: URL_ROOT + '/.*/runners',
+  RUNNER: URL_ROOT + '/.*/runners/((r|R)-[\\w]+)',
+  RUNNER_DEFAULT_SECURITY: URL_ROOT + '/.*/runners/((r|R)-[\\w]+)/security/default',
   RUNNER_SECURITY_ACL: URL_ROOT + '/.*/runners/((r|R)-[\\w]+)/security/access',
   RUNNER_SECURITY_USER_ACCESS: URL_ROOT + '/.*/runners/((r|R)-[\\w]+)/security/access/(.*)',
   WORKSPACES: URL_ROOT + '/.*/workspaces',
@@ -179,6 +182,9 @@ export const API_REGEX = {
   DATASET_REFRESH: new RegExp('^' + API_ENDPOINT.DATASET_REFRESH + '$'),
   DATASET_ROLLBACK: new RegExp('^' + API_ENDPOINT.DATASET_ROLLBACK + '$'),
   DATASET_STATUS: new RegExp('^' + API_ENDPOINT.DATASET_STATUS + '$'),
+  DATASET_TWINGRAPH: new RegExp('^' + API_ENDPOINT.DATASET_TWINGRAPH + '$'),
+  RUNNERS: new RegExp('^' + API_ENDPOINT.RUNNERS + '$'),
+  RUNNER: new RegExp('^' + API_ENDPOINT.RUNNER + '$'),
   RUNNER_DEFAULT_SECURITY: new RegExp('^' + API_ENDPOINT.RUNNER_DEFAULT_SECURITY + '$'),
   RUNNER_SECURITY_ACL: new RegExp('^' + API_ENDPOINT.RUNNER_SECURITY_ACL + '$'),
   RUNNER_SECURITY_USER_ACCESS: new RegExp('^' + API_ENDPOINT.RUNNER_SECURITY_USER_ACCESS + '$'),

cypress/commons/services/stubbing.js

@@ -5,6 +5,7 @@ import {
   DEFAULT_ORGANIZATION_PERMISSIONS,
   DEFAULT_SCENARIOS_LIST,
   DEFAULT_DATASETS_LIST,
+  DEFAULT_RUNNER,
   DEFAULT_WORKSPACE,
   DEFAULT_WORKSPACES_LIST,
   DEFAULT_ORGANIZATION,
@@ -25,7 +26,6 @@ const STUB_TYPES = [
   'LAUNCH_SCENARIO',
   'PERMISSIONS_MAPPING',
   'UPDATE_DATASET',
-  'UPDATE_RUNNER',
   'UPDATE_SCENARIO',
 ];
 
@@ -62,6 +62,7 @@ const DEFAULT_AUTH_DATA = {
 // scenarios runs, solutions and workspaces
 const DEFAULT_RESOURCES_DATA = {
   datasets: DEFAULT_DATASETS_LIST,
+  runners: [{ ...DEFAULT_RUNNER }],
   scenarioRuns: [],
   scenarios: DEFAULT_SCENARIOS_LIST,
   solutions: DEFAULT_SOLUTIONS_LIST,
@@ -300,6 +301,11 @@ class Stubbing {
     this.patchDatasetSecurity(datasetId, dataset.security?.default, newACL);
   };
 
+  getRunners = () => this._getResources('runners');
+  setRunners = (newRunners) => this._setResources('runners', newRunners);
+  addRunner = (newRunner) => this._addResource('runners', newRunner);
+  patchRunner = (runnerId, runnerPatch) => this._patchResourceById('runners', runnerId, runnerPatch);
+
   getSolutions = () => this._getResources('solutions');
   setSolutions = (newSolutions) => this._setResources('solutions', newSolutions);
   getSolutionById = (solutionId) => this._getResourceById('solutions', solutionId);

cypress/commons/utils/apiUtils.js

@@ -2,7 +2,12 @@
 // Licensed under the MIT license.
 import { POLLING_START_DELAY } from '../../../src/services/config/FunctionalConstants';
 import utils from '../../commons/TestUtils';
-import { DEFAULT_DATASET, SCENARIO_EXAMPLE, SCENARIO_RUN_EXAMPLE } from '../../fixtures/stubbing/default';
+import {
+  DEFAULT_DATASET,
+  DEFAULT_RUNNER,
+  SCENARIO_EXAMPLE,
+  SCENARIO_RUN_EXAMPLE,
+} from '../../fixtures/stubbing/default';
 import { API_ENDPOINT, API_REGEX, AUTH_QUERY_URL, URL_POWERBI, URL_ROOT } from '../constants/generic/TestConstants';
 import { stub } from '../services/stubbing';
 import { authUtils } from './authUtils';
@@ -282,7 +287,7 @@ const interceptUpdateRunnerDefaultSecurity = (expectedDefaultSecurity) => {
   cy.intercept({ method: 'POST', url: API_REGEX.RUNNER_DEFAULT_SECURITY, times: 1 }, (req) => {
     const newDefaultSecurity = req.body.role;
     if (expectedDefaultSecurity) expect(newDefaultSecurity).to.deep.equal(expectedDefaultSecurity);
-    if (stub.isEnabledFor('UPDATE_RUNNER')) req.reply(newDefaultSecurity);
+    if (stub.isEnabledFor('UPDATE_DATASET')) req.reply(newDefaultSecurity);
   }).as(alias);
   return alias;
 };
@@ -292,7 +297,7 @@ const interceptUpdateRunnerACLSecurity = (expectedACLSecurity) => {
   cy.intercept({ method: 'POST', url: API_REGEX.RUNNER_SECURITY_ACL, times: 1 }, (req) => {
     const newACLSecurityItem = req.body;
     if (expectedACLSecurity) expect(newACLSecurityItem).to.deep.equal(expectedACLSecurity);
-    if (stub.isEnabledFor('UPDATE_RUNNER')) req.reply(newACLSecurityItem);
+    if (stub.isEnabledFor('UPDATE_DATASET')) req.reply(newACLSecurityItem);
   }).as(alias);
   return alias;
 };
@@ -380,6 +385,21 @@ const interceptGetDatasetStatus = (times = 1) => {
   return alias;
 };
 
+// Parameters:
+//   - response (optional): JSON response to the twingraph query that is simulated if stubbing is enabled. Example:
+//       [{"id":"Dynamic value 1"},{"id":"Dynamic value 2"},{"id":"Dynamic value 3"}]
+//   - validateRequest (optional): a function, taking the request object as argument, that can be used to perform
+//       cypress checks on the content of the intercepted query
+const interceptPostDatasetTwingraphQuery = (response = {}, validateRequest) => {
+  const alias = forgeAlias('reqPostDatasetTwingraphQuery');
+  cy.intercept({ method: 'POST', url: API_REGEX.DATASET_TWINGRAPH, times: 1 }, (req) => {
+    if (validateRequest) validateRequest(req);
+    if (!stub.isEnabledFor('GET_DATASETS')) return;
+    req.reply(response);
+  }).as(alias);
+  return alias;
+};
+
 const interceptRollbackDatasetStatus = () => {
   const alias = forgeAlias('reqRollbackDatasetStatus');
   cy.intercept({ method: 'POST', url: API_REGEX.DATASET_ROLLBACK, times: 1 }, (req) => {
@@ -581,6 +601,57 @@ const interceptDeleteDataset = (datasetName) => {
   return alias;
 };
 
+// Parameters:
+//   - options: dict with properties:
+//     - id (optional): id of the runner to create
+//     - validateRequest (optional): a function, taking the request object as argument, that can be used to perform
+//       cypress checks on the content of the intercepted query
+//     - customRunnerPatch (optional): data to set in the request body, you can use this option to replace the original
+//       content of the query
+const interceptCreateRunner = (options = {}) => {
+  const alias = forgeAlias('reqCreateRunner');
+  cy.intercept({ method: 'POST', url: API_REGEX.RUNNERS, times: 1 }, (req) => {
+    if (options?.validateRequest) options?.validateRequest(req);
+    const runnerId = options.id ?? `r-stbd${utils.randomStr(4)}`;
+    if (stub.isEnabledFor('CREATE_DATASET')) {
+      const runner = {
+        ...DEFAULT_RUNNER,
+        ...req.body,
+        id: runnerId,
+        security: { default: 'none', accessControlList: [{ id: stub.getUser().email, role: 'admin' }] },
+        ...options?.customRunnerPatch,
+      };
+
+      stub.addRunner(runner);
+      req.reply(runner);
+    } else if (stub.isEnabledFor('GET_DATASETS')) {
+      req.continue((res) => stub.addRunner(res.body));
+    }
+  }).as(alias);
+  return alias;
+};
+
+// Parameters:
+//   - options: dict with properties:
+//     - validateRequest (optional): a function, taking the request object as argument, that can be used to perform
+//       cypress checks on the content of the intercepted query
+//     - customRunnerPatch (optional): data to set in the request body, you can use this option to replace the original
+//       content of the query
+const interceptUpdateRunner = (options = {}) => {
+  const alias = forgeAlias('reqUpdateRunner');
+  cy.intercept({ method: 'PATCH', url: API_REGEX.RUNNER, times: 1 }, (req) => {
+    if (options?.validateRequest) options?.validateRequest(req);
+    const runnerPatch = {
+      ...req.body,
+      ...options?.customRunnerPatch,
+    };
+    const runnerId = req.url.match(API_REGEX.RUNNER)?.[1];
+    if (stub.isEnabledFor('GET_DATASETS')) stub.patchRunner(runnerId, runnerPatch);
+    if (stub.isEnabledFor('UPDATE_DATASET')) req.reply(runnerPatch);
+  }).as(alias);
+  return alias;
+};
+
 const interceptDownloadWorkspaceFile = () => {
   const alias = forgeAlias('reqDownloadWorkspaceFile');
   cy.intercept({ method: 'GET', url: API_REGEX.FILE_DOWNLOAD, times: 1 }, (req) => {
@@ -691,6 +762,7 @@ export const apiUtils = {
   interceptGetDatasets,
   interceptGetDataset,
   interceptGetDatasetStatus,
+  interceptPostDatasetTwingraphQuery,
   interceptRollbackDatasetStatus,
   interceptCreateDataset,
   interceptLinkDataset,
@@ -703,6 +775,8 @@ export const apiUtils = {
   interceptRemoveDatasetAccessControl,
   interceptSetDatasetDefaultSecurity,
   interceptDeleteDataset,
+  interceptCreateRunner,
+  interceptUpdateRunner,
   interceptDownloadWorkspaceFile,
   interceptUploadWorkspaceFile,
   interceptGetOrganizationPermissions,

cypress/e2e/brewery/DatasetManager_DynamicValues.cy.js

@@ -0,0 +1,50 @@
+// Copyright (c) Cosmo Tech.
+// Licensed under the MIT license.
+import { Login, DatasetManager } from '../../commons/actions';
+import { stub } from '../../commons/services/stubbing';
+import {
+  DATASETS,
+  SOLUTION_WITH_DYNAMIC_VALUES,
+  WORKSPACE,
+  ORGANIZATION_WITH_DEFAULT_ROLE_USER,
+} from '../../fixtures/stubbing/DatasetManager';
+
+describe('Subdatasets creation', () => {
+  before(() => {
+    stub.start();
+    stub.setOrganizations([ORGANIZATION_WITH_DEFAULT_ROLE_USER]);
+    stub.setSolutions([SOLUTION_WITH_DYNAMIC_VALUES]);
+    stub.setWorkspaces([WORKSPACE]);
+    stub.setDatasets([...DATASETS]);
+  });
+  beforeEach(() => Login.login({ url: '/W-stbbdbrwryWithDM', workspaceId: 'W-stbbdbrwryWithDM' }));
+  after(stub.stop);
+
+  it('must show the parameter input fields associated to the selected data source', () => {
+    const DATASET_A = DATASETS[0];
+    const enumParameterSelector = '[data-cy=enum-input-select-etl_dynamic_values_enum_parameter]';
+    const enumOption2Selector = '[data-cy="Dynamic value 2"]';
+    const selectorForAllEnumOptions = '[data-cy^="Dynamic value"]';
+
+    const validateRequest = (req) =>
+      expect(req.body).to.deep.equal({ query: 'MATCH(n:Customer) RETURN n.id as customer_id' });
+    const queryResponse = [
+      { customer_id: 'Dynamic value 1' },
+      { customer_id: 'Dynamic value 2' },
+      { customer_id: 'Dynamic value 3' },
+    ];
+
+    DatasetManager.switchToDatasetManagerView();
+    DatasetManager.selectDatasetById(DATASET_A.id);
+    DatasetManager.startSubdatasetCreation();
+
+    const waitForTwingraphQuery = DatasetManager.expectDatasetTwingraphQuery(queryResponse, validateRequest);
+    DatasetManager.getDatasetCreationNextStep().click();
+    DatasetManager.selectNewDatasetSourceType('dynamic_values_enum_filter');
+    waitForTwingraphQuery();
+
+    cy.get(enumParameterSelector).should('be.visible').click();
+    cy.get(selectorForAllEnumOptions).should('have.length', 3);
+    cy.get(enumOption2Selector).click();
+  });
+});