fix: improve the Javascript and Lodash queries generated via the wizard

src/lib/plugins/query/javascriptQueryLanguage.js

@@ -1,3 +1,5 @@
+import { createPropertySelector } from '../../utils/pathUtils.js'
+
 const description = `
 <p>
   Enter a JavaScript function to filter, sort, or transform the data.
@@ -13,22 +15,6 @@ export const javascriptQueryLanguage = {
   executeQuery
 }
 
-/**
- * Turn a path like:
- *
- *   ['location', 'latitude']
- *
- * into a JavaScript selector (string) like:
- *
- *   '?.["location"]?.["latitude"]'
- *
- * @param {Path} path
- * @returns {string}
- */
-function createPropertySelector(path) {
-  return path.map((f) => `?.[${JSON.stringify(f)}]`).join('')
-}
-
 /**
  * @param {JSON} json
  * @param {QueryLanguageOptions} queryOptions
@@ -41,7 +27,7 @@ function createQuery(json, queryOptions) {
   if (filter && filter.path && filter.relation && filter.value) {
     // Note that the comparisons embrace type coercion,
     // so a filter value like '5' (text) will match numbers like 5 too.
-    const getActualValue = 'item => item' + createPropertySelector(filter.path)
+    const getActualValue = `item => item${createPropertySelector(filter.path)}`
 
     queryParts.push(
       `  data = data.filter(${getActualValue} ${filter.relation} '${filter.value}')\n`
@@ -77,13 +63,13 @@ function createQuery(json, queryOptions) {
     if (projection.paths.length > 1) {
       const paths = projection.paths.map((path) => {
         const name = path[path.length - 1] || 'item' // 'item' in case of having selected the whole item
-        const item = 'item' + createPropertySelector(path)
+        const item = `item${createPropertySelector(path)}`
         return `    ${JSON.stringify(name)}: ${item}`
       })
 
       queryParts.push(`  data = data.map(item => ({\n${paths.join(',\n')}})\n  )\n`)
     } else {
-      const item = 'item' + createPropertySelector(projection.paths[0])
+      const item = `item${createPropertySelector(projection.paths[0])}`
 
       queryParts.push(`  data = data.map(item => ${item})\n`)
     }

src/lib/plugins/query/javascriptQueryLanguage.test.js

@@ -32,7 +32,7 @@ describe('javascriptQueryLanguage', () => {
       assert.deepStrictEqual(
         query,
         'function query (data) {\n' +
-          '  data = data.filter(item => item?.["user"]?.["name"] == \'Bob\')\n' +
+          "  data = data.filter(item => item?.user?.name == 'Bob')\n" +
           '  return data\n' +
           '}'
       )
@@ -42,7 +42,7 @@ describe('javascriptQueryLanguage', () => {
       assert.deepStrictEqual(users, originalUsers) // must not touch the original data
     })
 
-    it('should create and execute a filter query for a property with sepcial characters in the name', () => {
+    it('should create and execute a filter query for a property with special characters in the name', () => {
       const data = users.map((item) => ({ 'user name!': item.user.name }))
       const originalData = cloneDeep(data)
 
@@ -101,8 +101,8 @@ describe('javascriptQueryLanguage', () => {
         'function query (data) {\n' +
           '  data = data.slice().sort((a, b) => {\n' +
           '    // sort ascending\n' +
-          '    const valueA = a?.["user"]?.["age"]\n' +
-          '    const valueB = b?.["user"]?.["age"]\n' +
+          '    const valueA = a?.user?.age\n' +
+          '    const valueB = b?.user?.age\n' +
           '    return valueA > valueB ? 1 : valueA < valueB ? -1 : 0\n' +
           '  })\n' +
           '  return data\n' +
@@ -126,8 +126,8 @@ describe('javascriptQueryLanguage', () => {
         'function query (data) {\n' +
           '  data = data.slice().sort((a, b) => {\n' +
           '    // sort descending\n' +
-          '    const valueA = a?.["user"]?.["age"]\n' +
-          '    const valueB = b?.["user"]?.["age"]\n' +
+          '    const valueA = a?.user?.age\n' +
+          '    const valueB = b?.user?.age\n' +
           '    return valueA > valueB ? -1 : valueA < valueB ? 1 : 0\n' +
           '  })\n' +
           '  return data\n' +
@@ -149,7 +149,7 @@ describe('javascriptQueryLanguage', () => {
       assert.deepStrictEqual(
         query,
         'function query (data) {\n' +
-          '  data = data.map(item => item?.["user"]?.["name"])\n' +
+          '  data = data.map(item => item?.user?.name)\n' +
           '  return data\n' +
           '}'
       )
@@ -170,8 +170,8 @@ describe('javascriptQueryLanguage', () => {
         query,
         'function query (data) {\n' +
           '  data = data.map(item => ({\n' +
-          '    "name": item?.["user"]?.["name"],\n' +
-          '    "_id": item?.["_id"]})\n' +
+          '    "name": item?.user?.name,\n' +
+          '    "_id": item?._id})\n' +
           '  )\n' +
           '  return data\n' +
           '}'
@@ -205,14 +205,14 @@ describe('javascriptQueryLanguage', () => {
       assert.deepStrictEqual(
         query,
         'function query (data) {\n' +
-          '  data = data.filter(item => item?.["user"]?.["age"] <= \'7\')\n' +
+          "  data = data.filter(item => item?.user?.age <= '7')\n" +
           '  data = data.slice().sort((a, b) => {\n' +
           '    // sort ascending\n' +
-          '    const valueA = a?.["user"]?.["name"]\n' +
-          '    const valueB = b?.["user"]?.["name"]\n' +
+          '    const valueA = a?.user?.name\n' +
+          '    const valueB = b?.user?.name\n' +
           '    return valueA > valueB ? 1 : valueA < valueB ? -1 : 0\n' +
           '  })\n' +
-          '  data = data.map(item => item?.["user"]?.["name"])\n' +
+          '  data = data.map(item => item?.user?.name)\n' +
           '  return data\n' +
           '}'
       )

src/lib/plugins/query/lodashQueryLanguage.js

@@ -1,5 +1,6 @@
 import * as _ from 'lodash-es'
-import { isEmpty, last } from 'lodash-es'
+import { last } from 'lodash-es'
+import { createPropertySelector, stringifyPath } from '../../utils/pathUtils.js'
 
 const description = `
 <p>
@@ -20,6 +21,10 @@ export const lodashQueryLanguage = {
   executeQuery
 }
 
+export function createLodashPropertySelector(path) {
+  return stringifyPath(path).replace(/^\./, '') // remove any leading dot
+}
+
 /**
  * @param {JSON} json
  * @param {QueryLanguageOptions} queryOptions
@@ -32,34 +37,34 @@ function createQuery(json, queryOptions) {
   if (filter && filter.path && filter.relation && filter.value) {
     // Note that the comparisons embrace type coercion,
     // so a filter value like '5' (text) will match numbers like 5 too.
-    const getActualValue = !isEmpty(filter.path)
-      ? `item => _.get(item, ${JSON.stringify(filter.path)})`
-      : 'item => item'
+    const getActualValue = `item => item${createPropertySelector(filter.path)}`
+
     queryParts.push(
       `  data = _.filter(data, ${getActualValue} ${filter.relation} '${filter.value}')\n`
     )
   }
 
   if (sort && sort.path && sort.direction) {
     queryParts.push(
-      `  data = _.orderBy(data, [${JSON.stringify(sort.path)}], ['${sort.direction}'])\n`
+      `  data = _.orderBy(data, ['${createLodashPropertySelector(sort.path)}'], ['${
+        sort.direction
+      }'])\n`
     )
   }
 
   if (projection && projection.paths) {
     // It is possible to make a util function "pickFlat"
     // and use that when building the query to make it more readable.
     if (projection.paths.length > 1) {
+      // Note that we do not use _.pick() here because this function doesn't flatten the results
       const paths = projection.paths.map((path) => {
         const name = last(path) || 'item' // 'item' in case of having selected the whole item
-        const item = !isEmpty(path) ? `_.get(item, ${JSON.stringify(path)})` : 'item'
-        return `    ${JSON.stringify(name)}: ${item}`
+        return `    ${JSON.stringify(name)}: item${createPropertySelector(path)}`
       })
-      queryParts.push(`  data = _.map(data, item => ({\n${paths.join(',\n')}})\n  )\n`)
+      queryParts.push(`  data = _.map(data, item => ({\n${paths.join(',\n')}\n  }))\n`)
     } else {
       const path = projection.paths[0]
-      const item = !isEmpty(path) ? `_.get(item, ${JSON.stringify(path)})` : 'item'
-      queryParts.push(`  data = _.map(data, item => ${item})\n`)
+      queryParts.push(`  data = _.map(data, item => item${createPropertySelector(path)})\n`)
     }
   }
 

src/lib/plugins/query/lodashQueryLanguage.test.js

@@ -35,7 +35,7 @@ describe('lodashQueryLanguage', () => {
       assert.deepStrictEqual(
         query,
         'function query (data) {\n' +
-          '  data = _.filter(data, item => _.get(item, ["user","name"]) == \'Bob\')\n' +
+          "  data = _.filter(data, item => item?.user?.name == 'Bob')\n" +
           '  return data\n' +
           '}'
       )
@@ -45,7 +45,7 @@ describe('lodashQueryLanguage', () => {
       assert.deepStrictEqual(users, originalUsers) // must not touch the original data
     })
 
-    it('should create and execute a filter query for a property with sepcial characters in the name', () => {
+    it('should create and execute a filter query for a property with special characters in the name', () => {
       const data = users.map((item) => ({ 'user name!': item.user.name }))
       const originalData = cloneDeep(data)
 
@@ -59,7 +59,7 @@ describe('lodashQueryLanguage', () => {
       assert.deepStrictEqual(
         query,
         'function query (data) {\n' +
-          '  data = _.filter(data, item => _.get(item, ["user name!"]) == \'Bob\')\n' +
+          '  data = _.filter(data, item => item?.["user name!"] == \'Bob\')\n' +
           '  return data\n' +
           '}'
       )
@@ -102,7 +102,7 @@ describe('lodashQueryLanguage', () => {
       assert.deepStrictEqual(
         query,
         'function query (data) {\n' +
-          '  data = _.orderBy(data, [["user","age"]], [\'asc\'])\n' +
+          "  data = _.orderBy(data, ['user.age'], ['asc'])\n" +
           '  return data\n' +
           '}'
       )
@@ -122,7 +122,7 @@ describe('lodashQueryLanguage', () => {
       assert.deepStrictEqual(
         query,
         'function query (data) {\n' +
-          '  data = _.orderBy(data, [["user","age"]], [\'desc\'])\n' +
+          "  data = _.orderBy(data, ['user.age'], ['desc'])\n" +
           '  return data\n' +
           '}'
       )
@@ -141,7 +141,7 @@ describe('lodashQueryLanguage', () => {
       assert.deepStrictEqual(
         query,
         'function query (data) {\n' +
-          '  data = _.map(data, item => _.get(item, ["user","name"]))\n' +
+          '  data = _.map(data, item => item?.user?.name)\n' +
           '  return data\n' +
           '}'
       )
@@ -161,9 +161,9 @@ describe('lodashQueryLanguage', () => {
         query,
         'function query (data) {\n' +
           '  data = _.map(data, item => ({\n' +
-          '    "name": _.get(item, ["user","name"]),\n' +
-          '    "_id": _.get(item, ["_id"])})\n' +
-          '  )\n' +
+          '    "name": item?.user?.name,\n' +
+          '    "_id": item?._id\n' +
+          '  }))\n' +
           '  return data\n' +
           '}'
       )
@@ -197,9 +197,9 @@ describe('lodashQueryLanguage', () => {
       assert.deepStrictEqual(
         query,
         'function query (data) {\n' +
-          '  data = _.filter(data, item => _.get(item, ["user","age"]) <= \'7\')\n' +
-          '  data = _.orderBy(data, [["user","name"]], [\'asc\'])\n' +
-          '  data = _.map(data, item => _.get(item, ["user","name"]))\n' +
+          "  data = _.filter(data, item => item?.user?.age <= '7')\n" +
+          "  data = _.orderBy(data, ['user.name'], ['asc'])\n" +
+          '  data = _.map(data, item => item?.user?.name)\n' +
           '  return data\n' +
           '}'
       )

src/lib/utils/pathUtils.js

@@ -23,6 +23,35 @@ export function stringifyPath(path) {
     .join('')
 }
 
+/**
+ * Create a JavaScript property selector
+ *
+ * Turn a paths like:
+ *
+ *   ['location', 'latitude']
+ *   ['address', 'full name']
+ *
+ * into a JavaScript selector (string) like:
+ *
+ *   '?.location?.latitude'
+ *   '?.address?.["full name"]'
+ *
+ * @param {Path} path
+ * @returns {string}
+ */
+export function createPropertySelector(path) {
+  return path
+    .map((prop) => {
+      return javaScriptPropertyRegex.test(prop) ? `?.${prop}` : `?.[${JSON.stringify(prop)}]`
+    })
+    .join('')
+}
+
+// https://developer.mozilla.org/en-US/docs/Glossary/Identifier
+// Note: We can extend this regex to allow unicode characters too.
+// I'm too lazy to figure that out right now
+const javaScriptPropertyRegex = /^[A-z$_][A-z$_\d]*$/i
+
 /**
  * Create a memoized function that will memoize the input path, and return
  * the memoized instance of the path when the stringified version is the same.

src/lib/utils/pathUtils.test.js

@@ -1,5 +1,5 @@
 import { notStrictEqual, strictEqual } from 'assert'
-import { createMemoizePath, stringifyPath } from './pathUtils.js'
+import { createMemoizePath, createPropertySelector, stringifyPath } from './pathUtils.js'
 
 describe('pathUtils', () => {
   it('stringifyPath', () => {
@@ -15,6 +15,16 @@ describe('pathUtils', () => {
     strictEqual(stringifyPath(['foo', 'prop with spaces']), '.foo["prop with spaces"]')
   })
 
+  it('createPropertySelector', () => {
+    strictEqual(createPropertySelector([]), '')
+    strictEqual(createPropertySelector(['location', 'latitude']), '?.location?.latitude')
+    strictEqual(createPropertySelector(['a', 'b']), '?.a?.b')
+    strictEqual(createPropertySelector(['A', 'B']), '?.A?.B')
+    strictEqual(createPropertySelector(['prop_$123']), '?.prop_$123')
+    strictEqual(createPropertySelector(['Hello World', 'b']), '?.["Hello World"]?.b')
+    strictEqual(createPropertySelector(['a', 2]), '?.a?.[2]')
+  })
+
   it('createMemoizePath', () => {
     const memoizePath = createMemoizePath()