From 6a153365d3eb5f73863963fe26a14b153a0a7a94 Mon Sep 17 00:00:00 2001 From: Riad Benguella Date: Mon, 7 Sep 2020 10:06:52 +0100 Subject: [PATCH 1/9] Improve the block and patterns search algorithm --- .../src/components/inserter/search-items.js | 93 +++++++++++++++---- .../components/inserter/test/search-items.js | 38 +++++++- 2 files changed, 112 insertions(+), 19 deletions(-) diff --git a/packages/block-editor/src/components/inserter/search-items.js b/packages/block-editor/src/components/inserter/search-items.js index 58b964d579bef1..8145ab85e4232f 100644 --- a/packages/block-editor/src/components/inserter/search-items.js +++ b/packages/block-editor/src/components/inserter/search-items.js @@ -11,13 +11,13 @@ import { } from 'lodash'; /** - * Converts the search term into a list of normalized terms. + * Sanitizes the search term string. * - * @param {string} term The search term to normalize. + * @param {string} term The search term to santize. * - * @return {string[]} The normalized list of search terms. + * @return {string} The sanitized search term. */ -export const normalizeSearchTerm = ( term = '' ) => { +function sanitizeTerm( term = '' ) { // Disregard diacritics. // Input: "média" term = deburr( term ); @@ -30,8 +30,19 @@ export const normalizeSearchTerm = ( term = '' ) => { // Input: "MEDIA" term = term.toLowerCase(); + return term; +} + +/** + * Converts the search term into a list of normalized terms. + * + * @param {string} term The search term to normalize. + * + * @return {string[]} The normalized list of search terms. + */ +export const normalizeSearchTerm = ( term = '' ) => { // Extract words. - return words( term ); + return words( sanitizeTerm( term ) ); }; const removeMatchingTerms = ( unmatchedTerms, unprocessedTerms ) => { @@ -116,12 +127,36 @@ export const searchItems = ( items = [], searchTerm = '', config = {} ) => { return items; } - const defaultGetTitle = ( item ) => item.title; - const defaultGetKeywords = ( item ) => item.keywords || []; - const defaultGetCategory = ( item ) => item.category; + const rankedItems = items + .map( ( item ) => { + return [ item, getItemSearchRank( item, searchTerm, config ) ]; + } ) + .filter( ( [ , rank ] ) => rank > 0 ); + + rankedItems.sort( ( [ , rank1 ], [ , rank2 ] ) => rank2 - rank1 ); + return rankedItems.map( ( [ item ] ) => item ); +}; + +/** + * Get the search rank for a given iotem and a specific search term. + * The higher is higher for items with the best match. + * If the rank equals 0, it should be excluded from the results. + * + * @param {Object} item Item to filter. + * @param {string} searchTerm Search term. + * @param {Object} config Search Config. + * @return {number} Search Rank. + */ +export function getItemSearchRank( item, searchTerm, config = {} ) { + const defaultGetName = ( it ) => it.name || ''; + const defaultGetTitle = ( it ) => it.title; + const defaultGetKeywords = ( it ) => it.keywords || []; + const defaultGetCategory = ( it ) => it.category; const defaultGetCollection = () => null; const defaultGetVariations = () => []; + const { + getName = defaultGetName, getTitle = defaultGetTitle, getKeywords = defaultGetKeywords, getCategory = defaultGetCategory, @@ -129,13 +164,26 @@ export const searchItems = ( items = [], searchTerm = '', config = {} ) => { getVariations = defaultGetVariations, } = config; - return items.filter( ( item ) => { - const title = getTitle( item ); - const keywords = getKeywords( item ); - const category = getCategory( item ); - const collection = getCollection( item ); - const variations = getVariations( item ); + const name = getName( item ); + const title = getTitle( item ); + const keywords = getKeywords( item ); + const category = getCategory( item ); + const collection = getCollection( item ); + const variations = getVariations( item ); + + const sanitizedSearchTerm = sanitizeTerm( searchTerm ); + const sanitizedTitle = sanitizeTerm( title ); + + let rank = 0; + // Prefers exact matchs + // Then prefers if the beginning of the title matches the search term + // Keywords, categories, collection, variations match come later. + if ( sanitizedSearchTerm === sanitizedTitle ) { + rank += 30; + } else if ( sanitizedTitle.indexOf( sanitizedSearchTerm ) === 0 ) { + rank += 20; + } else { const terms = [ title, ...keywords, @@ -143,12 +191,21 @@ export const searchItems = ( items = [], searchTerm = '', config = {} ) => { collection, ...variations, ].join( ' ' ); - + const normalizedSearchTerms = words( sanitizedSearchTerm ); const unmatchedTerms = removeMatchingTerms( normalizedSearchTerms, terms ); - return unmatchedTerms.length === 0; - } ); -}; + if ( unmatchedTerms.length === 0 ) { + rank += 10; + } + } + + // Give a better rank to "core" namespaced items. + if ( rank !== 0 && name.indexOf( 'core/' ) === 0 ) { + rank++; + } + + return rank; +} diff --git a/packages/block-editor/src/components/inserter/test/search-items.js b/packages/block-editor/src/components/inserter/test/search-items.js index ad20485987d169..131cb699f59934 100644 --- a/packages/block-editor/src/components/inserter/test/search-items.js +++ b/packages/block-editor/src/components/inserter/test/search-items.js @@ -10,7 +10,11 @@ import items, { youtubeItem, paragraphEmbedItem, } from './fixtures'; -import { normalizeSearchTerm, searchBlockItems } from '../search-items'; +import { + normalizeSearchTerm, + searchBlockItems, + getItemSearchRank, +} from '../search-items'; describe( 'normalizeSearchTerm', () => { it( 'should return an empty array when no words detected', () => { @@ -36,6 +40,38 @@ describe( 'normalizeSearchTerm', () => { } ); } ); +describe( 'getItemSearchRank', () => { + it( 'should return the highest rank for exact matches', () => { + expect( getItemSearchRank( { title: 'Button' }, 'button' ) ).toEqual( + 30 + ); + } ); + + it( 'should return a high rank if the start of title matches the search term', () => { + expect( + getItemSearchRank( { title: 'Button Advanced' }, 'button' ) + ).toEqual( 20 ); + } ); + + it( 'should add a bonus point to items with core namespaces', () => { + expect( + getItemSearchRank( + { name: 'core/button', title: 'Button' }, + 'button' + ) + ).toEqual( 31 ); + } ); + + it( 'should have a small rank if it matches keywords, category...', () => { + expect( + getItemSearchRank( + { title: 'link', keywords: [ 'button' ] }, + 'button' + ) + ).toEqual( 10 ); + } ); +} ); + describe( 'searchBlockItems', () => { it( 'should return back all items when no terms detected', () => { expect( From 296854f659b39858f1265f0d6a4c6a43b3dafd87 Mon Sep 17 00:00:00 2001 From: Riad Benguella Date: Mon, 7 Sep 2020 11:31:17 +0100 Subject: [PATCH 2/9] Allow using the block name to match search terms --- packages/block-editor/src/components/inserter/search-items.js | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/block-editor/src/components/inserter/search-items.js b/packages/block-editor/src/components/inserter/search-items.js index 8145ab85e4232f..279a7440fa4693 100644 --- a/packages/block-editor/src/components/inserter/search-items.js +++ b/packages/block-editor/src/components/inserter/search-items.js @@ -185,6 +185,7 @@ export function getItemSearchRank( item, searchTerm, config = {} ) { rank += 20; } else { const terms = [ + name, title, ...keywords, category, From 450b9a85485bb31df02c4e92a87b61a72f2ff019 Mon Sep 17 00:00:00 2001 From: Riad Benguella Date: Mon, 7 Sep 2020 11:32:25 +0100 Subject: [PATCH 3/9] clarify comments --- packages/block-editor/src/components/inserter/search-items.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/block-editor/src/components/inserter/search-items.js b/packages/block-editor/src/components/inserter/search-items.js index 279a7440fa4693..5edc8480e07f01 100644 --- a/packages/block-editor/src/components/inserter/search-items.js +++ b/packages/block-editor/src/components/inserter/search-items.js @@ -176,9 +176,9 @@ export function getItemSearchRank( item, searchTerm, config = {} ) { let rank = 0; - // Prefers exact matchs + // Prefers exact matches // Then prefers if the beginning of the title matches the search term - // Keywords, categories, collection, variations match come later. + // name, keywords, categories, collection, variations match come later. if ( sanitizedSearchTerm === sanitizedTitle ) { rank += 30; } else if ( sanitizedTitle.indexOf( sanitizedSearchTerm ) === 0 ) { From 16d37e85326cd853e98231f3b64d88d11f505104 Mon Sep 17 00:00:00 2001 From: Riad Benguella Date: Mon, 7 Sep 2020 11:32:57 +0100 Subject: [PATCH 4/9] typo Co-authored-by: Miguel Fonseca --- packages/block-editor/src/components/inserter/search-items.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/block-editor/src/components/inserter/search-items.js b/packages/block-editor/src/components/inserter/search-items.js index 5edc8480e07f01..84e16aa5e2f2a8 100644 --- a/packages/block-editor/src/components/inserter/search-items.js +++ b/packages/block-editor/src/components/inserter/search-items.js @@ -138,7 +138,7 @@ export const searchItems = ( items = [], searchTerm = '', config = {} ) => { }; /** - * Get the search rank for a given iotem and a specific search term. + * Get the search rank for a given item and a specific search term. * The higher is higher for items with the best match. * If the rank equals 0, it should be excluded from the results. * From 208aa496db757a2a10610a5c87392644a9868d29 Mon Sep 17 00:00:00 2001 From: Riad Benguella Date: Mon, 7 Sep 2020 11:33:10 +0100 Subject: [PATCH 5/9] copy Co-authored-by: Miguel Fonseca --- packages/block-editor/src/components/inserter/search-items.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/block-editor/src/components/inserter/search-items.js b/packages/block-editor/src/components/inserter/search-items.js index 84e16aa5e2f2a8..8d1fa5ff1d004c 100644 --- a/packages/block-editor/src/components/inserter/search-items.js +++ b/packages/block-editor/src/components/inserter/search-items.js @@ -139,7 +139,7 @@ export const searchItems = ( items = [], searchTerm = '', config = {} ) => { /** * Get the search rank for a given item and a specific search term. - * The higher is higher for items with the best match. + * The better the match, the higher the rank. * If the rank equals 0, it should be excluded from the results. * * @param {Object} item Item to filter. From 9687b7c9ed97812f903ceee7049cbeaaa10328e2 Mon Sep 17 00:00:00 2001 From: Riad Benguella Date: Mon, 7 Sep 2020 11:43:41 +0100 Subject: [PATCH 6/9] Rename --- .../src/components/inserter/search-items.js | 56 +++++++++---------- .../components/inserter/test/search-items.js | 18 +++--- 2 files changed, 39 insertions(+), 35 deletions(-) diff --git a/packages/block-editor/src/components/inserter/search-items.js b/packages/block-editor/src/components/inserter/search-items.js index 8d1fa5ff1d004c..45c3da672b78a0 100644 --- a/packages/block-editor/src/components/inserter/search-items.js +++ b/packages/block-editor/src/components/inserter/search-items.js @@ -11,44 +11,44 @@ import { } from 'lodash'; /** - * Sanitizes the search term string. + * Sanitizes the search input string. * - * @param {string} term The search term to santize. + * @param {string} input The search input to normalize. * - * @return {string} The sanitized search term. + * @return {string} The normalized search input. */ -function sanitizeTerm( term = '' ) { +function normalizeSearchInput( input = '' ) { // Disregard diacritics. // Input: "média" - term = deburr( term ); + input = deburr( input ); // Accommodate leading slash, matching autocomplete expectations. // Input: "/media" - term = term.replace( /^\//, '' ); + input = input.replace( /^\//, '' ); // Lowercase. // Input: "MEDIA" - term = term.toLowerCase(); + input = input.toLowerCase(); - return term; + return input; } /** * Converts the search term into a list of normalized terms. * - * @param {string} term The search term to normalize. + * @param {string} input The search term to normalize. * * @return {string[]} The normalized list of search terms. */ -export const normalizeSearchTerm = ( term = '' ) => { +export const getNormalizedSearchTerms = ( input = '' ) => { // Extract words. - return words( sanitizeTerm( term ) ); + return words( normalizeSearchInput( input ) ); }; const removeMatchingTerms = ( unmatchedTerms, unprocessedTerms ) => { return differenceWith( unmatchedTerms, - normalizeSearchTerm( unprocessedTerms ), + getNormalizedSearchTerms( unprocessedTerms ), ( unmatchedTerm, unprocessedTerm ) => unprocessedTerm.includes( unmatchedTerm ) ); @@ -58,9 +58,9 @@ export const searchBlockItems = ( items, categories, collections, - searchTerm + searchInput ) => { - const normalizedSearchTerms = normalizeSearchTerm( searchTerm ); + const normalizedSearchTerms = getNormalizedSearchTerms( searchInput ); if ( normalizedSearchTerms.length === 0 ) { return items; } @@ -84,7 +84,7 @@ export const searchBlockItems = ( ) ), }; - return searchItems( items, searchTerm, config ).map( ( item ) => { + return searchItems( items, searchInput, config ).map( ( item ) => { if ( isEmpty( item.variations ) ) { return item; } @@ -94,7 +94,7 @@ export const searchBlockItems = ( return ( intersectionWith( normalizedSearchTerms, - normalizeSearchTerm( title ).concat( keywords ), + getNormalizedSearchTerms( title ).concat( keywords ), ( termToMatch, labelTerm ) => labelTerm.includes( termToMatch ) ).length > 0 @@ -116,20 +116,20 @@ export const searchBlockItems = ( /** * Filters an item list given a search term. * - * @param {Array} items Item list - * @param {string} searchTerm Search term. - * @param {Object} config Search Config. - * @return {Array} Filtered item list. + * @param {Array} items Item list + * @param {string} searchInput Search input. + * @param {Object} config Search Config. + * @return {Array} Filtered item list. */ -export const searchItems = ( items = [], searchTerm = '', config = {} ) => { - const normalizedSearchTerms = normalizeSearchTerm( searchTerm ); +export const searchItems = ( items = [], searchInput = '', config = {} ) => { + const normalizedSearchTerms = getNormalizedSearchTerms( searchInput ); if ( normalizedSearchTerms.length === 0 ) { return items; } const rankedItems = items .map( ( item ) => { - return [ item, getItemSearchRank( item, searchTerm, config ) ]; + return [ item, getItemSearchRank( item, searchInput, config ) ]; } ) .filter( ( [ , rank ] ) => rank > 0 ); @@ -171,17 +171,17 @@ export function getItemSearchRank( item, searchTerm, config = {} ) { const collection = getCollection( item ); const variations = getVariations( item ); - const sanitizedSearchTerm = sanitizeTerm( searchTerm ); - const sanitizedTitle = sanitizeTerm( title ); + const normalizedSearchInput = normalizeSearchInput( searchTerm ); + const normalizedTitle = normalizeSearchInput( title ); let rank = 0; // Prefers exact matches // Then prefers if the beginning of the title matches the search term // name, keywords, categories, collection, variations match come later. - if ( sanitizedSearchTerm === sanitizedTitle ) { + if ( normalizedSearchInput === normalizedTitle ) { rank += 30; - } else if ( sanitizedTitle.indexOf( sanitizedSearchTerm ) === 0 ) { + } else if ( normalizedTitle.indexOf( normalizedSearchInput ) === 0 ) { rank += 20; } else { const terms = [ @@ -192,7 +192,7 @@ export function getItemSearchRank( item, searchTerm, config = {} ) { collection, ...variations, ].join( ' ' ); - const normalizedSearchTerms = words( sanitizedSearchTerm ); + const normalizedSearchTerms = words( normalizedSearchInput ); const unmatchedTerms = removeMatchingTerms( normalizedSearchTerms, terms diff --git a/packages/block-editor/src/components/inserter/test/search-items.js b/packages/block-editor/src/components/inserter/test/search-items.js index 131cb699f59934..ee41ee7ce0052a 100644 --- a/packages/block-editor/src/components/inserter/test/search-items.js +++ b/packages/block-editor/src/components/inserter/test/search-items.js @@ -11,31 +11,35 @@ import items, { paragraphEmbedItem, } from './fixtures'; import { - normalizeSearchTerm, + getNormalizedSearchTerms, searchBlockItems, getItemSearchRank, } from '../search-items'; -describe( 'normalizeSearchTerm', () => { +describe( 'getNormalizedSearchTerms', () => { it( 'should return an empty array when no words detected', () => { - expect( normalizeSearchTerm( ' - !? *** ' ) ).toEqual( [] ); + expect( getNormalizedSearchTerms( ' - !? *** ' ) ).toEqual( [] ); } ); it( 'should remove diacritics', () => { - expect( normalizeSearchTerm( 'média' ) ).toEqual( [ 'media' ] ); + expect( getNormalizedSearchTerms( 'média' ) ).toEqual( [ 'media' ] ); } ); it( 'should trim whitespace', () => { - expect( normalizeSearchTerm( ' média ' ) ).toEqual( [ 'media' ] ); + expect( getNormalizedSearchTerms( ' média ' ) ).toEqual( [ + 'media', + ] ); } ); it( 'should convert to lowercase', () => { - expect( normalizeSearchTerm( ' Média ' ) ).toEqual( [ 'media' ] ); + expect( getNormalizedSearchTerms( ' Média ' ) ).toEqual( [ + 'media', + ] ); } ); it( 'should extract only words', () => { expect( - normalizeSearchTerm( ' Média & Text Tag-Cloud > 123' ) + getNormalizedSearchTerms( ' Média & Text Tag-Cloud > 123' ) ).toEqual( [ 'media', 'text', 'tag', 'cloud', '123' ] ); } ); } ); From d8d4fe687ccc2045c09f5cb1d0e90178d6e5b619 Mon Sep 17 00:00:00 2001 From: Riad Benguella Date: Mon, 7 Sep 2020 11:49:29 +0100 Subject: [PATCH 7/9] Use startsWith instead of indexOf --- packages/block-editor/src/components/inserter/search-items.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/block-editor/src/components/inserter/search-items.js b/packages/block-editor/src/components/inserter/search-items.js index 45c3da672b78a0..44a751e99067ec 100644 --- a/packages/block-editor/src/components/inserter/search-items.js +++ b/packages/block-editor/src/components/inserter/search-items.js @@ -181,7 +181,7 @@ export function getItemSearchRank( item, searchTerm, config = {} ) { // name, keywords, categories, collection, variations match come later. if ( normalizedSearchInput === normalizedTitle ) { rank += 30; - } else if ( normalizedTitle.indexOf( normalizedSearchInput ) === 0 ) { + } else if ( normalizedTitle.startsWith( normalizedSearchInput ) ) { rank += 20; } else { const terms = [ @@ -204,7 +204,7 @@ export function getItemSearchRank( item, searchTerm, config = {} ) { } // Give a better rank to "core" namespaced items. - if ( rank !== 0 && name.indexOf( 'core/' ) === 0 ) { + if ( rank !== 0 && name.startsWith( 'core/' ) ) { rank++; } From b2643ffd169aad6a74a1d408683f332dd215d918 Mon Sep 17 00:00:00 2001 From: Riad Benguella Date: Mon, 7 Sep 2020 11:52:13 +0100 Subject: [PATCH 8/9] add unit test --- .../src/components/inserter/test/search-items.js | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/packages/block-editor/src/components/inserter/test/search-items.js b/packages/block-editor/src/components/inserter/test/search-items.js index ee41ee7ce0052a..3a785eb35e78fa 100644 --- a/packages/block-editor/src/components/inserter/test/search-items.js +++ b/packages/block-editor/src/components/inserter/test/search-items.js @@ -93,6 +93,16 @@ describe( 'searchBlockItems', () => { ] ); } ); + it( 'should use the ranking algorithm to order the blocks', () => { + expect( + searchBlockItems( items, categories, collections, 'a para' ) + ).toEqual( [ + paragraphEmbedItem, + paragraphItem, + advancedParagraphItem, + ] ); + } ); + it( 'should search items using the keywords and partial terms', () => { expect( searchBlockItems( items, categories, collections, 'GOOGL' ) From cab940b624649b8ffef330988e6e59ed7a51dd49 Mon Sep 17 00:00:00 2001 From: Riad Benguella Date: Mon, 7 Sep 2020 11:54:26 +0100 Subject: [PATCH 9/9] Extract default search helpers --- .../src/components/inserter/search-items.js | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/packages/block-editor/src/components/inserter/search-items.js b/packages/block-editor/src/components/inserter/search-items.js index 44a751e99067ec..543ddd273f9243 100644 --- a/packages/block-editor/src/components/inserter/search-items.js +++ b/packages/block-editor/src/components/inserter/search-items.js @@ -10,6 +10,14 @@ import { words, } from 'lodash'; +// Default search helpers +const defaultGetName = ( item ) => item.name || ''; +const defaultGetTitle = ( item ) => item.title; +const defaultGetKeywords = ( item ) => item.keywords || []; +const defaultGetCategory = ( item ) => item.category; +const defaultGetCollection = () => null; +const defaultGetVariations = () => []; + /** * Sanitizes the search input string. * @@ -148,13 +156,6 @@ export const searchItems = ( items = [], searchInput = '', config = {} ) => { * @return {number} Search Rank. */ export function getItemSearchRank( item, searchTerm, config = {} ) { - const defaultGetName = ( it ) => it.name || ''; - const defaultGetTitle = ( it ) => it.title; - const defaultGetKeywords = ( it ) => it.keywords || []; - const defaultGetCategory = ( it ) => it.category; - const defaultGetCollection = () => null; - const defaultGetVariations = () => []; - const { getName = defaultGetName, getTitle = defaultGetTitle,