diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8f7cc0f1537..f7db617ad9d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -276,9 +276,9 @@ npm run baseline mock_* ``` **IMPORTANT:** the `baseline`, `test-image` and `test-export` scripts do **not** bundle the source files before -running the image tests. We recommend running `npm run watch` or `npm start` in +running the image tests. We recommend running `npm start` in a separate tab to ensure that the most up-to-date code is used. -Also if you are adding a new mock, you may need to re-run `npm start` or `npm run watch` +Also if you are adding a new mock, you may need to re-run `npm start` to be able to find the new mock in the browser. To help ensure valid attributes are used in your new mock(s), please run `npm run test-mock` or `npm run test-mock mock_name(s)` after adding new mocks or implementing any new attributes. diff --git a/package-lock.json b/package-lock.json index 3e9f4bc2407..cee9a6d891b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,6 +18,7 @@ "@turf/centroid": "^7.1.0", "base64-arraybuffer": "^1.0.2", "canvas-fit": "^1.5.0", + "chart2music": "^1.13.0", "color-alpha": "1.0.4", "color-normalize": "1.5.0", "color-parse": "2.0.0", @@ -753,6 +754,92 @@ "node": ">=18" } }, + "node_modules/@formatjs/ecma402-abstract": { + "version": "1.18.2", + "resolved": "https://registry.npmjs.org/@formatjs/ecma402-abstract/-/ecma402-abstract-1.18.2.tgz", + "integrity": "sha512-+QoPW4csYALsQIl8GbN14igZzDbuwzcpWrku9nyMXlaqAlwRBgl5V+p0vWMGFqHOw37czNXaP/lEk4wbLgcmtA==", + "dependencies": { + "@formatjs/intl-localematcher": "0.5.4", + "tslib": "^2.4.0" + } + }, + "node_modules/@formatjs/fast-memoize": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@formatjs/fast-memoize/-/fast-memoize-2.2.0.tgz", + "integrity": "sha512-hnk/nY8FyrL5YxwP9e4r9dqeM6cAbo8PeU9UjyXojZMNvVad2Z06FAVHyR3Ecw6fza+0GH7vdJgiKIVXTMbSBA==", + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@formatjs/icu-messageformat-parser": { + "version": "2.7.6", + "resolved": "https://registry.npmjs.org/@formatjs/icu-messageformat-parser/-/icu-messageformat-parser-2.7.6.tgz", + "integrity": "sha512-etVau26po9+eewJKYoiBKP6743I1br0/Ie00Pb/S/PtmYfmjTcOn2YCh2yNkSZI12h6Rg+BOgQYborXk46BvkA==", + "dependencies": { + "@formatjs/ecma402-abstract": "1.18.2", + "@formatjs/icu-skeleton-parser": "1.8.0", + "tslib": "^2.4.0" + } + }, + "node_modules/@formatjs/icu-skeleton-parser": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/@formatjs/icu-skeleton-parser/-/icu-skeleton-parser-1.8.0.tgz", + "integrity": "sha512-QWLAYvM0n8hv7Nq5BEs4LKIjevpVpbGLAJgOaYzg9wABEoX1j0JO1q2/jVkO6CVlq0dbsxZCngS5aXbysYueqA==", + "dependencies": { + "@formatjs/ecma402-abstract": "1.18.2", + "tslib": "^2.4.0" + } + }, + "node_modules/@formatjs/intl": { + "version": "2.10.2", + "resolved": "https://registry.npmjs.org/@formatjs/intl/-/intl-2.10.2.tgz", + "integrity": "sha512-raPGWr3JRv3neXV78SqPFrGC05fIbhhNzVghHNxFde27ls2KkXiMhtP7HBybjGpikVSjjhdhaZto+4p1vmm9bQ==", + "dependencies": { + "@formatjs/ecma402-abstract": "1.18.2", + "@formatjs/fast-memoize": "2.2.0", + "@formatjs/icu-messageformat-parser": "2.7.6", + "@formatjs/intl-displaynames": "6.6.6", + "@formatjs/intl-listformat": "7.5.5", + "intl-messageformat": "10.5.12", + "tslib": "^2.4.0" + }, + "peerDependencies": { + "typescript": "^4.7 || 5" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/@formatjs/intl-displaynames": { + "version": "6.6.6", + "resolved": "https://registry.npmjs.org/@formatjs/intl-displaynames/-/intl-displaynames-6.6.6.tgz", + "integrity": "sha512-Dg5URSjx0uzF8VZXtHb6KYZ6LFEEhCbAbKoYChYHEOnMFTw/ZU3jIo/NrujzQD2EfKPgQzIq73LOUvW6Z/LpFA==", + "dependencies": { + "@formatjs/ecma402-abstract": "1.18.2", + "@formatjs/intl-localematcher": "0.5.4", + "tslib": "^2.4.0" + } + }, + "node_modules/@formatjs/intl-listformat": { + "version": "7.5.5", + "resolved": "https://registry.npmjs.org/@formatjs/intl-listformat/-/intl-listformat-7.5.5.tgz", + "integrity": "sha512-XoI52qrU6aBGJC9KJddqnacuBbPlb/bXFN+lIFVFhQ1RnFHpzuFrlFdjD9am2O7ZSYsyqzYRpkVcXeT1GHkwDQ==", + "dependencies": { + "@formatjs/ecma402-abstract": "1.18.2", + "@formatjs/intl-localematcher": "0.5.4", + "tslib": "^2.4.0" + } + }, + "node_modules/@formatjs/intl-localematcher": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/@formatjs/intl-localematcher/-/intl-localematcher-0.5.4.tgz", + "integrity": "sha512-zTwEpWOzZ2CiKcB93BLngUX59hQkuZjT2+SAQEscSm52peDW/getsawMcWF1rGRpMCX6D7nSJA3CzJ8gn13N/g==", + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@isaacs/cliui": { "version": "8.0.2", "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", @@ -2396,6 +2483,14 @@ "node": ">=4.0.0" } }, + "node_modules/chart2music": { + "version": "1.17.0", + "resolved": "https://registry.npmjs.org/chart2music/-/chart2music-1.17.0.tgz", + "integrity": "sha512-oDlISz51Mttx74cbA8REJDHxennlRxdafSSyimeqtsk/EUF3wO+KCuBMEGj9ZYTemlfe+GtfUa9kkV6tPScEvQ==", + "dependencies": { + "@formatjs/intl": "2.10.2" + } + }, "node_modules/check-node-version": { "version": "4.2.1", "resolved": "https://registry.npmjs.org/check-node-version/-/check-node-version-4.2.1.tgz", @@ -5538,6 +5633,17 @@ "node": ">= 0.4" } }, + "node_modules/intl-messageformat": { + "version": "10.5.12", + "resolved": "https://registry.npmjs.org/intl-messageformat/-/intl-messageformat-10.5.12.tgz", + "integrity": "sha512-izl0uxhy/melhw8gP2r8pGiVieviZmM4v5Oqx3c1/R7g9cwER2smmGfSjcIsp8Y3Q53bfciL/gkxacJRx/dUvg==", + "dependencies": { + "@formatjs/ecma402-abstract": "1.18.2", + "@formatjs/fast-memoize": "2.2.0", + "@formatjs/icu-messageformat-parser": "2.7.6", + "tslib": "^2.4.0" + } + }, "node_modules/into-stream": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/into-stream/-/into-stream-6.0.0.tgz", @@ -10074,7 +10180,7 @@ "version": "5.5.4", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.5.4.tgz", "integrity": "sha512-Mtq29sKDAEYP7aljRgtPOpTvOfbwRWlS6dPRzwjdE+C0R4brX/GUyhHSecbHMFLNBLcJIPt9nl9yG5TZ1weH+Q==", - "dev": true, + "devOptional": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" diff --git a/package.json b/package.json index f0c562db62a..fe8708ec84e 100644 --- a/package.json +++ b/package.json @@ -79,6 +79,7 @@ "@turf/centroid": "^7.1.0", "base64-arraybuffer": "^1.0.2", "canvas-fit": "^1.5.0", + "chart2music": "^1.13.0", "color-alpha": "1.0.4", "color-normalize": "1.5.0", "color-parse": "2.0.0", diff --git a/src/plot_api/plot_api.js b/src/plot_api/plot_api.js index e82bd29ebc1..1a24cf1922d 100644 --- a/src/plot_api/plot_api.js +++ b/src/plot_api/plot_api.js @@ -29,6 +29,7 @@ var manageArrays = require('./manage_arrays'); var helpers = require('./helpers'); var subroutines = require('./subroutines'); var editTypes = require('./edit_types'); +var sonification = require('../sonification/enable_sonification'); var AX_NAME_PATTERN = require('../plots/cartesian/constants').AX_NAME_PATTERN; @@ -381,10 +382,14 @@ function _doPlot(gd, data, layout, config) { // happens outside of marginPushers where all the other automargins are // calculated. Would be much better to separate margin calculations from // component drawing - see https://github.com/plotly/plotly.js/issues/2704 - Plots.doAutoMargin, - Plots.previousPromises + Plots.doAutoMargin ); + if(gd._context.sonification.enabled) seq.push(sonification.enable_sonification); + + seq.push(Plots.previousPromises); + + // even if everything we did was synchronous, return a promise // so that the caller doesn't care which route we took var plotDone = Lib.syncOrAsync(seq, gd); @@ -2667,7 +2672,6 @@ function react(gd, data, layout, config) { setPlotContext(gd, config); configChanged = diffConfig(oldConfig, gd._context); } - gd.data = data || []; helpers.cleanData(gd.data); gd.layout = layout || {}; @@ -2738,7 +2742,6 @@ function react(gd, data, layout, config) { Plots.doCalcdata(gd); subroutines.doAutoRangeAndConstraints(gd); - seq.push(function() { return Plots.transitionFromReact(gd, restyleFlags, relayoutFlags, oldFullLayout); }); @@ -3654,7 +3657,6 @@ function deleteFrames(gd, frameList) { */ function purge(gd) { gd = Lib.getGraphDiv(gd); - var fullLayout = gd._fullLayout || {}; var fullData = gd._fullData || []; @@ -3672,7 +3674,6 @@ function purge(gd) { // in contrast to _doPlots.purge which does NOT clear _context! delete gd._context; - return gd; } diff --git a/src/plot_api/plot_config.js b/src/plot_api/plot_config.js index 0a1f5f9fca0..4624a7b2306 100644 --- a/src/plot_api/plot_config.js +++ b/src/plot_api/plot_config.js @@ -471,7 +471,20 @@ var configAttributes = { 'instead of MM/DD/YYYY). Currently `grouping` and `currency` are ignored', 'for our automatic number formatting, but can be used in custom formats.' ].join(' ') - } + }, + + sonification: { + valType: 'any', + dflt: { + enabled: false, + options: {}, + info: {}, + closedCaptions: {generate: false, elId: 'c2m-plotly-cc', elClassname: 'c2m-plotly-closed_captions'} + }, + description: ['Sonification options: whether to enable, options to pass to the library, info to pass to the library, closedCaptions to control how plotly renders the closed-captions element.', + 'chart2music is supported and options here include Options and Info from https://www.chart2music.com/docs/API/Config. ' + ].join(' ') + }, }; var dfltConfig = {}; diff --git a/src/sonification/README.md b/src/sonification/README.md new file mode 100644 index 00000000000..77161f8d65f --- /dev/null +++ b/src/sonification/README.md @@ -0,0 +1,29 @@ +# `plotly.js` wrapper for `chart2music`, sonified charts + +This wrapper attaches a context object to `gd._context._c2m` with the following properties: + +* `.options`: the options object as required by c2m +* `.info`: the info object as required by c2m +* `.ccOptions`: information about where to place the closed caption div +* `.c2mHandler`: the object returned by calling the c2m library's initializer + +The first three have most of their values set by the defaults in *src/plot_api/plot_config.js*. + + +### index.js + +**index.js** exposes the following api: + +* `initC2M(gd, defaultConfig)` full resets the `c2mChart` object. + * `defaultConfig` is equal to `gd._context.sonification`, as defined in *src/plot_api/plot_config.js*. + +### all_codecs.js +**all_codecs.js** agregates all the individual **_codec.js* files, all are expect to export exactly two functions: `test` and `process`. + +I chose to aggregate all codecs in a separate file because it will ultimately be a 1-1 conversion if `plotly.js` adopts ES modules. + +## Writing a Codec + +This section is TODO + +chart2music supports N types of graphs. We should figure out how how to convert as many plotly types to chart2music types. I will write descriptions for each type in the folder, if not the actual code, along with what `test` must return and what `process` must return. diff --git a/src/sonification/all_codecs.js b/src/sonification/all_codecs.js new file mode 100644 index 00000000000..540f4de3a3f --- /dev/null +++ b/src/sonification/all_codecs.js @@ -0,0 +1,5 @@ +'use strict'; + +var scatterXY = require('./xy_scatter_codec'); + +exports.codecs = [ scatterXY ]; diff --git a/src/sonification/enable_sonification.js b/src/sonification/enable_sonification.js new file mode 100644 index 00000000000..22ebcc4ced2 --- /dev/null +++ b/src/sonification/enable_sonification.js @@ -0,0 +1,10 @@ +'use strict'; + +var c2mPlotly = require('.'); + +function enable_sonification(gd) { + // Collecting defaults + var defaultConfig = gd._context.sonification; + c2mPlotly.initC2M(gd, defaultConfig); +} +exports.enable_sonification = enable_sonification; diff --git a/src/sonification/index.js b/src/sonification/index.js new file mode 100644 index 00000000000..97c371299dd --- /dev/null +++ b/src/sonification/index.js @@ -0,0 +1,96 @@ +'use strict'; + +var c2m = require('chart2music'); +var Fx = require('../components/fx'); +var Lib = require('../lib'); + +var codecs = require('./all_codecs').codecs; + +/* initClosedCaptionDiv: Initialize the closed caption div with the given configuration. + * This function works by either creating a new div or returning the existing div +*/ +function initClosedCaptionDiv(gd, config) { + if(config.generate) { + var closedCaptions = document.createElement('div'); + closedCaptions.id = config.elId; + closedCaptions.className = config.elClassname; + gd.parentNode.insertBefore(closedCaptions, gd.nextSibling); // this really might not work + return closedCaptions; + } else { + return document.getElementById(config.elId); + } +} + +/* initC2M: Initialize the chart2music library with the given configuration. + * This function works by resetting the c2m context of the given graph div + */ +function initC2M(gd, defaultConfig) { + var c2mContext = gd._context._c2m = {}; + c2mContext.options = Lib.extendDeepAll({}, defaultConfig.options); + c2mContext.info = Lib.extendDeepAll({}, defaultConfig.info); + c2mContext.ccOptions = Lib.extendDeepAll({}, defaultConfig.closedCaptions); + + var labels = []; + // Set the onFocusCallback to highlight the hovered point + c2mContext.options.onFocusCallback = function(dataInfo) { + Fx.hover(gd, [{ + curveNumber: labels.indexOf(dataInfo.slice), + pointNumber: dataInfo.index + }]); + }; + + var ccElement = initClosedCaptionDiv(gd, c2mContext.ccOptions); + + // Get the chart, x, and y axis titles from the layout. + // This will be used for the closed captions. + var titleText = 'Chart'; + if((gd._fullLayout.title !== undefined) && (gd._fullLayout.title.text !== undefined)) { + titleText = gd._fullLayout.title.text; + } + var xAxisText = 'X Axis'; + if((gd._fullLayout.xaxis !== undefined) && + (gd._fullLayout.xaxis.title !== undefined) && + (gd._fullLayout.xaxis.title.text !== undefined)) { + xAxisText = gd._fullLayout.xaxis.title.text; + } + var yAxisText = 'Y Axis'; + if((gd._fullLayout.yaxis !== undefined) && + (gd._fullLayout.yaxis.title !== undefined) && + (gd._fullLayout.yaxis.title.text !== undefined)) { + yAxisText = gd._fullLayout.yaxis.title.text; + } + + // Convert the data to the format that c2m expects + var c2mData = {}; + var types = []; + var fullData = gd._fullData; + + // Iterate through the traces and find the codec that matches the trace + for (var trace of fullData) { + for (var codec of codecs) { + var test = codec.test(trace); + if (!test) continue; + + // Generate a unique label for the trace + var label = test.name; + labels.push(label); + types.push(test.type); + c2mData[label] = codec.process(trace); + } + } + + c2mContext.c2mHandler = c2m.c2mChart({ + title: titleText, + type: types, + axes: { + x: { label: xAxisText }, + y: { label: yAxisText }, + }, + element: gd, + cc: ccElement, + data: c2mData, + options: c2mContext.options, + info: c2mContext.info + }); +} +exports.initC2M = initC2M; diff --git a/src/sonification/xy_scatter_codec.js b/src/sonification/xy_scatter_codec.js new file mode 100644 index 00000000000..5fb5723a211 --- /dev/null +++ b/src/sonification/xy_scatter_codec.js @@ -0,0 +1,25 @@ +'use strict'; +// This codec serves for one x axis and one y axis + +function test(trace) { + if (!trace || !trace.type || trace.type !== 'scatter') return null; + if ((trace.y === undefined || trace.y.length === 0) && (trace.x === undefined || trace.x.length === 0)) return null; + return {type: 'scatter', name: trace.name}; +} +exports.test = test; + +function process(trace) { + var traceData = []; + var x = trace.x && trace.x.length !== 0 ? trace.x : []; + var y = trace.y && trace.y.length !== 0 ? trace.y : []; + + for(var p = 0; p < Math.max(x.length, y.length); p++) { + traceData.push({ + x: x[p] ? x[p] : p, + y: y[p] ? y[p] : p, + label: trace.text[p] ? trace.text[p] : p + }); + } + return traceData; +} +exports.process = process; diff --git a/test/image/baselines/zz-chart2music.png b/test/image/baselines/zz-chart2music.png new file mode 100644 index 00000000000..95b08021ecf Binary files /dev/null and b/test/image/baselines/zz-chart2music.png differ diff --git a/test/image/mocks/zz-chart2music.json b/test/image/mocks/zz-chart2music.json new file mode 100644 index 00000000000..ab9a3fa71d1 --- /dev/null +++ b/test/image/mocks/zz-chart2music.json @@ -0,0 +1,57 @@ +{ + "data": [ + { + "x": [1, 2, 3, 4], + "y": [10, 15, 13, 17], + "mode": "markers", + "type": "scatter", + "marker": { + "opacity": 0.5 + } + }, + { + "x": [2, 3, 4, 5], + "y": [16, 5, 11, 9], + "mode": "lines", + "type": "scatter", + "opacity": 0.5 + }, + { + "x": [1, 2, 3, 4], + "y": [12, 9, 15, 12], + "mode": "lines+markers", + "type": "scatter", + "opacity": 0.707, + "marker": { + "opacity": 0.707 + } + }, + { + "x": [1, 2, 3], + "y": [1, 2, 3], + "opacity": 0.2, + "line": { "width": 10, "color": "red" }, + "marker": { "size": 20, "color": "blue" } + } + ], + "config": { + "sonification": { + "enabled": true, + "info": { + "notes": [ + "Sample notes to show" + ], + "annotations": [ + { + "x": 6, + "label": "This is an annotation to show at x=6" + } + ] + }, + "closedCaptions": { + "generate": false, + "elId": "c2m-plotly-cc" + } + } + } +} diff --git a/test/plot-schema.json b/test/plot-schema.json index b2f43d0245d..c25d75fc833 100644 --- a/test/plot-schema.json +++ b/test/plot-schema.json @@ -355,6 +355,20 @@ "dflt": true, "valType": "boolean" }, + "sonification": { + "description": "Sonification options: whether to enable, options to pass to the library, info to pass to the library, closedCaptions to control how plotly renders the closed-captions element. chart2music is supported and options here include Options and Info from https://www.chart2music.com/docs/API/Config. ", + "dflt": { + "closedCaptions": { + "elClassname": "c2m-plotly-closed_captions", + "elId": "c2m-plotly-cc", + "generate": false + }, + "enabled": false, + "info": {}, + "options": {} + }, + "valType": "any" + }, "staticPlot": { "description": "Determines whether the graphs are interactive or not. If *false*, no interactivity, for export or image generation.", "dflt": false,