Values are now flattened in providers

package.json

@@ -45,7 +45,6 @@
     "lodash-es": "^4.17.21",
     "ndarray": "^1.0.19",
     "ndarray-ops": "^1.2.2",
-    "ndarray-unpack": "^1.0.0",
     "normalize.css": "^8.0.1",
     "react": "^17.0.2",
     "react-aria-menubutton": "^7.0.1",

src/h5web/providers/hsds/hsds-api.ts

@@ -21,7 +21,7 @@ import {
   Group,
   ProviderError,
 } from '../models';
-import { assertDefined, assertGroup } from '../../guards';
+import { assertDefined, assertGroup, hasArrayShape } from '../../guards';
 import { ProviderApi } from '../api';
 import {
   assertHsdsDataset,
@@ -100,7 +100,7 @@ export class HsdsApi extends ProviderApi {
   }
 
   public async getValue(params: ValueRequestParams): Promise<unknown> {
-    const { path, selection = '' } = params;
+    const { path } = params;
 
     const entity = await this.getEntity(path);
     assertHsdsDataset(entity);
@@ -109,14 +109,12 @@ export class HsdsApi extends ProviderApi {
 
     // https://github.com/HDFGroup/hsds/issues/88
     // HSDS does not reduce the number of dimensions when selecting indices
-    // This is an issue when trying to slice nD arrays
-    // Therefore the dimension reduction is done "manually" here with a flat
-    const numberIndices = selection.match(/\d+/gu)?.length;
-    if (!numberIndices) {
-      return value;
+    // Therefore the flattening must be done on all dimensions regardless of the selection
+    if (hasArrayShape(entity)) {
+      return (value as unknown[]).flat(entity.shape.length - 1);
     }
 
-    return (value as unknown[]).flat(numberIndices);
+    return value;
   }
 
   private async fetchRootId(): Promise<HsdsId> {

src/h5web/providers/jupyter/jupyter-api.ts

@@ -24,7 +24,7 @@ import type {
   JupyterMetaResponse,
 } from './models';
 import { makeStrAttr } from '../mock/metadata-utils';
-import { assertDataset, hasComplexType } from '../../guards';
+import { assertDataset, hasArrayShape, hasComplexType } from '../../guards';
 
 export class JupyterStableApi extends ProviderApi {
   /* API compatible with jupyterlab_hdf v0.5.1 */
@@ -37,18 +37,20 @@ export class JupyterStableApi extends ProviderApi {
   }
 
   public async getValue(params: ValueRequestParams): Promise<unknown> {
+    const { path, selection } = params;
     const [value, entity] = await Promise.all([
       this.fetchData(params),
-      this.getEntity(params.path),
+      this.getEntity(path),
     ]);
 
     assertDataset(entity);
+    const flatValue = this.flattenValue(value, entity, selection);
 
     if (hasComplexType(entity)) {
-      return parseComplex(value as JupyterComplex);
+      return parseComplex(flatValue as JupyterComplex);
     }
 
-    return value;
+    return flatValue;
   }
 
   protected async fetchAttributes(path: string): Promise<JupyterAttrsResponse> {
@@ -161,4 +163,14 @@ export class JupyterStableApi extends ProviderApi {
 
     return contents;
   }
+
+  protected flattenValue(value: unknown, entity: Dataset, selection?: string) {
+    if (!hasArrayShape(entity)) {
+      return value;
+    }
+
+    const slicedDims = selection?.split(',').filter((s) => s.includes(':'));
+    const dims = slicedDims || entity.shape;
+    return (value as unknown[]).flat(dims.length - 1);
+  }
 }

src/h5web/providers/jupyter/jupyter-dev-api.ts

@@ -7,6 +7,7 @@ import type {
   JupyterMetaResponse,
 } from './models';
 import { assertGroupContent, convertDtype } from './utils';
+import { assertDataset } from '../../guards';
 
 interface DevJupyterAttrMeta {
   name: string;
@@ -32,7 +33,15 @@ export class JupyterDevApi extends JupyterStableApi {
   }
 
   public async getValue(params: ValueRequestParams): Promise<unknown> {
-    return this.fetchData(params);
+    const { path, selection } = params;
+    const [value, entity] = await Promise.all([
+      this.fetchData(params),
+      this.getEntity(path),
+    ]);
+
+    assertDataset(entity);
+
+    return this.flattenValue(value, entity, selection);
   }
 
   protected async fetchMetadata(path: string): Promise<DevJupyterMetaResponse> {

src/h5web/providers/mock/mock-api.ts

@@ -1,9 +1,13 @@
 import axios from 'axios';
 import ndarray from 'ndarray';
-import unpack from 'ndarray-unpack';
-import { assertArrayShape, assertPrintableType } from '../../guards';
+import {
+  assertArrayShape,
+  assertPrintableType,
+  hasArrayShape,
+} from '../../guards';
+import { applyMapping } from '../../vis-packs/core/utils';
 import { ProviderApi } from '../api';
-import type { ValueRequestParams, Entity } from '../models';
+import type { ValueRequestParams, Entity, Primitive } from '../models';
 import { mockFilepath } from './metadata';
 import { assertMockDataset, findMockEntity } from './utils';
 
@@ -39,28 +43,27 @@ export class MockApi extends ProviderApi {
       await this.cancellableDelay(params);
     }
 
-    const { value } = dataset;
+    const { value: rawValue } = dataset;
+    const value = hasArrayShape(dataset)
+      ? (rawValue as unknown[]).flat(dataset.shape.length - 1)
+      : rawValue;
     if (!selection) {
       return value;
     }
 
     assertArrayShape(dataset);
     assertPrintableType(dataset);
 
-    const dataArray = ndarray(
-      (dataset.value as (number | string | boolean)[]).flat(
-        dataset.shape.length - 1
-      ),
-      dataset.shape
-    );
-
-    const dataView = dataArray.pick(
-      ...selection
+    const { shape, type } = dataset;
+    const dataArray = ndarray(value as Primitive<typeof type>[], shape);
+    const mappedArray = applyMapping(
+      dataArray,
+      selection
         .split(',')
-        .map((s) => (s === ':' ? null : Number.parseInt(s, 10)))
+        .map((s) => (s === ':' ? 'x' : Number.parseInt(s, 10)))
     );
 
-    return unpack(dataView);
+    return mappedArray.data;
   }
 
   private async cancellableDelay(params: ValueRequestParams) {

src/h5web/providers/mock/utils.ts

@@ -56,5 +56,5 @@ export function getMockDataArray(path: string): NdArray {
   assertNumericType(dataset);
   assertArrayShape(dataset);
 
-  return ndarray(value.flat(Infinity), dataset.shape);
+  return ndarray(value.flat(dataset.shape.length - 1), dataset.shape);
 }

src/h5web/vis-packs/core/complex/MappedComplexVis.tsx

@@ -1,8 +1,7 @@
-import { useEffect, useMemo } from 'react';
-import { useMappedArray } from '../hooks';
+import { useEffect } from 'react';
+import { useMappedArray, useSliceDimsAndMapping } from '../hooks';
 import type { AxisMapping, ScaleType } from '../models';
 import type { DimensionMapping } from '../../../dimension-mapper/models';
-import { isAxis } from '../../../dimension-mapper/utils';
 import shallow from 'zustand/shallow';
 import { useSafeDomain, useVisDomain } from '../heatmap/hooks';
 import type { H5WebComplex } from '../../../providers/models';
@@ -45,13 +44,7 @@ function MappedComplexVis(props: Props) {
 
   const { visType } = useComplexConfig((state) => state, shallow);
 
-  const [slicedDims, slicedMapping] = useMemo(
-    () => [
-      dims.filter((_, i) => isAxis(dimMapping[i])),
-      dimMapping.filter((dim) => isAxis(dim)),
-    ],
-    [dims, dimMapping]
-  );
+  const [slicedDims, slicedMapping] = useSliceDimsAndMapping(dims, dimMapping);
 
   const [mappedArray] = useMappedArray(value, slicedDims, slicedMapping);
 

src/h5web/vis-packs/core/heatmap/MappedHeatmapVis.tsx

@@ -1,11 +1,10 @@
-import { useEffect, useMemo } from 'react';
+import { useEffect } from 'react';
 import HeatmapVis from './HeatmapVis';
-import { useDomain, useMappedArray } from '../hooks';
+import { useDomain, useMappedArray, useSliceDimsAndMapping } from '../hooks';
 import { useHeatmapConfig } from './config';
 import type { AxisMapping, ScaleType } from '../models';
 import { DEFAULT_DOMAIN } from '../utils';
 import type { DimensionMapping } from '../../../dimension-mapper/models';
-import { isAxis } from '../../../dimension-mapper/utils';
 import shallow from 'zustand/shallow';
 import { useSafeDomain, useVisDomain } from './hooks';
 
@@ -39,13 +38,7 @@ function MappedHeatmapVis(props: Props) {
     invertColorMap,
   } = useHeatmapConfig((state) => state, shallow);
 
-  const [slicedDims, slicedMapping] = useMemo(
-    () => [
-      dims.filter((_, i) => isAxis(dimMapping[i])),
-      dimMapping.filter((dim) => isAxis(dim)),
-    ],
-    [dims, dimMapping]
-  );
+  const [slicedDims, slicedMapping] = useSliceDimsAndMapping(dims, dimMapping);
 
   const [dataArray] = useMappedArray(value, slicedDims, slicedMapping);
 

src/h5web/vis-packs/core/hooks.ts

@@ -16,15 +16,16 @@ import {
   getBounds,
   getCanvasScale,
   getCombinedDomain,
+  getSliceDimsAndMapping,
   getSliceSelection,
   getValidDomainForScale,
   getValueToIndexScale,
 } from './utils';
 import AxisSystemContext from './shared/AxisSystemContext';
 import { AxisScale, ScaleType } from './models';
 import { ProviderContext } from '../../providers/context';
-import type { Dataset, Value } from '../../providers/models';
 import { isDefined } from '../../guards';
+import type { Dataset, Value } from '../../providers/models';
 
 export function usePrefetchValues(
   datasets: (Dataset | undefined)[],
@@ -200,3 +201,13 @@ export function useCSSCustomProperties(...names: string[]): {
     refCallback,
   };
 }
+
+export function useSliceDimsAndMapping<T extends DimensionMapping | undefined>(
+  dims: number[],
+  dimMapping: T
+) {
+  return useMemo(
+    () => getSliceDimsAndMapping(dims, dimMapping),
+    [dimMapping, dims]
+  );
+}

src/h5web/vis-packs/core/matrix/MappedMatrixVis.tsx

@@ -1,9 +1,7 @@
-import { useMemo } from 'react';
 import MatrixVis from './MatrixVis';
-import { useMappedArray } from '../hooks';
 import type { DimensionMapping } from '../../../dimension-mapper/models';
 import type { Primitive } from '../../../providers/models';
-import { isAxis } from '../../../dimension-mapper/utils';
+import { useMappedArray, useSliceDimsAndMapping } from '../hooks';
 import type { PrintableType } from '../models';
 
 interface Props {
@@ -17,13 +15,7 @@ interface Props {
 function MappedMatrixVis(props: Props) {
   const { value, dims, dimMapping, formatter, cellWidth } = props;
 
-  const [slicedDims, slicedMapping] = useMemo(
-    () => [
-      dims.filter((_, i) => isAxis(dimMapping[i])),
-      dimMapping.filter((dim) => isAxis(dim)),
-    ],
-    [dimMapping, dims]
-  );
+  const [slicedDims, slicedMapping] = useSliceDimsAndMapping(dims, dimMapping);
 
   const [mappedArray] = useMappedArray(value, slicedDims, slicedMapping);
 

src/h5web/vis-packs/core/utils.ts

@@ -305,7 +305,7 @@ export function getBaseArray<T>(
   value: T[] | undefined,
   rawDims: number[]
 ): NdArray<T> | undefined {
-  return value && ndarray<T>(value.flat(rawDims.length - 1) as T[], rawDims);
+  return value && ndarray<T>(value, rawDims);
 }
 
 export function applyMapping<T extends NdArray<unknown> | undefined>(
@@ -363,3 +363,21 @@ export function getSliceSelection(
   // Create slice selection string from dim mapping - e.g. [0, 'y', 'x'] => "0,:,:"
   return dimMapping.map((dim) => (isAxis(dim) ? ':' : dim)).join(',');
 }
+
+export function getSliceDimsAndMapping<T extends DimensionMapping | undefined>(
+  dims: number[],
+  dimMapping: T
+): [number[], T];
+export function getSliceDimsAndMapping(
+  dims: number[],
+  dimMapping: DimensionMapping | undefined
+): unknown {
+  if (dimMapping === undefined) {
+    return [dims, undefined];
+  }
+
+  return [
+    dims.filter((_, i) => isAxis(dimMapping[i])),
+    dimMapping.filter(isAxis),
+  ];
+}

src/react-app-env.d.ts

@@ -1,12 +1,7 @@
 /// <reference types="react-scripts" />
 
-declare module 'ndarray-unpack' {
+declare module 'ndarray-ops' {
   import type { NdArray } from 'ndarray';
 
-  function unpack<LT = T>(a: NdArray<T>): LT[];
-  export = unpack;
-}
-
-declare module 'ndarray-ops' {
   export function assign(a: NdArray<T>, b: NdArray<T>);
 }