From b74242706f5135d4371178eed83926dd58363b05 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Wed, 5 Jun 2024 10:07:23 -0400 Subject: [PATCH 01/10] feat: prompt point per connected component in shape interpolation auto-sam --- .../control/modes/ShapeInterpolationMode.kt | 69 ++++++++++++++----- .../ShapeInterpolationSAMTool.kt | 6 +- .../ShapeInterpolationTool.kt | 4 +- 3 files changed, 57 insertions(+), 22 deletions(-) diff --git a/src/main/kotlin/org/janelia/saalfeldlab/paintera/control/modes/ShapeInterpolationMode.kt b/src/main/kotlin/org/janelia/saalfeldlab/paintera/control/modes/ShapeInterpolationMode.kt index e14b28114..65f5f7a32 100644 --- a/src/main/kotlin/org/janelia/saalfeldlab/paintera/control/modes/ShapeInterpolationMode.kt +++ b/src/main/kotlin/org/janelia/saalfeldlab/paintera/control/modes/ShapeInterpolationMode.kt @@ -1,6 +1,7 @@ package org.janelia.saalfeldlab.paintera.control.modes import bdv.fx.viewer.render.RenderUnitState +import bdv.util.BdvFunctions import de.jensd.fx.glyphs.fontawesome.FontAwesomeIconView import io.github.oshai.kotlinlogging.KotlinLogging import javafx.beans.value.ChangeListener @@ -10,12 +11,17 @@ import javafx.event.Event import javafx.scene.input.KeyEvent.KEY_PRESSED import javafx.scene.input.KeyEvent.KEY_RELEASED import net.imglib2.Interval +import net.imglib2.algorithm.labeling.ConnectedComponents import net.imglib2.algorithm.morphology.distance.DistanceTransform import net.imglib2.img.array.ArrayImgs +import net.imglib2.loops.LoopBuilder import net.imglib2.realtransform.AffineTransform3D import net.imglib2.type.logic.BoolType import net.imglib2.type.numeric.IntegerType +import net.imglib2.type.numeric.integer.UnsignedIntType import net.imglib2.type.numeric.integer.UnsignedLongType +import net.imglib2.type.volatiles.VolatileUnsignedIntType +import net.imglib2.util.Intervals import net.imglib2.view.IntervalView import net.imglib2.view.Views import org.janelia.saalfeldlab.control.mcu.MCUButtonControl @@ -375,10 +381,10 @@ class ShapeInterpolationMode>(val controller: ShapeInterpolat AffineTransform3D().also { viewerAndTransforms.viewer().state.getViewerTransform(it) } } - val (maxDistanceX, maxDistanceY) = controller.getInterpolationImg(globalToViewerTransform, closest = true)?.let { + val maxDistancePositions = controller.getInterpolationImg(globalToViewerTransform, closest = true)?.let { val interpolantInViewer = if (translate) alignTransformAndViewCenter(it, globalToViewerTransform, width, height) else it - interpolantInViewer.getPositionAtMaxDistance() - } ?: doubleArrayOf(width / 2.0, height / 2.0, 0.0) + interpolantInViewer.getComponentMaxDistancePosition() + } ?: listOf(doubleArrayOf(width / 2.0, height / 2.0, 0.0)) val maskInfo = MaskInfo(0, currentBestMipMapLevel) @@ -390,7 +396,7 @@ class ShapeInterpolationMode>(val controller: ShapeInterpolat .toList() val renderState = RenderUnitState(mask.initialGlobalToViewerTransform.copy(), mask.info.time, sources, width.toLong(), height.toLong()) - val predictionRequest = SamPredictor.SparsePrediction(listOf(renderState.getSamPoint(maxDistanceX, maxDistanceY, SamPredictor.SparseLabel.IN))) + val predictionRequest = SamPredictor.SparsePrediction(maxDistancePositions.map { (x,y) -> renderState.getSamPoint(x,y, SamPredictor.SparseLabel.IN) }) SamSliceInfo(renderState, mask, predictionRequest, null, false).also { SamEmbeddingLoaderCache.load(renderState) @@ -502,21 +508,48 @@ internal fun RenderUnitState.getSamPoint(screenX: Double, screenY: Double, label return SamPredictor.SamPoint(screenX * screenScaleFactor, screenY * screenScaleFactor, label) } -internal fun IntervalView.getPositionAtMaxDistance(): DoubleArray { +internal fun IntervalView.getComponentMaxDistancePosition(): List { /* find the max point to initialize with */ - val distances = ArrayImgs.doubles(*dimensionsAsLongArray()) - val binaryImg = convert(BoolType()) { source, target -> target.set(!(source.get() != Label.INVALID && source.get() != Label.TRANSPARENT)) }.zeroMin() - DistanceTransform.binaryTransform(binaryImg, distances, DistanceTransform.DISTANCE_TYPE.EUCLIDIAN) - var maxDistance = -Double.MAX_VALUE - var maxPos = center() - BundleView(distances).interval(distances).forEach { access -> - val distance = access.get().get() - if (distance > maxDistance) { - maxDistance = distance - maxPos = access.positionAsLongArray() + val invalidBorderRai = extendValue(Label.INVALID).interval(Intervals.expand(this, 1, 1, 0)) + val distances = ArrayImgs.doubles(*invalidBorderRai.dimensionsAsLongArray()) + val binaryImg = invalidBorderRai.convert(BoolType()) { source, target -> target.set((source.get() != Label.INVALID && source.get() != Label.TRANSPARENT)) }.zeroMin() + val invertedBinaryImg = binaryImg.convert(BoolType()) { source, target -> target.set(!source.get()) } + + val connectedComponents = ArrayImgs.unsignedInts(*binaryImg.dimensionsAsLongArray()) + ConnectedComponents.labelAllConnectedComponents( + binaryImg, + connectedComponents, + ConnectedComponents.StructuringElement.FOUR_CONNECTED + ) + + DistanceTransform.binaryTransform(invertedBinaryImg, distances, DistanceTransform.DISTANCE_TYPE.EUCLIDIAN) + + val distancePerComponent = mutableMapOf>() + + var backgroundId = -1; + + LoopBuilder.setImages(BundleView(distances).interval(distances), connectedComponents).forEachPixel { distanceRA, componentType -> + val thisDist = distanceRA.get().get() + val componentId = componentType.integer + + val curDist = distancePerComponent[componentId]?.first + + if (curDist != null) { + if (thisDist > curDist) + distancePerComponent[componentId] = thisDist to distanceRA.positionAsLongArray() + } else { + if (backgroundId == -1 && !binaryImg.getAt(distanceRA).get()) { + backgroundId = componentId + } else if (componentId != backgroundId) + distancePerComponent[componentId] = thisDist to distanceRA.positionAsLongArray() } } - return maxPos.mapIndexed { idx, value -> value.toDouble() + min(idx) }.toDoubleArray() + + return distancePerComponent.values.map { + it.second.mapIndexed { idx, value -> + (value + min(idx)).toDouble() + }.toDoubleArray() + }.toList() } internal class SamSliceCache : HashMap() { @@ -537,4 +570,8 @@ internal data class SamSliceInfo(val renderState: RenderUnitState, val mask: Vie fun updatePrediction(viewerX: Double, viewerY: Double, label: SamPredictor.SparseLabel = SamPredictor.SparseLabel.IN) { prediction = SamPredictor.SparsePrediction(listOf(renderState.getSamPoint(viewerX, viewerY, label))) } + + fun updatePrediction(viewerPositions : List, label: SamPredictor.SparseLabel = SamPredictor.SparseLabel.IN) { + prediction = SamPredictor.SparsePrediction(viewerPositions.map { (x,y) -> renderState.getSamPoint(x,y, label) }) + } } \ No newline at end of file diff --git a/src/main/kotlin/org/janelia/saalfeldlab/paintera/control/tools/shapeinterpolation/ShapeInterpolationSAMTool.kt b/src/main/kotlin/org/janelia/saalfeldlab/paintera/control/tools/shapeinterpolation/ShapeInterpolationSAMTool.kt index 47806584f..990873385 100644 --- a/src/main/kotlin/org/janelia/saalfeldlab/paintera/control/tools/shapeinterpolation/ShapeInterpolationSAMTool.kt +++ b/src/main/kotlin/org/janelia/saalfeldlab/paintera/control/tools/shapeinterpolation/ShapeInterpolationSAMTool.kt @@ -36,15 +36,13 @@ internal class ShapeInterpolationSAMTool(private val controller: ShapeInterpolat shapeInterpolationMode.samSliceCache[controller.currentDepth] ?: let { SamEmbeddingLoaderCache.cancelPendingRequests() } val info = shapeInterpolationMode.cacheLoadSamSliceInfo(controller.currentDepth) - if (!info.preGenerated) - temporaryPrompt = false - else - controller.deleteSliceAt(controller.currentDepth) + if (info.preGenerated) controller.deleteSliceAt(controller.currentDepth) maskedSource?.resetMasks(false) viewerMask = controller.getMask() super.activate() + if (!info.preGenerated) temporaryPrompt = false requestPrediction(info.prediction) } diff --git a/src/main/kotlin/org/janelia/saalfeldlab/paintera/control/tools/shapeinterpolation/ShapeInterpolationTool.kt b/src/main/kotlin/org/janelia/saalfeldlab/paintera/control/tools/shapeinterpolation/ShapeInterpolationTool.kt index ad110ec6e..f2b0adc6e 100644 --- a/src/main/kotlin/org/janelia/saalfeldlab/paintera/control/tools/shapeinterpolation/ShapeInterpolationTool.kt +++ b/src/main/kotlin/org/janelia/saalfeldlab/paintera/control/tools/shapeinterpolation/ShapeInterpolationTool.kt @@ -159,8 +159,8 @@ internal class ShapeInterpolationTool( val samSliceInfo = shapeInterpolationMode.cacheLoadSamSliceInfo(depth) if (!newPrediction && refresh) { - controller.getInterpolationImg(samSliceInfo.globalToViewerTransform, closest = true)?.getPositionAtMaxDistance()?.let { (x, y) -> - samSliceInfo.updatePrediction(x, y) + controller.getInterpolationImg(samSliceInfo.globalToViewerTransform, closest = true)?.getComponentMaxDistancePosition()?.let { positions -> + samSliceInfo.updatePrediction(positions) } } From 23b2285120defffbf6b6c17b98ea699dfab39931 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Wed, 5 Jun 2024 16:08:01 -0400 Subject: [PATCH 02/10] fix: initialize to label when paintera label type known from metadata --- .../saalfeldlab/paintera/state/metadata/MetadataState.kt | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/org/janelia/saalfeldlab/paintera/state/metadata/MetadataState.kt b/src/main/kotlin/org/janelia/saalfeldlab/paintera/state/metadata/MetadataState.kt index f776db58f..320fb3211 100644 --- a/src/main/kotlin/org/janelia/saalfeldlab/paintera/state/metadata/MetadataState.kt +++ b/src/main/kotlin/org/janelia/saalfeldlab/paintera/state/metadata/MetadataState.kt @@ -132,7 +132,7 @@ open class SingleScaleMetadataState( } -open class MultiScaleMetadataState constructor( +open class MultiScaleMetadataState ( override val n5ContainerState: N5ContainerState, final override val metadata: SpatialMultiscaleMetadata ) : MetadataState by SingleScaleMetadataState(n5ContainerState, metadata[0]) { @@ -140,7 +140,10 @@ open class MultiScaleMetadataState constructor( private val highestResMetadata: N5SpatialDatasetMetadata = metadata[0] final override var transform: AffineTransform3D = metadata.spatialTransform3d() final override var isLabelMultiset: Boolean = metadata[0].isLabelMultiset - override var isLabel: Boolean = isLabel(highestResMetadata.attributes.dataType) || isLabelMultiset + override var isLabel: Boolean = when { + metadata is N5PainteraLabelMultiScaleGroup -> metadata.isLabel + else -> isLabel(highestResMetadata.attributes.dataType) || isLabelMultiset + } override var resolution: DoubleArray = transform.run { doubleArrayOf(get(0, 0), get(1, 1), get(2, 2)) } override var translation: DoubleArray = transform.translation override var group: String = metadata.path From 20f661e957e3d549280cc2b1bfbc7fb5d2f70866 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Thu, 6 Jun 2024 16:34:17 -0400 Subject: [PATCH 03/10] fix: don't serialize null Properties necessary to ensure consistency regardless of backing N5Container storage format --- .../saalfeldlab/paintera/PainteraMainWindow.kt | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/org/janelia/saalfeldlab/paintera/PainteraMainWindow.kt b/src/main/kotlin/org/janelia/saalfeldlab/paintera/PainteraMainWindow.kt index 6919b5352..82596a661 100644 --- a/src/main/kotlin/org/janelia/saalfeldlab/paintera/PainteraMainWindow.kt +++ b/src/main/kotlin/org/janelia/saalfeldlab/paintera/PainteraMainWindow.kt @@ -1,6 +1,5 @@ package org.janelia.saalfeldlab.paintera -import org.janelia.saalfeldlab.bdv.fx.viewer.ViewerPanelFX import bdv.viewer.ViewerOptions import com.google.gson.Gson import com.google.gson.JsonElement @@ -16,6 +15,7 @@ import javafx.scene.image.Image import javafx.stage.Stage import net.imglib2.realtransform.AffineTransform3D import org.controlsfx.control.Notifications +import org.janelia.saalfeldlab.bdv.fx.viewer.ViewerPanelFX import org.janelia.saalfeldlab.fx.event.KeyTracker import org.janelia.saalfeldlab.fx.event.MouseTracker import org.janelia.saalfeldlab.fx.extensions.createNullableValueBinding @@ -239,7 +239,13 @@ class PainteraMainWindow(val gateway: PainteraGateway = PainteraGateway()) { .toFile() private fun deserialize(json: JsonObject?, gson: Gson, indexToState: MutableMap>) { - initProperties(gson.get(json) ?: Properties()) + /* Clear any errant null values from the properties. It shouldn't happen, but also is recoverable in case is does (did). */ + json?.entrySet() + ?.filter { (_, value) -> value == null || value.isJsonNull }?.toList() + ?.forEach { (key, _) -> json.remove(key) } + + val properties = gson.get(json) ?: Properties() + initProperties(properties) json?.get(SOURCES_KEY)?.let { sourcesJson -> SourceInfoSerializer.populate( { baseView.addState(it) }, @@ -273,7 +279,7 @@ class PainteraMainWindow(val gateway: PainteraGateway = PainteraGateway()) { stage.onHiding = EventHandler { if (!doSaveAndQuit()) it.consume() } } - internal fun askSaveAndQuit() : Boolean { + internal fun askSaveAndQuit(): Boolean { return when { wasQuit -> false !isSaveNecessary() -> true @@ -337,6 +343,7 @@ class PainteraMainWindow(val gateway: PainteraGateway = PainteraGateway()) { class Serializer : PainteraSerialization.PainteraSerializer { override fun serialize(mainWindow: PainteraMainWindow, typeOfSrc: Type, context: JsonSerializationContext): JsonElement { val map = context[mainWindow.properties].asJsonObject + map.entrySet().filter { (_, value) -> value == null || value.isJsonNull }.toList().forEach { (key, _) -> map.remove(key) } map.add(SOURCES_KEY, context[mainWindow.baseView.sourceInfo()]) map.addProperty(VERSION_KEY, VERSION_STRING) map.add(GLOBAL_TRANSFORM_KEY, context[AffineTransform3D().also { mainWindow.baseView.manager().getTransform(it) }]) From 8db27448e833de77bbc9ef6dc803018c6a118e6e Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 7 Jun 2024 13:24:04 -0400 Subject: [PATCH 04/10] feat: create scalar label sources --- .../ui/dialogs/create/MipMapLevel.java | 10 +++++++++- .../janelia/saalfeldlab/util/n5/N5Data.java | 19 ++++++++++++------- .../ui/dialogs/create/CreateDataset.kt | 16 ++++++++++++++-- 3 files changed, 35 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/paintera/ui/dialogs/create/MipMapLevel.java b/src/main/java/org/janelia/saalfeldlab/paintera/ui/dialogs/create/MipMapLevel.java index 918351c55..938ce2179 100644 --- a/src/main/java/org/janelia/saalfeldlab/paintera/ui/dialogs/create/MipMapLevel.java +++ b/src/main/java/org/janelia/saalfeldlab/paintera/ui/dialogs/create/MipMapLevel.java @@ -18,6 +18,7 @@ public class MipMapLevel { + final public SimpleBooleanProperty isLabelMultisetProperty = new SimpleBooleanProperty(true); final SpatialField baseDimensions; final SpatialField relativeDownsamplingFactors; @@ -80,7 +81,14 @@ public Node makeNode() { mipMapRow.getChildren().add(new VBox(resolutionHeader, resolution.getNode())); mipMapRow.getChildren().add(new VBox(dimensionHeader, dimensions.getNode())); } - mipMapRow.getChildren().add(new VBox(maxEntriesHeader, maxNumberOfEntriesPerSet.getTextField())); + final VBox numEntriesRow = new VBox(maxEntriesHeader, maxNumberOfEntriesPerSet.getTextField()); + + isLabelMultisetProperty.subscribe(isLabelMultiset -> { + if (isLabelMultiset) + mipMapRow.getChildren().add(numEntriesRow); + else + mipMapRow.getChildren().remove(numEntriesRow); + }); mipMapRow.setPadding(new Insets(0, 10.0, 0, 10.0)); mipMapRow.spacingProperty().setValue(10.0); diff --git a/src/main/java/org/janelia/saalfeldlab/util/n5/N5Data.java b/src/main/java/org/janelia/saalfeldlab/util/n5/N5Data.java index 75faf878d..11c6dc19e 100644 --- a/src/main/java/org/janelia/saalfeldlab/util/n5/N5Data.java +++ b/src/main/java/org/janelia/saalfeldlab/util/n5/N5Data.java @@ -50,6 +50,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nullable; import java.io.IOException; import java.lang.invoke.MethodHandles; import java.util.ArrayList; @@ -749,9 +750,10 @@ public static void createEmptyLabelDataset( final double[] resolution, final double[] offset, final double[][] relativeScaleFactors, - final int[] maxNumEntries) throws IOException { + @Nullable final int[] maxNumEntries, + final boolean labelMultiset) throws IOException { - createEmptyLabelDataset(container, group, dimensions, blockSize, resolution, offset, relativeScaleFactors, maxNumEntries, false); + createEmptyLabelDataset(container, group, dimensions, blockSize, resolution, offset, relativeScaleFactors, maxNumEntries, labelMultiset, false); } /** @@ -776,7 +778,8 @@ public static void createEmptyLabelDataset( final double[] resolution, final double[] offset, final double[][] relativeScaleFactors, - final int[] maxNumEntries, + @Nullable final int[] maxNumEntries, + final boolean labelMultisetType, final boolean ignoreExisiting) throws IOException { final Map pd = new HashMap<>(); @@ -806,7 +809,7 @@ public static void createEmptyLabelDataset( n5.setAttribute(dataGroup, N5Helpers.MULTI_SCALE_KEY, true); n5.setAttribute(dataGroup, N5Helpers.OFFSET_KEY, offset); n5.setAttribute(dataGroup, N5Helpers.RESOLUTION_KEY, resolution); - n5.setAttribute(dataGroup, N5Helpers.IS_LABEL_MULTISET_KEY, true); + n5.setAttribute(dataGroup, N5Helpers.IS_LABEL_MULTISET_KEY, labelMultisetType); n5.createGroup(uniqueLabelsGroup); n5.setAttribute(uniqueLabelsGroup, N5Helpers.MULTI_SCALE_KEY, true); @@ -825,12 +828,14 @@ public static void createEmptyLabelDataset( final String dataset = String.format(scaleDatasetPattern, scaleLevel); final String uniqeLabelsDataset = String.format(scaleUniqueLabelsPattern, scaleLevel); - final int maxNum = downscaledLevel < 0 ? -1 : maxNumEntries[downscaledLevel]; n5.createDataset(dataset, scaledDimensions, blockSize, DataType.UINT8, new GzipCompression()); n5.createDataset(uniqeLabelsDataset, scaledDimensions, blockSize, DataType.UINT64, new GzipCompression()); - n5.setAttribute(dataset, N5Helpers.MAX_NUM_ENTRIES_KEY, maxNum); - n5.setAttribute(dataset, N5Helpers.IS_LABEL_MULTISET_KEY, true); + if (labelMultisetType) { + final int maxNum = downscaledLevel < 0 ? -1 : maxNumEntries[downscaledLevel]; + n5.setAttribute(dataset, N5Helpers.MAX_NUM_ENTRIES_KEY, maxNum); + n5.setAttribute(dataset, N5Helpers.IS_LABEL_MULTISET_KEY, true); + } if (scaleLevel != 0) { n5.setAttribute(dataset, N5Helpers.DOWNSAMPLING_FACTORS_KEY, accumulatedFactors); diff --git a/src/main/kotlin/org/janelia/saalfeldlab/paintera/ui/dialogs/create/CreateDataset.kt b/src/main/kotlin/org/janelia/saalfeldlab/paintera/ui/dialogs/create/CreateDataset.kt index 40a17c2d6..de494515b 100644 --- a/src/main/kotlin/org/janelia/saalfeldlab/paintera/ui/dialogs/create/CreateDataset.kt +++ b/src/main/kotlin/org/janelia/saalfeldlab/paintera/ui/dialogs/create/CreateDataset.kt @@ -63,7 +63,15 @@ import java.util.* class CreateDataset(private val currentSource: Source<*>?, vararg allSources: SourceState<*, *>) { - private val mipmapLevels = FXCollections.observableArrayList() + private val mipmapLevels = FXCollections.observableArrayList().also { + it.addListener( ListChangeListener { change -> + if (change.next()) { + change.addedSubList.forEach { level -> + level.isLabelMultisetProperty.bind(labelMultiset.selectedProperty()) + } + } + }) + } private val mipmapLevelsNode by lazy { createMipMapLevelsNode(mipmapLevels, FIELD_WIDTH, NAME_WIDTH, *SubmitOn.entries.toTypedArray()) } @@ -124,7 +132,9 @@ class CreateDataset(private val currentSource: Source<*>?, vararg allSources: So private val blockSize = SpatialField.intField(1, { it > 0 }, FIELD_WIDTH, *SubmitOn.entries.toTypedArray()) private val resolution = SpatialField.doubleField(1.0, { it > 0 }, FIELD_WIDTH, *SubmitOn.entries.toTypedArray()) private val offset = SpatialField.doubleField(0.0, { true }, FIELD_WIDTH, *SubmitOn.entries.toTypedArray()) + private val labelMultiset = CheckBox().also { it.isSelected = true } private val scaleLevels = TitledPane("Scale Levels", mipmapLevelsNode) + //TODO Caleb: Use a proper grid layout instead of this... private val pane = VBox( nameIt("Name", NAME_WIDTH, true, nameField), nameIt("N5", NAME_WIDTH, true, n5Container.asNode()), @@ -133,6 +143,7 @@ class CreateDataset(private val currentSource: Source<*>?, vararg allSources: So nameIt("Block Size", NAME_WIDTH, false, bufferNode(), blockSize.node), nameIt("Resolution", NAME_WIDTH, false, bufferNode(), resolution.node), nameIt("Offset", NAME_WIDTH, false, bufferNode(), offset.node), + nameIt("", NAME_WIDTH, false, bufferNode(), HBox(Label("Label Multiset Type "), labelMultiset)), setFromCurrentBox, scaleLevels ) @@ -265,7 +276,8 @@ class CreateDataset(private val currentSource: Source<*>?, vararg allSources: So resolution.asDoubleArray(), offset.asDoubleArray(), scaleLevels.map { it.downsamplingFactors() }.toTypedArray(), - scaleLevels.stream().mapToInt { it.maxNumEntries() }.toArray() + if (labelMultiset.isSelected) scaleLevels.stream().mapToInt { it.maxNumEntries() }.toArray() else null, + labelMultiset.isSelected ) val writer = n5Factory.openWriter(container) From 598a944e0b9279ea636919f3b9efd410c11e8707 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 7 Jun 2024 15:02:13 -0400 Subject: [PATCH 05/10] fix: Handle SAM in ShapeInterpolation when label source is smaller than viewrer --- .../paintera/control/ShapeInterpolationController.kt | 9 ++++++--- .../saalfeldlab/paintera/control/tools/paint/SamTool.kt | 6 ++++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/main/kotlin/org/janelia/saalfeldlab/paintera/control/ShapeInterpolationController.kt b/src/main/kotlin/org/janelia/saalfeldlab/paintera/control/ShapeInterpolationController.kt index 12a47d045..f630126e7 100644 --- a/src/main/kotlin/org/janelia/saalfeldlab/paintera/control/ShapeInterpolationController.kt +++ b/src/main/kotlin/org/janelia/saalfeldlab/paintera/control/ShapeInterpolationController.kt @@ -41,7 +41,6 @@ import net.imglib2.util.* import net.imglib2.view.ExtendedRealRandomAccessibleRealInterval import net.imglib2.view.IntervalView import net.imglib2.view.Views -import org.checkerframework.common.reflection.qual.Invoke import org.janelia.saalfeldlab.fx.Tasks import org.janelia.saalfeldlab.fx.extensions.* import org.janelia.saalfeldlab.fx.util.InvokeOnJavaFXApplicationThread @@ -732,11 +731,14 @@ class ShapeInterpolationController>( /* get union of adjacent slices bounding boxes */ val unionInterval = let { + + val sourceIntervalInMaskSpace = viewerMask.run { currentMaskToSourceTransform.inverse().estimateBounds(source.getSource(0, info.level)) } val interpolantIntervalInMaskSpace = viewerMask.currentGlobalToMaskTransform.estimateBounds(interpolantInterval) val minZSlice = interpolantIntervalInMaskSpace.minAsDoubleArray().also { it[2] = 0.0 } val maxZSlice = interpolantIntervalInMaskSpace.maxAsDoubleArray().also { it[2] = 0.0 } - val interpolantIntervalSliceInMaskSpace = FinalRealInterval(minZSlice, maxZSlice) + val interpolantIntervalSliceInMaskSpace = FinalRealInterval(minZSlice, maxZSlice) intersect sourceIntervalInMaskSpace + val interpolatedMaskView = interpolant.dataInterpolant @@ -1215,8 +1217,9 @@ class ShapeInterpolationController>( } + val sourceInMaskInterval = mask.initialMaskToSourceTransform.inverse().estimateBounds(mask.source.getSource(0, mask.info.level)) selectionIntervals - .map { BundleView(mask.viewerImg).interval(it) } + .map { BundleView(mask.viewerImg).interval(it intersect sourceInMaskInterval) } .map { val shrinkingInterval = ShrinkingInterval(it.numDimensions()) LoopBuilder.setImages(it).forEachPixel { access -> diff --git a/src/main/kotlin/org/janelia/saalfeldlab/paintera/control/tools/paint/SamTool.kt b/src/main/kotlin/org/janelia/saalfeldlab/paintera/control/tools/paint/SamTool.kt index 32aaab3df..32ad62cb2 100644 --- a/src/main/kotlin/org/janelia/saalfeldlab/paintera/control/tools/paint/SamTool.kt +++ b/src/main/kotlin/org/janelia/saalfeldlab/paintera/control/tools/paint/SamTool.kt @@ -797,14 +797,16 @@ open class SamTool(activeSourceStateProperty: SimpleObjectProperty originalImage.set(currentImage.get()) } + val volatilePredictionMaxInterval = originalWritableVolatileBackingImage!!.intersect(maskInterval) LoopBuilder - .setImages(originalWritableVolatileBackingImage!!.interval(maskInterval), currentMask.volatileViewerImg.wrappedSource.interval(maskInterval)) + .setImages(originalWritableVolatileBackingImage!!.interval(volatilePredictionMaxInterval), currentMask.volatileViewerImg.wrappedSource.interval(volatilePredictionMaxInterval)) .multiThreaded() .forEachPixel { originalImage, currentImage -> originalImage.isValid = currentImage.isValid From 8ed7891494a933d5781a20caaedf487e4e7eb9db Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 7 Jun 2024 15:02:33 -0400 Subject: [PATCH 06/10] fix: invalidate key if cancelled --- .../saalfeldlab/paintera/cache/AsyncCacheWithLoader.kt | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/org/janelia/saalfeldlab/paintera/cache/AsyncCacheWithLoader.kt b/src/main/kotlin/org/janelia/saalfeldlab/paintera/cache/AsyncCacheWithLoader.kt index a6ac0fbed..bbb9e973e 100644 --- a/src/main/kotlin/org/janelia/saalfeldlab/paintera/cache/AsyncCacheWithLoader.kt +++ b/src/main/kotlin/org/janelia/saalfeldlab/paintera/cache/AsyncCacheWithLoader.kt @@ -45,12 +45,16 @@ open class AsyncCacheWithLoader( cache.get(key) { if (clear) cancelUnsubmittedLoadRequests() async(loaderContext) { loader(key) } - } + }.also { if (it.isCancelled) cache.invalidate(key) } } open fun load(key: K) : Job { - /* If it's already in the cache, deferred or not, just return a completed job and skipp the loader queue */ - if (cache.getIfPresent(key) != null) return Job().apply { complete() } + /* If it's already in the cache, deferred or not, just return a completed job and skip the loader queue */ + cache.getIfPresent(key)?.let { + if (it.isCancelled) + cache.invalidate(key) + else return Job().apply { complete() } + } return runBlocking { async(loaderQueueContext) { From db34acfe9843f2d75b51d7e586cddecdf795af34 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 7 Jun 2024 16:27:39 -0400 Subject: [PATCH 07/10] fix: refresh SI mask when leaving SAM mode --- .../saalfeldlab/paintera/control/tools/paint/SamTool.kt | 6 +++++- .../tools/shapeinterpolation/ShapeInterpolationSAMTool.kt | 5 +++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/org/janelia/saalfeldlab/paintera/control/tools/paint/SamTool.kt b/src/main/kotlin/org/janelia/saalfeldlab/paintera/control/tools/paint/SamTool.kt index 32ad62cb2..5f72559e5 100644 --- a/src/main/kotlin/org/janelia/saalfeldlab/paintera/control/tools/paint/SamTool.kt +++ b/src/main/kotlin/org/janelia/saalfeldlab/paintera/control/tools/paint/SamTool.kt @@ -564,7 +564,7 @@ open class SamTool(activeSourceStateProperty: SimpleObjectProperty Date: Fri, 7 Jun 2024 16:30:12 -0400 Subject: [PATCH 08/10] feat: replace existing slice with manual SAM if current slice was auto-predicted --- .../control/ShapeInterpolationController.kt | 26 ++++++++----------- .../control/modes/ShapeInterpolationMode.kt | 5 ++-- .../ShapeInterpolationSAMTool.kt | 15 +++++++---- 3 files changed, 24 insertions(+), 22 deletions(-) diff --git a/src/main/kotlin/org/janelia/saalfeldlab/paintera/control/ShapeInterpolationController.kt b/src/main/kotlin/org/janelia/saalfeldlab/paintera/control/ShapeInterpolationController.kt index f630126e7..f9ba1a39a 100644 --- a/src/main/kotlin/org/janelia/saalfeldlab/paintera/control/ShapeInterpolationController.kt +++ b/src/main/kotlin/org/janelia/saalfeldlab/paintera/control/ShapeInterpolationController.kt @@ -1,6 +1,5 @@ package org.janelia.saalfeldlab.paintera.control -import org.janelia.saalfeldlab.bdv.fx.viewer.ViewerPanelFX import bdv.viewer.TransformListener import io.github.oshai.kotlinlogging.KotlinLogging import javafx.application.Platform @@ -25,7 +24,6 @@ import net.imglib2.img.array.ArrayImgFactory import net.imglib2.img.array.ArrayImgs import net.imglib2.interpolation.randomaccess.NLinearInterpolatorFactory import net.imglib2.loops.LoopBuilder -import org.janelia.saalfeldlab.net.imglib2.outofbounds.RealOutOfBoundsConstantValueFactory import net.imglib2.realtransform.AffineTransform3D import net.imglib2.realtransform.Translation3D import net.imglib2.type.BooleanType @@ -41,9 +39,12 @@ import net.imglib2.util.* import net.imglib2.view.ExtendedRealRandomAccessibleRealInterval import net.imglib2.view.IntervalView import net.imglib2.view.Views +import org.janelia.saalfeldlab.bdv.fx.viewer.ViewerPanelFX import org.janelia.saalfeldlab.fx.Tasks import org.janelia.saalfeldlab.fx.extensions.* import org.janelia.saalfeldlab.fx.util.InvokeOnJavaFXApplicationThread +import org.janelia.saalfeldlab.net.imglib2.outofbounds.RealOutOfBoundsConstantValueFactory +import org.janelia.saalfeldlab.net.imglib2.view.BundleView import org.janelia.saalfeldlab.paintera.Paintera import org.janelia.saalfeldlab.paintera.PainteraBaseView import org.janelia.saalfeldlab.paintera.control.assignment.FragmentSegmentAssignment @@ -60,7 +61,6 @@ import org.janelia.saalfeldlab.paintera.util.IntervalHelpers import org.janelia.saalfeldlab.paintera.util.IntervalHelpers.Companion.extendBy import org.janelia.saalfeldlab.paintera.util.IntervalHelpers.Companion.smallestContainingInterval import org.janelia.saalfeldlab.util.* -import org.janelia.saalfeldlab.net.imglib2.view.BundleView import java.math.BigDecimal import java.math.RoundingMode import java.util.Collections @@ -191,19 +191,16 @@ class ShapeInterpolationController>( @JvmOverloads fun addSelection( maskIntervalOverSelection: Interval, - keepInterpolation: Boolean = true, + replaceExistingSlice: Boolean = false, globalTransform: AffineTransform3D, viewerMask: ViewerMask ): SliceInfo? { if (controllerState == ControllerState.Off) return null isBusy = true val selectionDepth = depthAt(globalTransform) - if (!keepInterpolation && slicesAndInterpolants.getSliceAtDepth(selectionDepth) != null) { + if (replaceExistingSlice && slicesAndInterpolants.getSliceAtDepth(selectionDepth) != null) slicesAndInterpolants.removeSliceAtDepth(selectionDepth) - updateSliceAndInterpolantsCompositeMask() - val slice = SliceInfo(viewerMask, globalTransform, maskIntervalOverSelection) - slicesAndInterpolants.add(selectionDepth, slice) - } + if (slicesAndInterpolants.getSliceAtDepth(selectionDepth) == null) { val slice = SliceInfo(viewerMask, globalTransform, maskIntervalOverSelection) slicesAndInterpolants.add(selectionDepth, slice) @@ -335,7 +332,7 @@ class ShapeInterpolationController>( .reduceOrNull(Intervals::union) } updateSliceAndInterpolantsCompositeMask() - requestRepaintAfterTasks(updateInterval) + requestRepaintAfterTasks(updateInterval) } } .onCancelled { _, _ -> LOG.debug { "Interpolation Cancelled" } } @@ -619,10 +616,10 @@ class ShapeInterpolationController>( activeViewer!!.state.getViewerTransform(it) } - fun getMask(targetMipMapLevel: Int = currentBestMipMapLevel): ViewerMask { + fun getMask(targetMipMapLevel: Int = currentBestMipMapLevel, ignoreExisting: Boolean = false): ViewerMask { /* If we have a mask, get it; else create a new one */ - currentViewerMask = sliceAtCurrentDepth?.let { oldSlice -> + currentViewerMask = (if (ignoreExisting) null else sliceAtCurrentDepth)?.let { oldSlice -> val oldSliceBoundingBox = oldSlice.maskBoundingBox ?: let { deleteSliceAt() return@let null @@ -740,7 +737,6 @@ class ShapeInterpolationController>( val interpolantIntervalSliceInMaskSpace = FinalRealInterval(minZSlice, maxZSlice) intersect sourceIntervalInMaskSpace - val interpolatedMaskView = interpolant.dataInterpolant .affine(viewerMask.currentGlobalToMaskTransform) .interval(interpolantIntervalSliceInMaskSpace) @@ -775,7 +771,7 @@ class ShapeInterpolationController>( } companion object { - private val LOG = KotlinLogging.logger { } + private val LOG = KotlinLogging.logger { } private fun paintera(): PainteraBaseView = Paintera.getPaintera().baseView private val Long.isInterpolationLabel @@ -1217,7 +1213,7 @@ class ShapeInterpolationController>( } - val sourceInMaskInterval = mask.initialMaskToSourceTransform.inverse().estimateBounds(mask.source.getSource(0, mask.info.level)) + val sourceInMaskInterval = mask.initialMaskToSourceTransform.inverse().estimateBounds(mask.source.getSource(0, mask.info.level)) selectionIntervals .map { BundleView(mask.viewerImg).interval(it intersect sourceInMaskInterval) } .map { diff --git a/src/main/kotlin/org/janelia/saalfeldlab/paintera/control/modes/ShapeInterpolationMode.kt b/src/main/kotlin/org/janelia/saalfeldlab/paintera/control/modes/ShapeInterpolationMode.kt index 65f5f7a32..0f0ec5a6e 100644 --- a/src/main/kotlin/org/janelia/saalfeldlab/paintera/control/modes/ShapeInterpolationMode.kt +++ b/src/main/kotlin/org/janelia/saalfeldlab/paintera/control/modes/ShapeInterpolationMode.kt @@ -462,7 +462,8 @@ class ShapeInterpolationMode>(val controller: ShapeInterpolat internal fun addSelection( selectionIntervalOverMask: Interval, globalTransform: AffineTransform3D = paintera.baseView.manager().transform, - viewerMask: ViewerMask = controller.currentViewerMask!! + viewerMask: ViewerMask = controller.currentViewerMask!!, + replaceExistingSlice : Boolean = false ): SamSliceInfo? { val globalToViewerTransform = viewerMask.initialGlobalToMaskTransform val sliceDepth = controller.depthAt(globalToViewerTransform) @@ -492,7 +493,7 @@ class ShapeInterpolationMode>(val controller: ShapeInterpolat } } - val slice = controller.addSelection(selectionIntervalOverMask, globalTransform = globalTransform, viewerMask = viewerMask) ?: return null + val slice = controller.addSelection(selectionIntervalOverMask, replaceExistingSlice, globalTransform, viewerMask) ?: return null return cacheLoadSamSliceInfo(sliceDepth).apply { sliceInfo = slice } diff --git a/src/main/kotlin/org/janelia/saalfeldlab/paintera/control/tools/shapeinterpolation/ShapeInterpolationSAMTool.kt b/src/main/kotlin/org/janelia/saalfeldlab/paintera/control/tools/shapeinterpolation/ShapeInterpolationSAMTool.kt index d91509d43..e610abbb6 100644 --- a/src/main/kotlin/org/janelia/saalfeldlab/paintera/control/tools/shapeinterpolation/ShapeInterpolationSAMTool.kt +++ b/src/main/kotlin/org/janelia/saalfeldlab/paintera/control/tools/shapeinterpolation/ShapeInterpolationSAMTool.kt @@ -29,6 +29,8 @@ internal class ShapeInterpolationSAMTool(private val controller: ShapeInterpolat } } + private var replaceExistingSlice = false + override fun activate() { /* If we are requesting a new embedding that isn't already pre-cached, * then likely the existing requests are no longer needed. @@ -36,13 +38,16 @@ internal class ShapeInterpolationSAMTool(private val controller: ShapeInterpolat shapeInterpolationMode.samSliceCache[controller.currentDepth] ?: let { SamEmbeddingLoaderCache.cancelPendingRequests() } val info = shapeInterpolationMode.cacheLoadSamSliceInfo(controller.currentDepth) - if (info.preGenerated) controller.deleteSliceAt(controller.currentDepth) - maskedSource?.resetMasks(false) - viewerMask = controller.getMask() + replaceExistingSlice = !info.locked + viewerMask = controller.getMask(ignoreExisting = replaceExistingSlice) super.activate() - if (!info.preGenerated) temporaryPrompt = false + temporaryPrompt = when { + !info.locked -> true + !info.preGenerated -> false + else -> temporaryPrompt + } requestPrediction(info.prediction) } @@ -55,7 +60,7 @@ internal class ShapeInterpolationSAMTool(private val controller: ShapeInterpolat lastPrediction?.apply { /* cache the prediction. lock the cached slice, since this was applied manually */ super.applyPrediction() - shapeInterpolationMode.addSelection(maskInterval)?.also { + shapeInterpolationMode.addSelection(maskInterval, replaceExistingSlice = replaceExistingSlice)?.also { it.prediction = predictionRequest it.locked = true } From 316d6fc912fc303d39246e939f85dad17b8c3788 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 7 Jun 2024 16:41:48 -0400 Subject: [PATCH 09/10] fix: reduce flickering and double-prediction during bisect-all auto-sam --- .../paintera/control/ShapeInterpolationController.kt | 6 ++++++ .../tools/shapeinterpolation/ShapeInterpolationTool.kt | 4 +++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/org/janelia/saalfeldlab/paintera/control/ShapeInterpolationController.kt b/src/main/kotlin/org/janelia/saalfeldlab/paintera/control/ShapeInterpolationController.kt index f9ba1a39a..efcabd24c 100644 --- a/src/main/kotlin/org/janelia/saalfeldlab/paintera/control/ShapeInterpolationController.kt +++ b/src/main/kotlin/org/janelia/saalfeldlab/paintera/control/ShapeInterpolationController.kt @@ -276,6 +276,7 @@ class ShapeInterpolationController>( } fun togglePreviewMode() { + freezeInterpolation = false preview = !preview if (preview) interpolateBetweenSlices(false) else updateSliceAndInterpolantsCompositeMask() @@ -298,6 +299,7 @@ class ShapeInterpolationController>( @Synchronized fun interpolateBetweenSlices(replaceExistingInterpolants: Boolean) { + if (freezeInterpolation) return if (slicesAndInterpolants.slices.size < 2) { updateSliceAndInterpolantsCompositeMask() isBusy = false @@ -468,9 +470,13 @@ class ShapeInterpolationController>( refreshMeshes() } + private val freezeInterpolationProperty = SimpleBooleanProperty(false) + var freezeInterpolation: Boolean by freezeInterpolationProperty.nonnull() + @Throws(MaskInUse::class) private fun setCompositeMask(includeInterpolant: Boolean = preview) { + if (freezeInterpolation) return synchronized(source) { source.resetMasks(false) /* If preview is on, hide all except the first and last fill mask */ diff --git a/src/main/kotlin/org/janelia/saalfeldlab/paintera/control/tools/shapeinterpolation/ShapeInterpolationTool.kt b/src/main/kotlin/org/janelia/saalfeldlab/paintera/control/tools/shapeinterpolation/ShapeInterpolationTool.kt index f2b0adc6e..8442bf36f 100644 --- a/src/main/kotlin/org/janelia/saalfeldlab/paintera/control/tools/shapeinterpolation/ShapeInterpolationTool.kt +++ b/src/main/kotlin/org/janelia/saalfeldlab/paintera/control/tools/shapeinterpolation/ShapeInterpolationTool.kt @@ -288,7 +288,8 @@ internal class ShapeInterpolationTool( val remainingRequest = SimpleIntegerProperty().apply { addListener { _, _, remaining -> if (remaining == 0) { - depths.forEach { requestSamPrediction(it, refresh = true) } + controller.freezeInterpolation = false + controller.setMaskOverlay() depths.sort() /* eagerly request the next embeddings */ @@ -303,6 +304,7 @@ internal class ShapeInterpolationTool( /* Do the prediction */ sortedSliceDepths.zipWithNext { before, after -> (before + after) / 2.0 }.forEach { depths += it + controller.freezeInterpolation = true remainingRequest.value++ requestSamPrediction(it) { remainingRequest.value-- From 37c06c5df529629af4cae5a8261abce4cd59dcf1 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 7 Jun 2024 17:03:00 -0400 Subject: [PATCH 10/10] build: misc installer changes --- pom.xml | 5 +++-- src/packaging/linux/control | 8 ++++---- src/packaging/windows/jpackage.txt | 1 + 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/pom.xml b/pom.xml index cb9f0f27c..64677d0ad 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ org.janelia.saalfeldlab paintera - 1.2.4-SNAPSHOT + 1.3.0-SNAPSHOT Paintera New Era Painting and annotation tool @@ -66,7 +66,8 @@ org.janelia.saalfeldlab.paintera.Paintera Paintera - 1.2.4 + paintera + 1.3.0 javafx.base,javafx.controls,javafx.fxml,javafx.media,javafx.swing,javafx.web,javafx.graphics,java.naming,java.management,java.sql UTF-8 f918b6f9-8685-4b50-9fbd-9be7a1209249 diff --git a/src/packaging/linux/control b/src/packaging/linux/control index e28e4c0a0..89d2eb184 100644 --- a/src/packaging/linux/control +++ b/src/packaging/linux/control @@ -1,10 +1,10 @@ -Package: paintera -Version: 1.2.3 +Package: APPLICATION_PACKAGE +Version: APPLICATION_VERSION_WITH_RELEASE Section: misc Maintainer: Caleb Hulbert Priority: optional Architecture: amd64 Provides: paintera Description: Paintera is a general visualization tool for 3D volumetric data and proof-reading in segmentation/reconstruction with a primary focus on neuron reconstruction from electron micrographs in connectomics. -Depends: libbz2-1.0, libc6, libcap2, libcom-err2, libdbus-1-3, libexpat1, libgcc-s1, libgpg-error0, libkeyutils1, liblzma5, libpcre3, libselinux1, xdg-utils, zlib1g, libblosc1 -Installed-Size: 411870 +Depends: PACKAGE_DEFAULT_DEPENDENCIES PACKAGE_CUSTOM_DEPENDENCIES +Installed-Size: APPLICATION_INSTALLED_SIZE diff --git a/src/packaging/windows/jpackage.txt b/src/packaging/windows/jpackage.txt index dd28f1ae3..41c4f219e 100644 --- a/src/packaging/windows/jpackage.txt +++ b/src/packaging/windows/jpackage.txt @@ -3,6 +3,7 @@ --app-version ${app.version} --win-menu --win-menu-group ${windows.vendor} +--win-shortcut --vendor ${windows.vendor} --icon "${project.basedir}/img/icons/icon-draft.ico" --dest "${project.build.directory}/installer-${matrix.os}"