Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added video playback offset support for MCAP logs #155

Merged
merged 15 commits into from
Feb 28, 2024
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,6 @@ public void readVideoFrameNow(long timestamp)
readers.forEach(reader ->
{
reader.readFrameAtTimestamp(timestamp);
// LogTools.info("Reading frame at {} out of {}\n", timestamp / 1000000000.0, reader.getVideoLengthInSeconds());
});
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ public class FFMPEGVideoDataReader
private final FFmpegFrameGrabber frameGrabber;
private Frame currentFrame = null;
private final long maxVideoTimestamp;
private final AtomicLong playbackOffset = new AtomicLong();

private final AtomicLong currentTimestamp = new AtomicLong(-1);

Expand All @@ -37,16 +38,21 @@ public Frame getCurrentFrame()
return currentFrame;
}

public long readFrameAtTimestamp(long timestamp)
/**
* reads the frame in the video at timestamp + playbackOffset
*
* @param timestamp nanosecond timestamp to specify time from start of video
*/
public void readFrameAtTimestamp(long timestamp)
{
// NOTE: timestamp passed in is in nanoseconds
if (timestamp != currentTimestamp.get())
{
// timestamp / 1000L converts a nanosecond timestamp to the video timestamp in time_base units
long clampedTime = Math.min(maxVideoTimestamp, Math.max(0, timestamp / 1000L));
currentTimestamp.set(clampedTime);
currentTimestamp.set(Math.min(maxVideoTimestamp, Math.max(0, (timestamp / 1000L))));
try
{
long clampedTime = Math.min(maxVideoTimestamp, Math.max(0, currentTimestamp.get() + playbackOffset.get()));
frameGrabber.setVideoTimestamp(clampedTime);
currentFrame = frameGrabber.grabFrame();
}
Expand All @@ -55,14 +61,42 @@ public long readFrameAtTimestamp(long timestamp)
throw new RuntimeException(e);
}
}
return currentTimestamp.get();
}

public void readCurrentFrame()
{
try
{
long clampedTime = Math.min(maxVideoTimestamp, Math.max(0, currentTimestamp.get() + playbackOffset.get()));
frameGrabber.setVideoTimestamp(clampedTime);
currentFrame = frameGrabber.grabFrame();
}
catch (FrameGrabber.Exception e)
{
throw new RuntimeException(e);
}
}

public long getVideoLengthInSeconds()
{
return maxVideoTimestamp / 1000000;
}

public void setPlaybackOffset(long offset)
{
this.playbackOffset.set(offset);
}

public long getPlaybackOffset()
{
return this.playbackOffset.get();
}

public long getCurrentTimestamp()
{
return currentTimestamp.get();
}

public void shutdown()
{
try
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,17 +9,22 @@
import javafx.beans.property.SimpleBooleanProperty;
import javafx.beans.property.SimpleObjectProperty;
import javafx.geometry.Insets;
import javafx.geometry.Pos;
import javafx.geometry.Rectangle2D;
import javafx.scene.Node;
import javafx.scene.Scene;
import javafx.scene.control.Spinner;
import javafx.scene.image.Image;
import javafx.scene.image.ImageView;
import javafx.scene.input.MouseEvent;
import javafx.scene.layout.*;
import javafx.scene.layout.AnchorPane;
import javafx.scene.layout.Pane;
import javafx.scene.layout.StackPane;
import javafx.stage.Screen;
import javafx.stage.Stage;
import javafx.stage.Window;
import javafx.util.Duration;
import javafx.util.StringConverter;
import org.bytedeco.javacv.Frame;
import org.bytedeco.javacv.JavaFXFrameConverter;
import us.ihmc.log.LogTools;
Expand All @@ -32,7 +37,8 @@ public class FFMPEGVideoViewer
private static final double THUMBNAIL_HIGHLIGHT_SCALE = 1.05;

private final ImageView thumbnail = new ImageView();
private final StackPane thumbnailContainer = new StackPane(thumbnail);
private final Spinner<Double> offsetSpinner = new Spinner<>(-1.0e7, 1.0e7, 0.0, 0.001);
private final StackPane thumbnailContainer = new StackPane(thumbnail, offsetSpinner);
private final ImageView videoView = new ImageView();

private final BooleanProperty updateVideoView = new SimpleBooleanProperty(this, "updateVideoView", false);
Expand All @@ -47,23 +53,44 @@ public FFMPEGVideoViewer(Window owner, FFMPEGVideoDataReader reader, double defa
{
this.reader = reader;
this.defaultThumbnailSize = defaultThumbnailSize;

StackPane.setAlignment(offsetSpinner, Pos.BOTTOM_CENTER);
StackPane.setMargin(offsetSpinner, new Insets(0.0, 0.0, 5.0, 0.0));
offsetSpinner.setEditable(true);
offsetSpinner.getValueFactory().setConverter(new StringConverter<Double>()
{
@Override
public String toString(Double object)
{
return object.toString() + "s";
}

@Override
public Double fromString(String string)
{
return Double.valueOf(string.replaceAll("s", "").trim());
}
});

thumbnail.setPreserveRatio(true);
videoView.setPreserveRatio(true);
thumbnail.setFitWidth(defaultThumbnailSize);
thumbnail.setOnMouseEntered(e ->
{
Timeline timeline = new Timeline(new KeyFrame(Duration.seconds(0.1),
new KeyValue(thumbnail.fitWidthProperty(),
THUMBNAIL_HIGHLIGHT_SCALE * defaultThumbnailSize,
Interpolator.EASE_BOTH)));
timeline.playFromStart();
});
{
Timeline timeline = new Timeline(new KeyFrame(Duration.seconds(0.1),
new KeyValue(thumbnail.fitWidthProperty(),
THUMBNAIL_HIGHLIGHT_SCALE * defaultThumbnailSize,
Interpolator.EASE_BOTH)));
timeline.playFromStart();
});
thumbnail.setOnMouseExited(e ->
{
Timeline timeline = new Timeline(new KeyFrame(Duration.seconds(0.1),
new KeyValue(thumbnail.fitWidthProperty(), defaultThumbnailSize, Interpolator.EASE_BOTH)));
timeline.playFromStart();
});
{
Timeline timeline = new Timeline(new KeyFrame(Duration.seconds(0.1),
new KeyValue(thumbnail.fitWidthProperty(),
defaultThumbnailSize,
Interpolator.EASE_BOTH)));
timeline.playFromStart();
});

thumbnail.addEventHandler(MouseEvent.MOUSE_CLICKED, e ->
{
Expand Down Expand Up @@ -110,6 +137,12 @@ public FFMPEGVideoViewer(Window owner, FFMPEGVideoDataReader reader, double defa
stage.toFront();
stage.show();
});

offsetSpinner.valueProperty().addListener((observable, oldValue, newValue) ->
{
reader.setPlaybackOffset(Math.round(newValue * 1000000L));
reader.readCurrentFrame();
});
}

private static Pane createImageViewPane(ImageView imageView)
Expand Down Expand Up @@ -147,7 +180,8 @@ public void update()
try
{
currentImage = this.frameConverter.convert(currentFrame);
} catch (RuntimeException e)
}
catch (RuntimeException e)
{
LogTools.error("Frame has {} image channels", currentFrame.imageChannels);
}
Expand All @@ -166,7 +200,7 @@ public void update()

if (imageViewRootPane.get() != null)
{
imageViewRootPane.get().setPadding(new Insets(16,16,16,16));
imageViewRootPane.get().setPadding(new Insets(16, 16, 16, 16));
}
}
}
Expand Down
Loading