diff --git a/.gitignore b/.gitignore
index 42c698c..e32ed45 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,8 +5,10 @@ target/
.settings
*.iml
dependency-reduced-pom.xml
+.ipynb_checkpoints
# pip
+.venv
build/
dist/
.egg/
diff --git a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/BaseKernel.java b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/BaseKernel.java
index 0b50d0b..55be0d0 100644
--- a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/BaseKernel.java
+++ b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/BaseKernel.java
@@ -7,6 +7,7 @@
import org.dflib.jjava.jupyter.kernel.display.DisplayData;
import org.dflib.jjava.jupyter.kernel.display.Renderer;
import org.dflib.jjava.jupyter.kernel.display.common.Image;
+import org.dflib.jjava.jupyter.kernel.display.common.Swing;
import org.dflib.jjava.jupyter.kernel.display.common.Text;
import org.dflib.jjava.jupyter.kernel.display.common.Url;
import org.dflib.jjava.jupyter.kernel.history.HistoryEntry;
@@ -136,6 +137,7 @@ protected BaseKernel(
this.executionCount = new AtomicInteger(1);
Image.registerAll(this.renderer);
+ Swing.registerAll(this.renderer);
Url.registerAll(this.renderer);
Text.registerAll(this.renderer);
}
diff --git a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/BaseNotebookStatics.java b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/BaseNotebookStatics.java
index 8acdf85..1110fe2 100644
--- a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/BaseNotebookStatics.java
+++ b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/BaseNotebookStatics.java
@@ -1,8 +1,11 @@
package org.dflib.jjava.jupyter.kernel;
import org.dflib.jjava.jupyter.kernel.display.DisplayData;
+import org.dflib.jjava.jupyter.kernel.display.interactive.FxBridge;
+import org.dflib.jjava.jupyter.kernel.display.interactive.InteractiveSwing;
import org.dflib.jjava.jupyter.kernel.magic.UndefinedMagicException;
+import java.awt.Component;
import java.util.List;
import java.util.UUID;
@@ -79,6 +82,31 @@ public static String display(Object o, String... as) {
return id;
}
+ /**
+ * Display an AWT/Swing component or a JavaFX node as an interactive widget: it is streamed to the browser as a live
+ * image and mouse/keyboard events from the browser are dispatched back into it. Requires the JJava JupyterLab
+ * extension for interactivity; without it the output falls back to a static image snapshot.
+ *
+ *
A {@code javafx.scene.Node} is wrapped in a {@code JFXPanel} so it rides the same pipeline (JavaFX must be on
+ * the classpath — it is an optional, user-supplied dependency).
+ */
+ public static void displayInteractive(Object component) {
+ BaseKernel kernel = BaseKernel.notebookKernel();
+ if (component instanceof Component) {
+ InteractiveSwing.show(kernel, (Component) component);
+ } else if (FxBridge.isNode(component)) {
+ try {
+ InteractiveSwing.show(kernel, FxBridge.wrap(component));
+ } catch (ReflectiveOperationException e) {
+ throw new RuntimeException("Failed to embed JavaFX node (is JavaFX on the classpath?)", e);
+ }
+ } else {
+ throw new IllegalArgumentException(
+ "displayInteractive expects a java.awt.Component or a javafx.scene.Node, got: "
+ + (component == null ? "null" : component.getClass().getName()));
+ }
+ }
+
public static void updateDisplay(String id, Object o) {
DisplayData data = render(o);
BaseKernel.notebookKernel().getIO().display.updateDisplay(id, data);
diff --git a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/comm/CommManager.java b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/comm/CommManager.java
index f39b510..aed583f 100644
--- a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/comm/CommManager.java
+++ b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/comm/CommManager.java
@@ -94,11 +94,20 @@ public Comm unregisterComm(String id) {
* The latter may happen if the manager is not connected to the frontend
*/
public T openComm(String targetName, CommFactory factory) {
+ return openComm(targetName, new JsonObject(), factory);
+ }
+
+ /**
+ * A {@link #openComm(String, CommFactory)} variant that attaches an initial {@code data} payload to the
+ * {@code comm_open} message. The frontend target handler receives this as the open command's data, which is useful
+ * for correlating the new comm with something the frontend already knows about.
+ */
+ public T openComm(String targetName, JsonObject data, CommFactory factory) {
if (this.iopub == null)
return null;
String id = UUID.randomUUID().toString();
- CommOpenCommand content = new CommOpenCommand(id, targetName, new JsonObject());
+ CommOpenCommand content = new CommOpenCommand(id, targetName, data);
Message message = new Message<>(this.context, CommOpenCommand.MESSAGE_TYPE, content);
T comm = factory.produce(this, id, targetName, message);
diff --git a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/common/Swing.java b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/common/Swing.java
new file mode 100644
index 0000000..729d0c0
--- /dev/null
+++ b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/common/Swing.java
@@ -0,0 +1,82 @@
+package org.dflib.jjava.jupyter.kernel.display.common;
+
+import org.dflib.jjava.jupyter.kernel.display.RenderContext;
+import org.dflib.jjava.jupyter.kernel.display.Renderer;
+import org.dflib.jjava.jupyter.kernel.display.mime.MIMEType;
+
+import java.awt.Component;
+import java.awt.Container;
+import java.awt.Dimension;
+import java.awt.Graphics2D;
+import java.awt.RenderingHints;
+import java.awt.image.BufferedImage;
+
+/**
+ * Renders an {@link java.awt.Component AWT/Swing component} as an image by painting it off-screen into a
+ * {@link BufferedImage}, which is then encoded by {@link Image}. This works without a display (headless) since no native
+ * window is ever created.
+ */
+public class Swing {
+ public static final MIMEType PNG = MIMEType.IMAGE_PNG;
+ public static final MIMEType JPEG = MIMEType.IMAGE_JPEG;
+ public static final MIMEType GIF = MIMEType.IMAGE_GIF;
+
+ public static void registerAll(Renderer renderer) {
+ renderer.createRegistration(Component.class)
+ .preferring(PNG)
+ .supporting(JPEG, GIF)
+ .register(Swing::renderComponent);
+ }
+
+ public static void renderComponent(Component component, RenderContext context) {
+ Image.renderImage(snapshot(component), context);
+ }
+
+ /**
+ * Paint a component using its current size, falling back to its {@link Component#getPreferredSize() preferred size}
+ * when it has not been laid out yet (which is the usual case for a component that was never added to a window).
+ */
+ public static BufferedImage snapshot(Component component) {
+ Dimension size = component.getSize();
+ if (size.width <= 0 || size.height <= 0) {
+ size = component.getPreferredSize();
+ }
+ return snapshot(component, size.width, size.height);
+ }
+
+ public static BufferedImage snapshot(Component component, int width, int height) {
+ width = Math.max(width, 1);
+ height = Math.max(height, 1);
+
+ component.setSize(width, height);
+ layoutTree(component);
+
+ BufferedImage image = new BufferedImage(width, height, BufferedImage.TYPE_INT_ARGB);
+ Graphics2D g = image.createGraphics();
+ try {
+ g.setRenderingHint(RenderingHints.KEY_ANTIALIASING, RenderingHints.VALUE_ANTIALIAS_ON);
+ g.setRenderingHint(RenderingHints.KEY_TEXT_ANTIALIASING, RenderingHints.VALUE_TEXT_ANTIALIAS_ON);
+ // print() rather than paint(): it bypasses Swing's double-buffering, which is the right behaviour for
+ // off-screen rendering of a component that has no native peer.
+ component.print(g);
+ } finally {
+ g.dispose();
+ }
+ return image;
+ }
+
+ /**
+ * A component that was never realized in a window is not laid out, so its children have zero bounds and would not
+ * paint (nor hit-test). Forcing a layout pass over the whole subtree gives every child a position and size.
+ */
+ public static void layoutTree(Component component) {
+ synchronized (component.getTreeLock()) {
+ component.doLayout();
+ if (component instanceof Container) {
+ for (Component child : ((Container) component).getComponents()) {
+ layoutTree(child);
+ }
+ }
+ }
+ }
+}
diff --git a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/interactive/FxBridge.java b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/interactive/FxBridge.java
new file mode 100644
index 0000000..ff83018
Binary files /dev/null and b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/interactive/FxBridge.java differ
diff --git a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/interactive/InteractiveSwing.java b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/interactive/InteractiveSwing.java
new file mode 100644
index 0000000..eb8242a
--- /dev/null
+++ b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/interactive/InteractiveSwing.java
@@ -0,0 +1,91 @@
+package org.dflib.jjava.jupyter.kernel.display.interactive;
+
+import com.google.gson.JsonObject;
+import org.dflib.jjava.jupyter.kernel.BaseKernel;
+import org.dflib.jjava.jupyter.kernel.comm.CommManager;
+import org.dflib.jjava.jupyter.kernel.display.DisplayData;
+import org.dflib.jjava.jupyter.kernel.display.Renderer;
+import org.dflib.jjava.jupyter.kernel.display.mime.MIMEType;
+
+import java.awt.Component;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.UUID;
+
+/**
+ * Entry point for embedding an interactive {@link Component} in a notebook.
+ *
+ * It emits an output carrying the {@link #MIME_TYPE custom MIME type} so the JupyterLab extension can mount a canvas,
+ * then opens a {@value #TARGET_NAME} comm whose {@code comm_open} data carries a {@code token}. The extension matches
+ * that token to the canvas it just created and starts exchanging frames/events with the
+ * {@link InteractiveSwingSession}.
+ *
+ *
The same output also carries a static {@code image/png} snapshot, so a frontend without the extension
+ * (or any non-Lab frontend) degrades gracefully to a still image instead of a placeholder message.
+ */
+public final class InteractiveSwing {
+
+ /** Comm target the JupyterLab extension registers a handler for. */
+ public static final String TARGET_NAME = "jjava.swing.v1";
+
+ /** MIME type of the placeholder output the JupyterLab extension renders into a canvas. */
+ public static final String MIME_TYPE = "application/vnd.jjava.swing.v1+json";
+
+ private InteractiveSwing() {
+ }
+
+ public static InteractiveSwingSession show(BaseKernel kernel, Component component) {
+ // A JFXPanel needs a window ancestor (for its GraphicsConfiguration) to map input; no-op for pure Swing.
+ FxBridge.ensureScreened(component);
+
+ String token = UUID.randomUUID().toString();
+ InteractiveSwingSession session = new InteractiveSwingSession(token, component);
+
+ DisplayData data = buildDisplayData(
+ kernel.getRenderer(), token, session.getWidth(), session.getHeight(), component);
+ kernel.display(data);
+
+ CommManager commManager = kernel.getCommManager();
+ JsonObject openData = new JsonObject();
+ openData.addProperty("token", token);
+ openData.addProperty("width", session.getWidth());
+ openData.addProperty("height", session.getHeight());
+
+ InteractiveSwingComm comm = commManager.openComm(
+ TARGET_NAME, openData,
+ (manager, id, target, message) -> new InteractiveSwingComm(manager, id, target, session));
+
+ if (comm != null) {
+ session.bind(comm);
+ }
+
+ return session;
+ }
+
+ /**
+ * Build the output bundle: the interactive {@link #MIME_TYPE} payload, plus a static {@code image/png} snapshot as a
+ * fallback for frontends without the extension, plus a {@code text/plain} description.
+ */
+ static DisplayData buildDisplayData(Renderer renderer, String token, int width, int height, Component component) {
+ Map payload = new LinkedHashMap<>();
+ payload.put("token", token);
+ payload.put("width", width);
+ payload.put("height", height);
+
+ DisplayData data = new DisplayData();
+ data.putData(MIME_TYPE, payload);
+
+ try {
+ Object png = renderer.renderAs(component, MIMEType.IMAGE_PNG.toString()).getData(MIMEType.IMAGE_PNG);
+ if (png != null) {
+ data.putData(MIMEType.IMAGE_PNG, png);
+ }
+ } catch (Exception e) {
+ // The static snapshot is best-effort; the interactive payload is still emitted.
+ }
+
+ data.putText("Interactive Swing component — install the jjava-swing JupyterLab extension for interactivity "
+ + "(a static snapshot is shown otherwise).");
+ return data;
+ }
+}
diff --git a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/interactive/InteractiveSwingComm.java b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/interactive/InteractiveSwingComm.java
new file mode 100644
index 0000000..0260fb4
--- /dev/null
+++ b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/interactive/InteractiveSwingComm.java
@@ -0,0 +1,36 @@
+package org.dflib.jjava.jupyter.kernel.display.interactive;
+
+import org.dflib.jjava.jupyter.kernel.comm.Comm;
+import org.dflib.jjava.jupyter.kernel.comm.CommManager;
+import org.dflib.jjava.jupyter.messages.Message;
+import org.dflib.jjava.jupyter.messages.comm.CommCloseCommand;
+import org.dflib.jjava.jupyter.messages.comm.CommMsgCommand;
+
+/**
+ * The kernel side of the {@value InteractiveSwing#TARGET_NAME} comm. It is opened by the kernel (see
+ * {@link InteractiveSwing#show}) and forwards frontend messages to its {@link InteractiveSwingSession}, tearing the
+ * session down when the comm closes.
+ */
+class InteractiveSwingComm extends Comm {
+
+ private final InteractiveSwingSession session;
+
+ InteractiveSwingComm(CommManager manager, String id, String targetName, InteractiveSwingSession session) {
+ super(manager, id, targetName);
+ this.session = session;
+ }
+
+ @Override
+ protected void onMessage(Message message) {
+ if (session != null) {
+ session.handleMessage(message.getContent().getData());
+ }
+ }
+
+ @Override
+ protected void onClose(Message closeMessage, boolean sending) {
+ if (session != null) {
+ session.dispose();
+ }
+ }
+}
diff --git a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/interactive/InteractiveSwingSession.java b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/interactive/InteractiveSwingSession.java
new file mode 100644
index 0000000..31252cb
--- /dev/null
+++ b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/interactive/InteractiveSwingSession.java
@@ -0,0 +1,430 @@
+package org.dflib.jjava.jupyter.kernel.display.interactive;
+
+import com.google.gson.JsonObject;
+import org.dflib.jjava.jupyter.kernel.comm.Comm;
+import org.dflib.jjava.jupyter.kernel.display.common.Swing;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.imageio.ImageIO;
+import javax.swing.SwingUtilities;
+import java.awt.AlphaComposite;
+import java.awt.Component;
+import java.awt.Dimension;
+import java.awt.Graphics2D;
+import java.awt.KeyboardFocusManager;
+import java.awt.Point;
+import java.awt.RenderingHints;
+import java.awt.event.FocusEvent;
+import java.awt.event.InputEvent;
+import java.awt.event.KeyEvent;
+import java.awt.event.MouseEvent;
+import java.awt.event.MouseWheelEvent;
+import java.awt.image.BufferedImage;
+import java.awt.image.DataBufferInt;
+import java.io.ByteArrayOutputStream;
+import java.util.Collections;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * Hosts a single {@link Component} off-screen and bridges it to a browser canvas: it streams PNG frames of the component
+ * over a {@link Comm} and dispatches the mouse/keyboard events that arrive back from the frontend into the component's
+ * Swing event handling. No native window is ever created, so this works headless.
+ *
+ * All component manipulation (layout, painting, event dispatch) happens on the AWT event dispatch thread (EDT).
+ */
+public class InteractiveSwingSession {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(InteractiveSwingSession.class);
+
+ /** Frames are coalesced to roughly this cadence so a burst of repaints produces one frame. */
+ private static final long FRAME_INTERVAL_MS = 33;
+
+ private static final ScheduledExecutorService SCHEDULER = Executors.newSingleThreadScheduledExecutor(runnable -> {
+ Thread thread = new Thread(runnable, "jjava-swing-frames");
+ thread.setDaemon(true);
+ return thread;
+ });
+
+ private final String token;
+ private final Component component;
+
+ private volatile int width;
+ private volatile int height;
+ private volatile Comm comm;
+ private volatile boolean disposed;
+
+ private final AtomicBoolean framePending = new AtomicBoolean(false);
+
+ // Reused across frames to avoid per-frame allocation; touched only on the EDT during painting.
+ private BufferedImage frameImage;
+ // Layout is expensive, so only redo it when something actually changed (resize or an invalidated component) rather
+ // than on every repaint. Touched only on the EDT.
+ private boolean needsLayout = true;
+ // Hash of the last frame actually streamed; lets us drop frames whose pixels are unchanged. Scheduler-thread only.
+ private long lastFrameHash;
+
+ // Accessed only on the EDT. The component that should receive keyboard events, tracked from the last mouse press.
+ private Component focusOwner;
+
+ InteractiveSwingSession(String token, Component component) {
+ this.token = token;
+ this.component = component;
+
+ Dimension size = component.getSize();
+ if (size.width <= 0 || size.height <= 0) {
+ size = component.getPreferredSize();
+ }
+ // Fall back to a usable default when the component has neither a size nor a preferred size (e.g. a bare
+ // JFXPanel, which reports 0x0 until its scene drives a size).
+ this.width = size.width > 0 ? size.width : 400;
+ this.height = size.height > 0 ? size.height : 300;
+
+ // Size and lay out eagerly so the component (and its children) have real bounds for hit-testing the very first
+ // events, rather than waiting for the first frame to be rendered.
+ component.setSize(width, height);
+ Swing.layoutTree(component);
+
+ SwingRepaintManager.install();
+ SwingRepaintManager.register(component, this);
+ }
+
+ public String getToken() {
+ return token;
+ }
+
+ public int getWidth() {
+ return width;
+ }
+
+ public int getHeight() {
+ return height;
+ }
+
+ /** Bind the comm opened by the frontend and push the first frame. */
+ void bind(Comm comm) {
+ this.comm = comm;
+ requestFrame();
+ }
+
+ /** Handle a message coming from the frontend canvas. */
+ void handleMessage(JsonObject data) {
+ if (data == null || disposed) {
+ return;
+ }
+ String type = optString(data, "type", "");
+ switch (type) {
+ case "mouse":
+ dispatchOnEdt(() -> dispatchMouse(data));
+ break;
+ case "key":
+ dispatchOnEdt(() -> dispatchKey(data));
+ break;
+ case "resize":
+ dispatchOnEdt(() -> resize(data));
+ break;
+ case "refresh":
+ requestFrame();
+ break;
+ default:
+ break;
+ }
+ }
+
+ /**
+ * Run an event-applying action on the EDT, then request a frame. A failing handler (e.g. a toolkit that can't map a
+ * synthetic event off-screen) is logged and swallowed so it never kills the EDT or the session.
+ */
+ private void dispatchOnEdt(Runnable action) {
+ SwingUtilities.invokeLater(() -> {
+ try {
+ action.run();
+ } catch (Throwable t) {
+ LOGGER.debug("Interactive event dispatch failed", t);
+ }
+ requestFrame();
+ });
+ }
+
+ /** Schedule a (coalesced) frame to be rendered and streamed. Safe to call from any thread. */
+ void requestFrame() {
+ if (disposed) {
+ return;
+ }
+ if (framePending.compareAndSet(false, true)) {
+ SCHEDULER.schedule(this::renderAndSend, FRAME_INTERVAL_MS, TimeUnit.MILLISECONDS);
+ }
+ }
+
+ /** Flag that the component tree must be laid out again before the next frame (e.g. a child was invalidated). */
+ void markNeedsLayout() {
+ needsLayout = true;
+ }
+
+ void dispose() {
+ disposed = true;
+ SwingRepaintManager.unregister(component);
+ }
+
+ // --- frame streaming -------------------------------------------------------------------------------------------
+
+ private void renderAndSend() {
+ framePending.set(false);
+ Comm target = this.comm;
+ if (target == null || target.isClosed() || disposed) {
+ return;
+ }
+ try {
+ BufferedImage image = paintFrame();
+ long hash = framePixelHash(image);
+ if (hash == lastFrameHash) {
+ // Nothing changed since the last streamed frame (e.g. a timer called repaint() without altering
+ // anything) — skip the PNG encode and the send entirely.
+ return;
+ }
+ lastFrameHash = hash;
+
+ ByteArrayOutputStream out = new ByteArrayOutputStream(32 * 1024);
+ ImageIO.write(image, "png", out);
+
+ JsonObject message = new JsonObject();
+ message.addProperty("type", "frame");
+ message.addProperty("w", image.getWidth());
+ message.addProperty("h", image.getHeight());
+ target.send(message, null, Collections.singletonList(out.toByteArray()));
+ } catch (Exception e) {
+ LOGGER.debug("Failed to render/stream interactive Swing frame", e);
+ }
+ }
+
+ private BufferedImage paintFrame() throws Exception {
+ BufferedImage[] holder = new BufferedImage[1];
+ Runnable paint = () -> holder[0] = paintOnEdt();
+ if (SwingUtilities.isEventDispatchThread()) {
+ paint.run();
+ } else {
+ SwingUtilities.invokeAndWait(paint);
+ }
+ return holder[0];
+ }
+
+ /** Paint the component into the reused buffer, laying out only when needed. Must run on the EDT. */
+ private BufferedImage paintOnEdt() {
+ int w = width;
+ int h = height;
+ if (frameImage == null || frameImage.getWidth() != w || frameImage.getHeight() != h) {
+ frameImage = new BufferedImage(w, h, BufferedImage.TYPE_INT_ARGB);
+ needsLayout = true;
+ }
+ if (needsLayout) {
+ component.setSize(w, h);
+ Swing.layoutTree(component);
+ needsLayout = false;
+ }
+
+ Graphics2D g = frameImage.createGraphics();
+ try {
+ // Clear the reused buffer to fully transparent before repainting.
+ g.setComposite(AlphaComposite.Clear);
+ g.fillRect(0, 0, w, h);
+ g.setComposite(AlphaComposite.SrcOver);
+ g.setRenderingHint(RenderingHints.KEY_ANTIALIASING, RenderingHints.VALUE_ANTIALIAS_ON);
+ g.setRenderingHint(RenderingHints.KEY_TEXT_ANTIALIASING, RenderingHints.VALUE_TEXT_ANTIALIAS_ON);
+ component.print(g);
+ } finally {
+ g.dispose();
+ }
+ return frameImage;
+ }
+
+ private static long framePixelHash(BufferedImage image) {
+ int[] pixels = ((DataBufferInt) image.getRaster().getDataBuffer()).getData();
+ long hash = 1125899906842597L;
+ for (int pixel : pixels) {
+ hash = 31 * hash + pixel;
+ }
+ return hash;
+ }
+
+ // --- event dispatch (EDT) --------------------------------------------------------------------------------------
+
+ private void resize(JsonObject data) {
+ this.width = Math.max(1, optInt(data, "width", width));
+ this.height = Math.max(1, optInt(data, "height", height));
+ component.setSize(width, height);
+ needsLayout = true;
+ }
+
+ private void dispatchMouse(JsonObject data) {
+ String action = optString(data, "action", "");
+ int x = optInt(data, "x", 0);
+ int y = optInt(data, "y", 0);
+ int button = optInt(data, "button", 1);
+ long when = System.currentTimeMillis();
+ int modifiers = mouseModifiers(data, action, button);
+
+ Component target = componentAt(x, y);
+ Point p = SwingUtilities.convertPoint(component, x, y, target);
+
+ if ("wheel".equals(action)) {
+ int rotation = optInt(data, "deltaY", 0) >= 0 ? 1 : -1;
+ MouseWheelEvent wheel = new MouseWheelEvent(target, MouseEvent.MOUSE_WHEEL, when, modifiers,
+ p.x, p.y, 0, false, MouseWheelEvent.WHEEL_UNIT_SCROLL, 3, rotation);
+ target.dispatchEvent(wheel);
+ return;
+ }
+
+ int id;
+ switch (action) {
+ case "down": id = MouseEvent.MOUSE_PRESSED; break;
+ case "up": id = MouseEvent.MOUSE_RELEASED; break;
+ case "click": id = MouseEvent.MOUSE_CLICKED; break;
+ case "move": id = MouseEvent.MOUSE_MOVED; break;
+ case "drag": id = MouseEvent.MOUSE_DRAGGED; break;
+ default: return;
+ }
+
+ if (id == MouseEvent.MOUSE_PRESSED) {
+ focusOn(target);
+ }
+
+ boolean buttoned = id != MouseEvent.MOUSE_MOVED && id != MouseEvent.MOUSE_DRAGGED;
+ int clickCount = buttoned ? Math.max(1, optInt(data, "clickCount", 1)) : 0;
+ int awtButton = buttoned ? awtButton(button) : MouseEvent.NOBUTTON;
+
+ MouseEvent event = new MouseEvent(target, id, when, modifiers, p.x, p.y, clickCount, false, awtButton);
+ target.dispatchEvent(event);
+ }
+
+ private void dispatchKey(JsonObject data) {
+ Component target = focusOwner != null ? focusOwner : component;
+
+ // JavaFX: AWT key forwarding through an off-screen JFXPanel doesn't work (no real keyboard focus), so deliver
+ // the key straight into the FX scene's focused node instead.
+ if (FxBridge.isJFXPanel(target) && FxBridge.injectKey(target, data)) {
+ return;
+ }
+
+ String action = optString(data, "action", "");
+ long when = System.currentTimeMillis();
+ int modifiers = keyboardModifiers(data);
+
+ if ("press".equals(action)) {
+ String text = optString(data, "char", "");
+ if (text.isEmpty()) {
+ return;
+ }
+ KeyEvent typed = new KeyEvent(target, KeyEvent.KEY_TYPED, when, modifiers,
+ KeyEvent.VK_UNDEFINED, text.charAt(0));
+ redispatchKey(target, typed);
+ return;
+ }
+
+ int id;
+ switch (action) {
+ case "down": id = KeyEvent.KEY_PRESSED; break;
+ case "up": id = KeyEvent.KEY_RELEASED; break;
+ default: return;
+ }
+ int keyCode = awtKeyCode(optInt(data, "keyCode", 0));
+ String text = optString(data, "char", "");
+ char keyChar = text.isEmpty() ? KeyEvent.CHAR_UNDEFINED : text.charAt(0);
+ KeyEvent event = new KeyEvent(target, id, when, modifiers, keyCode, keyChar);
+ redispatchKey(target, event);
+ }
+
+ /**
+ * Deliver a key event via {@link KeyboardFocusManager#redispatchEvent}, which bypasses the focus manager's
+ * type-ahead handling. A plain {@code dispatchEvent} would be swallowed here because no Java window holds focus
+ * (the browser does), so the key would never reach the component's key bindings.
+ */
+ private static void redispatchKey(Component target, KeyEvent event) {
+ KeyboardFocusManager.getCurrentKeyboardFocusManager().redispatchEvent(target, event);
+ }
+
+ /**
+ * Move synthetic focus to {@code target}: there is no real keyboard-focus manager off-screen, so we hand-deliver
+ * FOCUS_LOST/FOCUS_GAINED. This activates a Swing text caret, and — for a JFXPanel — activates the embedded JavaFX
+ * scene so its focused node accepts typing.
+ */
+ private void focusOn(Component target) {
+ if (focusOwner == target) {
+ return;
+ }
+ if (focusOwner != null) {
+ focusOwner.dispatchEvent(new FocusEvent(focusOwner, FocusEvent.FOCUS_LOST));
+ }
+ focusOwner = target;
+ target.dispatchEvent(new FocusEvent(target, FocusEvent.FOCUS_GAINED));
+ }
+
+ private Component componentAt(int x, int y) {
+ Component deepest = SwingUtilities.getDeepestComponentAt(component, x, y);
+ return deepest != null ? deepest : component;
+ }
+
+ private int mouseModifiers(JsonObject data, String action, int button) {
+ int modifiers = keyboardModifiers(data);
+ boolean buttonInvolved = "down".equals(action) || "up".equals(action)
+ || "drag".equals(action) || "click".equals(action);
+ if (buttonInvolved) {
+ // Keep the button-down mask set even on release so SwingUtilities.isLeftMouseButton(e) and friends — which
+ // gate button activation — recognise the gesture.
+ modifiers |= buttonDownMask(button);
+ }
+ return modifiers;
+ }
+
+ private static int keyboardModifiers(JsonObject data) {
+ int modifiers = 0;
+ if (optBool(data, "shift")) modifiers |= InputEvent.SHIFT_DOWN_MASK;
+ if (optBool(data, "ctrl")) modifiers |= InputEvent.CTRL_DOWN_MASK;
+ if (optBool(data, "alt")) modifiers |= InputEvent.ALT_DOWN_MASK;
+ if (optBool(data, "meta")) modifiers |= InputEvent.META_DOWN_MASK;
+ return modifiers;
+ }
+
+ private static int buttonDownMask(int button) {
+ switch (button) {
+ case 2: return InputEvent.BUTTON2_DOWN_MASK;
+ case 3: return InputEvent.BUTTON3_DOWN_MASK;
+ default: return InputEvent.BUTTON1_DOWN_MASK;
+ }
+ }
+
+ private static int awtButton(int button) {
+ switch (button) {
+ case 2: return MouseEvent.BUTTON2;
+ case 3: return MouseEvent.BUTTON3;
+ default: return MouseEvent.BUTTON1;
+ }
+ }
+
+ /**
+ * Browser {@code KeyboardEvent.keyCode} values mostly coincide with AWT virtual key codes for letters, digits,
+ * arrows, etc. The notable exception is Enter (13 in the browser, {@link KeyEvent#VK_ENTER} 10 in AWT).
+ */
+ private static int awtKeyCode(int browserKeyCode) {
+ if (browserKeyCode == 13) {
+ return KeyEvent.VK_ENTER;
+ }
+ return browserKeyCode;
+ }
+
+ // --- json helpers ----------------------------------------------------------------------------------------------
+
+ private static String optString(JsonObject data, String key, String fallback) {
+ return data.has(key) && !data.get(key).isJsonNull() ? data.get(key).getAsString() : fallback;
+ }
+
+ private static int optInt(JsonObject data, String key, int fallback) {
+ return data.has(key) && !data.get(key).isJsonNull() ? data.get(key).getAsInt() : fallback;
+ }
+
+ private static boolean optBool(JsonObject data, String key) {
+ return data.has(key) && !data.get(key).isJsonNull() && data.get(key).getAsBoolean();
+ }
+}
diff --git a/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/interactive/SwingRepaintManager.java b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/interactive/SwingRepaintManager.java
new file mode 100644
index 0000000..35bd08e
--- /dev/null
+++ b/jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/interactive/SwingRepaintManager.java
@@ -0,0 +1,73 @@
+package org.dflib.jjava.jupyter.kernel.display.interactive;
+
+import javax.swing.JComponent;
+import javax.swing.RepaintManager;
+import java.awt.Component;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * A {@link RepaintManager} that, in addition to the default behaviour, notices when a component belonging to an
+ * {@link InteractiveSwingSession} is repainted and asks that session to stream a fresh frame. This is what lets
+ * asynchronous changes (animation timers, model updates, etc.) reach the browser, not just the repaints that follow an
+ * input event.
+ *
+ *
It is installed globally (one per JVM/AppContext) but delegates to the superclass for everything, so ordinary Swing
+ * components that are not hosted in a session behave exactly as before.
+ */
+class SwingRepaintManager extends RepaintManager {
+
+ private static volatile boolean installed = false;
+
+ // Maps each session root component to its session. A repainted component is matched to a session by walking up its
+ // parent chain until a registered root is found.
+ private static final Map ROOTS = new ConcurrentHashMap<>();
+
+ static synchronized void install() {
+ if (!installed) {
+ RepaintManager.setCurrentManager(new SwingRepaintManager());
+ installed = true;
+ }
+ }
+
+ static void register(Component root, InteractiveSwingSession session) {
+ ROOTS.put(root, session);
+ }
+
+ static void unregister(Component root) {
+ ROOTS.remove(root);
+ }
+
+ private static InteractiveSwingSession sessionFor(Component component) {
+ if (ROOTS.isEmpty()) {
+ return null;
+ }
+ for (Component current = component; current != null; current = current.getParent()) {
+ InteractiveSwingSession session = ROOTS.get(current);
+ if (session != null) {
+ return session;
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public void addDirtyRegion(JComponent c, int x, int y, int w, int h) {
+ super.addDirtyRegion(c, x, y, w, h);
+ InteractiveSwingSession session = sessionFor(c);
+ if (session != null) {
+ session.requestFrame();
+ }
+ }
+
+ @Override
+ public void addInvalidComponent(JComponent invalidComponent) {
+ super.addInvalidComponent(invalidComponent);
+ InteractiveSwingSession session = sessionFor(invalidComponent);
+ if (session != null) {
+ // An invalidated component means the layout may have changed, so the next frame must re-lay-out.
+ session.markNeedsLayout();
+ session.requestFrame();
+ }
+ }
+}
diff --git a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/common/SwingTest.java b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/common/SwingTest.java
new file mode 100644
index 0000000..a3fbbe3
--- /dev/null
+++ b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/common/SwingTest.java
@@ -0,0 +1,94 @@
+package org.dflib.jjava.jupyter.kernel.display.common;
+
+import org.dflib.jjava.jupyter.kernel.display.DisplayData;
+import org.dflib.jjava.jupyter.kernel.display.Renderer;
+import org.dflib.jjava.jupyter.kernel.display.mime.MIMEType;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import javax.imageio.ImageIO;
+import javax.swing.JButton;
+import javax.swing.JLabel;
+import javax.swing.JPanel;
+import java.awt.BorderLayout;
+import java.awt.Dimension;
+import java.awt.image.BufferedImage;
+import java.io.ByteArrayInputStream;
+import java.util.Base64;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class SwingTest {
+
+ static {
+ // Snapshotting works with or without a display, but force headless so the suite behaves identically on CI
+ // servers and developer machines.
+ System.setProperty("java.awt.headless", "true");
+ }
+
+ private Renderer renderer;
+
+ @BeforeEach
+ public void setUp() {
+ this.renderer = new Renderer();
+ Swing.registerAll(this.renderer);
+ }
+
+ @Test
+ public void rendersComponentAsPng() throws Exception {
+ JButton button = new JButton("Click me");
+ button.setSize(120, 40);
+
+ DisplayData data = this.renderer.render(button);
+
+ BufferedImage image = decodePng(data);
+ assertEquals(120, image.getWidth());
+ assertEquals(40, image.getHeight());
+ }
+
+ @Test
+ public void fallsBackToPreferredSizeWhenNotSized() throws Exception {
+ JLabel label = new JLabel("hello");
+ Dimension preferred = label.getPreferredSize();
+
+ DisplayData data = this.renderer.render(label);
+
+ BufferedImage image = decodePng(data);
+ assertEquals(preferred.width, image.getWidth());
+ assertEquals(preferred.height, image.getHeight());
+ }
+
+ @Test
+ public void snapshotHonoursRequestedSize() {
+ JPanel panel = new JPanel();
+
+ BufferedImage image = Swing.snapshot(panel, 200, 100);
+
+ assertEquals(200, image.getWidth());
+ assertEquals(100, image.getHeight());
+ }
+
+ @Test
+ public void laysOutChildrenBeforePainting() {
+ // A nested component that was never realized in a window: layout() must give the child non-zero bounds.
+ JPanel panel = new JPanel(new BorderLayout());
+ JButton child = new JButton("child");
+ panel.add(child, BorderLayout.CENTER);
+
+ Swing.snapshot(panel, 300, 80);
+
+ assertTrue(child.getWidth() > 0, "child should have been laid out with a non-zero width");
+ assertTrue(child.getHeight() > 0, "child should have been laid out with a non-zero height");
+ }
+
+ private static BufferedImage decodePng(DisplayData data) throws Exception {
+ Object png = data.getData(MIMEType.IMAGE_PNG);
+ assertNotNull(png, "expected an image/png rendering");
+ byte[] bytes = Base64.getDecoder().decode(png.toString());
+ BufferedImage image = ImageIO.read(new ByteArrayInputStream(bytes));
+ assertNotNull(image, "image/png payload should decode to an image");
+ return image;
+ }
+}
diff --git a/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/interactive/InteractiveSwingSessionTest.java b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/interactive/InteractiveSwingSessionTest.java
new file mode 100644
index 0000000..a8ff87e
--- /dev/null
+++ b/jjava-jupyter/src/test/java/org/dflib/jjava/jupyter/kernel/display/interactive/InteractiveSwingSessionTest.java
@@ -0,0 +1,200 @@
+package org.dflib.jjava.jupyter.kernel.display.interactive;
+
+import com.google.gson.JsonObject;
+import org.dflib.jjava.jupyter.kernel.comm.Comm;
+import org.dflib.jjava.jupyter.kernel.comm.CommManager;
+import org.dflib.jjava.jupyter.kernel.display.DisplayData;
+import org.dflib.jjava.jupyter.kernel.display.Renderer;
+import org.dflib.jjava.jupyter.kernel.display.common.Swing;
+import org.dflib.jjava.jupyter.kernel.display.mime.MIMEType;
+import org.junit.jupiter.api.Test;
+
+import javax.swing.JButton;
+import javax.swing.JLabel;
+import javax.swing.JTextField;
+import javax.swing.SwingUtilities;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class InteractiveSwingSessionTest {
+
+ static {
+ System.setProperty("java.awt.headless", "true");
+ }
+
+ /** A {@link Comm} that records frames instead of putting them on a socket. */
+ private static class RecordingComm extends Comm {
+ final List frames = new ArrayList<>();
+ final CountDownLatch firstFrame = new CountDownLatch(1);
+
+ RecordingComm() {
+ super(new CommManager(), "test-comm", InteractiveSwing.TARGET_NAME);
+ }
+
+ @Override
+ public void send(JsonObject data, Map metadata, List blobs) {
+ if (blobs != null && !blobs.isEmpty()) {
+ frames.add(blobs.get(0));
+ firstFrame.countDown();
+ }
+ }
+
+ @Override
+ protected void onMessage(org.dflib.jjava.jupyter.messages.Message message) {
+ }
+
+ @Override
+ protected void onClose(org.dflib.jjava.jupyter.messages.Message closeMessage, boolean sending) {
+ }
+ }
+
+ private static JsonObject key(String action, String ch) {
+ JsonObject o = new JsonObject();
+ o.addProperty("type", "key");
+ o.addProperty("action", action);
+ o.addProperty("char", ch);
+ o.addProperty("keyCode", ch.isEmpty() ? 0 : Character.toUpperCase(ch.charAt(0)));
+ return o;
+ }
+
+ private static void typeChar(InteractiveSwingSession session, String ch) {
+ session.handleMessage(key("down", ch));
+ session.handleMessage(key("press", ch));
+ session.handleMessage(key("up", ch));
+ }
+
+ private static JsonObject mouse(String action, int x, int y) {
+ JsonObject o = new JsonObject();
+ o.addProperty("type", "mouse");
+ o.addProperty("action", action);
+ o.addProperty("x", x);
+ o.addProperty("y", y);
+ o.addProperty("button", 1);
+ return o;
+ }
+
+ private static void flushEventQueue() throws Exception {
+ // The EDT is single-threaded and FIFO, so anything queued by handleMessage() has run once this returns.
+ SwingUtilities.invokeAndWait(() -> { });
+ }
+
+ @Test
+ public void streamsAFrameOnBind() throws Exception {
+ JButton button = new JButton("Hi");
+ button.setSize(120, 40);
+ InteractiveSwingSession session = new InteractiveSwingSession("tok-1", button);
+ RecordingComm comm = new RecordingComm();
+
+ session.bind(comm);
+
+ assertTrue(comm.firstFrame.await(2, TimeUnit.SECONDS), "expected an initial frame to be streamed");
+ assertTrue(comm.frames.get(0).length > 0, "frame should be non-empty PNG bytes");
+ }
+
+ @Test
+ public void firesActionListenerOnClickGesture() throws Exception {
+ AtomicInteger clicks = new AtomicInteger();
+ JButton button = new JButton("Click");
+ button.addActionListener(e -> clicks.incrementAndGet());
+
+ InteractiveSwingSession session = new InteractiveSwingSession("tok-2", button);
+ session.bind(new RecordingComm());
+
+ // Press then release inside the button's bounds: the button fires its action on release.
+ session.handleMessage(mouse("down", 30, 15));
+ session.handleMessage(mouse("up", 30, 15));
+ flushEventQueue();
+
+ assertEquals(1, clicks.get(), "the button's ActionListener should have fired once");
+ }
+
+ @Test
+ public void skipsUnchangedFramesButStreamsChangedOnes() throws Exception {
+ JLabel label = new JLabel("v1");
+ label.setSize(140, 30);
+ InteractiveSwingSession session = new InteractiveSwingSession("tok-dup", label);
+ RecordingComm comm = new RecordingComm();
+
+ session.bind(comm);
+ assertTrue(comm.firstFrame.await(2, TimeUnit.SECONDS));
+ int afterFirst = comm.frames.size();
+
+ // Re-request with no change: the identical frame must be suppressed (no encode/send).
+ session.requestFrame();
+ Thread.sleep(200);
+ assertEquals(afterFirst, comm.frames.size(), "an unchanged frame should be skipped");
+
+ // Change the visible content: a new frame must be streamed.
+ SwingUtilities.invokeAndWait(() -> label.setText("v2 has changed"));
+ session.requestFrame();
+ Thread.sleep(200);
+ assertTrue(comm.frames.size() > afterFirst, "a changed frame should be streamed");
+ }
+
+ @Test
+ public void outputBundleCarriesInteractiveAndStaticFallback() {
+ Renderer renderer = new Renderer();
+ Swing.registerAll(renderer);
+ JButton button = new JButton("Hi");
+ button.setSize(120, 40);
+
+ DisplayData data = InteractiveSwing.buildDisplayData(renderer, "tok", 120, 40, button);
+
+ // Interactive payload for the extension...
+ assertTrue(data.hasDataForType(MIMEType.parse(InteractiveSwing.MIME_TYPE)),
+ "interactive MIME payload should be present");
+ // ...and a static PNG fallback for frontends without it.
+ assertNotNull(data.getData(MIMEType.IMAGE_PNG), "static PNG fallback should be present");
+ }
+
+ @Test
+ public void typesIntoSwingTextField() throws Exception {
+ JTextField field = new JTextField();
+ field.setSize(160, 24);
+ InteractiveSwingSession session = new InteractiveSwingSession("tok-type", field);
+ session.bind(new RecordingComm());
+
+ // Click to focus the field, then type "hi".
+ session.handleMessage(mouse("down", 10, 10));
+ session.handleMessage(mouse("up", 10, 10));
+ typeChar(session, "h");
+ typeChar(session, "i");
+ flushEventQueue();
+
+ assertEquals("hi", field.getText(), "typed characters should be inserted into the text field");
+ }
+
+ @Test
+ public void fxBridgeRejectsNonNodes() {
+ // Name-based detection must not false-positive on ordinary objects (and must not require JavaFX to be present).
+ assertFalse(FxBridge.isNode(null));
+ assertFalse(FxBridge.isNode("not a node"));
+ assertFalse(FxBridge.isNode(new JButton()));
+ }
+
+ @Test
+ public void resizeUpdatesDimensions() throws Exception {
+ JButton button = new JButton("Resize");
+ InteractiveSwingSession session = new InteractiveSwingSession("tok-3", button);
+ session.bind(new RecordingComm());
+
+ JsonObject resize = new JsonObject();
+ resize.addProperty("type", "resize");
+ resize.addProperty("width", 321);
+ resize.addProperty("height", 123);
+ session.handleMessage(resize);
+ flushEventQueue();
+
+ assertEquals(321, session.getWidth());
+ assertEquals(123, session.getHeight());
+ }
+}
diff --git a/jupyterlab-extension/.gitignore b/jupyterlab-extension/.gitignore
new file mode 100644
index 0000000..c2d4841
--- /dev/null
+++ b/jupyterlab-extension/.gitignore
@@ -0,0 +1,8 @@
+node_modules/
+lib/
+tsconfig.tsbuildinfo
+jjava_swing/labextension/
+*.egg-info/
+dist/
+build/
+__pycache__/
diff --git a/jupyterlab-extension/README.md b/jupyterlab-extension/README.md
new file mode 100644
index 0000000..fd4e1e8
--- /dev/null
+++ b/jupyterlab-extension/README.md
@@ -0,0 +1,101 @@
+# jjava-swing — interactive Swing for JupyterLab
+
+A JupyterLab 4 extension that makes `displayInteractive(component)` in the
+[JJava](https://github.com/dflib/jjava) Java kernel render a **live, interactive** Swing/AWT
+component in a notebook cell: the kernel streams PNG frames of the component and the extension
+forwards mouse/keyboard events back to it.
+
+## How it works
+
+1. The kernel emits an output of MIME type `application/vnd.jjava.swing.v1+json` carrying a `token`
+ and the component's size. The extension's **mime renderer** mounts a `