Interactive Swing & JavaFX rendering in notebooks

.gitignore

@@ -5,8 +5,10 @@ target/
 .settings
 *.iml
 dependency-reduced-pom.xml
+.ipynb_checkpoints
 
 # pip
+.venv
 build/
 dist/
 .egg/

jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/BaseKernel.java

@@ -7,6 +7,7 @@
 import org.dflib.jjava.jupyter.kernel.display.DisplayData;
 import org.dflib.jjava.jupyter.kernel.display.Renderer;
 import org.dflib.jjava.jupyter.kernel.display.common.Image;
+import org.dflib.jjava.jupyter.kernel.display.common.Swing;
 import org.dflib.jjava.jupyter.kernel.display.common.Text;
 import org.dflib.jjava.jupyter.kernel.display.common.Url;
 import org.dflib.jjava.jupyter.kernel.history.HistoryEntry;
@@ -136,6 +137,7 @@ protected BaseKernel(
         this.executionCount = new AtomicInteger(1);
 
         Image.registerAll(this.renderer);
+        Swing.registerAll(this.renderer);
         Url.registerAll(this.renderer);
         Text.registerAll(this.renderer);
     }

jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/BaseNotebookStatics.java

@@ -1,8 +1,11 @@
 package org.dflib.jjava.jupyter.kernel;
 
 import org.dflib.jjava.jupyter.kernel.display.DisplayData;
+import org.dflib.jjava.jupyter.kernel.display.interactive.FxBridge;
+import org.dflib.jjava.jupyter.kernel.display.interactive.InteractiveSwing;
 import org.dflib.jjava.jupyter.kernel.magic.UndefinedMagicException;
 
+import java.awt.Component;
 import java.util.List;
 import java.util.UUID;
 
@@ -79,6 +82,31 @@ public static String display(Object o, String... as) {
         return id;
     }
 
+    /**
+     * Display an AWT/Swing component or a JavaFX node as an interactive widget: it is streamed to the browser as a live
+     * image and mouse/keyboard events from the browser are dispatched back into it. Requires the JJava JupyterLab
+     * extension for interactivity; without it the output falls back to a static image snapshot.
+     *
+     * <p>A {@code javafx.scene.Node} is wrapped in a {@code JFXPanel} so it rides the same pipeline (JavaFX must be on
+     * the classpath — it is an optional, user-supplied dependency).
+     */
+    public static void displayInteractive(Object component) {
+        BaseKernel kernel = BaseKernel.notebookKernel();
+        if (component instanceof Component) {
+            InteractiveSwing.show(kernel, (Component) component);
+        } else if (FxBridge.isNode(component)) {
+            try {
+                InteractiveSwing.show(kernel, FxBridge.wrap(component));
+            } catch (ReflectiveOperationException e) {
+                throw new RuntimeException("Failed to embed JavaFX node (is JavaFX on the classpath?)", e);
+            }
+        } else {
+            throw new IllegalArgumentException(
+                    "displayInteractive expects a java.awt.Component or a javafx.scene.Node, got: "
+                            + (component == null ? "null" : component.getClass().getName()));
+        }
+    }
+
     public static void updateDisplay(String id, Object o) {
         DisplayData data = render(o);
         BaseKernel.notebookKernel().getIO().display.updateDisplay(id, data);

jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/comm/CommManager.java

@@ -94,11 +94,20 @@ public Comm unregisterComm(String id) {
      * The latter may happen if the manager is not connected to the frontend
      */
     public <T extends Comm> T openComm(String targetName, CommFactory<T> factory) {
+        return openComm(targetName, new JsonObject(), factory);
+    }
+
+    /**
+     * A {@link #openComm(String, CommFactory)} variant that attaches an initial {@code data} payload to the
+     * {@code comm_open} message. The frontend target handler receives this as the open command's data, which is useful
+     * for correlating the new comm with something the frontend already knows about.
+     */
+    public <T extends Comm> T openComm(String targetName, JsonObject data, CommFactory<T> factory) {
         if (this.iopub == null)
             return null;
         String id = UUID.randomUUID().toString();
 
-        CommOpenCommand content = new CommOpenCommand(id, targetName, new JsonObject());
+        CommOpenCommand content = new CommOpenCommand(id, targetName, data);
         Message<CommOpenCommand> message = new Message<>(this.context, CommOpenCommand.MESSAGE_TYPE, content);
 
         T comm = factory.produce(this, id, targetName, message);

jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/common/Swing.java

@@ -0,0 +1,82 @@
+package org.dflib.jjava.jupyter.kernel.display.common;
+
+import org.dflib.jjava.jupyter.kernel.display.RenderContext;
+import org.dflib.jjava.jupyter.kernel.display.Renderer;
+import org.dflib.jjava.jupyter.kernel.display.mime.MIMEType;
+
+import java.awt.Component;
+import java.awt.Container;
+import java.awt.Dimension;
+import java.awt.Graphics2D;
+import java.awt.RenderingHints;
+import java.awt.image.BufferedImage;
+
+/**
+ * Renders an {@link java.awt.Component AWT/Swing component} as an image by painting it off-screen into a
+ * {@link BufferedImage}, which is then encoded by {@link Image}. This works without a display (headless) since no native
+ * window is ever created.
+ */
+public class Swing {
+    public static final MIMEType PNG = MIMEType.IMAGE_PNG;
+    public static final MIMEType JPEG = MIMEType.IMAGE_JPEG;
+    public static final MIMEType GIF = MIMEType.IMAGE_GIF;
+
+    public static void registerAll(Renderer renderer) {
+        renderer.createRegistration(Component.class)
+                .preferring(PNG)
+                .supporting(JPEG, GIF)
+                .register(Swing::renderComponent);
+    }
+
+    public static void renderComponent(Component component, RenderContext context) {
+        Image.renderImage(snapshot(component), context);
+    }
+
+    /**
+     * Paint a component using its current size, falling back to its {@link Component#getPreferredSize() preferred size}
+     * when it has not been laid out yet (which is the usual case for a component that was never added to a window).
+     */
+    public static BufferedImage snapshot(Component component) {
+        Dimension size = component.getSize();
+        if (size.width <= 0 || size.height <= 0) {
+            size = component.getPreferredSize();
+        }
+        return snapshot(component, size.width, size.height);
+    }
+
+    public static BufferedImage snapshot(Component component, int width, int height) {
+        width = Math.max(width, 1);
+        height = Math.max(height, 1);
+
+        component.setSize(width, height);
+        layoutTree(component);
+
+        BufferedImage image = new BufferedImage(width, height, BufferedImage.TYPE_INT_ARGB);
+        Graphics2D g = image.createGraphics();
+        try {
+            g.setRenderingHint(RenderingHints.KEY_ANTIALIASING, RenderingHints.VALUE_ANTIALIAS_ON);
+            g.setRenderingHint(RenderingHints.KEY_TEXT_ANTIALIASING, RenderingHints.VALUE_TEXT_ANTIALIAS_ON);
+            // print() rather than paint(): it bypasses Swing's double-buffering, which is the right behaviour for
+            // off-screen rendering of a component that has no native peer.
+            component.print(g);
+        } finally {
+            g.dispose();
+        }
+        return image;
+    }
+
+    /**
+     * A component that was never realized in a window is not laid out, so its children have zero bounds and would not
+     * paint (nor hit-test). Forcing a layout pass over the whole subtree gives every child a position and size.
+     */
+    public static void layoutTree(Component component) {
+        synchronized (component.getTreeLock()) {
+            component.doLayout();
+            if (component instanceof Container) {
+                for (Component child : ((Container) component).getComponents()) {
+                    layoutTree(child);
+                }
+            }
+        }
+    }
+}

jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/interactive/InteractiveSwing.java

@@ -0,0 +1,91 @@
+package org.dflib.jjava.jupyter.kernel.display.interactive;
+
+import com.google.gson.JsonObject;
+import org.dflib.jjava.jupyter.kernel.BaseKernel;
+import org.dflib.jjava.jupyter.kernel.comm.CommManager;
+import org.dflib.jjava.jupyter.kernel.display.DisplayData;
+import org.dflib.jjava.jupyter.kernel.display.Renderer;
+import org.dflib.jjava.jupyter.kernel.display.mime.MIMEType;
+
+import java.awt.Component;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.UUID;
+
+/**
+ * Entry point for embedding an interactive {@link Component} in a notebook.
+ *
+ * <p>It emits an output carrying the {@link #MIME_TYPE custom MIME type} so the JupyterLab extension can mount a canvas,
+ * then opens a {@value #TARGET_NAME} comm whose {@code comm_open} data carries a {@code token}. The extension matches
+ * that token to the canvas it just created and starts exchanging frames/events with the
+ * {@link InteractiveSwingSession}.
+ *
+ * <p>The same output also carries a static {@code image/png} snapshot, so a frontend <em>without</em> the extension
+ * (or any non-Lab frontend) degrades gracefully to a still image instead of a placeholder message.
+ */
+public final class InteractiveSwing {
+
+    /** Comm target the JupyterLab extension registers a handler for. */
+    public static final String TARGET_NAME = "jjava.swing.v1";
+
+    /** MIME type of the placeholder output the JupyterLab extension renders into a canvas. */
+    public static final String MIME_TYPE = "application/vnd.jjava.swing.v1+json";
+
+    private InteractiveSwing() {
+    }
+
+    public static InteractiveSwingSession show(BaseKernel kernel, Component component) {
+        // A JFXPanel needs a window ancestor (for its GraphicsConfiguration) to map input; no-op for pure Swing.
+        FxBridge.ensureScreened(component);
+
+        String token = UUID.randomUUID().toString();
+        InteractiveSwingSession session = new InteractiveSwingSession(token, component);
+
+        DisplayData data = buildDisplayData(
+                kernel.getRenderer(), token, session.getWidth(), session.getHeight(), component);
+        kernel.display(data);
+
+        CommManager commManager = kernel.getCommManager();
+        JsonObject openData = new JsonObject();
+        openData.addProperty("token", token);
+        openData.addProperty("width", session.getWidth());
+        openData.addProperty("height", session.getHeight());
+
+        InteractiveSwingComm comm = commManager.openComm(
+                TARGET_NAME, openData,
+                (manager, id, target, message) -> new InteractiveSwingComm(manager, id, target, session));
+
+        if (comm != null) {
+            session.bind(comm);
+        }
+
+        return session;
+    }
+
+    /**
+     * Build the output bundle: the interactive {@link #MIME_TYPE} payload, plus a static {@code image/png} snapshot as a
+     * fallback for frontends without the extension, plus a {@code text/plain} description.
+     */
+    static DisplayData buildDisplayData(Renderer renderer, String token, int width, int height, Component component) {
+        Map<String, Object> payload = new LinkedHashMap<>();
+        payload.put("token", token);
+        payload.put("width", width);
+        payload.put("height", height);
+
+        DisplayData data = new DisplayData();
+        data.putData(MIME_TYPE, payload);
+
+        try {
+            Object png = renderer.renderAs(component, MIMEType.IMAGE_PNG.toString()).getData(MIMEType.IMAGE_PNG);
+            if (png != null) {
+                data.putData(MIMEType.IMAGE_PNG, png);
+            }
+        } catch (Exception e) {
+            // The static snapshot is best-effort; the interactive payload is still emitted.
+        }
+
+        data.putText("Interactive Swing component — install the jjava-swing JupyterLab extension for interactivity "
+                + "(a static snapshot is shown otherwise).");
+        return data;
+    }
+}

jjava-jupyter/src/main/java/org/dflib/jjava/jupyter/kernel/display/interactive/InteractiveSwingComm.java

@@ -0,0 +1,36 @@
+package org.dflib.jjava.jupyter.kernel.display.interactive;
+
+import org.dflib.jjava.jupyter.kernel.comm.Comm;
+import org.dflib.jjava.jupyter.kernel.comm.CommManager;
+import org.dflib.jjava.jupyter.messages.Message;
+import org.dflib.jjava.jupyter.messages.comm.CommCloseCommand;
+import org.dflib.jjava.jupyter.messages.comm.CommMsgCommand;
+
+/**
+ * The kernel side of the {@value InteractiveSwing#TARGET_NAME} comm. It is opened by the kernel (see
+ * {@link InteractiveSwing#show}) and forwards frontend messages to its {@link InteractiveSwingSession}, tearing the
+ * session down when the comm closes.
+ */
+class InteractiveSwingComm extends Comm {
+
+    private final InteractiveSwingSession session;
+
+    InteractiveSwingComm(CommManager manager, String id, String targetName, InteractiveSwingSession session) {
+        super(manager, id, targetName);
+        this.session = session;
+    }
+
+    @Override
+    protected void onMessage(Message<CommMsgCommand> message) {
+        if (session != null) {
+            session.handleMessage(message.getContent().getData());
+        }
+    }
+
+    @Override
+    protected void onClose(Message<CommCloseCommand> closeMessage, boolean sending) {
+        if (session != null) {
+            session.dispose();
+        }
+    }
+}