From 86e60df13e0c4de56f81df40f32d8ba0ceb61ce1 Mon Sep 17 00:00:00 2001 From: Duc Trung LE Date: Tue, 17 Sep 2024 18:48:12 +0200 Subject: [PATCH 01/19] wip --- packages/voila/src/plugins/widget/index.ts | 136 +++++++++++++----- .../templates/base/skeleton.macro.html.j2 | 61 ++++++++ .../templates/base/spinner.macro.html.j2 | 2 +- .../templates/base/voila_setup.macro.html.j2 | 2 +- .../jupyter/voila/templates/lab/index.html.j2 | 9 ++ voila/app.py | 12 +- voila/execution_request_handler.py | 132 +++++++++++++++++ voila/handler.py | 8 ++ voila/notebook_renderer.py | 88 ++++++------ voila/request_info_handler.py | 6 +- 10 files changed, 376 insertions(+), 80 deletions(-) create mode 100644 share/jupyter/voila/templates/base/skeleton.macro.html.j2 create mode 100644 voila/execution_request_handler.py diff --git a/packages/voila/src/plugins/widget/index.ts b/packages/voila/src/plugins/widget/index.ts index 23e090e5c..505ad8f3c 100644 --- a/packages/voila/src/plugins/widget/index.ts +++ b/packages/voila/src/plugins/widget/index.ts @@ -34,6 +34,11 @@ import { VoilaApp } from '../../app'; import { Widget } from '@lumino/widgets'; import { RenderedCells } from './renderedcells'; +import { + // OutputArea, + OutputAreaModel, + SimplifiedOutputArea +} from '@jupyterlab/outputarea'; const WIDGET_MIMETYPE = 'application/vnd.jupyter.widget-view+json'; @@ -123,40 +128,107 @@ export const renderOutputsPlugin: JupyterFrontEndPlugin = { rendermime.latexTypesetter?.typeset(md as HTMLElement); }); // Render code cell - const cellOutputs = document.body.querySelectorAll( - 'script[type="application/vnd.voila.cell-output+json"]' - ); - cellOutputs.forEach(async (cellOutput) => { - const model = JSON.parse(cellOutput.innerHTML); - - const mimeType = rendermime.preferredMimeType(model.data, 'any'); - - if (!mimeType) { - return null; + // const cellOutputs = document.body.querySelectorAll( + // 'script[type="application/vnd.voila.cell-output+json"]' + // ); + // cellOutputs.forEach(async (cellOutput) => { + // const model = JSON.parse(cellOutput.innerHTML); + + // const mimeType = rendermime.preferredMimeType(model.data, 'any'); + + // if (!mimeType) { + // return null; + // } + // const output = rendermime.createRenderer(mimeType); + // output.renderModel(model).catch((error) => { + // // Manually append error message to output + // const pre = document.createElement('pre'); + // pre.textContent = `Javascript Error: ${error.message}`; + // output.node.appendChild(pre); + + // // Remove mime-type-specific CSS classes + // pre.className = 'lm-Widget jp-RenderedText'; + // pre.setAttribute('data-mime-type', 'application/vnd.jupyter.stderr'); + // }); + + // output.addClass('jp-OutputArea-output'); + + // if (cellOutput.parentElement) { + // const container = cellOutput.parentElement; + + // container.removeChild(cellOutput); + + // // Attach output + // Widget.attach(output, container); + // } + // }); + const kernelId = (app as VoilaApp).widgetManager?.kernel.id; + console.log('using kernel', kernelId); + const ws = new WebSocket(`ws://localhost:8866/voila/execution/${kernelId}`); + ws.onmessage = (msg) => { + const { action, payload } = JSON.parse(msg.data); + if (action === 'execution_result') { + const { cell_index, output_cell, request_kernel_id } = payload; + const element = document.querySelector( + `[cell-index="${cell_index + 1}"]` + ); + if (element) { + const model = new OutputAreaModel({ trusted: true }); + const area = new SimplifiedOutputArea({ + model, + rendermime + }); + + const wrapper = document.createElement('div'); + wrapper.classList.add('jp-Cell-outputWrapper'); + const collapser = document.createElement('div'); + collapser.classList.add( + 'jp-Collapser', + 'jp-OutputCollapser', + 'jp-Cell-outputCollapser' + ); + wrapper.appendChild(collapser); + element.lastElementChild?.appendChild(wrapper); + + area.node.classList.add('jp-Cell-outputArea'); + + // Why do we need this? Are we missing a CSS class? + area.node.style.display = 'flex'; + area.node.style.flexDirection = 'column'; + + Widget.attach(area, wrapper); + const skeleton = element + .getElementsByClassName('voila-skeleton-container') + .item(0); + if (skeleton) { + element.removeChild(skeleton); + } + const outputData = output_cell.outputs[0]; + if (outputData) { + console.log( + 'adding', + outputData, + 'request_kernel_id', + request_kernel_id, + 'kernelId', + kernelId + ); + element.lastElementChild?.classList.remove( + 'jp-mod-noOutputs', + 'jp-mod-noInput' + ); + model.add(outputData); + } + } } - const output = rendermime.createRenderer(mimeType); - output.renderModel(model).catch((error) => { - // Manually append error message to output - const pre = document.createElement('pre'); - pre.textContent = `Javascript Error: ${error.message}`; - output.node.appendChild(pre); - - // Remove mime-type-specific CSS classes - pre.className = 'lm-Widget jp-RenderedText'; - pre.setAttribute('data-mime-type', 'application/vnd.jupyter.stderr'); - }); - - output.addClass('jp-OutputArea-output'); - - if (cellOutput.parentElement) { - const container = cellOutput.parentElement; - - container.removeChild(cellOutput); + }; + ws.onopen = () => { + console.log('opened'); + ws.send( + JSON.stringify({ action: 'execute', payload: { kernel_id: kernelId } }) + ); + }; - // Attach output - Widget.attach(output, container); - } - }); const node = document.getElementById('rendered_cells'); if (node) { const cells = new RenderedCells({ node }); diff --git a/share/jupyter/voila/templates/base/skeleton.macro.html.j2 b/share/jupyter/voila/templates/base/skeleton.macro.html.j2 new file mode 100644 index 000000000..460858a76 --- /dev/null +++ b/share/jupyter/voila/templates/base/skeleton.macro.html.j2 @@ -0,0 +1,61 @@ +{% macro css() %} + + +{% endmacro %} diff --git a/share/jupyter/voila/templates/base/spinner.macro.html.j2 b/share/jupyter/voila/templates/base/spinner.macro.html.j2 index 627c8ee14..b84e384d7 100644 --- a/share/jupyter/voila/templates/base/spinner.macro.html.j2 +++ b/share/jupyter/voila/templates/base/spinner.macro.html.j2 @@ -33,7 +33,7 @@ .voila-spinner-color2{ fill:#f8e14b; - } + } {% endmacro %} diff --git a/share/jupyter/voila/templates/base/voila_setup.macro.html.j2 b/share/jupyter/voila/templates/base/voila_setup.macro.html.j2 index 8da579fab..191a4bc08 100644 --- a/share/jupyter/voila/templates/base/voila_setup.macro.html.j2 +++ b/share/jupyter/voila/templates/base/voila_setup.macro.html.j2 @@ -19,7 +19,7 @@ spinner.style.display="flex"; } var el = document.getElementById("loading_text"); - innterText = text ?? `Executing ${cell_index} of ${cell_count}` + innterText = text ?? `Reading ${cell_index} of ${cell_count}` if(el){ el.innerHTML = innterText; } diff --git a/share/jupyter/voila/templates/lab/index.html.j2 b/share/jupyter/voila/templates/lab/index.html.j2 index 1e3df3083..ad513f67d 100644 --- a/share/jupyter/voila/templates/lab/index.html.j2 +++ b/share/jupyter/voila/templates/lab/index.html.j2 @@ -1,5 +1,6 @@ {%- extends 'nbconvert/templates/lab/index.html.j2' -%} {% import "spinner.macro.html.j2" as spinner %} +{% import "skeleton.macro.html.j2" as skeleton %} {% import "log.macro.html.j2" as log %} {% from 'voila_setup.macro.html.j2' import voila_setup_helper_functions, voila_setup_labextensions with context %} @@ -15,6 +16,7 @@ {% endif %} {{ spinner.css() }} + {{ skeleton.css() }} {% endmacro %} diff --git a/share/jupyter/voila/templates/base/spinner.macro.html.j2 b/share/jupyter/voila/templates/base/spinner.macro.html.j2 index b84e384d7..627c8ee14 100644 --- a/share/jupyter/voila/templates/base/spinner.macro.html.j2 +++ b/share/jupyter/voila/templates/base/spinner.macro.html.j2 @@ -33,7 +33,7 @@ .voila-spinner-color2{ fill:#f8e14b; - } + } {% endmacro %} diff --git a/voila/notebook_renderer.py b/voila/notebook_renderer.py index e40fc37d7..9f6231104 100644 --- a/voila/notebook_renderer.py +++ b/voila/notebook_renderer.py @@ -9,8 +9,6 @@ import os -import sys -import traceback from functools import partial from copy import deepcopy from typing import Generator, List, Tuple, Union @@ -19,13 +17,12 @@ import tornado.web from jupyter_core.utils import ensure_async from jupyter_server.config_manager import recursive_update -from nbclient.exceptions import CellExecutionError from nbconvert.preprocessors.clearoutput import ClearOutputPreprocessor from traitlets.config.configurable import LoggingConfigurable from voila.configuration import VoilaConfiguration -from .execute import VoilaExecutor, strip_code_cell_warnings +from .execute import VoilaExecutor from .exporter import VoilaExporter from .paths import collect_template_paths from .utils import ENV_VARIABLE @@ -242,7 +239,7 @@ async def _jinja_kernel_start(self, nb, kernel_id, kernel_future): return kernel_id async def _jinja_notebook_execute(self, nb, kernel_id): - print('VVVVVVVVVVVVVVVV _jinja_notebook_execute' ) + print("VVVVVVVVVVVVVVVV _jinja_notebook_execute") result = await self.executor.async_execute(cleanup_kc=False) # we modify the notebook in place, since the nb variable cannot be # reassigned it seems in jinja2 e.g. if we do {% with nb = notebook_execute(nb, kernel_id) %} @@ -254,7 +251,7 @@ async def _jinja_notebook_execute(self, nb, kernel_id): async def _jinja_cell_generator(self, nb, kernel_id): """Generator that will execute a single notebook cell at a time""" - print('VVVVVVVVVVVVVVVV _jinja_cell_generator' ) + print("VVVVVVVVVVVVVVVV _jinja_cell_generator") nb, _ = ClearOutputPreprocessor().preprocess( nb, {"metadata": {"path": self.cwd}} ) From bc8b5e73c602cb909d29e5972e2231d23bba527a Mon Sep 17 00:00:00 2001 From: Duc Trung Le Date: Fri, 27 Sep 2024 21:35:32 +0200 Subject: [PATCH 03/19] Bump actions/upload-artifact --- .github/workflows/packaging.yml | 2 +- .github/workflows/ui-tests.yml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml index bc3c1ab07..b3bd45800 100644 --- a/.github/workflows/packaging.yml +++ b/.github/workflows/packaging.yml @@ -46,7 +46,7 @@ jobs: sha256sum * | tee SHA256SUMS - name: Upload distributions - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: dist ${{ github.run_number }} path: ./dist diff --git a/.github/workflows/ui-tests.yml b/.github/workflows/ui-tests.yml index 068c76898..99d8f901e 100644 --- a/.github/workflows/ui-tests.yml +++ b/.github/workflows/ui-tests.yml @@ -50,7 +50,7 @@ jobs: - name: Upload Playwright Test assets if: always() - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: voila-test-assets path: | @@ -58,7 +58,7 @@ jobs: - name: Upload Playwright Benchmark report if: always() - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: voila-benchmark-report path: | @@ -66,7 +66,7 @@ jobs: - name: Upload Playwright Test report if: always() - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: voila-test-report path: | From acdece6e201e5f8bfffd20151c7ae96b94918139 Mon Sep 17 00:00:00 2001 From: Duc Trung Le Date: Fri, 27 Sep 2024 21:36:38 +0200 Subject: [PATCH 04/19] Create render plugin --- packages/voila/src/app.ts | 12 +- packages/voila/src/plugins/widget/index.ts | 249 +++++++++++-------- packages/voila/src/plugins/widget/manager.ts | 22 ++ packages/voila/src/voilaplugins.ts | 3 + voila/execution_request_handler.py | 32 +-- voila/notebook_renderer.py | 2 - 6 files changed, 185 insertions(+), 135 deletions(-) create mode 100644 packages/voila/src/plugins/widget/manager.ts diff --git a/packages/voila/src/app.ts b/packages/voila/src/app.ts index f06399f2e..3db7d0851 100644 --- a/packages/voila/src/app.ts +++ b/packages/voila/src/app.ts @@ -10,7 +10,7 @@ import { PageConfig } from '@jupyterlab/coreutils'; import { IRenderMime } from '@jupyterlab/rendermime'; -import { KernelWidgetManager } from '@jupyter-widgets/jupyterlab-manager'; +import { VoilaWidgetManager } from './plugins/widget'; import { IShell, VoilaShell } from './shell'; @@ -121,23 +121,23 @@ export class VoilaApp extends JupyterFrontEnd { /** * A promise that resolves when the Voila Widget Manager is created */ - get widgetManagerPromise(): PromiseDelegate { + get widgetManagerPromise(): PromiseDelegate { return this._widgetManagerPromise; } - set widgetManager(manager: KernelWidgetManager | null) { + set widgetManager(manager: VoilaWidgetManager | null) { this._widgetManager = manager; if (this._widgetManager) { this._widgetManagerPromise.resolve(this._widgetManager); } } - get widgetManager(): KernelWidgetManager | null { + get widgetManager(): VoilaWidgetManager | null { return this._widgetManager; } - protected _widgetManager: KernelWidgetManager | null = null; - protected _widgetManagerPromise = new PromiseDelegate(); + protected _widgetManager: VoilaWidgetManager | null = null; + protected _widgetManagerPromise = new PromiseDelegate(); } /** diff --git a/packages/voila/src/plugins/widget/index.ts b/packages/voila/src/plugins/widget/index.ts index 505ad8f3c..8ad1402cf 100644 --- a/packages/voila/src/plugins/widget/index.ts +++ b/packages/voila/src/plugins/widget/index.ts @@ -6,39 +6,26 @@ * * * The full license is in the file LICENSE, distributed with this software. * ****************************************************************************/ - +import { + IJupyterWidgetRegistry, + IWidgetRegistryData +} from '@jupyter-widgets/base'; +import { WidgetRenderer } from '@jupyter-widgets/jupyterlab-manager'; import { JupyterFrontEnd, JupyterFrontEndPlugin } from '@jupyterlab/application'; - import { PageConfig } from '@jupyterlab/coreutils'; - +import { IOutput } from '@jupyterlab/nbformat'; +import { OutputAreaModel, SimplifiedOutputArea } from '@jupyterlab/outputarea'; import { IRenderMimeRegistry } from '@jupyterlab/rendermime'; - import { KernelAPI, ServerConnection } from '@jupyterlab/services'; - import { KernelConnection } from '@jupyterlab/services/lib/kernel/default'; - -import { - WidgetRenderer, - KernelWidgetManager -} from '@jupyter-widgets/jupyterlab-manager'; - -import { - IJupyterWidgetRegistry, - IWidgetRegistryData -} from '@jupyter-widgets/base'; +import { Widget } from '@lumino/widgets'; import { VoilaApp } from '../../app'; - -import { Widget } from '@lumino/widgets'; +import { VoilaWidgetManager } from './manager'; import { RenderedCells } from './renderedcells'; -import { - // OutputArea, - OutputAreaModel, - SimplifiedOutputArea -} from '@jupyterlab/outputarea'; const WIDGET_MIMETYPE = 'application/vnd.jupyter.widget-view+json'; @@ -72,7 +59,7 @@ export const widgetManager: JupyterFrontEndPlugin = { }; } const kernel = new KernelConnection({ model, serverSettings }); - const manager = new KernelWidgetManager(kernel, rendermime); + const manager = new VoilaWidgetManager(kernel, rendermime); app.widgetManager = manager; rendermime.removeMimeType(WIDGET_MIMETYPE); @@ -128,111 +115,169 @@ export const renderOutputsPlugin: JupyterFrontEndPlugin = { rendermime.latexTypesetter?.typeset(md as HTMLElement); }); // Render code cell - // const cellOutputs = document.body.querySelectorAll( - // 'script[type="application/vnd.voila.cell-output+json"]' - // ); - // cellOutputs.forEach(async (cellOutput) => { - // const model = JSON.parse(cellOutput.innerHTML); - - // const mimeType = rendermime.preferredMimeType(model.data, 'any'); - - // if (!mimeType) { - // return null; - // } - // const output = rendermime.createRenderer(mimeType); - // output.renderModel(model).catch((error) => { - // // Manually append error message to output - // const pre = document.createElement('pre'); - // pre.textContent = `Javascript Error: ${error.message}`; - // output.node.appendChild(pre); - - // // Remove mime-type-specific CSS classes - // pre.className = 'lm-Widget jp-RenderedText'; - // pre.setAttribute('data-mime-type', 'application/vnd.jupyter.stderr'); - // }); - - // output.addClass('jp-OutputArea-output'); - - // if (cellOutput.parentElement) { - // const container = cellOutput.parentElement; - - // container.removeChild(cellOutput); - - // // Attach output - // Widget.attach(output, container); - // } - // }); + const cellOutputs = document.body.querySelectorAll( + 'script[type="application/vnd.voila.cell-output+json"]' + ); + cellOutputs.forEach(async (cellOutput) => { + const model = JSON.parse(cellOutput.innerHTML); + + const mimeType = rendermime.preferredMimeType(model.data, 'any'); + + if (!mimeType) { + return null; + } + const output = rendermime.createRenderer(mimeType); + output.renderModel(model).catch((error) => { + // Manually append error message to output + const pre = document.createElement('pre'); + pre.textContent = `Javascript Error: ${error.message}`; + output.node.appendChild(pre); + + // Remove mime-type-specific CSS classes + pre.className = 'lm-Widget jp-RenderedText'; + pre.setAttribute('data-mime-type', 'application/vnd.jupyter.stderr'); + }); + + output.addClass('jp-OutputArea-output'); + + if (cellOutput.parentElement) { + const container = cellOutput.parentElement; + + container.removeChild(cellOutput); + + // Attach output + Widget.attach(output, container); + } + }); + + const node = document.getElementById('rendered_cells'); + if (node) { + const cells = new RenderedCells({ node }); + app.shell.add(cells, 'main'); + } + } +}; + +function createOutputArea({ + rendermime, + parent +}: { + rendermime: IRenderMimeRegistry; + parent: Element; +}): OutputAreaModel { + const model = new OutputAreaModel({ trusted: true }); + const area = new SimplifiedOutputArea({ + model, + rendermime + }); + + const wrapper = document.createElement('div'); + wrapper.classList.add('jp-Cell-outputWrapper'); + const collapser = document.createElement('div'); + collapser.classList.add( + 'jp-Collapser', + 'jp-OutputCollapser', + 'jp-Cell-outputCollapser' + ); + wrapper.appendChild(collapser); + parent.lastElementChild?.appendChild(wrapper); + area.node.classList.add('jp-Cell-outputArea'); + + area.node.style.display = 'flex'; + area.node.style.flexDirection = 'column'; + + Widget.attach(area, wrapper); + return model; +} + +/** + * The plugin that renders outputs. + */ +export const renderOutputsProgressivelyPlugin: JupyterFrontEndPlugin = { + id: '@voila-dashboards/voila:render-outputs-progressively', + autoStart: true, + requires: [IRenderMimeRegistry, IJupyterWidgetRegistry], + activate: async ( + app: JupyterFrontEnd, + rendermime: IRenderMimeRegistry + ): Promise => { + const widgetManager = (app as VoilaApp).widgetManager; + if (!widgetManager) { + return; + } + const kernelId = (app as VoilaApp).widgetManager?.kernel.id; - console.log('using kernel', kernelId); + + const receivedWidgetModel: { + [modelId: string]: { + outputModel: OutputAreaModel; + executionModel: IOutput; + }; + } = {}; + const modelRegisteredHandler = (_: VoilaWidgetManager, modelId: string) => { + if (receivedWidgetModel[modelId]) { + const { outputModel, executionModel } = receivedWidgetModel[modelId]; + console.log('render later'); + outputModel.add(executionModel); + widgetManager.removeRegisteredModel(modelId); + } + }; + widgetManager.modelRegistered.connect(modelRegisteredHandler); + const ws = new WebSocket(`ws://localhost:8866/voila/execution/${kernelId}`); - ws.onmessage = (msg) => { + + ws.onmessage = async (msg) => { const { action, payload } = JSON.parse(msg.data); if (action === 'execution_result') { - const { cell_index, output_cell, request_kernel_id } = payload; + const { cell_index, output_cell } = payload; const element = document.querySelector( `[cell-index="${cell_index + 1}"]` ); if (element) { - const model = new OutputAreaModel({ trusted: true }); - const area = new SimplifiedOutputArea({ - model, - rendermime - }); - - const wrapper = document.createElement('div'); - wrapper.classList.add('jp-Cell-outputWrapper'); - const collapser = document.createElement('div'); - collapser.classList.add( - 'jp-Collapser', - 'jp-OutputCollapser', - 'jp-Cell-outputCollapser' - ); - wrapper.appendChild(collapser); - element.lastElementChild?.appendChild(wrapper); - - area.node.classList.add('jp-Cell-outputArea'); - - // Why do we need this? Are we missing a CSS class? - area.node.style.display = 'flex'; - area.node.style.flexDirection = 'column'; - - Widget.attach(area, wrapper); const skeleton = element .getElementsByClassName('voila-skeleton-container') .item(0); if (skeleton) { element.removeChild(skeleton); } - const outputData = output_cell.outputs[0]; - if (outputData) { - console.log( - 'adding', - outputData, - 'request_kernel_id', - request_kernel_id, - 'kernelId', - kernelId - ); + const model = createOutputArea({ rendermime, parent: element }); + + if (output_cell.outputs.length > 0) { element.lastElementChild?.classList.remove( 'jp-mod-noOutputs', 'jp-mod-noInput' ); - model.add(outputData); + } + for (const outputData of output_cell.outputs) { + const modelId = + outputData?.data?.['application/vnd.jupyter.widget-view+json'] + ?.model_id; + if (modelId) { + if (widgetManager.has_model(modelId)) { + console.log('render immediatly'); + model.add(outputData); + } else { + receivedWidgetModel[modelId] = { + outputModel: model, + executionModel: outputData + }; + } + } else { + model.add(outputData); + } } } + } else if (action === 'finished') { + widgetManager.modelRegistered.disconnect(modelRegisteredHandler); + ws.close(); } }; ws.onopen = () => { - console.log('opened'); ws.send( JSON.stringify({ action: 'execute', payload: { kernel_id: kernelId } }) ); }; - - const node = document.getElementById('rendered_cells'); - if (node) { - const cells = new RenderedCells({ node }); - app.shell.add(cells, 'main'); - } } }; + +export { VoilaWidgetManager }; diff --git a/packages/voila/src/plugins/widget/manager.ts b/packages/voila/src/plugins/widget/manager.ts new file mode 100644 index 000000000..32132ca7c --- /dev/null +++ b/packages/voila/src/plugins/widget/manager.ts @@ -0,0 +1,22 @@ +import { WidgetModel } from '@jupyter-widgets/base'; +import { KernelWidgetManager } from '@jupyter-widgets/jupyterlab-manager'; +import { ISignal, Signal } from '@lumino/signaling'; + +export class VoilaWidgetManager extends KernelWidgetManager { + register_model(model_id: string, modelPromise: Promise): void { + super.register_model(model_id, modelPromise); + this._registeredModels.add(model_id); + this._modelRegistered.emit(model_id); + } + get registeredModels(): ReadonlySet { + return this._registeredModels; + } + get modelRegistered(): ISignal { + return this._modelRegistered; + } + removeRegisteredModel(modelId: string) { + this._registeredModels.delete(modelId); + } + private _modelRegistered = new Signal(this); + private _registeredModels = new Set(); +} diff --git a/packages/voila/src/voilaplugins.ts b/packages/voila/src/voilaplugins.ts index 7e52e7113..dfcb67526 100644 --- a/packages/voila/src/voilaplugins.ts +++ b/packages/voila/src/voilaplugins.ts @@ -12,6 +12,7 @@ import { pathsPlugin } from './plugins/path'; import { translatorPlugin } from './plugins/translator'; import { renderOutputsPlugin, widgetManager } from './plugins/widget'; import { themePlugin, themesManagerPlugin } from './plugins/themes'; +import { renderOutputsProgressivelyPlugin } from './plugins/widget/index'; /** * Export the plugins as default. @@ -21,6 +22,7 @@ const plugins: JupyterFrontEndPlugin[] = [ translatorPlugin, widgetManager, renderOutputsPlugin, + renderOutputsProgressivelyPlugin, themesManagerPlugin, themePlugin ]; @@ -32,6 +34,7 @@ export { translatorPlugin, widgetManager, renderOutputsPlugin, + renderOutputsProgressivelyPlugin, themesManagerPlugin, themePlugin }; diff --git a/voila/execution_request_handler.py b/voila/execution_request_handler.py index 9bacec16f..8d492da5c 100644 --- a/voila/execution_request_handler.py +++ b/voila/execution_request_handler.py @@ -1,6 +1,5 @@ import json -import logging -from typing import Awaitable, Dict +from typing import Awaitable from jupyter_server.base.handlers import JupyterHandler from tornado.websocket import WebSocketHandler from jupyter_server.base.websocket import WebSocketMixin @@ -19,7 +18,6 @@ class ExecutionRequestHandler(WebSocketMixin, WebSocketHandler, JupyterHandler): def initialize(self, **kwargs): super().initialize() - print("cccc", self.kernel_manager) def open(self, kernel_id: str) -> None: """Create a new websocket connection, this connection is @@ -30,7 +28,6 @@ def open(self, kernel_id: str) -> None: the websocket connection. """ super().open() - print("self", self) self._kernel_id = kernel_id ExecutionRequestHandler._kernels[kernel_id] = self self.write_message({"action": "initialized", "payload": {}}) @@ -43,20 +40,20 @@ async def on_message(self, message_str: str | bytes) -> Awaitable[None] | None: payload = message.get("payload", {}) if action == "execute": request_kernel_id = payload.get("kernel_id") - print("RECEIVEDDDDDDDD", request_kernel_id, self._kernel_id) kernel_future = self.kernel_manager.get_kernel(self._kernel_id) km = await ensure_async(kernel_future) execution_data = self._execution_data.get(self._kernel_id) nb = execution_data["nb"] - executor = VoilaExecutor( + self._executor = executor = VoilaExecutor( nb, km=km, config=execution_data["config"], show_tracebacks=execution_data["show_tracebacks"], ) executor.kc = await executor.async_start_new_kernel_client() + for cell_idx, input_cell in enumerate(nb.cells): try: output_cell = await executor.execute_cell( @@ -64,7 +61,7 @@ async def on_message(self, message_str: str | bytes) -> Awaitable[None] | None: ) except TimeoutError: output_cell = input_cell - break + except CellExecutionError: self.log.exception( "Error at server while executing cell: %r", input_cell @@ -73,7 +70,7 @@ async def on_message(self, message_str: str | bytes) -> Awaitable[None] | None: strip_code_cell_warnings(input_cell) executor.strip_code_cell_errors(input_cell) output_cell = input_cell - break + except Exception as e: self.log.exception( "Error at server while executing cell: %r", input_cell @@ -113,20 +110,5 @@ async def on_message(self, message_str: str | bytes) -> Awaitable[None] | None: ) def on_close(self) -> None: - for k_id, waiter in ExecutionRequestHandler._kernels.items(): - if waiter == self: - break - del ExecutionRequestHandler._kernels[k_id] - - @classmethod - def send_updates(cls: "ExecutionRequestHandler", msg: Dict) -> None: - kernel_id = msg["kernel_id"] - payload = msg["payload"] - waiter = cls._kernels.get(kernel_id, None) - if waiter is not None: - try: - waiter.write_message(payload) - except Exception: - logging.error("Error sending message", exc_info=True) - else: - cls._cache[kernel_id] = payload + if self._executor: + del self._executor.kc diff --git a/voila/notebook_renderer.py b/voila/notebook_renderer.py index 9f6231104..a2d39d4c8 100644 --- a/voila/notebook_renderer.py +++ b/voila/notebook_renderer.py @@ -239,7 +239,6 @@ async def _jinja_kernel_start(self, nb, kernel_id, kernel_future): return kernel_id async def _jinja_notebook_execute(self, nb, kernel_id): - print("VVVVVVVVVVVVVVVV _jinja_notebook_execute") result = await self.executor.async_execute(cleanup_kc=False) # we modify the notebook in place, since the nb variable cannot be # reassigned it seems in jinja2 e.g. if we do {% with nb = notebook_execute(nb, kernel_id) %} @@ -251,7 +250,6 @@ async def _jinja_notebook_execute(self, nb, kernel_id): async def _jinja_cell_generator(self, nb, kernel_id): """Generator that will execute a single notebook cell at a time""" - print("VVVVVVVVVVVVVVVV _jinja_cell_generator") nb, _ = ClearOutputPreprocessor().preprocess( nb, {"metadata": {"path": self.cwd}} ) From a870523676e0469f04e4ff3871bb7da6bb112d31 Mon Sep 17 00:00:00 2001 From: Duc Trung Le Date: Fri, 27 Sep 2024 22:03:23 +0200 Subject: [PATCH 05/19] Bump action --- .github/workflows/check-release.yml | 2 +- .github/workflows/packaging.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/check-release.yml b/.github/workflows/check-release.yml index 6c2d0da54..59af32d22 100644 --- a/.github/workflows/check-release.yml +++ b/.github/workflows/check-release.yml @@ -27,7 +27,7 @@ jobs: version_spec: next - name: Upload Distributions - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: voila-releaser-dist-${{ github.run_number }} path: .jupyter_releaser_checkout/dist diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml index b3bd45800..34ae13bc2 100644 --- a/.github/workflows/packaging.yml +++ b/.github/workflows/packaging.yml @@ -76,7 +76,7 @@ jobs: with: python-version: ${{ matrix.python }} architecture: 'x64' - - uses: actions/download-artifact@v2 + - uses: actions/download-artifact@v4 with: name: dist ${{ github.run_number }} path: ./dist From 30f844370bfe2a05bb537b09419850c060b4dbba Mon Sep 17 00:00:00 2001 From: Duc Trung Le Date: Mon, 30 Sep 2024 13:53:38 +0200 Subject: [PATCH 06/19] Refactor plugins --- packages/voila/src/plugins/widget/index.ts | 292 +------------------ packages/voila/src/plugins/widget/plugins.ts | 230 +++++++++++++++ packages/voila/src/plugins/widget/tools.ts | 137 +++++++++ voila/execution_request_handler.py | 33 ++- 4 files changed, 400 insertions(+), 292 deletions(-) create mode 100644 packages/voila/src/plugins/widget/plugins.ts create mode 100644 packages/voila/src/plugins/widget/tools.ts diff --git a/packages/voila/src/plugins/widget/index.ts b/packages/voila/src/plugins/widget/index.ts index 8ad1402cf..2819a5a51 100644 --- a/packages/voila/src/plugins/widget/index.ts +++ b/packages/voila/src/plugins/widget/index.ts @@ -1,283 +1,13 @@ -/*************************************************************************** - * Copyright (c) 2018, Voilà contributors * - * Copyright (c) 2018, QuantStack * - * * - * Distributed under the terms of the BSD 3-Clause License. * - * * - * The full license is in the file LICENSE, distributed with this software. * - ****************************************************************************/ -import { - IJupyterWidgetRegistry, - IWidgetRegistryData -} from '@jupyter-widgets/base'; -import { WidgetRenderer } from '@jupyter-widgets/jupyterlab-manager'; -import { - JupyterFrontEnd, - JupyterFrontEndPlugin -} from '@jupyterlab/application'; -import { PageConfig } from '@jupyterlab/coreutils'; -import { IOutput } from '@jupyterlab/nbformat'; -import { OutputAreaModel, SimplifiedOutputArea } from '@jupyterlab/outputarea'; -import { IRenderMimeRegistry } from '@jupyterlab/rendermime'; -import { KernelAPI, ServerConnection } from '@jupyterlab/services'; -import { KernelConnection } from '@jupyterlab/services/lib/kernel/default'; -import { Widget } from '@lumino/widgets'; - -import { VoilaApp } from '../../app'; import { VoilaWidgetManager } from './manager'; -import { RenderedCells } from './renderedcells'; - -const WIDGET_MIMETYPE = 'application/vnd.jupyter.widget-view+json'; - -/** - * The Voila widgets manager plugin. - */ -export const widgetManager: JupyterFrontEndPlugin = { - id: '@voila-dashboards/voila:widget-manager', - autoStart: true, - requires: [IRenderMimeRegistry], - provides: IJupyterWidgetRegistry, - activate: async ( - app: JupyterFrontEnd, - rendermime: IRenderMimeRegistry - ): Promise => { - if (!(app instanceof VoilaApp)) { - throw Error( - 'The Voila Widget Manager plugin must be activated in a VoilaApp' - ); - } - const baseUrl = PageConfig.getBaseUrl(); - const kernelId = PageConfig.getOption('kernelId'); - const serverSettings = ServerConnection.makeSettings({ baseUrl }); - - const model = await KernelAPI.getKernelModel(kernelId, serverSettings); - if (!model) { - return { - registerWidget(data: IWidgetRegistryData): void { - throw Error(`The model for kernel id ${kernelId} does not exist`); - } - }; - } - const kernel = new KernelConnection({ model, serverSettings }); - const manager = new VoilaWidgetManager(kernel, rendermime); - app.widgetManager = manager; - - rendermime.removeMimeType(WIDGET_MIMETYPE); - rendermime.addFactory( - { - safe: false, - mimeTypes: [WIDGET_MIMETYPE], - createRenderer: (options) => new WidgetRenderer(options, manager) - }, - -10 - ); - window.addEventListener('beforeunload', (e) => { - const data = new FormData(); - // it seems if we attach this to early, it will not be called - const matches = document.cookie.match('\\b_xsrf=([^;]*)\\b'); - const xsrfToken = (matches && matches[1]) || ''; - data.append('_xsrf', xsrfToken); - window.navigator.sendBeacon( - `${baseUrl}voila/api/shutdown/${kernel.id}`, - data - ); - kernel.dispose(); - }); - - return { - registerWidget: async (data: IWidgetRegistryData) => { - const manager = await app.widgetManagerPromise.promise; - - manager.register(data); - } - }; - } -}; - -/** - * The plugin that renders outputs. - */ -export const renderOutputsPlugin: JupyterFrontEndPlugin = { - id: '@voila-dashboards/voila:render-outputs', - autoStart: true, - requires: [IRenderMimeRegistry, IJupyterWidgetRegistry], - activate: async ( - app: JupyterFrontEnd, - rendermime: IRenderMimeRegistry - ): Promise => { - // TODO: Typeset a fake element to get MathJax loaded, remove this hack once - // MathJax 2 is removed. - await rendermime.latexTypesetter?.typeset(document.createElement('div')); - - // Render latex in markdown cells - const mdOutput = document.body.querySelectorAll('div.jp-MarkdownOutput'); - mdOutput.forEach((md) => { - rendermime.latexTypesetter?.typeset(md as HTMLElement); - }); - // Render code cell - const cellOutputs = document.body.querySelectorAll( - 'script[type="application/vnd.voila.cell-output+json"]' - ); - cellOutputs.forEach(async (cellOutput) => { - const model = JSON.parse(cellOutput.innerHTML); - - const mimeType = rendermime.preferredMimeType(model.data, 'any'); - - if (!mimeType) { - return null; - } - const output = rendermime.createRenderer(mimeType); - output.renderModel(model).catch((error) => { - // Manually append error message to output - const pre = document.createElement('pre'); - pre.textContent = `Javascript Error: ${error.message}`; - output.node.appendChild(pre); - - // Remove mime-type-specific CSS classes - pre.className = 'lm-Widget jp-RenderedText'; - pre.setAttribute('data-mime-type', 'application/vnd.jupyter.stderr'); - }); - - output.addClass('jp-OutputArea-output'); - - if (cellOutput.parentElement) { - const container = cellOutput.parentElement; - - container.removeChild(cellOutput); - - // Attach output - Widget.attach(output, container); - } - }); - - const node = document.getElementById('rendered_cells'); - if (node) { - const cells = new RenderedCells({ node }); - app.shell.add(cells, 'main'); - } - } -}; - -function createOutputArea({ - rendermime, - parent -}: { - rendermime: IRenderMimeRegistry; - parent: Element; -}): OutputAreaModel { - const model = new OutputAreaModel({ trusted: true }); - const area = new SimplifiedOutputArea({ - model, - rendermime - }); - - const wrapper = document.createElement('div'); - wrapper.classList.add('jp-Cell-outputWrapper'); - const collapser = document.createElement('div'); - collapser.classList.add( - 'jp-Collapser', - 'jp-OutputCollapser', - 'jp-Cell-outputCollapser' - ); - wrapper.appendChild(collapser); - parent.lastElementChild?.appendChild(wrapper); - area.node.classList.add('jp-Cell-outputArea'); - - area.node.style.display = 'flex'; - area.node.style.flexDirection = 'column'; - - Widget.attach(area, wrapper); - return model; -} - -/** - * The plugin that renders outputs. - */ -export const renderOutputsProgressivelyPlugin: JupyterFrontEndPlugin = { - id: '@voila-dashboards/voila:render-outputs-progressively', - autoStart: true, - requires: [IRenderMimeRegistry, IJupyterWidgetRegistry], - activate: async ( - app: JupyterFrontEnd, - rendermime: IRenderMimeRegistry - ): Promise => { - const widgetManager = (app as VoilaApp).widgetManager; - if (!widgetManager) { - return; - } - - const kernelId = (app as VoilaApp).widgetManager?.kernel.id; - - const receivedWidgetModel: { - [modelId: string]: { - outputModel: OutputAreaModel; - executionModel: IOutput; - }; - } = {}; - const modelRegisteredHandler = (_: VoilaWidgetManager, modelId: string) => { - if (receivedWidgetModel[modelId]) { - const { outputModel, executionModel } = receivedWidgetModel[modelId]; - console.log('render later'); - outputModel.add(executionModel); - widgetManager.removeRegisteredModel(modelId); - } - }; - widgetManager.modelRegistered.connect(modelRegisteredHandler); - - const ws = new WebSocket(`ws://localhost:8866/voila/execution/${kernelId}`); - - ws.onmessage = async (msg) => { - const { action, payload } = JSON.parse(msg.data); - if (action === 'execution_result') { - const { cell_index, output_cell } = payload; - const element = document.querySelector( - `[cell-index="${cell_index + 1}"]` - ); - if (element) { - const skeleton = element - .getElementsByClassName('voila-skeleton-container') - .item(0); - if (skeleton) { - element.removeChild(skeleton); - } - const model = createOutputArea({ rendermime, parent: element }); - - if (output_cell.outputs.length > 0) { - element.lastElementChild?.classList.remove( - 'jp-mod-noOutputs', - 'jp-mod-noInput' - ); - } - for (const outputData of output_cell.outputs) { - const modelId = - outputData?.data?.['application/vnd.jupyter.widget-view+json'] - ?.model_id; - if (modelId) { - if (widgetManager.has_model(modelId)) { - console.log('render immediatly'); - model.add(outputData); - } else { - receivedWidgetModel[modelId] = { - outputModel: model, - executionModel: outputData - }; - } - } else { - model.add(outputData); - } - } - } - } else if (action === 'finished') { - widgetManager.modelRegistered.disconnect(modelRegisteredHandler); - ws.close(); - } - }; - ws.onopen = () => { - ws.send( - JSON.stringify({ action: 'execute', payload: { kernel_id: kernelId } }) - ); - }; - } +import { + widgetManager, + renderOutputsPlugin, + renderOutputsProgressivelyPlugin +} from './plugins'; + +export { + VoilaWidgetManager, + widgetManager, + renderOutputsPlugin, + renderOutputsProgressivelyPlugin }; - -export { VoilaWidgetManager }; diff --git a/packages/voila/src/plugins/widget/plugins.ts b/packages/voila/src/plugins/widget/plugins.ts new file mode 100644 index 000000000..fb18dffa4 --- /dev/null +++ b/packages/voila/src/plugins/widget/plugins.ts @@ -0,0 +1,230 @@ +/*************************************************************************** + * Copyright (c) 2018, Voilà contributors * + * Copyright (c) 2018, QuantStack * + * * + * Distributed under the terms of the BSD 3-Clause License. * + * * + * The full license is in the file LICENSE, distributed with this software. * + ****************************************************************************/ +import { + IJupyterWidgetRegistry, + IWidgetRegistryData +} from '@jupyter-widgets/base'; +import { WidgetRenderer } from '@jupyter-widgets/jupyterlab-manager'; +import { + JupyterFrontEnd, + JupyterFrontEndPlugin +} from '@jupyterlab/application'; +import { PageConfig } from '@jupyterlab/coreutils'; +import { IRenderMimeRegistry } from '@jupyterlab/rendermime'; +import { KernelAPI, ServerConnection } from '@jupyterlab/services'; +import { KernelConnection } from '@jupyterlab/services/lib/kernel/default'; +import { Widget } from '@lumino/widgets'; + +import { VoilaApp } from '../../app'; +import { VoilaWidgetManager } from './manager'; +import { RenderedCells } from './renderedcells'; +import { + handleExecutionResult, + IExecutionMessage, + IReceivedWidgetModel +} from './tools'; + +const WIDGET_MIMETYPE = 'application/vnd.jupyter.widget-view+json'; + +/** + * The Voila widgets manager plugin. + */ +export const widgetManager: JupyterFrontEndPlugin = { + id: '@voila-dashboards/voila:widget-manager', + autoStart: true, + requires: [IRenderMimeRegistry], + provides: IJupyterWidgetRegistry, + activate: async ( + app: JupyterFrontEnd, + rendermime: IRenderMimeRegistry + ): Promise => { + if (!(app instanceof VoilaApp)) { + throw Error( + 'The Voila Widget Manager plugin must be activated in a VoilaApp' + ); + } + const baseUrl = PageConfig.getBaseUrl(); + const kernelId = PageConfig.getOption('kernelId'); + const serverSettings = ServerConnection.makeSettings({ baseUrl }); + + const model = await KernelAPI.getKernelModel(kernelId, serverSettings); + if (!model) { + return { + registerWidget(data: IWidgetRegistryData): void { + throw Error(`The model for kernel id ${kernelId} does not exist`); + } + }; + } + const kernel = new KernelConnection({ model, serverSettings }); + const manager = new VoilaWidgetManager(kernel, rendermime); + app.widgetManager = manager; + + rendermime.removeMimeType(WIDGET_MIMETYPE); + rendermime.addFactory( + { + safe: false, + mimeTypes: [WIDGET_MIMETYPE], + createRenderer: (options) => new WidgetRenderer(options, manager) + }, + -10 + ); + window.addEventListener('beforeunload', (e) => { + const data = new FormData(); + // it seems if we attach this to early, it will not be called + const matches = document.cookie.match('\\b_xsrf=([^;]*)\\b'); + const xsrfToken = (matches && matches[1]) || ''; + data.append('_xsrf', xsrfToken); + window.navigator.sendBeacon( + `${baseUrl}voila/api/shutdown/${kernel.id}`, + data + ); + kernel.dispose(); + }); + + return { + registerWidget: async (data: IWidgetRegistryData) => { + const manager = await app.widgetManagerPromise.promise; + + manager.register(data); + } + }; + } +}; + +/** + * The plugin that renders outputs. + */ +export const renderOutputsPlugin: JupyterFrontEndPlugin = { + id: '@voila-dashboards/voila:render-outputs', + autoStart: true, + requires: [IRenderMimeRegistry, IJupyterWidgetRegistry], + activate: async ( + app: JupyterFrontEnd, + rendermime: IRenderMimeRegistry + ): Promise => { + // TODO: Typeset a fake element to get MathJax loaded, remove this hack once + // MathJax 2 is removed. + await rendermime.latexTypesetter?.typeset(document.createElement('div')); + + // Render latex in markdown cells + const mdOutput = document.body.querySelectorAll('div.jp-MarkdownOutput'); + mdOutput.forEach((md) => { + rendermime.latexTypesetter?.typeset(md as HTMLElement); + }); + // Render code cell + const cellOutputs = document.body.querySelectorAll( + 'script[type="application/vnd.voila.cell-output+json"]' + ); + cellOutputs.forEach(async (cellOutput) => { + const model = JSON.parse(cellOutput.innerHTML); + + const mimeType = rendermime.preferredMimeType(model.data, 'any'); + + if (!mimeType) { + return null; + } + const output = rendermime.createRenderer(mimeType); + output.renderModel(model).catch((error) => { + // Manually append error message to output + const pre = document.createElement('pre'); + pre.textContent = `Javascript Error: ${error.message}`; + output.node.appendChild(pre); + + // Remove mime-type-specific CSS classes + pre.className = 'lm-Widget jp-RenderedText'; + pre.setAttribute('data-mime-type', 'application/vnd.jupyter.stderr'); + }); + + output.addClass('jp-OutputArea-output'); + + if (cellOutput.parentElement) { + const container = cellOutput.parentElement; + + container.removeChild(cellOutput); + + // Attach output + Widget.attach(output, container); + } + }); + + const node = document.getElementById('rendered_cells'); + if (node) { + const cells = new RenderedCells({ node }); + app.shell.add(cells, 'main'); + } + } +}; + +/** + * The plugin that renders outputs. + */ +export const renderOutputsProgressivelyPlugin: JupyterFrontEndPlugin = { + id: '@voila-dashboards/voila:render-outputs-progressively', + autoStart: true, + requires: [IRenderMimeRegistry, IJupyterWidgetRegistry], + activate: async ( + app: JupyterFrontEnd, + rendermime: IRenderMimeRegistry + ): Promise => { + const widgetManager = (app as VoilaApp).widgetManager; + if (!widgetManager) { + return; + } + + const kernelId = widgetManager.kernel.id; + + const receivedWidgetModel: IReceivedWidgetModel = {}; + const modelRegisteredHandler = (_: VoilaWidgetManager, modelId: string) => { + if (receivedWidgetModel[modelId]) { + const { outputModel, executionModel } = receivedWidgetModel[modelId]; + outputModel.add(executionModel); + widgetManager.removeRegisteredModel(modelId); + } + }; + widgetManager.modelRegistered.connect(modelRegisteredHandler); + + const ws = new WebSocket(`ws://localhost:8866/voila/execution/${kernelId}`); + + ws.onmessage = async (msg) => { + const { action, payload }: IExecutionMessage = JSON.parse(msg.data); + switch (action) { + case 'execution_result': { + const result = handleExecutionResult({ + payload, + rendermime, + widgetManager + }); + if (result) { + Object.entries(result).forEach(([key, val]) => { + receivedWidgetModel[key] = val; + }); + } + const { cell_index, total_cell } = payload; + if (cell_index + 1 === total_cell) { + // Executed all cells + ws.close(); + } + + break; + } + case 'execution_error': { + console.error(`Execution error: ${payload.error}`); + break; + } + default: + break; + } + }; + ws.onopen = () => { + ws.send( + JSON.stringify({ action: 'execute', payload: { kernel_id: kernelId } }) + ); + }; + } +}; diff --git a/packages/voila/src/plugins/widget/tools.ts b/packages/voila/src/plugins/widget/tools.ts new file mode 100644 index 000000000..505f91500 --- /dev/null +++ b/packages/voila/src/plugins/widget/tools.ts @@ -0,0 +1,137 @@ +import { IOutput } from '@jupyterlab/nbformat'; +import { OutputAreaModel, SimplifiedOutputArea } from '@jupyterlab/outputarea'; +import { IRenderMimeRegistry } from '@jupyterlab/rendermime'; +import { Widget } from '@lumino/widgets'; +import { VoilaWidgetManager } from './manager'; + +/** + * Interface representing the structure of an execution result message. + */ +export interface IExecutionResultMessage { + action: 'execution_result'; + payload: { + output_cell: { outputs: IOutput[] }; + cell_index: number; + total_cell: number; + }; +} + +/** + * Interface representing the structure of an execution error message. + */ +export interface IExecutionErrorMessage { + action: 'execution_error'; + payload: { + error: string; + }; +} + +/** + * Interface representing a received widget model + * containing output and execution models. + */ +export interface IReceivedWidgetModel { + [modelId: string]: { + outputModel: OutputAreaModel; + executionModel: IOutput; + }; +} +export type IExecutionMessage = + | IExecutionResultMessage + | IExecutionErrorMessage; + +/** + * Handles the execution result by rendering the output area and managing widget models. + * + * @param payload - The payload from the execution result message, + * including the output cell and cell index. + * @param rendermime - A render mime registry to render the output. + * @param widgetManager - The Voila widget manager to manage Jupyter widgets. + * @returns An object containing a model ID and its corresponding output and + * execution models if the model is not ready to be rendered, undefined otherwise. + */ +export function handleExecutionResult({ + payload, + rendermime, + widgetManager +}: { + payload: IExecutionResultMessage['payload']; + rendermime: IRenderMimeRegistry; + widgetManager: VoilaWidgetManager; +}): IReceivedWidgetModel | undefined { + const { cell_index, output_cell } = payload; + const element = document.querySelector(`[cell-index="${cell_index + 1}"]`); + if (element) { + const skeleton = element + .getElementsByClassName('voila-skeleton-container') + .item(0); + if (skeleton) { + element.removeChild(skeleton); + } + const model = createOutputArea({ rendermime, parent: element }); + + if (output_cell.outputs.length > 0) { + element.lastElementChild?.classList.remove( + 'jp-mod-noOutputs', + 'jp-mod-noInput' + ); + } + const key = 'application/vnd.jupyter.widget-view+json'; + for (const outputData of output_cell.outputs) { + const modelId = (outputData?.data as any)?.[key]?.model_id; + if (modelId) { + if (widgetManager.registeredModels.has(modelId)) { + model.add(outputData); + } else { + return { + [modelId]: { + outputModel: model, + executionModel: outputData + } + }; + } + } else { + model.add(outputData); + } + } + } +} + +/** + * Creates an output area model and attaches the output area to a specified parent element. + * + * @param rendermime - The render mime registry. + * @param parent - The parent HTML element where the output area will be appended. + * @returns The created OutputAreaModel. + */ +export function createOutputArea({ + rendermime, + parent +}: { + rendermime: IRenderMimeRegistry; + parent: Element; +}): OutputAreaModel { + const model = new OutputAreaModel({ trusted: true }); + const area = new SimplifiedOutputArea({ + model, + rendermime + }); + + const wrapper = document.createElement('div'); + wrapper.classList.add('jp-Cell-outputWrapper'); + const collapser = document.createElement('div'); + collapser.classList.add( + 'jp-Collapser', + 'jp-OutputCollapser', + 'jp-Cell-outputCollapser' + ); + wrapper.appendChild(collapser); + parent.lastElementChild?.appendChild(wrapper); + area.node.classList.add('jp-Cell-outputArea'); + + area.node.style.display = 'flex'; + area.node.style.flexDirection = 'column'; + + Widget.attach(area, wrapper); + return model; +} diff --git a/voila/execution_request_handler.py b/voila/execution_request_handler.py index 8d492da5c..3036873c7 100644 --- a/voila/execution_request_handler.py +++ b/voila/execution_request_handler.py @@ -1,3 +1,4 @@ +import asyncio import json from typing import Awaitable from jupyter_server.base.handlers import JupyterHandler @@ -14,7 +15,6 @@ class ExecutionRequestHandler(WebSocketMixin, WebSocketHandler, JupyterHandler): _kernels = {} _execution_data = {} - _cache = {} def initialize(self, **kwargs): super().initialize() @@ -29,10 +29,7 @@ def open(self, kernel_id: str) -> None: """ super().open() self._kernel_id = kernel_id - ExecutionRequestHandler._kernels[kernel_id] = self self.write_message({"action": "initialized", "payload": {}}) - if kernel_id in self._cache: - self.write_message(self._cache[kernel_id]) async def on_message(self, message_str: str | bytes) -> Awaitable[None] | None: message = json.loads(message_str) @@ -40,11 +37,25 @@ async def on_message(self, message_str: str | bytes) -> Awaitable[None] | None: payload = message.get("payload", {}) if action == "execute": request_kernel_id = payload.get("kernel_id") - + if request_kernel_id != self._kernel_id: + await self.write_message( + { + "action": "execution_error", + "payload": {"error": "Kernel ID does not match"}, + } + ) + return kernel_future = self.kernel_manager.get_kernel(self._kernel_id) km = await ensure_async(kernel_future) - execution_data = self._execution_data.get(self._kernel_id) - + execution_data = self._execution_data.pop(self._kernel_id, None) + if execution_data is None: + await self.write_message( + { + "action": "execution_error", + "payload": {"error": "Missing notebook data"}, + } + ) + return nb = execution_data["nb"] self._executor = executor = VoilaExecutor( nb, @@ -53,7 +64,7 @@ async def on_message(self, message_str: str | bytes) -> Awaitable[None] | None: show_tracebacks=execution_data["show_tracebacks"], ) executor.kc = await executor.async_start_new_kernel_client() - + total_cell = len(nb.cells) for cell_idx, input_cell in enumerate(nb.cells): try: output_cell = await executor.execute_cell( @@ -102,13 +113,13 @@ async def on_message(self, message_str: str | bytes) -> Awaitable[None] | None: { "action": "execution_result", "payload": { - "request_kernel_id": request_kernel_id, "output_cell": output_cell, "cell_index": cell_idx, + "total_cell": total_cell, }, } ) def on_close(self) -> None: - if self._executor: - del self._executor.kc + if self._executor and self._executor.kc: + asyncio.create_task(ensure_async(self._executor.kc.stop_channels())) From 1752477f4f51126c9a917eb58d9abc88b426dcbe Mon Sep 17 00:00:00 2001 From: Duc Trung Le Date: Mon, 30 Sep 2024 21:23:01 +0200 Subject: [PATCH 07/19] Add progressive rendering flag --- packages/voila/src/plugins/widget/plugins.ts | 3 ++- packages/voila/src/plugins/widget/tools.ts | 9 +++++++++ voila/app.py | 8 ++++++++ voila/configuration.py | 8 ++++++++ voila/execution_request_handler.py | 1 - voila/utils.py | 2 ++ 6 files changed, 29 insertions(+), 2 deletions(-) diff --git a/packages/voila/src/plugins/widget/plugins.ts b/packages/voila/src/plugins/widget/plugins.ts index fb18dffa4..11422673b 100644 --- a/packages/voila/src/plugins/widget/plugins.ts +++ b/packages/voila/src/plugins/widget/plugins.ts @@ -25,6 +25,7 @@ import { VoilaApp } from '../../app'; import { VoilaWidgetManager } from './manager'; import { RenderedCells } from './renderedcells'; import { + getExecutionURL, handleExecutionResult, IExecutionMessage, IReceivedWidgetModel @@ -190,7 +191,7 @@ export const renderOutputsProgressivelyPlugin: JupyterFrontEndPlugin = { widgetManager.modelRegistered.connect(modelRegisteredHandler); const ws = new WebSocket(`ws://localhost:8866/voila/execution/${kernelId}`); - + getExecutionURL(); ws.onmessage = async (msg) => { const { action, payload }: IExecutionMessage = JSON.parse(msg.data); switch (action) { diff --git a/packages/voila/src/plugins/widget/tools.ts b/packages/voila/src/plugins/widget/tools.ts index 505f91500..dd3b5dd34 100644 --- a/packages/voila/src/plugins/widget/tools.ts +++ b/packages/voila/src/plugins/widget/tools.ts @@ -3,6 +3,7 @@ import { OutputAreaModel, SimplifiedOutputArea } from '@jupyterlab/outputarea'; import { IRenderMimeRegistry } from '@jupyterlab/rendermime'; import { Widget } from '@lumino/widgets'; import { VoilaWidgetManager } from './manager'; +import { PageConfig, URLExt } from '@jupyterlab/coreutils'; /** * Interface representing the structure of an execution result message. @@ -40,6 +41,14 @@ export type IExecutionMessage = | IExecutionResultMessage | IExecutionErrorMessage; +export function getExecutionURL(): string { + const executeProtocol = + PageConfig.getOption('progressiveRenderingProtocol') ?? 'ws'; + const baseUrl = PageConfig.getOption('baseUrl') ?? '/'; + console.log('cccc', executeProtocol, URLExt.normalize(baseUrl)); + return `${URLExt.join(baseUrl)}`; +} + /** * Handles the execution result by rendering the output area and managing widget models. * diff --git a/voila/app.py b/voila/app.py index 71a864b46..42726a840 100644 --- a/voila/app.py +++ b/voila/app.py @@ -173,6 +173,7 @@ class Voila(Application): "pool_size": "VoilaConfiguration.default_pool_size", "show_tracebacks": "VoilaConfiguration.show_tracebacks", "preheat_kernel": "VoilaConfiguration.preheat_kernel", + "progressive_rendering": "VoilaConfiguration.progressive_rendering", "strip_sources": "VoilaConfiguration.strip_sources", "template": "VoilaConfiguration.template", "theme": "VoilaConfiguration.theme", @@ -592,6 +593,12 @@ def init_settings(self) -> Dict: if preheat_kernel and self.prelaunch_hook: raise Exception("`preheat_kernel` and `prelaunch_hook` are incompatible") + progressive_rendering = self.voila_configuration.progressive_rendering + if preheat_kernel and progressive_rendering: + raise Exception( + "`preheat_kernel` and `progressive_rendering` are incompatible" + ) + kernel_manager_class = voila_kernel_manager_factory( self.voila_configuration.multi_kernel_manager_class, preheat_kernel, @@ -668,6 +675,7 @@ def init_settings(self) -> Dict: login_url=url_path_join(self.base_url, "/login"), mathjax_config=self.mathjax_config, mathjax_url=self.mathjax_url, + progressive_rendering=progressive_rendering, ) settings[self.name] = self # Why??? diff --git a/voila/configuration.py b/voila/configuration.py index e44fb338f..229aa1ec9 100644 --- a/voila/configuration.py +++ b/voila/configuration.py @@ -217,3 +217,11 @@ def _valid_file_blacklist(self, proposal): config=True, help="""Whether or not voila should attempt to fix and resolve a notebooks kernelspec metadata""", ) + + progressive_rendering = Bool( + False, + config=True, + help="""Flag to enable or disable progressive rendering option. + This option is not compatible with preheat-kernel option. + """, + ) diff --git a/voila/execution_request_handler.py b/voila/execution_request_handler.py index 3036873c7..5777101fd 100644 --- a/voila/execution_request_handler.py +++ b/voila/execution_request_handler.py @@ -13,7 +13,6 @@ class ExecutionRequestHandler(WebSocketMixin, WebSocketHandler, JupyterHandler): - _kernels = {} _execution_data = {} def initialize(self, **kwargs): diff --git a/voila/utils.py b/voila/utils.py index 7100a02af..b3f0c299c 100644 --- a/voila/utils.py +++ b/voila/utils.py @@ -90,6 +90,7 @@ async def _get_request_info(ws_url: str) -> Awaitable: def get_page_config(base_url, settings, log, voila_configuration: VoilaConfiguration): + progressive_rendering = settings.get("progressive_rendering", False) page_config = { "appVersion": __version__, "appUrl": "voila/", @@ -99,6 +100,7 @@ def get_page_config(base_url, settings, log, voila_configuration: VoilaConfigura "fullStaticUrl": url_path_join(base_url, "voila/static"), "fullLabextensionsUrl": url_path_join(base_url, "voila/labextensions"), "extensionConfig": voila_configuration.extension_config, + "progressiveRendering": progressive_rendering, } mathjax_config = settings.get("mathjax_config", "TeX-AMS_CHTML-full,Safe") mathjax_url = settings.get( From 4bba4c3e82be0a8795af9369bfc0cda490c36096 Mon Sep 17 00:00:00 2001 From: Duc Trung Le Date: Wed, 2 Oct 2024 10:29:36 +0200 Subject: [PATCH 08/19] Disable kernel message --- packages/voila/src/plugins/widget/plugins.ts | 4 +-- packages/voila/src/plugins/widget/tools.ts | 9 ++--- voila/app.py | 5 +-- voila/execution_request_handler.py | 9 ++++- voila/tornado/kernelwebsockethandler.py | 36 ++++++++++++++++++++ 5 files changed, 52 insertions(+), 11 deletions(-) create mode 100644 voila/tornado/kernelwebsockethandler.py diff --git a/packages/voila/src/plugins/widget/plugins.ts b/packages/voila/src/plugins/widget/plugins.ts index 11422673b..df1e7b7eb 100644 --- a/packages/voila/src/plugins/widget/plugins.ts +++ b/packages/voila/src/plugins/widget/plugins.ts @@ -189,8 +189,8 @@ export const renderOutputsProgressivelyPlugin: JupyterFrontEndPlugin = { } }; widgetManager.modelRegistered.connect(modelRegisteredHandler); - - const ws = new WebSocket(`ws://localhost:8866/voila/execution/${kernelId}`); + const wsUrl = getExecutionURL(kernelId); + const ws = new WebSocket(wsUrl); getExecutionURL(); ws.onmessage = async (msg) => { const { action, payload }: IExecutionMessage = JSON.parse(msg.data); diff --git a/packages/voila/src/plugins/widget/tools.ts b/packages/voila/src/plugins/widget/tools.ts index dd3b5dd34..2af92faae 100644 --- a/packages/voila/src/plugins/widget/tools.ts +++ b/packages/voila/src/plugins/widget/tools.ts @@ -41,12 +41,9 @@ export type IExecutionMessage = | IExecutionResultMessage | IExecutionErrorMessage; -export function getExecutionURL(): string { - const executeProtocol = - PageConfig.getOption('progressiveRenderingProtocol') ?? 'ws'; - const baseUrl = PageConfig.getOption('baseUrl') ?? '/'; - console.log('cccc', executeProtocol, URLExt.normalize(baseUrl)); - return `${URLExt.join(baseUrl)}`; +export function getExecutionURL(kernelId?: string): string { + const wsUrl = PageConfig.getWsUrl(); + return URLExt.join(wsUrl, 'voila/execution', kernelId ?? ''); } /** diff --git a/voila/app.py b/voila/app.py index 42726a840..5601d450f 100644 --- a/voila/app.py +++ b/voila/app.py @@ -20,6 +20,8 @@ import threading import webbrowser +from .tornado.kernelwebsockethandler import VoilaKernelWebsocketHandler + from .execution_request_handler import ExecutionRequestHandler from .tornado.contentshandler import VoilaContentsHandler @@ -38,7 +40,6 @@ from jupyter_server.services.config.manager import ConfigManager from jupyter_server.services.contents.largefilemanager import LargeFileManager from jupyter_server.services.kernels.handlers import KernelHandler -from jupyter_server.services.kernels.websocket import KernelWebsocketHandler from jupyter_server.auth.authorizer import AllowAllAuthorizer, Authorizer from jupyter_server.auth.identity import PasswordIdentityProvider from jupyter_server import DEFAULT_TEMPLATE_PATH_LIST, DEFAULT_STATIC_FILES_PATH @@ -697,7 +698,7 @@ def init_handlers(self) -> List: url_path_join( self.server_url, r"/api/kernels/%s/channels" % _kernel_id_regex ), - KernelWebsocketHandler, + VoilaKernelWebsocketHandler, ), ( url_path_join(self.server_url, r"/voila/templates/(.*)"), diff --git a/voila/execution_request_handler.py b/voila/execution_request_handler.py index 5777101fd..6a41649f1 100644 --- a/voila/execution_request_handler.py +++ b/voila/execution_request_handler.py @@ -3,6 +3,7 @@ from typing import Awaitable from jupyter_server.base.handlers import JupyterHandler from tornado.websocket import WebSocketHandler +from tornado.web import HTTPError from jupyter_server.base.websocket import WebSocketMixin from jupyter_core.utils import ensure_async from nbclient.exceptions import CellExecutionError @@ -18,7 +19,7 @@ class ExecutionRequestHandler(WebSocketMixin, WebSocketHandler, JupyterHandler): def initialize(self, **kwargs): super().initialize() - def open(self, kernel_id: str) -> None: + async def open(self, kernel_id: str) -> None: """Create a new websocket connection, this connection is identified by the kernel id. @@ -26,7 +27,12 @@ def open(self, kernel_id: str) -> None: kernel_id (str): Kernel id used by the notebook when it opens the websocket connection. """ + identity_provider = self.settings.get("identity_provider") + user = await ensure_async(identity_provider.get_user(self)) + if user is None: + raise HTTPError(403, "Unauthenticated") super().open() + self._kernel_id = kernel_id self.write_message({"action": "initialized", "payload": {}}) @@ -108,6 +114,7 @@ async def on_message(self, message_str: str | bytes) -> Awaitable[None] | None: } ] finally: + output_cell.pop("source", None) await self.write_message( { "action": "execution_result", diff --git a/voila/tornado/kernelwebsockethandler.py b/voila/tornado/kernelwebsockethandler.py new file mode 100644 index 000000000..f4eae542e --- /dev/null +++ b/voila/tornado/kernelwebsockethandler.py @@ -0,0 +1,36 @@ +import json +from typing import Any, Dict, Optional +from jupyter_server.services.kernels.websocket import KernelWebsocketHandler + + +def read_header_from_binary_message(ws_msg: bytes) -> Optional[Dict]: + """Read message header using the v1 protocol.""" + + offset_number = int.from_bytes(ws_msg[:8], "little") + offsets = [ + int.from_bytes(ws_msg[8 * (i + 1) : 8 * (i + 2)], "little") + for i in range(offset_number) + ] + try: + header = ws_msg[offsets[1] : offsets[2]].decode("utf-8") + return json.loads(header) + except Exception: + return + + +class VoilaKernelWebsocketHandler(KernelWebsocketHandler): + def write_message( + self, message: bytes | str | Dict[str, Any], binary: bool = False + ): + # if '"msg_type": "execute_input"' in message: + if isinstance(message, bytes): + header = read_header_from_binary_message(message) + elif isinstance(message, dict): + header = message.get("header", None) + else: + header = None + + if header and header.get("msg_type", None) == "execute_input": + return # Ignore execute_input message + + return super().write_message(message, binary) From b6d6d0c042c33758c00c0c9124c5bc1b8f393e79 Mon Sep 17 00:00:00 2001 From: Duc Trung Le Date: Wed, 2 Oct 2024 12:32:14 +0200 Subject: [PATCH 09/19] Simplify template --- packages/voila/src/plugins/widget/plugins.ts | 12 +- packages/voila/src/plugins/widget/tools.ts | 15 +++ .../templates/base/voila_setup.macro.html.j2 | 6 +- .../jupyter/voila/templates/lab/index.html.j2 | 7 -- voila/app.py | 20 ++-- voila/handler.py | 14 +-- voila/notebook_renderer.py | 113 ++++++++++-------- .../execution_request_handler.py | 0 voila/utils.py | 3 +- 9 files changed, 114 insertions(+), 76 deletions(-) rename voila/{ => tornado}/execution_request_handler.py (100%) diff --git a/packages/voila/src/plugins/widget/plugins.ts b/packages/voila/src/plugins/widget/plugins.ts index df1e7b7eb..8236e75c1 100644 --- a/packages/voila/src/plugins/widget/plugins.ts +++ b/packages/voila/src/plugins/widget/plugins.ts @@ -25,6 +25,7 @@ import { VoilaApp } from '../../app'; import { VoilaWidgetManager } from './manager'; import { RenderedCells } from './renderedcells'; import { + createSkeleton, getExecutionURL, handleExecutionResult, IExecutionMessage, @@ -173,7 +174,16 @@ export const renderOutputsProgressivelyPlugin: JupyterFrontEndPlugin = { app: JupyterFrontEnd, rendermime: IRenderMimeRegistry ): Promise => { - const widgetManager = (app as VoilaApp).widgetManager; + const progressiveRendering = + PageConfig.getOption('progressiveRendering') === 'true'; + if (!progressiveRendering) { + return; + } + + createSkeleton(); + + const widgetManager = await (app as VoilaApp).widgetManagerPromise.promise; + if (!widgetManager) { return; } diff --git a/packages/voila/src/plugins/widget/tools.ts b/packages/voila/src/plugins/widget/tools.ts index 2af92faae..aecc98727 100644 --- a/packages/voila/src/plugins/widget/tools.ts +++ b/packages/voila/src/plugins/widget/tools.ts @@ -141,3 +141,18 @@ export function createOutputArea({ Widget.attach(area, wrapper); return model; } + +export function createSkeleton(): void { + const innerHtml = `
+
+
+
+
`; + const elements = document.querySelectorAll('[cell-index]'); + elements.forEach((it) => { + const element = document.createElement('div'); + element.className = 'voila-skeleton-container'; + element.innerHTML = innerHtml; + it.appendChild(element); + }); +} diff --git a/share/jupyter/voila/templates/base/voila_setup.macro.html.j2 b/share/jupyter/voila/templates/base/voila_setup.macro.html.j2 index 191a4bc08..cc6afbdb1 100644 --- a/share/jupyter/voila/templates/base/voila_setup.macro.html.j2 +++ b/share/jupyter/voila/templates/base/voila_setup.macro.html.j2 @@ -19,7 +19,11 @@ spinner.style.display="flex"; } var el = document.getElementById("loading_text"); - innterText = text ?? `Reading ${cell_index} of ${cell_count}` + let defaultText = `Executing ${cell_index} of ${cell_count}` + if("{{ progressive_rendering | default("False", true) }}" === "True"){ + defaultText = `Reading ${cell_index} of ${cell_count}` + } + innterText = text ?? defaultText if(el){ el.innerHTML = innterText; } diff --git a/share/jupyter/voila/templates/lab/index.html.j2 b/share/jupyter/voila/templates/lab/index.html.j2 index ad513f67d..d000b2615 100644 --- a/share/jupyter/voila/templates/lab/index.html.j2 +++ b/share/jupyter/voila/templates/lab/index.html.j2 @@ -110,13 +110,6 @@ -
-
-
-
-
-
-
{{ super() }} {%- endblock any_cell -%} diff --git a/voila/app.py b/voila/app.py index 5601d450f..ca75c2f0c 100644 --- a/voila/app.py +++ b/voila/app.py @@ -22,7 +22,7 @@ from .tornado.kernelwebsockethandler import VoilaKernelWebsocketHandler -from .execution_request_handler import ExecutionRequestHandler +from .tornado.execution_request_handler import ExecutionRequestHandler from .tornado.contentshandler import VoilaContentsHandler @@ -676,7 +676,6 @@ def init_settings(self) -> Dict: login_url=url_path_join(self.base_url, "/login"), mathjax_config=self.mathjax_config, mathjax_url=self.mathjax_url, - progressive_rendering=progressive_rendering, ) settings[self.name] = self # Why??? @@ -726,7 +725,7 @@ def init_handlers(self) -> List: ] ) handlers.extend(self.identity_provider.get_handlers()) - if self.voila_configuration.preheat_kernel or True: + if self.voila_configuration.preheat_kernel: handlers.append( ( url_path_join( @@ -735,14 +734,15 @@ def init_handlers(self) -> List: RequestInfoSocketHandler, ) ) - handlers.append( - ( - url_path_join( - self.server_url, r"/voila/execution/%s" % _kernel_id_regex - ), - ExecutionRequestHandler, + if self.voila_configuration.progressive_rendering: + handlers.append( + ( + url_path_join( + self.server_url, r"/voila/execution/%s" % _kernel_id_regex + ), + ExecutionRequestHandler, + ) ) - ) # Serving JupyterLab extensions handlers.append( ( diff --git a/voila/handler.py b/voila/handler.py index 2b6fec472..059e23ca8 100644 --- a/voila/handler.py +++ b/voila/handler.py @@ -19,7 +19,7 @@ from tornado.httputil import split_host_and_port from traitlets.traitlets import Bool -from voila.execution_request_handler import ExecutionRequestHandler +from voila.tornado.execution_request_handler import ExecutionRequestHandler from .configuration import VoilaConfiguration @@ -238,12 +238,12 @@ def time_out(): ) kernel_future = self.kernel_manager.get_kernel(kernel_id) queue = asyncio.Queue() - - ExecutionRequestHandler._execution_data[kernel_id] = { - "nb": gen.notebook, - "config": self.traitlet_config, - "show_tracebacks": self.voila_configuration.show_tracebacks, - } + if self.voila_configuration.progressive_rendering: + ExecutionRequestHandler._execution_data[kernel_id] = { + "nb": gen.notebook, + "config": self.traitlet_config, + "show_tracebacks": self.voila_configuration.show_tracebacks, + } async def put_html(): async for html_snippet, _ in gen.generate_content_generator( diff --git a/voila/notebook_renderer.py b/voila/notebook_renderer.py index a2d39d4c8..1bb6fc34f 100644 --- a/voila/notebook_renderer.py +++ b/voila/notebook_renderer.py @@ -9,20 +9,23 @@ import os +import sys +import traceback from functools import partial from copy import deepcopy -from typing import Generator, List, Tuple, Union +from typing import AsyncGenerator, Generator, List, Tuple, Union import nbformat import tornado.web from jupyter_core.utils import ensure_async from jupyter_server.config_manager import recursive_update +from nbclient.exceptions import CellExecutionError from nbconvert.preprocessors.clearoutput import ClearOutputPreprocessor from traitlets.config.configurable import LoggingConfigurable from voila.configuration import VoilaConfiguration -from .execute import VoilaExecutor +from .execute import VoilaExecutor, strip_code_cell_warnings from .exporter import VoilaExporter from .paths import collect_template_paths from .utils import ENV_VARIABLE @@ -153,7 +156,7 @@ def generate_content_generator( self, kernel_id: Union[str, None] = None, kernel_future=None, - ) -> Generator: + ) -> AsyncGenerator: inner_kernel_start = partial( self._jinja_kernel_start, kernel_id=kernel_id, kernel_future=kernel_future ) @@ -168,9 +171,16 @@ def generate_content_generator( "frontend": "voila", "main_js": "voila.js", "kernel_start": inner_kernel_start, - "cell_generator": self._jinja_cell_generator, "notebook_execute": self._jinja_notebook_execute, + "progressive_rendering": self.voila_configuration.progressive_rendering, } + if self.voila_configuration.progressive_rendering: + extra_context["cell_generator"] = ( + self._jinja_cell_generator_without_execution + ) + else: + extra_context["cell_generator"] = self._jinja_cell_generator + # render notebook in snippets, then return an iterator so we can flush # them out to the browser progressively. return self.exporter.generate_from_notebook_node( @@ -248,56 +258,63 @@ async def _jinja_notebook_execute(self, nb, kernel_id): await self._cleanup_resources() + async def _jinja_cell_generator_without_execution(self, nb, kernel_id): + nb, _ = ClearOutputPreprocessor().preprocess( + nb, {"metadata": {"path": self.cwd}} + ) + for input_cell in nb.cells: + output = input_cell.copy() + yield output + await self._cleanup_resources() + async def _jinja_cell_generator(self, nb, kernel_id): """Generator that will execute a single notebook cell at a time""" nb, _ = ClearOutputPreprocessor().preprocess( nb, {"metadata": {"path": self.cwd}} ) for cell_idx, input_cell in enumerate(nb.cells): - output = input_cell.copy() - yield output - # try: - # output_cell = await self.executor.execute_cell( - # input_cell, None, cell_idx, store_history=False - # ) - # except TimeoutError: - # output_cell = input_cell - # break - # except CellExecutionError: - # self.log.exception( - # "Error at server while executing cell: %r", input_cell - # ) - # if self.executor.should_strip_error(): - # strip_code_cell_warnings(input_cell) - # self.executor.strip_code_cell_errors(input_cell) - # output_cell = input_cell - # break - # except Exception as e: - # self.log.exception( - # "Error at server while executing cell: %r", input_cell - # ) - # output_cell = nbformat.v4.new_code_cell() - # if self.executor.should_strip_error(): - # output_cell.outputs = [ - # { - # "output_type": "stream", - # "name": "stderr", - # "text": "An exception occurred at the server (not the notebook). {}".format( - # self.executor.cell_error_instruction - # ), - # } - # ] - # else: - # output_cell.outputs = [ - # { - # "output_type": "error", - # "ename": type(e).__name__, - # "evalue": str(e), - # "traceback": traceback.format_exception(*sys.exc_info()), - # } - # ] - # finally: - # yield output_cell + try: + output_cell = await self.executor.execute_cell( + input_cell, None, cell_idx, store_history=False + ) + except TimeoutError: + output_cell = input_cell + break + except CellExecutionError: + self.log.exception( + "Error at server while executing cell: %r", input_cell + ) + if self.executor.should_strip_error(): + strip_code_cell_warnings(input_cell) + self.executor.strip_code_cell_errors(input_cell) + output_cell = input_cell + break + except Exception as e: + self.log.exception( + "Error at server while executing cell: %r", input_cell + ) + output_cell = nbformat.v4.new_code_cell() + if self.executor.should_strip_error(): + output_cell.outputs = [ + { + "output_type": "stream", + "name": "stderr", + "text": "An exception occurred at the server (not the notebook). {}".format( + self.executor.cell_error_instruction + ), + } + ] + else: + output_cell.outputs = [ + { + "output_type": "error", + "ename": type(e).__name__, + "evalue": str(e), + "traceback": traceback.format_exception(*sys.exc_info()), + } + ] + finally: + yield output_cell await self._cleanup_resources() diff --git a/voila/execution_request_handler.py b/voila/tornado/execution_request_handler.py similarity index 100% rename from voila/execution_request_handler.py rename to voila/tornado/execution_request_handler.py diff --git a/voila/utils.py b/voila/utils.py index b3f0c299c..073ed50a1 100644 --- a/voila/utils.py +++ b/voila/utils.py @@ -90,7 +90,6 @@ async def _get_request_info(ws_url: str) -> Awaitable: def get_page_config(base_url, settings, log, voila_configuration: VoilaConfiguration): - progressive_rendering = settings.get("progressive_rendering", False) page_config = { "appVersion": __version__, "appUrl": "voila/", @@ -100,7 +99,7 @@ def get_page_config(base_url, settings, log, voila_configuration: VoilaConfigura "fullStaticUrl": url_path_join(base_url, "voila/static"), "fullLabextensionsUrl": url_path_join(base_url, "voila/labextensions"), "extensionConfig": voila_configuration.extension_config, - "progressiveRendering": progressive_rendering, + "progressiveRendering": voila_configuration.progressive_rendering, } mathjax_config = settings.get("mathjax_config", "TeX-AMS_CHTML-full,Safe") mathjax_url = settings.get( From 2ee122095f322f38d0be6622107c9af80d78d894 Mon Sep 17 00:00:00 2001 From: Duc Trung Le Date: Wed, 2 Oct 2024 17:11:12 +0200 Subject: [PATCH 10/19] Update for reveal template --- .../templates/base/skeleton.macro.html.j2 | 61 --------- .../templates/base/spinner.macro.html.j2 | 57 ++++++++ .../jupyter/voila/templates/lab/index.html.j2 | 2 - voila/app.py | 2 +- voila/notebook_renderer.py | 8 +- voila/server_extension.py | 129 ++++++++++-------- ...handler.py => kernel_websocket_handler.py} | 0 7 files changed, 134 insertions(+), 125 deletions(-) delete mode 100644 share/jupyter/voila/templates/base/skeleton.macro.html.j2 rename voila/tornado/{kernelwebsockethandler.py => kernel_websocket_handler.py} (100%) diff --git a/share/jupyter/voila/templates/base/skeleton.macro.html.j2 b/share/jupyter/voila/templates/base/skeleton.macro.html.j2 deleted file mode 100644 index 8335cad60..000000000 --- a/share/jupyter/voila/templates/base/skeleton.macro.html.j2 +++ /dev/null @@ -1,61 +0,0 @@ -{% macro css() %} - - -{% endmacro %} diff --git a/share/jupyter/voila/templates/base/spinner.macro.html.j2 b/share/jupyter/voila/templates/base/spinner.macro.html.j2 index 627c8ee14..a3ebb98ba 100644 --- a/share/jupyter/voila/templates/base/spinner.macro.html.j2 +++ b/share/jupyter/voila/templates/base/spinner.macro.html.j2 @@ -34,6 +34,63 @@ .voila-spinner-color2{ fill:#f8e14b; } + + .voila-skeleton-container { + display: flex; + flex-direction: column; + justify-content: center; + align-items: center; + } + + .voila-skeleton-post { + width: 220px; + height: 80px; + } + .voila-skeleton-post .voila-skeleton-avatar { + float: left; + width: 52px; + height: 52px; + background-color: var(--jp-layout-color1); + border-radius: 25%; + margin: 8px; + background-image: linear-gradient(90deg, var(--jp-layout-color2) 0px, var(--jp-layout-color3) 40px, var(--jp-layout-color2) 80px); + background-size: 600px; + animation: shine-avatar 1.6s infinite linear; + } + .voila-skeleton-post .voila-skeleton-line { + float: left; + width: 140px; + height: 16px; + margin-top: 12px; + border-radius: 7px; + background-image: linear-gradient(90deg, var(--jp-layout-color2) 0px, var(--jp-layout-color3) 40px, var(--jp-layout-color2) 80px); + background-size: 600px; + animation: shine-lines 1.6s infinite linear; + } + .voila-skeleton-post .voila-skeleton-avatar + .voila-skeleton-line { + margin-top: 11px; + width: 100px; + } + .voila-skeleton-post .voila-skeleton-line ~ .voila-skeleton-line { + background-color: var(--jp-layout-color2); + } + + @keyframes shine-lines { + 0% { + background-position: -100px; + } + 40%, 100% { + background-position: 140px; + } + } + @keyframes shine-avatar { + 0% { + background-position: -32px; + } + 40%, 100% { + background-position: 208px; + } + } {% endmacro %} diff --git a/share/jupyter/voila/templates/lab/index.html.j2 b/share/jupyter/voila/templates/lab/index.html.j2 index d000b2615..1e3df3083 100644 --- a/share/jupyter/voila/templates/lab/index.html.j2 +++ b/share/jupyter/voila/templates/lab/index.html.j2 @@ -1,6 +1,5 @@ {%- extends 'nbconvert/templates/lab/index.html.j2' -%} {% import "spinner.macro.html.j2" as spinner %} -{% import "skeleton.macro.html.j2" as skeleton %} {% import "log.macro.html.j2" as log %} {% from 'voila_setup.macro.html.j2' import voila_setup_helper_functions, voila_setup_labextensions with context %} @@ -16,7 +15,6 @@ {% endif %} {{ spinner.css() }} - {{ skeleton.css() }}