diff --git a/.github/workflows/check-release.yml b/.github/workflows/check-release.yml index 6c2d0da54..59af32d22 100644 --- a/.github/workflows/check-release.yml +++ b/.github/workflows/check-release.yml @@ -27,7 +27,7 @@ jobs: version_spec: next - name: Upload Distributions - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: voila-releaser-dist-${{ github.run_number }} path: .jupyter_releaser_checkout/dist diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml index bc3c1ab07..81bcaec95 100644 --- a/.github/workflows/packaging.yml +++ b/.github/workflows/packaging.yml @@ -46,29 +46,29 @@ jobs: sha256sum * | tee SHA256SUMS - name: Upload distributions - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: dist ${{ github.run_number }} path: ./dist install: - runs-on: ${{ matrix.os }}-latest + runs-on: ${{ matrix.os }} needs: [build] strategy: fail-fast: false matrix: - os: [ubuntu, macos, windows] + os: [ubuntu-latest, macos-12, windows-latest] python: ['3.8', '3.10'] include: - python: '3.8' dist: 'voila*.tar.gz' - python: '3.10' dist: 'voila*.whl' - - os: windows + - os: windows-latest py_cmd: python - - os: macos + - os: macos-12 py_cmd: python3 - - os: ubuntu + - os: ubuntu-latest py_cmd: python steps: - name: Install Python @@ -76,7 +76,7 @@ jobs: with: python-version: ${{ matrix.python }} architecture: 'x64' - - uses: actions/download-artifact@v2 + - uses: actions/download-artifact@v4 with: name: dist ${{ github.run_number }} path: ./dist diff --git a/.github/workflows/ui-tests.yml b/.github/workflows/ui-tests.yml index 068c76898..6aaaf1a00 100644 --- a/.github/workflows/ui-tests.yml +++ b/.github/workflows/ui-tests.yml @@ -8,8 +8,9 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.8] + python-version: ['3.10'] node-version: [18.x] + progressive_rendering: [false, true] fail-fast: false steps: @@ -31,7 +32,7 @@ jobs: - name: Launch Voila run: | # Mount a volume to overwrite the server configuration - jlpm start 2>&1 > /tmp/voila_server.log & + jlpm start --progressive_rendering=${{ matrix.progressive_rendering }} 2>&1 > /tmp/voila_server.log & working-directory: ui-tests - name: Install browser @@ -45,30 +46,46 @@ jobs: timeout: 360000 - name: Test + env: + PROGRESSIVE_RENDERING: ${{ matrix.progressive_rendering }} run: jlpm run test working-directory: ui-tests + - name: Set test report name for progressive rendering + if: always() && matrix.progressive_rendering == true + run: | + echo "TEST_REPORT_NAME=progressive-voila-test-report" >> $GITHUB_ENV + echo "TEST_ASSETS_NAME=progressive-voila-test-assets" >> $GITHUB_ENV + echo "TEST_BENCHMARK_NAME=progressive-voila-test-benchmark" >> $GITHUB_ENV + + - name: Set test report name for non-progressive rendering + if: always() && matrix.progressive_rendering == false + run: | + echo "TEST_REPORT_NAME=voila-test-report" >> $GITHUB_ENV + echo "TEST_ASSETS_NAME=voila-test-assets" >> $GITHUB_ENV + echo "TEST_BENCHMARK_NAME=voila-test-benchmark" >> $GITHUB_ENV + - name: Upload Playwright Test assets if: always() - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: - name: voila-test-assets + name: ${{ env.TEST_ASSETS_NAME }} path: | ui-tests/test-results - name: Upload Playwright Benchmark report if: always() - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: - name: voila-benchmark-report + name: ${{ env.TEST_BENCHMARK_NAME }} path: | ui-tests/benchmark-results - name: Upload Playwright Test report if: always() - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: - name: voila-test-report + name: ${{ env.TEST_REPORT_NAME }} path: | ui-tests/playwright-report diff --git a/.github/workflows/update_galata_references.yaml b/.github/workflows/update_galata_references.yaml index 76171b165..4c73b9855 100644 --- a/.github/workflows/update_galata_references.yaml +++ b/.github/workflows/update_galata_references.yaml @@ -15,29 +15,70 @@ defaults: jobs: update-reference-screenshots: name: Update Galata References - if: ${{ github.event.issue.pull_request && contains(github.event.comment.body, 'update galata references') }} + if: > + ( + github.event.comment.author_association == 'OWNER' || + github.event.comment.author_association == 'COLLABORATOR' || + github.event.comment.author_association == 'MEMBER' + ) && github.event.issue.pull_request && contains(github.event.comment.body, 'please update snapshots') runs-on: ubuntu-latest strategy: matrix: - python-version: [3.8] - node-version: [16.x] + python-version: [3.10] + node-version: [18.x] steps: - - name: Base Setup - uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 + - name: React to the triggering comment + run: | + gh api repos/${{ github.repository }}/issues/comments/${{ github.event.comment.id }}/reactions --raw-field 'content=+1' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: token: ${{ secrets.GITHUB_TOKEN }} - - name: Configure git to use https - run: git config --global hub.protocol https + - name: Get PR Info + id: pr + env: + PR_NUMBER: ${{ github.event.issue.number }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + COMMENT_AT: ${{ github.event.comment.created_at }} + run: | + pr="$(gh api /repos/${GH_REPO}/pulls/${PR_NUMBER})" + head_sha="$(echo "$pr" | jq -r .head.sha)" + pushed_at="$(echo "$pr" | jq -r .pushed_at)" + + if [[ $(date -d "$pushed_at" +%s) -gt $(date -d "$COMMENT_AT" +%s) ]]; then + echo "Updating is not allowed because the PR was pushed to (at $pushed_at) after the triggering comment was issued (at $COMMENT_AT)" + exit 1 + fi + + echo "head_sha=$head_sha" >> $GITHUB_OUTPUT - name: Checkout the branch from the PR that triggered the job - run: hub pr checkout ${{ github.event.issue.number }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: gh pr checkout ${{ github.event.issue.number }} + + - name: Validate the fetched branch HEAD revision + env: + EXPECTED_SHA: ${{ steps.pr.outputs.head_sha }} + run: | + actual_sha="$(git rev-parse HEAD)" + + if [[ "$actual_sha" != "$EXPECTED_SHA" ]]; then + echo "The HEAD of the checked out branch ($actual_sha) differs from the HEAD commit available at the time when trigger comment was submitted ($EXPECTED_SHA)" + exit 1 + fi + + - name: Base Setup + uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 + + - name: Configure git to use https + run: git config --global hub.protocol https - name: Install dependencies run: | diff --git a/docs/customize.md b/docs/customize.md index fe3fd59fc..d5bb143ce 100644 --- a/docs/customize.md +++ b/docs/customize.md @@ -665,3 +665,33 @@ By default, Voilà will attempt to resolve a kernel spec to the best fit, based ```py c.VoilaConfiguration.attempt_fix_notebook = False ``` + +## Changing dashboard rendering technique + +There are two dashboard rendering techniques in Voilà: + +- **Blocking rendering** (default): In this method, Voilà completes the execution of the entire notebook before displaying the dashboard. It is ideal for lightweight notebooks, as a loading spinner is shown until execution finishes.. +- **Progressive rendering** (introduced in Voilà 0.6): With this method, the dashboard appears immediately, with placeholders filling the cell outputs. These outputs are updated as each cell is executed by the kernel. + +To start Voilà with progressive rendering mode using CLI: + +```bash +voila ... --progressive_rendering=True +``` + +or using `voila.json` file + +```python +# voila.json +{ + ... + "VoilaConfiguration": { + "progressive_rendering": true, + ... + } +} +``` + +:::{warning} +Progressive rendering mode is incompatible with the preheated kernels functionality. +::: diff --git a/packages/voila/src/app.ts b/packages/voila/src/app.ts index f06399f2e..3db7d0851 100644 --- a/packages/voila/src/app.ts +++ b/packages/voila/src/app.ts @@ -10,7 +10,7 @@ import { PageConfig } from '@jupyterlab/coreutils'; import { IRenderMime } from '@jupyterlab/rendermime'; -import { KernelWidgetManager } from '@jupyter-widgets/jupyterlab-manager'; +import { VoilaWidgetManager } from './plugins/widget'; import { IShell, VoilaShell } from './shell'; @@ -121,23 +121,23 @@ export class VoilaApp extends JupyterFrontEnd { /** * A promise that resolves when the Voila Widget Manager is created */ - get widgetManagerPromise(): PromiseDelegate { + get widgetManagerPromise(): PromiseDelegate { return this._widgetManagerPromise; } - set widgetManager(manager: KernelWidgetManager | null) { + set widgetManager(manager: VoilaWidgetManager | null) { this._widgetManager = manager; if (this._widgetManager) { this._widgetManagerPromise.resolve(this._widgetManager); } } - get widgetManager(): KernelWidgetManager | null { + get widgetManager(): VoilaWidgetManager | null { return this._widgetManager; } - protected _widgetManager: KernelWidgetManager | null = null; - protected _widgetManagerPromise = new PromiseDelegate(); + protected _widgetManager: VoilaWidgetManager | null = null; + protected _widgetManagerPromise = new PromiseDelegate(); } /** diff --git a/packages/voila/src/plugins/widget/index.ts b/packages/voila/src/plugins/widget/index.ts index 23e090e5c..2819a5a51 100644 --- a/packages/voila/src/plugins/widget/index.ts +++ b/packages/voila/src/plugins/widget/index.ts @@ -1,166 +1,13 @@ -/*************************************************************************** - * Copyright (c) 2018, Voilà contributors * - * Copyright (c) 2018, QuantStack * - * * - * Distributed under the terms of the BSD 3-Clause License. * - * * - * The full license is in the file LICENSE, distributed with this software. * - ****************************************************************************/ - -import { - JupyterFrontEnd, - JupyterFrontEndPlugin -} from '@jupyterlab/application'; - -import { PageConfig } from '@jupyterlab/coreutils'; - -import { IRenderMimeRegistry } from '@jupyterlab/rendermime'; - -import { KernelAPI, ServerConnection } from '@jupyterlab/services'; - -import { KernelConnection } from '@jupyterlab/services/lib/kernel/default'; - +import { VoilaWidgetManager } from './manager'; import { - WidgetRenderer, - KernelWidgetManager -} from '@jupyter-widgets/jupyterlab-manager'; - -import { - IJupyterWidgetRegistry, - IWidgetRegistryData -} from '@jupyter-widgets/base'; - -import { VoilaApp } from '../../app'; - -import { Widget } from '@lumino/widgets'; -import { RenderedCells } from './renderedcells'; - -const WIDGET_MIMETYPE = 'application/vnd.jupyter.widget-view+json'; - -/** - * The Voila widgets manager plugin. - */ -export const widgetManager: JupyterFrontEndPlugin = { - id: '@voila-dashboards/voila:widget-manager', - autoStart: true, - requires: [IRenderMimeRegistry], - provides: IJupyterWidgetRegistry, - activate: async ( - app: JupyterFrontEnd, - rendermime: IRenderMimeRegistry - ): Promise => { - if (!(app instanceof VoilaApp)) { - throw Error( - 'The Voila Widget Manager plugin must be activated in a VoilaApp' - ); - } - const baseUrl = PageConfig.getBaseUrl(); - const kernelId = PageConfig.getOption('kernelId'); - const serverSettings = ServerConnection.makeSettings({ baseUrl }); - - const model = await KernelAPI.getKernelModel(kernelId, serverSettings); - if (!model) { - return { - registerWidget(data: IWidgetRegistryData): void { - throw Error(`The model for kernel id ${kernelId} does not exist`); - } - }; - } - const kernel = new KernelConnection({ model, serverSettings }); - const manager = new KernelWidgetManager(kernel, rendermime); - app.widgetManager = manager; - - rendermime.removeMimeType(WIDGET_MIMETYPE); - rendermime.addFactory( - { - safe: false, - mimeTypes: [WIDGET_MIMETYPE], - createRenderer: (options) => new WidgetRenderer(options, manager) - }, - -10 - ); - window.addEventListener('beforeunload', (e) => { - const data = new FormData(); - // it seems if we attach this to early, it will not be called - const matches = document.cookie.match('\\b_xsrf=([^;]*)\\b'); - const xsrfToken = (matches && matches[1]) || ''; - data.append('_xsrf', xsrfToken); - window.navigator.sendBeacon( - `${baseUrl}voila/api/shutdown/${kernel.id}`, - data - ); - kernel.dispose(); - }); - - return { - registerWidget: async (data: IWidgetRegistryData) => { - const manager = await app.widgetManagerPromise.promise; - - manager.register(data); - } - }; - } -}; - -/** - * The plugin that renders outputs. - */ -export const renderOutputsPlugin: JupyterFrontEndPlugin = { - id: '@voila-dashboards/voila:render-outputs', - autoStart: true, - requires: [IRenderMimeRegistry, IJupyterWidgetRegistry], - activate: async ( - app: JupyterFrontEnd, - rendermime: IRenderMimeRegistry - ): Promise => { - // TODO: Typeset a fake element to get MathJax loaded, remove this hack once - // MathJax 2 is removed. - await rendermime.latexTypesetter?.typeset(document.createElement('div')); - - // Render latex in markdown cells - const mdOutput = document.body.querySelectorAll('div.jp-MarkdownOutput'); - mdOutput.forEach((md) => { - rendermime.latexTypesetter?.typeset(md as HTMLElement); - }); - // Render code cell - const cellOutputs = document.body.querySelectorAll( - 'script[type="application/vnd.voila.cell-output+json"]' - ); - cellOutputs.forEach(async (cellOutput) => { - const model = JSON.parse(cellOutput.innerHTML); - - const mimeType = rendermime.preferredMimeType(model.data, 'any'); - - if (!mimeType) { - return null; - } - const output = rendermime.createRenderer(mimeType); - output.renderModel(model).catch((error) => { - // Manually append error message to output - const pre = document.createElement('pre'); - pre.textContent = `Javascript Error: ${error.message}`; - output.node.appendChild(pre); - - // Remove mime-type-specific CSS classes - pre.className = 'lm-Widget jp-RenderedText'; - pre.setAttribute('data-mime-type', 'application/vnd.jupyter.stderr'); - }); - - output.addClass('jp-OutputArea-output'); - - if (cellOutput.parentElement) { - const container = cellOutput.parentElement; - - container.removeChild(cellOutput); - - // Attach output - Widget.attach(output, container); - } - }); - const node = document.getElementById('rendered_cells'); - if (node) { - const cells = new RenderedCells({ node }); - app.shell.add(cells, 'main'); - } - } + widgetManager, + renderOutputsPlugin, + renderOutputsProgressivelyPlugin +} from './plugins'; + +export { + VoilaWidgetManager, + widgetManager, + renderOutputsPlugin, + renderOutputsProgressivelyPlugin }; diff --git a/packages/voila/src/plugins/widget/manager.ts b/packages/voila/src/plugins/widget/manager.ts new file mode 100644 index 000000000..32132ca7c --- /dev/null +++ b/packages/voila/src/plugins/widget/manager.ts @@ -0,0 +1,22 @@ +import { WidgetModel } from '@jupyter-widgets/base'; +import { KernelWidgetManager } from '@jupyter-widgets/jupyterlab-manager'; +import { ISignal, Signal } from '@lumino/signaling'; + +export class VoilaWidgetManager extends KernelWidgetManager { + register_model(model_id: string, modelPromise: Promise): void { + super.register_model(model_id, modelPromise); + this._registeredModels.add(model_id); + this._modelRegistered.emit(model_id); + } + get registeredModels(): ReadonlySet { + return this._registeredModels; + } + get modelRegistered(): ISignal { + return this._modelRegistered; + } + removeRegisteredModel(modelId: string) { + this._registeredModels.delete(modelId); + } + private _modelRegistered = new Signal(this); + private _registeredModels = new Set(); +} diff --git a/packages/voila/src/plugins/widget/plugins.ts b/packages/voila/src/plugins/widget/plugins.ts new file mode 100644 index 000000000..1aa7312c6 --- /dev/null +++ b/packages/voila/src/plugins/widget/plugins.ts @@ -0,0 +1,240 @@ +/*************************************************************************** + * Copyright (c) 2018, Voilà contributors * + * Copyright (c) 2018, QuantStack * + * * + * Distributed under the terms of the BSD 3-Clause License. * + * * + * The full license is in the file LICENSE, distributed with this software. * + ****************************************************************************/ +import { + IJupyterWidgetRegistry, + IWidgetRegistryData +} from '@jupyter-widgets/base'; +import { WidgetRenderer } from '@jupyter-widgets/jupyterlab-manager'; +import { + JupyterFrontEnd, + JupyterFrontEndPlugin +} from '@jupyterlab/application'; +import { PageConfig } from '@jupyterlab/coreutils'; +import { IRenderMimeRegistry } from '@jupyterlab/rendermime'; +import { KernelAPI, ServerConnection } from '@jupyterlab/services'; +import { KernelConnection } from '@jupyterlab/services/lib/kernel/default'; +import { Widget } from '@lumino/widgets'; + +import { VoilaApp } from '../../app'; +import { VoilaWidgetManager } from './manager'; +import { RenderedCells } from './renderedcells'; +import { + createSkeleton, + getExecutionURL, + handleExecutionResult, + IExecutionMessage, + IReceivedWidgetModel +} from './tools'; + +const WIDGET_MIMETYPE = 'application/vnd.jupyter.widget-view+json'; + +/** + * The Voila widgets manager plugin. + */ +export const widgetManager: JupyterFrontEndPlugin = { + id: '@voila-dashboards/voila:widget-manager', + autoStart: true, + requires: [IRenderMimeRegistry], + provides: IJupyterWidgetRegistry, + activate: async ( + app: JupyterFrontEnd, + rendermime: IRenderMimeRegistry + ): Promise => { + if (!(app instanceof VoilaApp)) { + throw Error( + 'The Voila Widget Manager plugin must be activated in a VoilaApp' + ); + } + const baseUrl = PageConfig.getBaseUrl(); + const kernelId = PageConfig.getOption('kernelId'); + const serverSettings = ServerConnection.makeSettings({ baseUrl }); + + const model = await KernelAPI.getKernelModel(kernelId, serverSettings); + if (!model) { + return { + registerWidget(data: IWidgetRegistryData): void { + throw Error(`The model for kernel id ${kernelId} does not exist`); + } + }; + } + const kernel = new KernelConnection({ model, serverSettings }); + const manager = new VoilaWidgetManager(kernel, rendermime); + app.widgetManager = manager; + + rendermime.removeMimeType(WIDGET_MIMETYPE); + rendermime.addFactory( + { + safe: false, + mimeTypes: [WIDGET_MIMETYPE], + createRenderer: (options) => new WidgetRenderer(options, manager) + }, + -10 + ); + window.addEventListener('beforeunload', (e) => { + const data = new FormData(); + // it seems if we attach this to early, it will not be called + const matches = document.cookie.match('\\b_xsrf=([^;]*)\\b'); + const xsrfToken = (matches && matches[1]) || ''; + data.append('_xsrf', xsrfToken); + window.navigator.sendBeacon( + `${baseUrl}voila/api/shutdown/${kernel.id}`, + data + ); + kernel.dispose(); + }); + + return { + registerWidget: async (data: IWidgetRegistryData) => { + const manager = await app.widgetManagerPromise.promise; + + manager.register(data); + } + }; + } +}; + +/** + * The plugin that renders outputs. + */ +export const renderOutputsPlugin: JupyterFrontEndPlugin = { + id: '@voila-dashboards/voila:render-outputs', + autoStart: true, + requires: [IRenderMimeRegistry, IJupyterWidgetRegistry], + activate: async ( + app: JupyterFrontEnd, + rendermime: IRenderMimeRegistry + ): Promise => { + // TODO: Typeset a fake element to get MathJax loaded, remove this hack once + // MathJax 2 is removed. + await rendermime.latexTypesetter?.typeset(document.createElement('div')); + + // Render latex in markdown cells + const mdOutput = document.body.querySelectorAll('div.jp-MarkdownOutput'); + mdOutput.forEach((md) => { + rendermime.latexTypesetter?.typeset(md as HTMLElement); + }); + // Render code cell + const cellOutputs = document.body.querySelectorAll( + 'script[type="application/vnd.voila.cell-output+json"]' + ); + cellOutputs.forEach(async (cellOutput) => { + const model = JSON.parse(cellOutput.innerHTML); + + const mimeType = rendermime.preferredMimeType(model.data, 'any'); + + if (!mimeType) { + return null; + } + const output = rendermime.createRenderer(mimeType); + output.renderModel(model).catch((error) => { + // Manually append error message to output + const pre = document.createElement('pre'); + pre.textContent = `Javascript Error: ${error.message}`; + output.node.appendChild(pre); + + // Remove mime-type-specific CSS classes + pre.className = 'lm-Widget jp-RenderedText'; + pre.setAttribute('data-mime-type', 'application/vnd.jupyter.stderr'); + }); + + output.addClass('jp-OutputArea-output'); + + if (cellOutput.parentElement) { + const container = cellOutput.parentElement; + + container.removeChild(cellOutput); + + // Attach output + Widget.attach(output, container); + } + }); + + const node = document.getElementById('rendered_cells'); + if (node) { + const cells = new RenderedCells({ node }); + app.shell.add(cells, 'main'); + } + } +}; + +/** + * The plugin that renders outputs. + */ +export const renderOutputsProgressivelyPlugin: JupyterFrontEndPlugin = { + id: '@voila-dashboards/voila:render-outputs-progressively', + autoStart: true, + requires: [IRenderMimeRegistry, IJupyterWidgetRegistry], + activate: async ( + app: JupyterFrontEnd, + rendermime: IRenderMimeRegistry + ): Promise => { + const progressiveRendering = + PageConfig.getOption('progressiveRendering') === 'true'; + if (!progressiveRendering) { + return; + } + + createSkeleton(); + + const widgetManager = await (app as VoilaApp).widgetManagerPromise.promise; + + if (!widgetManager) { + return; + } + + const kernelId = widgetManager.kernel.id; + + const receivedWidgetModel: IReceivedWidgetModel = {}; + const modelRegisteredHandler = (_: VoilaWidgetManager, modelId: string) => { + if (receivedWidgetModel[modelId]) { + const { outputModel, executionModel } = receivedWidgetModel[modelId]; + outputModel.add(executionModel); + widgetManager.removeRegisteredModel(modelId); + } + }; + widgetManager.modelRegistered.connect(modelRegisteredHandler); + const wsUrl = getExecutionURL(kernelId); + const ws = new WebSocket(wsUrl); + ws.onmessage = async (msg) => { + const { action, payload }: IExecutionMessage = JSON.parse(msg.data); + switch (action) { + case 'execution_result': { + const result = handleExecutionResult({ + payload, + rendermime, + widgetManager + }); + if (result) { + Object.entries(result).forEach(([key, val]) => { + receivedWidgetModel[key] = val; + }); + } + const { cell_index, total_cell } = payload; + if (cell_index + 1 === total_cell) { + // Executed all cells + ws.close(); + } + + break; + } + case 'execution_error': { + console.error(`Execution error: ${payload.error}`); + break; + } + default: + break; + } + }; + ws.onopen = () => { + ws.send( + JSON.stringify({ action: 'execute', payload: { kernel_id: kernelId } }) + ); + }; + } +}; diff --git a/packages/voila/src/plugins/widget/tools.ts b/packages/voila/src/plugins/widget/tools.ts new file mode 100644 index 000000000..5e1bf41a8 --- /dev/null +++ b/packages/voila/src/plugins/widget/tools.ts @@ -0,0 +1,160 @@ +import { IOutput } from '@jupyterlab/nbformat'; +import { OutputAreaModel, SimplifiedOutputArea } from '@jupyterlab/outputarea'; +import { IRenderMimeRegistry } from '@jupyterlab/rendermime'; +import { Widget } from '@lumino/widgets'; +import { VoilaWidgetManager } from './manager'; +import { PageConfig, URLExt } from '@jupyterlab/coreutils'; + +/** + * Interface representing the structure of an execution result message. + */ +export interface IExecutionResultMessage { + action: 'execution_result'; + payload: { + output_cell: { outputs: IOutput[] }; + cell_index: number; + total_cell: number; + }; +} + +/** + * Interface representing the structure of an execution error message. + */ +export interface IExecutionErrorMessage { + action: 'execution_error'; + payload: { + error: string; + }; +} + +/** + * Interface representing a received widget model + * containing output and execution models. + */ +export interface IReceivedWidgetModel { + [modelId: string]: { + outputModel: OutputAreaModel; + executionModel: IOutput; + }; +} +export type IExecutionMessage = + | IExecutionResultMessage + | IExecutionErrorMessage; + +export function getExecutionURL(kernelId?: string): string { + const wsUrl = PageConfig.getWsUrl(); + return URLExt.join(wsUrl, 'voila/execution', kernelId ?? ''); +} + +/** + * Handles the execution result by rendering the output area and managing widget models. + * + * @param payload - The payload from the execution result message, + * including the output cell and cell index. + * @param rendermime - A render mime registry to render the output. + * @param widgetManager - The Voila widget manager to manage Jupyter widgets. + * @returns An object containing a model ID and its corresponding output and + * execution models if the model is not ready to be rendered, undefined otherwise. + */ +export function handleExecutionResult({ + payload, + rendermime, + widgetManager +}: { + payload: IExecutionResultMessage['payload']; + rendermime: IRenderMimeRegistry; + widgetManager: VoilaWidgetManager; +}): IReceivedWidgetModel | undefined { + const { cell_index, output_cell } = payload; + const element = document.querySelector(`[cell-index="${cell_index + 1}"]`); + if (element) { + const skeleton = element + .getElementsByClassName('voila-skeleton-container') + .item(0); + if (skeleton) { + element.removeChild(skeleton); + } + const model = createOutputArea({ rendermime, parent: element }); + if (!output_cell.outputs) { + return; + } + if (output_cell.outputs.length > 0) { + element.lastElementChild?.classList.remove( + 'jp-mod-noOutputs', + 'jp-mod-noInput' + ); + } + const key = 'application/vnd.jupyter.widget-view+json'; + for (const outputData of output_cell.outputs) { + const modelId = (outputData?.data as any)?.[key]?.model_id; + if (modelId) { + if (widgetManager.registeredModels.has(modelId)) { + model.add(outputData); + } else { + return { + [modelId]: { + outputModel: model, + executionModel: outputData + } + }; + } + } else { + model.add(outputData); + } + } + } +} + +/** + * Creates an output area model and attaches the output area to a specified parent element. + * + * @param rendermime - The render mime registry. + * @param parent - The parent HTML element where the output area will be appended. + * @returns The created OutputAreaModel. + */ +export function createOutputArea({ + rendermime, + parent +}: { + rendermime: IRenderMimeRegistry; + parent: Element; +}): OutputAreaModel { + const model = new OutputAreaModel({ trusted: true }); + const area = new SimplifiedOutputArea({ + model, + rendermime + }); + + const wrapper = document.createElement('div'); + wrapper.classList.add('jp-Cell-outputWrapper'); + const collapser = document.createElement('div'); + collapser.classList.add( + 'jp-Collapser', + 'jp-OutputCollapser', + 'jp-Cell-outputCollapser' + ); + wrapper.appendChild(collapser); + parent.lastElementChild?.appendChild(wrapper); + area.node.classList.add('jp-Cell-outputArea'); + + area.node.style.display = 'flex'; + area.node.style.flexDirection = 'column'; + + Widget.attach(area, wrapper); + return model; +} + +export function createSkeleton(): void { + const innerHtml = `
+
+
+
+
`; + const elements = document.querySelectorAll('[cell-index]'); + elements.forEach((it) => { + const element = document.createElement('div'); + element.className = 'voila-skeleton-container'; + element.innerHTML = innerHtml; + it.appendChild(element); + }); +} diff --git a/packages/voila/src/voilaplugins.ts b/packages/voila/src/voilaplugins.ts index 7e52e7113..dfcb67526 100644 --- a/packages/voila/src/voilaplugins.ts +++ b/packages/voila/src/voilaplugins.ts @@ -12,6 +12,7 @@ import { pathsPlugin } from './plugins/path'; import { translatorPlugin } from './plugins/translator'; import { renderOutputsPlugin, widgetManager } from './plugins/widget'; import { themePlugin, themesManagerPlugin } from './plugins/themes'; +import { renderOutputsProgressivelyPlugin } from './plugins/widget/index'; /** * Export the plugins as default. @@ -21,6 +22,7 @@ const plugins: JupyterFrontEndPlugin[] = [ translatorPlugin, widgetManager, renderOutputsPlugin, + renderOutputsProgressivelyPlugin, themesManagerPlugin, themePlugin ]; @@ -32,6 +34,7 @@ export { translatorPlugin, widgetManager, renderOutputsPlugin, + renderOutputsProgressivelyPlugin, themesManagerPlugin, themePlugin }; diff --git a/requirements-visual-test.txt b/requirements-visual-test.txt index becdee9d0..3c1f23956 100644 --- a/requirements-visual-test.txt +++ b/requirements-visual-test.txt @@ -1,4 +1,5 @@ bokeh +bokeh_sampledata bqplot ipympl==0.9.2 jupyterlab~=4.0 diff --git a/share/jupyter/voila/templates/base/spinner.macro.html.j2 b/share/jupyter/voila/templates/base/spinner.macro.html.j2 index 627c8ee14..a3ebb98ba 100644 --- a/share/jupyter/voila/templates/base/spinner.macro.html.j2 +++ b/share/jupyter/voila/templates/base/spinner.macro.html.j2 @@ -34,6 +34,63 @@ .voila-spinner-color2{ fill:#f8e14b; } + + .voila-skeleton-container { + display: flex; + flex-direction: column; + justify-content: center; + align-items: center; + } + + .voila-skeleton-post { + width: 220px; + height: 80px; + } + .voila-skeleton-post .voila-skeleton-avatar { + float: left; + width: 52px; + height: 52px; + background-color: var(--jp-layout-color1); + border-radius: 25%; + margin: 8px; + background-image: linear-gradient(90deg, var(--jp-layout-color2) 0px, var(--jp-layout-color3) 40px, var(--jp-layout-color2) 80px); + background-size: 600px; + animation: shine-avatar 1.6s infinite linear; + } + .voila-skeleton-post .voila-skeleton-line { + float: left; + width: 140px; + height: 16px; + margin-top: 12px; + border-radius: 7px; + background-image: linear-gradient(90deg, var(--jp-layout-color2) 0px, var(--jp-layout-color3) 40px, var(--jp-layout-color2) 80px); + background-size: 600px; + animation: shine-lines 1.6s infinite linear; + } + .voila-skeleton-post .voila-skeleton-avatar + .voila-skeleton-line { + margin-top: 11px; + width: 100px; + } + .voila-skeleton-post .voila-skeleton-line ~ .voila-skeleton-line { + background-color: var(--jp-layout-color2); + } + + @keyframes shine-lines { + 0% { + background-position: -100px; + } + 40%, 100% { + background-position: 140px; + } + } + @keyframes shine-avatar { + 0% { + background-position: -32px; + } + 40%, 100% { + background-position: 208px; + } + } {% endmacro %} diff --git a/share/jupyter/voila/templates/base/voila_setup.macro.html.j2 b/share/jupyter/voila/templates/base/voila_setup.macro.html.j2 index 8da579fab..cc6afbdb1 100644 --- a/share/jupyter/voila/templates/base/voila_setup.macro.html.j2 +++ b/share/jupyter/voila/templates/base/voila_setup.macro.html.j2 @@ -19,7 +19,11 @@ spinner.style.display="flex"; } var el = document.getElementById("loading_text"); - innterText = text ?? `Executing ${cell_index} of ${cell_count}` + let defaultText = `Executing ${cell_index} of ${cell_count}` + if("{{ progressive_rendering | default("False", true) }}" === "True"){ + defaultText = `Reading ${cell_index} of ${cell_count}` + } + innterText = text ?? defaultText if(el){ el.innerHTML = innterText; } diff --git a/tests/app/conftest.py b/tests/app/conftest.py index 333d6c997..62a10738a 100644 --- a/tests/app/conftest.py +++ b/tests/app/conftest.py @@ -47,15 +47,31 @@ def preheat_mode(): return False +@pytest.fixture +def progressive_rendering_mode(): + """Fixture used to activate/deactivate progressive rendering mode. + Override this fixture in test file if you want to activate + progressive rendering mode. + """ + return False + + @pytest.fixture def preheat_config(preheat_mode): return f"--preheat_kernel={preheat_mode}" @pytest.fixture -def voila_app(voila_args, voila_config, preheat_config): +def progressive_rendering_config(progressive_rendering_mode): + return f"--progressive_rendering={progressive_rendering_mode}" + + +@pytest.fixture +def voila_app(voila_args, voila_config, preheat_config, progressive_rendering_config): voila_app = VoilaTest.instance() - voila_app.initialize([*voila_args, "--no-browser", preheat_config]) + voila_app.initialize( + [*voila_args, "--no-browser", preheat_config, progressive_rendering_config] + ) voila_config(voila_app) voila_app.start() yield voila_app diff --git a/tests/app/progressive_rendering_activation_test.py b/tests/app/progressive_rendering_activation_test.py new file mode 100644 index 000000000..7df48d211 --- /dev/null +++ b/tests/app/progressive_rendering_activation_test.py @@ -0,0 +1,37 @@ +import os +import voila.app +import pytest + + +@pytest.fixture +def progressive_rendering_mode(): + return True + + +@pytest.fixture +def preheat_mode(): + return True + + +@pytest.fixture +def voila_notebook(notebook_directory): + return os.path.join(notebook_directory, "preheat", "pre_heat.ipynb") + + +class VoilaTest(voila.app.Voila): + def listen(self): + pass + + +def test_voila(voila_args, voila_config, preheat_config, progressive_rendering_config): + with pytest.raises(Exception) as e_info: + voila_app = VoilaTest.instance() + voila_app.initialize( + [*voila_args, "--no-browser", preheat_config, progressive_rendering_config] + ) + voila_config(voila_app) + voila_app.start() + assert ( + str(e_info.value) + == "`preheat_kernel` and `progressive_rendering` are incompatible" + ) diff --git a/tests/app/progressive_rendering_test.py b/tests/app/progressive_rendering_test.py new file mode 100644 index 000000000..0208696d5 --- /dev/null +++ b/tests/app/progressive_rendering_test.py @@ -0,0 +1,39 @@ +import asyncio +import os +import time + +import pytest + + +@pytest.fixture +def progressive_rendering_mode(): + return True + + +@pytest.fixture +def voila_notebook(notebook_directory): + return os.path.join(notebook_directory, "preheat", "pre_heat.ipynb") + + +NOTEBOOK_EXECUTION_TIME = 2 +TIME_THRESHOLD = NOTEBOOK_EXECUTION_TIME + + +async def send_request(sc, url, wait=0): + await asyncio.sleep(wait) + real_time = time.time() + response = await sc.fetch(url) + real_time = time.time() - real_time + html_text = response.body.decode("utf-8") + return real_time, html_text + + +async def test_request(http_server_client, base_url): + """ + We send a request to server immediately, the returned HTML should + not contain the output. + """ + time, text = await send_request(sc=http_server_client, url=base_url) + assert '"progressiveRendering": true' in text + assert "hello world" not in text + assert time < NOTEBOOK_EXECUTION_TIME diff --git a/ui-tests/package.json b/ui-tests/package.json index 40e01f3bd..f87cdbd08 100644 --- a/ui-tests/package.json +++ b/ui-tests/package.json @@ -5,11 +5,13 @@ "private": true, "scripts": { "start": "voila ../notebooks --no-browser --show_tracebacks True --Voila.mathjax_url=https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.7/MathJax.js", + "start:progressive_rendering": "voila ../notebooks --no-browser --show_tracebacks True --Voila.mathjax_url=https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.7/MathJax.js --progressive_rendering=true", "start:detached": "yarn run start&", "test": "npx playwright test", "test:debug": "PWDEBUG=1 playwright test", "test:report": "http-server ./playwright-report -a localhost -o", - "test:update": "npx playwright test --update-snapshots" + "test:update": "npx playwright test --update-snapshots", + "test:update:progressive_rendering": "PROGRESSIVE_RENDERING=true && npx playwright test --update-snapshots" }, "author": "Project Jupyter", "license": "BSD-3-Clause", diff --git a/ui-tests/playwright.config.js b/ui-tests/playwright.config.js index 70ad94266..59ea65364 100644 --- a/ui-tests/playwright.config.js +++ b/ui-tests/playwright.config.js @@ -18,7 +18,7 @@ module.exports = { // Try one retry as some tests are flaky retries: 1, expect: { - toMatchSnapshot: { + toHaveScreenshot: { maxDiffPixelRatio: 0.05 } } diff --git a/ui-tests/tests/voila.test.ts b/ui-tests/tests/voila.test.ts index db486fc19..fcca4b364 100644 --- a/ui-tests/tests/voila.test.ts +++ b/ui-tests/tests/voila.test.ts @@ -4,6 +4,7 @@ import { expect, test } from '@playwright/test'; import { addBenchmarkToTest } from './utils'; +const PROGRESSIVE_RENDERING = process.env.PROGRESSIVE_RENDERING === 'true'; test.describe('Voila performance Tests', () => { test.beforeEach(({ page }) => { page.setDefaultTimeout(120000); @@ -24,7 +25,8 @@ test.describe('Voila performance Tests', () => { browserName ); - expect(await page.screenshot()).toMatchSnapshot('voila-tree-classic.png'); + // await expect(page).toHaveScreenshot('voila-tree-classic.png'); + await expect(page).toHaveScreenshot('voila-tree-classic.png'); }); test('Render tree light theme', async ({ page, browserName }, testInfo) => { @@ -40,7 +42,7 @@ test.describe('Voila performance Tests', () => { browserName ); - expect(await page.screenshot()).toMatchSnapshot('voila-tree-light.png'); + await expect(page).toHaveScreenshot('voila-tree-light.png'); }); test('Render tree dark theme', async ({ page, browserName }, testInfo) => { @@ -56,7 +58,7 @@ test.describe('Voila performance Tests', () => { browserName ); - expect(await page.screenshot()).toMatchSnapshot('voila-tree-dark.png'); + await expect(page).toHaveScreenshot('voila-tree-dark.png'); }); test('Render tree miami theme', async ({ page, browserName }, testInfo) => { @@ -72,7 +74,7 @@ test.describe('Voila performance Tests', () => { browserName ); - expect(await page.screenshot()).toMatchSnapshot('voila-tree-miami.png'); + await expect(page).toHaveScreenshot('voila-tree-miami.png'); }); test('Render and benchmark basics.ipynb with classic template', async ({ @@ -90,9 +92,7 @@ test.describe('Voila performance Tests', () => { // wait for the final MathJax message to be hidden await page.$('text=Typesetting math: 100%'); await page.waitForSelector('#MathJax_Message', { state: 'hidden' }); - expect(await page.screenshot()).toMatchSnapshot( - `${notebookName}-classic.png` - ); + await expect(page).toHaveScreenshot(`${notebookName}-classic.png`); }); test('Render and benchmark basics.ipynb', async ({ @@ -117,7 +117,7 @@ test.describe('Voila performance Tests', () => { // wait for the final MathJax message to be hidden await page.$('text=Typesetting math: 100%'); await page.waitForSelector('#MathJax_Message', { state: 'hidden' }); - expect(await page.screenshot()).toMatchSnapshot(`${notebookName}.png`); + await expect(page).toHaveScreenshot(`${notebookName}.png`); }); test('Render basics.ipynb with dark theme', async ({ @@ -135,7 +135,7 @@ test.describe('Voila performance Tests', () => { // wait for the final MathJax message to be hidden await page.$('text=Typesetting math: 100%'); await page.waitForSelector('#MathJax_Message', { state: 'hidden' }); - expect(await page.screenshot()).toMatchSnapshot(`${notebookName}-dark.png`); + await expect(page).toHaveScreenshot(`${notebookName}-dark.png`); }); test('Render basics.ipynb with miami theme', async ({ @@ -155,30 +155,28 @@ test.describe('Voila performance Tests', () => { // wait for the final MathJax message to be hidden await page.$('text=Typesetting math: 100%'); await page.waitForSelector('#MathJax_Message', { state: 'hidden' }); - expect(await page.screenshot()).toMatchSnapshot( - `${notebookName}-miami.png` - ); + await expect(page).toHaveScreenshot(`${notebookName}-miami.png`); }); test('Render 404 error', async ({ page }) => { await page.goto('/voila/render/unknown.ipynb'); await page.waitForSelector('.voila-error'); - expect(await page.screenshot()).toMatchSnapshot('404.png'); + await expect(page).toHaveScreenshot('404.png'); }); test('Render 404 error with classic template', async ({ page }) => { await page.goto('/voila/render/unknown.ipynb?template=classic'); await page.waitForSelector('.voila-error'); - expect(await page.screenshot()).toMatchSnapshot('404-classic.png'); + await expect(page).toHaveScreenshot('404-classic.png'); }); test('Render 404 error with dark theme', async ({ page }) => { await page.goto('/voila/render/unknown.ipynb?theme=dark'); await page.waitForSelector('.voila-error'); - expect(await page.screenshot()).toMatchSnapshot('404-dark.png'); + await expect(page).toHaveScreenshot('404-dark.png'); }); test('Render and benchmark bqplot.ipynb', async ({ @@ -191,7 +189,7 @@ test.describe('Voila performance Tests', () => { await page.waitForSelector('svg.svg-figure'); }; await addBenchmarkToTest(notebookName, testFunction, testInfo, browserName); - expect(await page.screenshot()).toMatchSnapshot(`${notebookName}.png`); + await expect(page).toHaveScreenshot(`${notebookName}.png`); }); test('Render and benchmark dashboard.ipynb', async ({ @@ -222,7 +220,7 @@ test.describe('Voila performance Tests', () => { ); }; await addBenchmarkToTest(notebookName, testFunction, testInfo, browserName); - expect(await page.screenshot()).toMatchSnapshot(`${notebookName}.png`); + await expect(page).toHaveScreenshot(`${notebookName}.png`); }); test('Render and benchmark interactive.ipynb', async ({ @@ -240,7 +238,7 @@ test.describe('Voila performance Tests', () => { await page.mouse.click(0, 0); }; await addBenchmarkToTest(notebookName, testFunction, testInfo, browserName); - expect(await page.screenshot()).toMatchSnapshot(`${notebookName}.png`); + await expect(page).toHaveScreenshot(`${notebookName}.png`); }); test('Render and benchmark ipympl.ipynb', async ({ @@ -253,7 +251,7 @@ test.describe('Voila performance Tests', () => { await page.waitForSelector('div.jupyter-matplotlib-figure'); }; await addBenchmarkToTest(notebookName, testFunction, testInfo, browserName); - expect(await page.screenshot()).toMatchSnapshot(`${notebookName}.png`); + await expect(page).toHaveScreenshot(`${notebookName}.png`); }); test('Render and benchmark mimerenderers.ipynb', async ({ @@ -268,7 +266,7 @@ test.describe('Voila performance Tests', () => { await page.waitForTimeout(2000); }; await addBenchmarkToTest(notebookName, testFunction, testInfo, browserName); - expect(await page.screenshot()).toMatchSnapshot(`${notebookName}.png`); + await expect(page).toHaveScreenshot(`${notebookName}.png`); }); test('Render and benchmark bokeh.ipynb', async ({ @@ -281,7 +279,7 @@ test.describe('Voila performance Tests', () => { await page.waitForSelector('.bk-Canvas'); }; await addBenchmarkToTest(notebookName, testFunction, testInfo, browserName); - expect(await page.screenshot()).toMatchSnapshot(`${notebookName}.png`); + await expect(page).toHaveScreenshot(`${notebookName}.png`); }); test('Benchmark the multiple widgets notebook', async ({ @@ -301,7 +299,7 @@ test.describe('Voila performance Tests', () => { testInfo, browserName ); - expect(await page.screenshot()).toMatchSnapshot(`${notebookName}.png`); + await expect(page).toHaveScreenshot(`${notebookName}.png`); }); test('Render and benchmark query-strings.ipynb', async ({ @@ -311,6 +309,15 @@ test.describe('Voila performance Tests', () => { const notebookName = 'query-strings'; const testFunction = async () => { await page.goto(`/voila/render/${notebookName}.ipynb`); + if (PROGRESSIVE_RENDERING) { + await page.waitForSelector( + 'div.lm-Widget[data-mime-type="application/vnd.jupyter.stdout"]' + ); + } else { + await page.waitForSelector( + 'div.jp-OutputArea-output[data-mime-type="text/plain"]' + ); + } const userName = await page.$$( 'div.jp-RenderedText.jp-OutputArea-output > pre' ); @@ -318,13 +325,22 @@ test.describe('Voila performance Tests', () => { }; await addBenchmarkToTest(notebookName, testFunction, testInfo, browserName); await page.goto(`/voila/render/${notebookName}.ipynb?username=Riley`); + if (PROGRESSIVE_RENDERING) { + await page.waitForSelector( + 'div.lm-Widget[data-mime-type="application/vnd.jupyter.stdout"]' + ); + } else { + await page.waitForSelector( + 'div.jp-OutputArea-output[data-mime-type="text/plain"]' + ); + } const userName = await page.$$( 'div.jp-RenderedText.jp-OutputArea-output > pre' ); expect(await userName[1].innerHTML()).toContain('Hi Riley'); await page.$('text=Typesetting math: 100%'); await page.waitForSelector('#MathJax_Message', { state: 'hidden' }); - expect(await page.screenshot()).toMatchSnapshot(`${notebookName}.png`); + await expect(page).toHaveScreenshot(`${notebookName}.png`); }); test('Render and benchmark reveal.ipynb', async ({ @@ -337,13 +353,13 @@ test.describe('Voila performance Tests', () => { await page.waitForSelector('span[role="presentation"] >> text=x'); }; await addBenchmarkToTest(notebookName, testFunction, testInfo, browserName); - expect(await page.screenshot()).toMatchSnapshot(`${notebookName}.png`); + await expect(page).toHaveScreenshot(`${notebookName}.png`); }); test('Render yaml.ipynb', async ({ page, browserName }, testInfo) => { const notebookName = 'yaml'; await page.goto(`/voila/render/${notebookName}.ipynb`); await page.waitForSelector('span >> text=hey'); - expect(await page.screenshot()).toMatchSnapshot(`${notebookName}.png`); + await expect(page).toHaveScreenshot(`${notebookName}.png`); }); }); diff --git a/ui-tests/tests/voila.test.ts-snapshots/404-dark-linux.png b/ui-tests/tests/voila.test.ts-snapshots/404-dark-linux.png index b81e5d451..5e5731036 100644 Binary files a/ui-tests/tests/voila.test.ts-snapshots/404-dark-linux.png and b/ui-tests/tests/voila.test.ts-snapshots/404-dark-linux.png differ diff --git a/ui-tests/tests/voila.test.ts-snapshots/404-linux.png b/ui-tests/tests/voila.test.ts-snapshots/404-linux.png index 7ac9b6054..40151eb1e 100644 Binary files a/ui-tests/tests/voila.test.ts-snapshots/404-linux.png and b/ui-tests/tests/voila.test.ts-snapshots/404-linux.png differ diff --git a/ui-tests/tests/voila.test.ts-snapshots/bokeh-linux.png b/ui-tests/tests/voila.test.ts-snapshots/bokeh-linux.png index e5fda33b9..3cdd25e63 100644 Binary files a/ui-tests/tests/voila.test.ts-snapshots/bokeh-linux.png and b/ui-tests/tests/voila.test.ts-snapshots/bokeh-linux.png differ diff --git a/ui-tests/tests/voila.test.ts-snapshots/gridspecLayout-linux.png b/ui-tests/tests/voila.test.ts-snapshots/gridspecLayout-linux.png index eb77fc2a5..d8950e4f5 100644 Binary files a/ui-tests/tests/voila.test.ts-snapshots/gridspecLayout-linux.png and b/ui-tests/tests/voila.test.ts-snapshots/gridspecLayout-linux.png differ diff --git a/ui-tests/tests/voila.test.ts-snapshots/ipympl-linux.png b/ui-tests/tests/voila.test.ts-snapshots/ipympl-linux.png index 5b93301dc..adfea8303 100644 Binary files a/ui-tests/tests/voila.test.ts-snapshots/ipympl-linux.png and b/ui-tests/tests/voila.test.ts-snapshots/ipympl-linux.png differ diff --git a/ui-tests/tests/voila.test.ts-snapshots/voila-tree-classic-linux.png b/ui-tests/tests/voila.test.ts-snapshots/voila-tree-classic-linux.png index 62320aaea..c7df7f70f 100644 Binary files a/ui-tests/tests/voila.test.ts-snapshots/voila-tree-classic-linux.png and b/ui-tests/tests/voila.test.ts-snapshots/voila-tree-classic-linux.png differ diff --git a/ui-tests/tests/voila.test.ts-snapshots/voila-tree-dark-linux.png b/ui-tests/tests/voila.test.ts-snapshots/voila-tree-dark-linux.png index 833bd75d1..158400b58 100644 Binary files a/ui-tests/tests/voila.test.ts-snapshots/voila-tree-dark-linux.png and b/ui-tests/tests/voila.test.ts-snapshots/voila-tree-dark-linux.png differ diff --git a/ui-tests/tests/voila.test.ts-snapshots/voila-tree-light-linux.png b/ui-tests/tests/voila.test.ts-snapshots/voila-tree-light-linux.png index c082fa0f2..0a95bca04 100644 Binary files a/ui-tests/tests/voila.test.ts-snapshots/voila-tree-light-linux.png and b/ui-tests/tests/voila.test.ts-snapshots/voila-tree-light-linux.png differ diff --git a/voila/app.py b/voila/app.py index f83fc7690..32f26a3ce 100644 --- a/voila/app.py +++ b/voila/app.py @@ -20,6 +20,10 @@ import threading import webbrowser +from .tornado.kernel_websocket_handler import VoilaKernelWebsocketHandler + +from .tornado.execution_request_handler import ExecutionRequestHandler + from .tornado.contentshandler import VoilaContentsHandler from .voila_identity_provider import VoilaLoginHandler @@ -36,7 +40,6 @@ from jupyter_server.services.config.manager import ConfigManager from jupyter_server.services.contents.largefilemanager import LargeFileManager from jupyter_server.services.kernels.handlers import KernelHandler -from jupyter_server.services.kernels.websocket import KernelWebsocketHandler from jupyter_server.auth.authorizer import AllowAllAuthorizer, Authorizer from jupyter_server.auth.identity import PasswordIdentityProvider from jupyter_server import DEFAULT_TEMPLATE_PATH_LIST, DEFAULT_STATIC_FILES_PATH @@ -171,6 +174,7 @@ class Voila(Application): "pool_size": "VoilaConfiguration.default_pool_size", "show_tracebacks": "VoilaConfiguration.show_tracebacks", "preheat_kernel": "VoilaConfiguration.preheat_kernel", + "progressive_rendering": "VoilaConfiguration.progressive_rendering", "strip_sources": "VoilaConfiguration.strip_sources", "template": "VoilaConfiguration.template", "theme": "VoilaConfiguration.theme", @@ -590,6 +594,12 @@ def init_settings(self) -> Dict: if preheat_kernel and self.prelaunch_hook: raise Exception("`preheat_kernel` and `prelaunch_hook` are incompatible") + progressive_rendering = self.voila_configuration.progressive_rendering + if preheat_kernel and progressive_rendering: + raise Exception( + "`preheat_kernel` and `progressive_rendering` are incompatible" + ) + kernel_manager_class = voila_kernel_manager_factory( self.voila_configuration.multi_kernel_manager_class, preheat_kernel, @@ -687,7 +697,7 @@ def init_handlers(self) -> List: url_path_join( self.server_url, r"/api/kernels/%s/channels" % _kernel_id_regex ), - KernelWebsocketHandler, + VoilaKernelWebsocketHandler, ), ( url_path_join(self.server_url, r"/voila/templates/(.*)"), @@ -724,6 +734,15 @@ def init_handlers(self) -> List: RequestInfoSocketHandler, ) ) + if self.voila_configuration.progressive_rendering: + handlers.append( + ( + url_path_join( + self.server_url, r"/voila/execution/%s" % _kernel_id_regex + ), + ExecutionRequestHandler, + ) + ) # Serving JupyterLab extensions handlers.append( ( @@ -835,6 +854,8 @@ def listen(self): try: self.app.listen(port, self.ip) self.port = port + if self.voila_configuration.progressive_rendering: + self.log.info("Progressive rendering is activated") self.log.info("Voilà is running at:\n%s" % self.display_url) except OSError as e: if e.errno == errno.EADDRINUSE: diff --git a/voila/configuration.py b/voila/configuration.py index e44fb338f..229aa1ec9 100644 --- a/voila/configuration.py +++ b/voila/configuration.py @@ -217,3 +217,11 @@ def _valid_file_blacklist(self, proposal): config=True, help="""Whether or not voila should attempt to fix and resolve a notebooks kernelspec metadata""", ) + + progressive_rendering = Bool( + False, + config=True, + help="""Flag to enable or disable progressive rendering option. + This option is not compatible with preheat-kernel option. + """, + ) diff --git a/voila/handler.py b/voila/handler.py index 22a541524..059e23ca8 100644 --- a/voila/handler.py +++ b/voila/handler.py @@ -19,6 +19,8 @@ from tornado.httputil import split_host_and_port from traitlets.traitlets import Bool +from voila.tornado.execution_request_handler import ExecutionRequestHandler + from .configuration import VoilaConfiguration from ._version import __version__ @@ -236,6 +238,12 @@ def time_out(): ) kernel_future = self.kernel_manager.get_kernel(kernel_id) queue = asyncio.Queue() + if self.voila_configuration.progressive_rendering: + ExecutionRequestHandler._execution_data[kernel_id] = { + "nb": gen.notebook, + "config": self.traitlet_config, + "show_tracebacks": self.voila_configuration.show_tracebacks, + } async def put_html(): async for html_snippet, _ in gen.generate_content_generator( diff --git a/voila/notebook_renderer.py b/voila/notebook_renderer.py index 2ba094e59..c1a8791d3 100644 --- a/voila/notebook_renderer.py +++ b/voila/notebook_renderer.py @@ -13,7 +13,7 @@ import traceback from functools import partial from copy import deepcopy -from typing import Generator, List, Tuple, Union +from typing import AsyncGenerator, Generator, List, Tuple, Union import nbformat import tornado.web @@ -156,7 +156,7 @@ def generate_content_generator( self, kernel_id: Union[str, None] = None, kernel_future=None, - ) -> Generator: + ) -> AsyncGenerator: inner_kernel_start = partial( self._jinja_kernel_start, kernel_id=kernel_id, kernel_future=kernel_future ) @@ -171,9 +171,16 @@ def generate_content_generator( "frontend": "voila", "main_js": "voila.js", "kernel_start": inner_kernel_start, - "cell_generator": self._jinja_cell_generator, "notebook_execute": self._jinja_notebook_execute, + "progressive_rendering": self.voila_configuration.progressive_rendering, } + if self.voila_configuration.progressive_rendering: + extra_context["cell_generator"] = ( + self._jinja_cell_generator_without_execution + ) + else: + extra_context["cell_generator"] = self._jinja_cell_generator + # render notebook in snippets, then return an iterator so we can flush # them out to the browser progressively. return self.exporter.generate_from_notebook_node( @@ -242,15 +249,30 @@ async def _jinja_kernel_start(self, nb, kernel_id, kernel_future): return kernel_id async def _jinja_notebook_execute(self, nb, kernel_id): - result = await self.executor.async_execute(cleanup_kc=False) # we modify the notebook in place, since the nb variable cannot be # reassigned it seems in jinja2 e.g. if we do {% with nb = notebook_execute(nb, kernel_id) %} # ,the base template/blocks will not see the updated variable # (it seems to be local to our block) + if self.voila_configuration.progressive_rendering: + result, _ = ClearOutputPreprocessor().preprocess( + nb, {"metadata": {"path": self.cwd}} + ) + else: + result = await self.executor.async_execute(cleanup_kc=False) + nb.cells = result.cells await self._cleanup_resources() + async def _jinja_cell_generator_without_execution(self, nb, kernel_id): + nb, _ = ClearOutputPreprocessor().preprocess( + nb, {"metadata": {"path": self.cwd}} + ) + for input_cell in nb.cells: + output = input_cell.copy() + yield output + await self._cleanup_resources() + async def _jinja_cell_generator(self, nb, kernel_id): """Generator that will execute a single notebook cell at a time""" nb, _ = ClearOutputPreprocessor().preprocess( diff --git a/voila/request_info_handler.py b/voila/request_info_handler.py index 57e8ac610..8e1596b85 100644 --- a/voila/request_info_handler.py +++ b/voila/request_info_handler.py @@ -44,7 +44,7 @@ def send_updates(cls: "RequestInfoSocketHandler", msg: Dict) -> None: notebook. This method is called in `VoilaHandler` when the request info becomes available. If this method is called before the opening of websocket connection, - `msg` is stored in `_cache0` and the message will be dispatched when + `msg` is stored in `_cache` and the message will be dispatched when a notebook with corresponding kernel id is connected. Args: @@ -59,5 +59,5 @@ def send_updates(cls: "RequestInfoSocketHandler", msg: Dict) -> None: waiter.write_message(payload) except Exception: logging.error("Error sending message", exc_info=True) - - cls._cache[kernel_id] = payload + else: + cls._cache[kernel_id] = payload diff --git a/voila/server_extension.py b/voila/server_extension.py index 6a5a99b47..2fab17506 100644 --- a/voila/server_extension.py +++ b/voila/server_extension.py @@ -16,6 +16,8 @@ from jupyterlab_server.themes_handler import ThemesHandler from jupyter_core.paths import jupyter_config_path from jupyter_server.serverapp import ServerApp + +from .tornado.execution_request_handler import ExecutionRequestHandler from .tornado.contentshandler import VoilaContentsHandler from traitlets.config import ( JSONFileConfigLoader, @@ -35,6 +37,8 @@ from .tornado.treehandler import TornadoVoilaTreeHandler from .utils import get_data_dir, get_server_root_dir, pjoin +_kernel_id_regex = r"(?P\w+-\w+-\w+-\w+-\w+)" + def _jupyter_server_extension_points(): """ @@ -105,67 +109,72 @@ def _load_jupyter_server_extension(server_app: ServerApp): tree_handler_conf = {"voila_configuration": voila_configuration} themes_dir = pjoin(get_data_dir(), "themes") - web_app.add_handlers( - host_pattern, - [ - ( - url_path_join(base_url, "/voila/render/(.*)"), - TornadoVoilaHandler, - { - "config": server_app.config, - "template_paths": template_paths, - "voila_configuration": voila_configuration, - }, - ), - ( - url_path_join(base_url, "/voila"), - TornadoVoilaTreeHandler, - tree_handler_conf, - ), + handlers = [ + ( + url_path_join(base_url, "/voila/render/(.*)"), + TornadoVoilaHandler, + { + "config": server_app.config, + "template_paths": template_paths, + "voila_configuration": voila_configuration, + }, + ), + ( + url_path_join(base_url, "/voila"), + TornadoVoilaTreeHandler, + tree_handler_conf, + ), + ( + url_path_join(base_url, "/voila/tree" + path_regex), + TornadoVoilaTreeHandler, + tree_handler_conf, + ), + ( + url_path_join(base_url, "/voila/templates/(.*)"), + TemplateStaticFileHandler, + ), + ( + url_path_join(base_url, r"/voila/api/themes/(.*)"), + ThemesHandler, + { + "themes_url": "/voila/api/themes", + "path": themes_dir, + "labextensions_path": jupyter_path("labextensions"), + "no_cache_paths": ["/"], + }, + ), + ( + url_path_join(base_url, "/voila/static/(.*)"), + MultiStaticFileHandler, + {"paths": static_paths}, + ), + ( + url_path_join(base_url, r"/voila/api/shutdown/(.*)"), + VoilaShutdownKernelHandler, + ), + ( + url_path_join(base_url, r"/voila/files/(.*)"), + AllowListFileHandler, + { + "allowlist": voila_configuration.file_allowlist, + "denylist": voila_configuration.file_denylist, + "path": os.path.expanduser(get_server_root_dir(web_app.settings)), + }, + ), + ( + url_path_join(base_url, r"/voila/api/contents%s" % path_regex), + VoilaContentsHandler, + tree_handler_conf, + ), + ] + if voila_configuration.progressive_rendering: + handlers.append( ( - url_path_join(base_url, "/voila/tree" + path_regex), - TornadoVoilaTreeHandler, - tree_handler_conf, - ), - ( - url_path_join(base_url, "/voila/templates/(.*)"), - TemplateStaticFileHandler, - ), - ( - url_path_join(base_url, r"/voila/api/themes/(.*)"), - ThemesHandler, - { - "themes_url": "/voila/api/themes", - "path": themes_dir, - "labextensions_path": jupyter_path("labextensions"), - "no_cache_paths": ["/"], - }, - ), - ( - url_path_join(base_url, "/voila/static/(.*)"), - MultiStaticFileHandler, - {"paths": static_paths}, - ), - ( - url_path_join(base_url, r"/voila/api/shutdown/(.*)"), - VoilaShutdownKernelHandler, - ), - ( - url_path_join(base_url, r"/voila/files/(.*)"), - AllowListFileHandler, - { - "allowlist": voila_configuration.file_allowlist, - "denylist": voila_configuration.file_denylist, - "path": os.path.expanduser(get_server_root_dir(web_app.settings)), - }, - ), - ( - url_path_join(base_url, r"/voila/api/contents%s" % path_regex), - VoilaContentsHandler, - tree_handler_conf, - ), - ], - ) + url_path_join(base_url, r"/voila/execution/%s" % _kernel_id_regex), + ExecutionRequestHandler, + ) + ) + web_app.add_handlers(host_pattern, handlers) # Serving lab extensions # TODO: reuse existing lab server endpoint? diff --git a/voila/tornado/execution_request_handler.py b/voila/tornado/execution_request_handler.py new file mode 100644 index 000000000..36210f974 --- /dev/null +++ b/voila/tornado/execution_request_handler.py @@ -0,0 +1,133 @@ +import asyncio +import json +from typing import Awaitable, Union +from jupyter_server.base.handlers import JupyterHandler +from tornado.websocket import WebSocketHandler +from tornado.web import HTTPError +from jupyter_server.base.websocket import WebSocketMixin +from jupyter_core.utils import ensure_async +from nbclient.exceptions import CellExecutionError +from voila.execute import VoilaExecutor, strip_code_cell_warnings +import nbformat +import traceback +import sys + + +class ExecutionRequestHandler(WebSocketMixin, WebSocketHandler, JupyterHandler): + _execution_data = {} + + def initialize(self, **kwargs): + super().initialize() + + async def open(self, kernel_id: str) -> None: + """Create a new websocket connection, this connection is + identified by the kernel id. + + Args: + kernel_id (str): Kernel id used by the notebook when it opens + the websocket connection. + """ + identity_provider = self.settings.get("identity_provider") + user = await ensure_async(identity_provider.get_user(self)) + if user is None: + raise HTTPError(403, "Unauthenticated") + super().open() + + self._kernel_id = kernel_id + self.write_message({"action": "initialized", "payload": {}}) + + async def on_message( + self, message_str: Union[str, bytes] + ) -> Union[Awaitable[None], None]: + message = json.loads(message_str) + action = message.get("action", None) + payload = message.get("payload", {}) + if action == "execute": + request_kernel_id = payload.get("kernel_id") + if request_kernel_id != self._kernel_id: + await self.write_message( + { + "action": "execution_error", + "payload": {"error": "Kernel ID does not match"}, + } + ) + return + kernel_future = self.kernel_manager.get_kernel(self._kernel_id) + km = await ensure_async(kernel_future) + execution_data = self._execution_data.pop(self._kernel_id, None) + if execution_data is None: + await self.write_message( + { + "action": "execution_error", + "payload": {"error": "Missing notebook data"}, + } + ) + return + nb = execution_data["nb"] + self._executor = executor = VoilaExecutor( + nb, + km=km, + config=execution_data["config"], + show_tracebacks=execution_data["show_tracebacks"], + ) + executor.kc = await executor.async_start_new_kernel_client() + total_cell = len(nb.cells) + for cell_idx, input_cell in enumerate(nb.cells): + try: + output_cell = await executor.execute_cell( + input_cell, None, cell_idx, store_history=False + ) + except TimeoutError: + output_cell = input_cell + + except CellExecutionError: + self.log.exception( + "Error at server while executing cell: %r", input_cell + ) + if executor.should_strip_error(): + strip_code_cell_warnings(input_cell) + executor.strip_code_cell_errors(input_cell) + output_cell = input_cell + + except Exception as e: + self.log.exception( + "Error at server while executing cell: %r", input_cell + ) + output_cell = nbformat.v4.new_code_cell() + if executor.should_strip_error(): + output_cell.outputs = [ + { + "output_type": "stream", + "name": "stderr", + "text": "An exception occurred at the server (not the notebook). {}".format( + executor.cell_error_instruction + ), + } + ] + else: + output_cell.outputs = [ + { + "output_type": "error", + "ename": type(e).__name__, + "evalue": str(e), + "traceback": traceback.format_exception( + *sys.exc_info() + ), + } + ] + finally: + output_cell.pop("source", None) + await self.write_message( + { + "action": "execution_result", + "payload": { + "output_cell": output_cell, + "cell_index": cell_idx, + "total_cell": total_cell, + }, + } + ) + + def on_close(self) -> None: + if self._executor and self._executor.kc: + asyncio.create_task(ensure_async(self._executor.kc.stop_channels())) diff --git a/voila/tornado/kernel_websocket_handler.py b/voila/tornado/kernel_websocket_handler.py new file mode 100644 index 000000000..60b879e6c --- /dev/null +++ b/voila/tornado/kernel_websocket_handler.py @@ -0,0 +1,35 @@ +import json +from typing import Any, Dict, Optional, Union +from jupyter_server.services.kernels.websocket import KernelWebsocketHandler + + +def read_header_from_binary_message(ws_msg: bytes) -> Optional[Dict]: + """Read message header using the v1 protocol.""" + + offset_number = int.from_bytes(ws_msg[:8], "little") + offsets = [ + int.from_bytes(ws_msg[8 * (i + 1) : 8 * (i + 2)], "little") + for i in range(offset_number) + ] + try: + header = ws_msg[offsets[1] : offsets[2]].decode("utf-8") + return json.loads(header) + except Exception: + return + + +class VoilaKernelWebsocketHandler(KernelWebsocketHandler): + def write_message( + self, message: Union[bytes, Dict[str, Any]], binary: bool = False + ): + if isinstance(message, bytes): + header = read_header_from_binary_message(message) + elif isinstance(message, dict): + header = message.get("header", None) + else: + header = None + + if header and header.get("msg_type", None) == "execute_input": + return # Ignore execute_input message + + return super().write_message(message, binary) diff --git a/voila/utils.py b/voila/utils.py index 7100a02af..073ed50a1 100644 --- a/voila/utils.py +++ b/voila/utils.py @@ -99,6 +99,7 @@ def get_page_config(base_url, settings, log, voila_configuration: VoilaConfigura "fullStaticUrl": url_path_join(base_url, "voila/static"), "fullLabextensionsUrl": url_path_join(base_url, "voila/labextensions"), "extensionConfig": voila_configuration.extension_config, + "progressiveRendering": voila_configuration.progressive_rendering, } mathjax_config = settings.get("mathjax_config", "TeX-AMS_CHTML-full,Safe") mathjax_url = settings.get(