Add new Jupyter kernel supervisor (#4910)

This change adds a new and currently optional Jupyter kernel supervisor
to Positron.

When enabled, all R and Python kernels are run under a supervisor
process (codenamed 'Kallichore'). This process is a native binary that
takes care of the minutiae of kernel lifecyle management (process
output, startup, shutdown, restart, status reporting, etc.). It also
takes care of the low-level ZeroMQ socket connectivity and exposes a
simple WebSocket based interface that proxies messages to and from the
ZeroMQ sockets.

```mermaid
graph TD
R[Positron R Extension] -- Jupyter Adapter API --> ka[Kallichore Adapter] 
ka -- WebSocket --> k[Kallichore Supervisor]
k -- ZeroMQ --> ark[R Jupyter Kernel - ark]
py[Positron Python Extension] -- Jupyter Adapter API --> ka
k -- ZeroMQ --> ipy[Python Jupyter Kernel - ipykernel]
```

The implementation is a drop-in replacement for the Jupyter Adapter and
currently exposes an API that is fully compatible with the Jupyter
Adapter; the only changes to the R and Python extensions are switches
that determine which extension the Jupyter Adapter API is loaded from.
The change is otherwise totally transparent to the R and Python
extensions since they already are behind a layer of abstraction from the
Jupyter protocol.

The current plan of record is for the Jupyter Adapter to be fully
deprecated in favor of this new supervisor when it reaches feature
parity, stability, and performance goals. This will allow us to get rid
of our troublesome ZeroMQ bindings in Node and will also improve
stability and performance around session management.

The goal of this PR is to start exposing the new supervisor to builds
and to make it possible to start collaborating on work associated with
the supervisor. While most everything works today with the new
supervisor, there are a lot of edge cases that aren't finished yet, so
the supervisor is disabled by default and will continue to be until it's
ready to take over.

Lint-level issues are aggregated here:
#4912

### QA Notes

This change isn't ready for QA yet. It tries very hard to avoid touching
any code outside the new adapter extension; the only thing worth testing
is probably R and Python restarts.

.eslintignore

@@ -7,6 +7,10 @@
 **/extensions/html-language-features/server/lib/jquery.d.ts
 **/extensions/html-language-features/server/src/test/pathCompletionFixtures/**
 **/extensions/ipynb/notebook-out/**
+# --- Start Positron ---
+# Ignore OpenAPI generated files
+**/extensions/kallichore-adapter/src/kcclient/**
+# --- End Positron ---
 **/extensions/markdown-language-features/media/**
 **/extensions/markdown-language-features/notebook-out/**
 **/extensions/markdown-math/notebook-out/**

.vscode-test.js

@@ -67,6 +67,11 @@ const extensions = [
 		workspaceFolder: 'extensions/positron-run-app/test-workspace',
 		mocha: { timeout: 60_000 }
 	},
+	{
+		label: 'kallichore-adapter',
+		workspaceFolder: path.join(os.tmpdir(), `kallichore-adapter-${Math.floor(Math.random() * 100000)}`),
+		mocha: { timeout: 60_000 }
+	},
 	// --- End Positron ---
 	{
 		label: 'microsoft-authentication',

build/darwin/create-universal-app.ts

@@ -50,6 +50,8 @@ const stashPatterns = [
 	'**/pydevd/**',             // Cython pre-built binaries for Python debugging
 	// Exclusions from R language pack (positron-r)
 	'**/ark',                   // Compiled R kernel and LSP
+	// Exclusions from Kallichore Jupyter supervisor
+	'**/kcserver',              // Compiled Jupyter supervisor
 	// Exclusions from Quarto
 	'**/quarto/bin/tools/**',
 ];

build/filters.js

@@ -28,9 +28,19 @@ module.exports.all = [
 	'!**/node_modules/**',
 
 	// --- Start Positron ---
+	// Excluded since it's generated code (an OpenAPI client)
+	'!extensions/kallichore-adapter/src/kcclient/**/*',
+
+	// Excluded since it comes from an external source with its own hygiene
+	// rules
 	'!extensions/positron-python/**/*',
+
+	// Excluded since it comes from an external source with its own hygiene
+	// rules
 	'!extensions/open-remote-ssh/**/*',
-	'!test/smoke/test-repo/**/*'
+
+	// Excluded since it isn't shipping code
+	'!test/smoke/test-repo/**/*',
 	// --- End Positron ---
 ];
 

build/gulpfile.extensions.js

@@ -29,6 +29,7 @@ const ext = require('./lib/extensions');
 // });
 const compilations = [
 	// --- Start Positron ---
+	'extensions/kallichore-adapter/tsconfig.json',
 	'extensions/open-remote-ssh/tsconfig.json',
 	'extensions/positron-code-cells/tsconfig.json',
 	'extensions/positron-connections/tsconfig.json',

build/npm/dirs.js

@@ -48,6 +48,7 @@ const dirs = [
 	'extensions/json-language-features/server',
 	// --- Start Positron ---
 	'extensions/jupyter-adapter',
+	'extensions/kallichore-adapter',
 	// --- End Positron ---
 	'extensions/markdown-language-features',
 	'extensions/markdown-math',

extensions/jupyter-adapter/src/Api.ts

@@ -35,16 +35,16 @@ export class JupyterAdapterApiImpl implements JupyterAdapterApi {
 		kernel: JupyterKernelSpec,
 		dynState: positron.LanguageRuntimeDynState,
 		extra: JupyterKernelExtra,
-	): JupyterLanguageRuntimeSession {
-		return new LanguageRuntimeSessionAdapter(
+	): Promise<JupyterLanguageRuntimeSession> {
+		return Promise.resolve(new LanguageRuntimeSessionAdapter(
 			runtimeMetadata,
 			sessionMetadata,
 			this._context,
 			this._channel,
 			kernel,
 			dynState,
 			extra
-		);
+		));
 	}
 
 	/**
@@ -59,7 +59,7 @@ export class JupyterAdapterApiImpl implements JupyterAdapterApi {
 	restoreSession(
 		runtimeMetadata: positron.LanguageRuntimeMetadata,
 		sessionMetadata: positron.RuntimeSessionMetadata
-	): JupyterLanguageRuntimeSession {
+	): Promise<JupyterLanguageRuntimeSession> {
 
 		// Get the serialized session from the workspace state. This state
 		// contains the information we need to reconnect to the session, such as
@@ -90,7 +90,7 @@ export class JupyterAdapterApiImpl implements JupyterAdapterApi {
 		// happens later when the session is started.
 		adapter.restoreSession(serialized.sessionState);
 
-		return adapter;
+		return Promise.resolve(adapter);
 	}
 
 	/**

extensions/jupyter-adapter/src/jupyter-adapter.d.ts

@@ -147,7 +147,7 @@ export interface JupyterAdapterApi extends vscode.Disposable {
 		kernel: JupyterKernelSpec,
 		dynState: positron.LanguageRuntimeDynState,
 		extra?: JupyterKernelExtra | undefined,
-	): JupyterLanguageRuntimeSession;
+	): Promise<JupyterLanguageRuntimeSession>;
 
 	/**
 	 * Restore a session for a Jupyter-compatible kernel.
@@ -161,7 +161,7 @@ export interface JupyterAdapterApi extends vscode.Disposable {
 	restoreSession(
 		runtimeMetadata: positron.LanguageRuntimeMetadata,
 		sessionMetadata: positron.RuntimeSessionMetadata
-	): JupyterLanguageRuntimeSession;
+	): Promise<JupyterLanguageRuntimeSession>;
 
 	/**
 	 * Finds an available TCP port for a server

extensions/kallichore-adapter/extension.webpack.config.js

@@ -0,0 +1,25 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (C) 2024 Posit Software, PBC. All rights reserved.
+ *  Licensed under the Elastic License 2.0. See LICENSE.txt for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+//@ts-check
+
+'use strict';
+
+const path = require('path');
+const withDefaults = require('../shared.webpack.config');
+
+module.exports = withDefaults({
+	context: __dirname,
+	entry: {
+		extension: './src/extension.ts',
+	},
+	node: {
+		__dirname: false
+	},
+	externals: {
+		'bufferutil': 'commonjs bufferutil',
+		'utf-8-validate': 'commonjs utf-8-validate'
+	}
+});