Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 8c5475f

Browse files
antocunihoodmanemadhur-tandon
authored
Move pyodide to a web worker (pyscript#1333)
This PR adds support for optionally running pyodide in a web worker: - add a new option config.execution_thread, which can be `main` or `worker`. The default is `main` - improve the test machinery so that we run all tests twice, once for `main` and once for `worker` - add a new esbuild target which builds the code for the worker The support for workers is not complete and many features are still missing: there are 71 tests which are marked as `@skip_worker`, but we can fix them in subsequent PRs. The vast majority of tests fail because js.document is unavailable: for it to run transparently, we need the "auto-syncify" feature of synclink. Co-authored-by: Hood Chatham <[email protected]> Co-authored-by: Madhur Tandon <[email protected]>
1 parent dfa116e commit 8c5475f

28 files changed

+495
-97
lines changed

‎docs/changelog.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,10 @@ Features
2626
### Plugins
2727
- Plugins may now implement the `beforePyReplExec()` and `afterPyReplExec()` hooks, which are called immediately before and after code in a `py-repl` tag is executed. ([#1106](https://github.com/pyscript/pyscript/pull/1106))
2828

29+
### Web worker support
30+
- introduced the new experimental `execution_thread` config option: if you set `execution_thread = "worker"`, the python interpreter runs inside a web worker
31+
- worker support is still **very** experimental: not everything works, use it at your own risk
32+
2933
Bug fixes
3034
---------
3135

‎docs/development/developing.md‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -190,6 +190,16 @@ $ pytest test_01_basic.py -k test_pyscript_hello -s --dev
190190
`--dev` implies `--headed --no-fake-server`. In addition, it also
191191
automatically open chrome dev tools.
192192

193+
#### To run only main thread or worker tests
194+
195+
By default, we run each test twice: one with `execution_thread = "main"` and
196+
one with `execution_thread = "worker"`. If you want to run only half of them,
197+
you can use `-m`:
198+
199+
```
200+
$ pytest -m main # run only the tests in the main thread
201+
$ pytest -m worker # ron only the tests in the web worker
202+
```
193203

194204
## Fake server, HTTP cache
195205

‎pyscriptjs/.eslintrc.js‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -16,20 +16,18 @@ module.exports = {
1616
browser: true,
1717
},
1818
plugins: ['@typescript-eslint'],
19-
ignorePatterns: ['node_modules'],
19+
ignorePatterns: ['node_modules', 'src/interpreter_worker/*'],
2020
rules: {
2121
// ts-ignore is already an explicit override, no need to have a second lint
2222
'@typescript-eslint/ban-ts-comment': 'off',
2323

24-
// any-related lints
25-
// These two come up a lot, so they probably aren't worth it
24+
// any related lints
2625
'@typescript-eslint/no-explicit-any': 'off',
2726
'@typescript-eslint/no-unsafe-assignment': 'off',
28-
// encourage people to cast "any" to a more specific type before using it
29-
'@typescript-eslint/no-unsafe-call': 'error',
30-
'@typescript-eslint/no-unsafe-member-access': 'error',
31-
'@typescript-eslint/no-unsafe-argument': 'error',
32-
'@typescript-eslint/no-unsafe-return': 'error',
27+
'@typescript-eslint/no-unsafe-call': 'off',
28+
'@typescript-eslint/no-unsafe-member-access': 'off',
29+
'@typescript-eslint/no-unsafe-argument': 'off',
30+
'@typescript-eslint/no-unsafe-return': 'off',
3331

3432
// other rules
3533
'no-prototype-builtins': 'error',

‎pyscriptjs/Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ build:
5959
npm run build
6060

6161
build-fast:
62-
node esbuild.js
62+
node esbuild.mjs
6363

6464
# use the following rule to do all the checks done by precommit: in
6565
# particular, use this if you want to run eslint.

‎pyscriptjs/environment.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ dependencies:
1414
- pillow
1515
- numpy
1616
- markdown
17+
- toml
1718
- pip:
1819
- playwright
1920
- pytest-playwright

‎pyscriptjs/esbuild.mjs‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,14 @@ const pyScriptConfig = {
6161
plugins: [bundlePyscriptPythonPlugin()],
6262
};
6363

64+
const interpreterWorkerConfig = {
65+
entryPoints: ['src/interpreter_worker/worker.ts'],
66+
loader: { '.py': 'text' },
67+
bundle: true,
68+
format: 'iife',
69+
plugins: [bundlePyscriptPythonPlugin()],
70+
};
71+
6472
const copyPath = (source, dest, ...rest) => cp(join(__dirname, source), join(__dirname, dest), ...rest);
6573

6674
const esbuild = async () => {
@@ -80,6 +88,14 @@ const esbuild = async () => {
8088
minify: true,
8189
outfile: 'build/pyscript.min.js',
8290
}),
91+
// XXX I suppose we should also build a minified version
92+
// TODO (HC): Simplify config a bit
93+
build({
94+
...interpreterWorkerConfig,
95+
sourcemap: false,
96+
minify: false,
97+
outfile: 'build/interpreter_worker.js',
98+
}),
8399
]);
84100

85101
const copy = [];

‎pyscriptjs/src/interpreter_client.ts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,6 @@ export class InterpreterClient extends Object {
4040
*/
4141
async initializeRemote(): Promise<void> {
4242
await this._remote.loadInterpreter(this.config, Synclink.proxy(this.stdio));
43-
// await this._remote.loadInterpreter(this.config, Synclink.proxy(this.stdio));
4443
this.globals = this._remote.globals;
4544
}
4645

‎pyscriptjs/src/interpreter_worker/worker.ts‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
// XXX: what about code duplications?
2+
// With the current build configuration, the code for logger,
3+
// remote_interpreter and everything which is included from there is
4+
// bundled/fetched/executed twice, once in pyscript.js and once in
5+
// worker_interpreter.js.
6+
7+
import { getLogger } from '../logger';
8+
import { RemoteInterpreter } from '../remote_interpreter';
9+
import * as Synclink from 'synclink';
10+
11+
const logger = getLogger('worker');
12+
logger.info('Interpreter worker starting...');
13+
14+
async function worker_initialize(cfg) {
15+
const remote_interpreter = new RemoteInterpreter(cfg.src);
16+
// this is the equivalent of await import(interpreterURL)
17+
logger.info(`Downloading ${cfg.name}...`); // XXX we should use logStatus
18+
importScripts(cfg.src);
19+
20+
logger.info('worker_initialize() complete');
21+
return Synclink.proxy(remote_interpreter);
22+
}
23+
24+
Synclink.expose(worker_initialize);
25+
26+
export type { worker_initialize };

‎pyscriptjs/src/main.ts‎

Lines changed: 53 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import './styles/pyscript_base.css';
22

33
import { loadConfigFromElement } from './pyconfig';
4-
import type { AppConfig } from './pyconfig';
4+
import type { AppConfig, InterpreterConfig } from './pyconfig';
55
import { InterpreterClient } from './interpreter_client';
66
import { PluginManager, Plugin, PythonPlugin } from './plugin';
77
import { make_PyScript, initHandlers, mountElements } from './components/pyscript';
@@ -59,16 +59,6 @@ throwHandler.serialize = new_error_transfer_handler;
5959
user scripts
6060
6161
8. initialize the rest of web components such as py-button, py-repl, etc.
62-
63-
More concretely:
64-
65-
- Points 1-4 are implemented sequentially in PyScriptApp.main().
66-
67-
- PyScriptApp.loadInterpreter adds a <script> tag to the document to initiate
68-
the download, and then adds an event listener for the 'load' event, which
69-
in turns calls PyScriptApp.afterInterpreterLoad().
70-
71-
- PyScriptApp.afterInterpreterLoad() implements all the points >= 5.
7262
*/
7363

7464
export let interpreter;
@@ -173,34 +163,26 @@ export class PyScriptApp {
173163
logger.info('config loaded:\n' + JSON.stringify(this.config, null, 2));
174164
}
175165

176-
// lifecycle (4)
177-
async loadInterpreter() {
178-
logger.info('Initializing interpreter');
179-
if (this.config.interpreters.length == 0) {
180-
throw new UserError(ErrorCode.BAD_CONFIG, 'Fatal error: config.interpreter is empty');
181-
}
182-
183-
if (this.config.interpreters.length > 1) {
184-
showWarning('Multiple interpreters are not supported yet.<br />Only the first will be used', 'html');
185-
}
186-
187-
const interpreter_cfg = this.config.interpreters[0];
166+
_get_base_url(): string {
167+
// Note that this requires that pyscript is loaded via a <script>
168+
// tag. If we want to allow loading via an ES6 module in the future,
169+
// we need to think about some other strategy
170+
const elem = document.currentScript as HTMLScriptElement;
171+
const slash = elem.src.lastIndexOf('/');
172+
return elem.src.slice(0, slash);
173+
}
188174

175+
async _startInterpreter_main(interpreter_cfg: InterpreterConfig) {
176+
logger.info('Starting the interpreter in the main thread');
177+
// this is basically equivalent to worker_initialize()
189178
const remote_interpreter = new RemoteInterpreter(interpreter_cfg.src);
190179
const { port1, port2 } = new Synclink.FakeMessageChannel() as unknown as MessageChannel;
191180
port1.start();
192181
port2.start();
193182
Synclink.expose(remote_interpreter, port2);
194183
const wrapped_remote_interpreter = Synclink.wrap(port1);
195-
this.interpreter = new InterpreterClient(
196-
this.config,
197-
this._stdioMultiplexer,
198-
wrapped_remote_interpreter as Synclink.Remote<RemoteInterpreter>,
199-
remote_interpreter,
200-
);
201184

202185
this.logStatus(`Downloading ${interpreter_cfg.name}...`);
203-
204186
/* Dynamically download and import pyodide: the import() puts a
205187
loadPyodide() function into globalThis, which is later called by
206188
RemoteInterpreter.
@@ -211,8 +193,48 @@ export class PyScriptApp {
211193
support ES modules in workers:
212194
https://caniuse.com/mdn-api_worker_worker_ecmascript_modules
213195
*/
214-
const interpreterURL = await this.interpreter._remote.src;
196+
const interpreterURL = interpreter_cfg.src;
215197
await import(interpreterURL);
198+
return { remote_interpreter, wrapped_remote_interpreter };
199+
}
200+
201+
async _startInterpreter_worker(interpreter_cfg: InterpreterConfig) {
202+
logger.warn('execution_thread = "worker" is still VERY experimental, use it at your own risk');
203+
logger.info('Starting the interpreter in a web worker');
204+
const base_url = this._get_base_url();
205+
const worker = new Worker(base_url + '/interpreter_worker.js');
206+
const worker_initialize: any = Synclink.wrap(worker);
207+
const wrapped_remote_interpreter = await worker_initialize(interpreter_cfg);
208+
const remote_interpreter = undefined; // this is _unwrapped_remote
209+
return { remote_interpreter, wrapped_remote_interpreter };
210+
}
211+
212+
// lifecycle (4)
213+
async loadInterpreter() {
214+
logger.info('Initializing interpreter');
215+
if (this.config.interpreters.length == 0) {
216+
throw new UserError(ErrorCode.BAD_CONFIG, 'Fatal error: config.interpreter is empty');
217+
}
218+
219+
if (this.config.interpreters.length > 1) {
220+
showWarning('Multiple interpreters are not supported yet.<br />Only the first will be used', 'html');
221+
}
222+
223+
const cfg = this.config.interpreters[0];
224+
let x;
225+
if (this.config.execution_thread == 'worker') {
226+
x = await this._startInterpreter_worker(cfg);
227+
} else {
228+
x = await this._startInterpreter_main(cfg);
229+
}
230+
const { remote_interpreter, wrapped_remote_interpreter } = x;
231+
232+
this.interpreter = new InterpreterClient(
233+
this.config,
234+
this._stdioMultiplexer,
235+
wrapped_remote_interpreter as Synclink.Remote<RemoteInterpreter>,
236+
remote_interpreter,
237+
);
216238
await this.afterInterpreterLoad(this.interpreter);
217239
}
218240

‎pyscriptjs/src/pyconfig.ts‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ export interface AppConfig extends Record<string, any> {
2222
fetch?: FetchConfig[];
2323
plugins?: string[];
2424
pyscript?: PyScriptMetadata;
25+
execution_thread?: string; // "main" or "worker"
2526
}
2627

2728
export type FetchConfig = {
@@ -43,7 +44,7 @@ export type PyScriptMetadata = {
4344
};
4445

4546
const allKeys = Object.entries({
46-
string: ['name', 'description', 'version', 'type', 'author_name', 'author_email', 'license'],
47+
string: ['name', 'description', 'version', 'type', 'author_name', 'author_email', 'license', 'execution_thread'],
4748
number: ['schema_version'],
4849
array: ['runtimes', 'interpreters', 'packages', 'fetch', 'plugins'],
4950
});
@@ -63,6 +64,7 @@ export const defaultConfig: AppConfig = {
6364
packages: [],
6465
fetch: [],
6566
plugins: [],
67+
execution_thread: 'main',
6668
};
6769

6870
export function loadConfigFromElement(el: Element): AppConfig {
@@ -237,6 +239,15 @@ function validateConfig(configText: string, configType = 'toml') {
237239
}
238240
finalConfig[item].push(eachFetchConfig);
239241
});
242+
} else if (item == 'execution_thread') {
243+
const value = config[item];
244+
if (value !== 'main' && value !== 'worker') {
245+
throw new UserError(
246+
ErrorCode.BAD_CONFIG,
247+
`"${value}" is not a valid value for the property "execution_thread". The only valid values are "main" and "worker"`,
248+
);
249+
}
250+
finalConfig[item] = value;
240251
} else {
241252
finalConfig[item] = config[item];
242253
}

0 commit comments

Comments
 (0)