10000 Allow pyscript package to contain multiple files (#1309) · trofimander/pyscript@26f0724 · GitHub
[go: up one dir, main page]
Skip to content

Commit 26f0724

Browse files
hoodmanehoodmane
authored
Allow pyscript package to contain multiple files (pyscript#1309)
Followup to pyscript#1232. Closes pyscript#1226. Use node to make a manifest of the src/python dir and then use an esbuild plugin to resolve an import called `pyscript_python_package.esbuild_injected.json` to an object indicating the directories and files in the package folder. This object is then used to govern runtime installation of the package.
1 parent 3ae4b3c commit 26f0724

File tree

4 files changed

+92
-19
lines changed

4 files changed

+92
-19
lines changed

‎pyscriptjs/esbuild.js

Lines changed: 73 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ const { build } = require('esbuild');
22
const { spawn } = require('child_process');
33
const { join } = require('path');
44
const { watchFile } = require('fs');
5-
const { cp, lstat, readdir } = require('fs/promises');
5+
const { cp, lstat, readdir, opendir, readFile } = require('fs/promises');
66

77
const production = !process.env.NODE_WATCH || process.env.NODE_ENV === 'production';
88

@@ -15,12 +15,84 @@ if (!production) {
1515
copy_targets.push({ src: 'build/*', dest: 'examples/build' });
1616
}
1717

18+
/**
19+
* List out everything in a directory, but skip __pycache__ directory. Used to
20+
* list out the directory paths and the [file path, file contents] pairs in the
21+
* Python package. All paths are relative to the directory we are listing. The
22+
* directories are sorted topologically so that a parent directory always
23+
* appears before its children.
24+
*
25+
* This is consumed in main.ts which calls mkdir for each directory and then
26+
* writeFile to create each file.
27+
*
28+
* @param {string} dir The path to the directory we want to list out
29+
* @returns {dirs: string[], files: [string, string][]}
30+
*/
31+
async function directoryManifest(dir) {
32+
const result = { dirs: [], files: [] };
33+
await _directoryManifestHelper(dir, '.', result);
34+
return result;
35+
}
36+
37+
/**
38+
* Recursive helper function for directoryManifest
39+
*/
40+
async function _directoryManifestHelper(root, dir, result) {
41+
const dirObj = await opendir(join(root, dir));
42+
for await (const d of dirObj) {
43+
const entry = join(dir, d.name);
44+
if (d.isDirectory()) {
45+
if (d.name === '__pycache__') {
46+
continue;
47+
}
48+
result.dirs.push(entry);
49+
await _directoryManifestHelper(root, entry, result);
50+
} else if (d.isFile()) {
51+
result.files.push([entry, await readFile(join(root, entry), { encoding: 'utf-8' })]);
52+
}
53+
}
54+
}
55+
56+
/**
57+
* An esbuild plugin that injects the Pyscript Python package.
58+
*
59+
* It uses onResolve to attach our custom namespace to the import and then uses
60+
* onLoad to inject the file contents.
61+
*/
62+
function bundlePyscriptPythonPlugin() {
63+
const namespace = 'bundlePyscriptPythonPlugin';
64+
return {
65+
name: namespace,
66+
setup(build) {
67+
// Resolve the pyscript_package to our custom namespace
68+
// The path doesn't really matter, but we need a separate namespace
69+
// or else the file system resolver will raise an error.
70+
build.onResolve({ filter: /^pyscript_python_package.esbuild_injected.json$/ }, args => {
71+
return { path: 'dummy', namespace };
72+
});
73+
// Inject our manifest as JSON contents, and use the JSON loader.
74+
// Also tell esbuild to watch the files & directories we've listed
75+
// for updates.
76+
build.onLoad({ filter: /^dummy$/, namespace }, async args => {
77+
const manifest = await directoryManifest('./src/python');
78+
return {
79+
contents: JSON.stringify(manifest),
80+
loader: 'json',
81+
watchFiles: manifest.files.map(([k, v]) => k),
82+
watchDirs: manifest.dirs,
83+
};
84+
});
85+
},
86+
};
87+
}
88+
1889
const pyScriptConfig = {
1990
entryPoints: ['src/main.ts'],
2091
loader: { '.py': 'text' },
2192
bundle: true,
2293
format: 'iife',
2394
globalName: 'pyscript',
95+
plugins: [bundlePyscriptPythonPlugin()],
2496
};
2597 341A

2698
const copyPath = (source, dest, ...rest) => cp(join(__dirname, source), join(__dirname, dest), ...rest);

‎pyscriptjs/src/interpreter_client.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -75,11 +75,11 @@ export class InterpreterClient extends Object {
7575
return this._remote.pyimport(mod_name);
7676
}
7777

78-
async mkdirTree(path: string) {
79-
await this._remote.mkdirTree(path);
78+
async mkdir(path: string) {
79+
await this._remote.FS.mkdir(path);
8080
}
8181

8282
async writeFile(path: string, content: string) {
83-
await this._remote.writeFile(path, content);
83+
await this._remote.FS.writeFile(path, content, { encoding: 'utf8' });
8484
}
8585
}

‎pyscriptjs/src/main.ts

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,17 @@ import { SplashscreenPlugin } from './plugins/splashscreen';
1717
import { ImportmapPlugin } from './plugins/importmap';
1818
import { StdioDirector as StdioDirector } from './plugins/stdiodirector';
1919
import type { PyProxy } from 'pyodide';
20+
import { RemoteInterpreter } from './remote_interpreter';
21+
import { robustFetch } from './fetch';
2022
import * as Synclink from 'synclink';
21-
// eslint-disable-next-line
23+
24+
// pyscript_package is injected from src/python by bundlePyscriptPythonPlugin in
25+
// esbuild.js
26+
2227
// @ts-ignore
23-
import pyscript from './python/pyscript/__init__.py';
24-
import { robustFetch } from './fetch';
25-
import { RemoteInterpreter } from './remote_interpreter';
28+
import python_package from 'pyscript_python_package.esbuild_injected.json';
29+
30+
declare const python_package: { dirs: string[]; files: [string, string] };
2631

2732
const logger = getLogger('pyscript/main');
2833

@@ -268,9 +273,13 @@ export class PyScriptApp {
268273
// compatible with the old behavior.
269274
logger.info('importing pyscript');
270275

271-
// Save and load pyscript.py from FS
272-
await interpreter.mkdirTree('/home/pyodide/pyscript');
273-
await interpreter.writeFile('pyscript/__init__.py', pyscript as string);
276+
// Write pyscript package into file system
277+
for (const dir of python_package.dirs) {
278+
await interpreter._remote.FS.mkdir('/home/pyodide/' + dir);
279+
}
280+
for (const [path, value] of python_package.files) {
281+
await interpreter._remote.FS.writeFile('/home/pyodide/' + path, value);
282+
}
274283
//Refresh the module cache so Python consistently finds pyscript module
275284
await interpreter._remote.invalidate_module_path_cache();
276285

‎pyscriptjs/src/remote_interpreter.ts

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -272,14 +272,6 @@ export class RemoteInterpreter extends Object {
272272
return Synclink.proxy(this.interface.pyimport(mod_name));
273273
}
274274

275-
mkdirTree(path: string) {
276-
this.FS.mkdirTree(path);
277-
}
278-
279-
writeFile(path: string, content: string) {
280-
this.FS.writeFile(path, content, { encoding: 'utf8' });
281-
}
282-
283275
// eslint-disable-next-line @typescript-eslint/no-explicit-any
284276
setHandler(func_name: string, handler: any): void {
285277
const pyscript_module = this.interface.pyimport('pyscript');

0 commit comments

Comments
 (0)
0