Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 49434e2

Browse files
IanMatthewHuffIanMatthewHuff
authored
Non-jupyter kernel finder (microsoft#11538)
1 parent 700caad commit 49434e2

File tree

4 files changed

+648
-311
lines changed

4 files changed

+648
-311
lines changed

‎src/client/datascience/jupyter/kernels/kernelSelections.ts

+3-3
Original file line numberDiff line numberDiff line change
@@ -122,10 +122,10 @@ export class InstalledJupyterKernelSelectionListProvider implements IKernelSelec
122122
export class InstalledRawKernelSelectionListProvider implements IKernelSelectionListProvider {
123123
constructor(private readonly kernelFinder: IKernelFinder, private readonly pathUtils: IPathUtils) {}
124124
public async getKernelSelections(
125-
_resource: Resource,
126-
cancelToken?: CancellationToken
125+
resource: Resource,
126+
_cancelToken?: CancellationToken
127127
): Promise<IKernelSpecQuickPickItem[]> {
128-
const items = await this.kernelFinder.listKernelSpecs(cancelToken);
128+
const items = await this.kernelFinder.listKernelSpecs(resource);
129129
return items
130130
.filter((item) => (item.language || '').toLowerCase() === PYTHON_LANGUAGE.toLowerCase())
131131
.map((item) => getQuickPickItemForKernelSpec(item, this.pathUtils));

‎src/client/datascience/kernel-launcher/kernelFinder.ts

+196-71
Original file line numberDiff line numberDiff line change
@@ -6,29 +6,26 @@ import { Kernel } from '@jupyterlab/services';
66
import { inject, injectable, named } from 'inversify';
77
import * as path from 'path';
88
import { CancellationToken, CancellationTokenSource } from 'vscode';
9+
import { IWorkspaceService } from '../../common/application/types';
910
import { wrapCancellationTokens } from '../../common/cancellation';
10-
import { traceInfo } from '../../common/logger';
11+
import { traceError, traceInfo } from '../../common/logger';
1112
import { IFileSystem, IPlatformService } from '../../common/platform/types';
1213
import { IExtensionContext, IInstaller, InstallerResponse, IPathUtils, Product, Resource } from '../../common/types';
13-
import {
14-
IInterpreterLocatorService,
15-
IInterpreterService,
16-
KNOWN_PATH_SERVICE,
17-
PythonInterpreter
18-
} from '../../interpreter/contracts';
14+
import { IInterpreterLocatorService, IInterpreterService, KNOWN_PATH_SERVICE } from '../../interpreter/contracts';
1915
import { captureTelemetry } from '../../telemetry';
2016
import { Telemetry } from '../constants';
2117
import { JupyterKernelSpec } from '../jupyter/kernels/jupyterKernelSpec';
2218
import { IJupyterKernelSpec } from '../types';
2319
import { getKernelInterpreter } from './helpers';
2420
import { IKernelFinder } from './types';
21+
// tslint:disable-next-line:no-require-imports no-var-requires
22+
const flatten = require('lodash/flatten') as typeof import('lodash/flatten');
23+
24+
const winJupyterPath = path.join('AppData', 'Roaming', 'jupyter', 'kernels');
25+
const linuxJupyterPath = path.join('.local', 'share', 'jupyter', 'kernels');
26+
const macJupyterPath = path.join('Library', 'Jupyter', 'kernels');
27+
const baseKernelPath = path.join('share', 'jupyter', 'kernels');
2528

26-
const kernelPaths = new Map([
27-
['winJupyterPath', path.join('AppData', 'Roaming', 'jupyter', 'kernels')],
28-
['linuxJupyterPath', path.join('.local', 'share', 'jupyter', 'kernels')],
29-
['macJupyterPath', path.join('Library', 'Jupyter', 'kernels')],
30-
['kernel', path.join('share', 'jupyter', 'kernels')]
31-
]);
3229
const cacheFile = 'kernelSpecPathCache.json';
3330
const defaultSpecName = 'python_defaultSpec_';
3431

@@ -47,6 +44,13 @@ export function findIndexOfConnectionFile(kernelSpec: Readonly<IJupyterKernelSpe
4744
@injectable()
4845
export class KernelFinder implements IKernelFinder {
4946
private cache: string[] = [];
47+
private cacheDirty = false;
48+
49+
// Store our results when listing all possible kernelspecs for a resource
50+
private workspaceToKernels = new Map<string, Promise<IJupyterKernelSpec[]>>();
51+
52+
// Store any json file that we have loaded from disk before
53+
private pathToKernelSpec = new Map<string, Promise<IJupyterKernelSpec | undefined>>();
5054

5155
constructor(
5256
@inject(IInterpreterService) private interpreterService: IInterpreterService,
@@ -57,7 +61,8 @@ export class KernelFinder implements IKernelFinder {
5761
@inject(IFileSystem) private file: IFileSystem,
5862
@inject(IPathUtils) private readonly pathUtils: IPathUtils,
5963
@inject(IInstaller) private installer: IInstaller,
60-
@inject(IExtensionContext) private readonly context: IExtensionContext
64+
@inject(IExtensionContext) private readonly context: IExtensionContext,
65+
@inject(IWorkspaceService) private readonly workspaceService: IWorkspaceService
6166
) {}
6267

6368
@captureTelemetry(Telemetry.KernelFinderPerf)
@@ -68,7 +73,6 @@ export class KernelFinder implements IKernelFinder {
6873
): Promise<IJupyterKernelSpec> {
6974
this.cache = await this.readCache();
7075
let foundKernel: IJupyterKernelSpec | undefined;
71-
const activeInterpreter = await this.interpreterService.getActiveInterpreter(resource);
7276

7377
if (kernelName && !kernelName.includes(defaultSpecName)) {
7478
let kernelSpec = await this.searchCache(kernelName);
@@ -78,32 +82,27 @@ export class KernelFinder implements IKernelFinder {
7882
}
7983

8084
// Check in active interpreter first
81-
if (activeInterpreter) {
82-
kernelSpec = await this.getKernelSpecFromActiveInterpreter(kernelName, activeInterpreter);
83-
}
85+
kernelSpec = await this.getKernelSpecFromActiveInterpreter(kernelName, resource);
8486

8587
if (kernelSpec) {
8688
this.writeCache(this.cache).ignoreErrors();
8789
return kernelSpec;
8890
}
8991

9092
const diskSearch = this.findDiskPath(kernelName);
91-
const interpreterSearch = this.interpreterLocator
92-
.getInterpreters(resource, { ignoreCache: false })
93-
.then((interpreters) => {
94-
const interpreterPaths = interpreters.map((interp) => interp.sysPrefix);
95-
return this.findInterpreterPath(interpreterPaths, kernelName);
96-
});
93+
const interpreterSearch = this.getInterpreterPaths(resource).then((interpreterPaths) => {
94+
return this.findInterpreterPath(interpreterPaths, kernelName);
95+
});
9796

9897
let result = await Promise.race([diskSearch, interpreterSearch]);
9998
if (!result) {
10099
const both = await Promise.all([diskSearch, interpreterSearch]);
101100
result = both[0] ? both[0] : both[1];
102101
}
103102

104-
foundKernel = result ? result : await this.getDefaultKernelSpec(activeInterpreter);
103+
foundKernel = result ? result : await this.getDefaultKernelSpec(resource);
105104
} else {
106-
foundKernel = await this.getDefaultKernelSpec(activeInterpreter);
105+
foundKernel = await this.getDefaultKernelSpec(resource);
107106
}
108107

109108
this.writeCache(this.cache).ignoreErrors();
@@ -113,8 +112,149 @@ export class KernelFinder implements IKernelFinder {
113112
}
114113

115114
// Search all our local file system locations for installed kernel specs and return them
116-
public async listKernelSpecs(_cancelToken?: CancellationToken): Promise<IJupyterKernelSpec[]> {
117-
throw new Error('Not yet implmented');
115+
public async listKernelSpecs(resource: Resource): Promise<IJupyterKernelSpec[]> {
116+
if (!resource) {
117+
// We need a resource to search for related kernel specs
118+
return [];
119+
}
120+
121+
// Get an id for the workspace folder, if we don't have one, use the fsPath of the resource
122+
const workspaceFolderId = this.workspaceService.getWorkspaceFolderIdentifier(resource, resource.fsPath);
123+
124+
// If we have not already searched for this resource, then generate the search
125+
if (!this.workspaceToKernels.has(workspaceFolderId)) {
126+
this.workspaceToKernels.set(workspaceFolderId, this.findResourceKernelSpecs(resource));
127+
}
128+
129+
this.writeCache(this.cache).ignoreErrors();
130+
131+
// ! as the has and set above verify that we have a return here
132+
return this.workspaceToKernels.get(workspaceFolderId)!;
133+
}
134+
135+
private async findResourceKernelSpecs(resource: Resource): Promise<IJupyterKernelSpec[]> {
136+
const results: IJupyterKernelSpec[] = [];
137+
138+
// Find all the possible places to look for this resource
139+
const paths = await this.findAllResourcePossibleKernelPaths(resource);
140+
141+
const searchResults = await this.kernelGlobSearch(paths);
142+
143+
await Promise.all(
144+
searchResults.map(async (resultPath) => {
145+
// Add these into our path cache to speed up later finds
146+
this.updateCache(resultPath);
147+
const kernelspec = await this.getKernelSpec(resultPath);
148+
149+
if (kernelspec) {
150+
results.push(kernelspec);
151+
}
152+
})
153+
);
154+
155+
return results;
156+
}
157+
158+
// Load the IJupyterKernelSpec for a given spec path, check the ones that we have already loaded first
159+
private async getKernelSpec(specPath: string): Promise<IJupyterKernelSpec | undefined> {
160+
// If we have not already loaded this kernel spec, then load it
161+
if (!this.pathToKernelSpec.has(specPath)) {
162+
this.pathToKernelSpec.set(specPath, this.loadKernelSpec(specPath));
163+
}
164+
165+
// ! as the has and set above verify that we have a return here
166+
return this.pathToKernelSpec.get(specPath)!.then((value) => {
167+
if (value) {
168+
return value;
169+
}
170+
171+
// If we failed to get a kernelspec pull path from our cache and loaded list
172+
this.pathToKernelSpec.delete(specPath);
173+
this.cache = this.cache.filter((itempath) => itempath !== specPath);
174+
return undefined;
175+
});
176+
}
177+
178+
// Load kernelspec json from disk
179+
private async loadKernelSpec(specPath: string): Promise<IJupyterKernelSpec | undefined> {
180+
let kernelJson;
181+
try {
182+
kernelJson = JSON.parse(await this.file.readFile(specPath));
183+
} catch {
184+
traceError(`Failed to parse kernelspec ${specPath}`);
185+
return undefined;
186+
}
187+
return new JupyterKernelSpec(kernelJson, specPath);
188+
}
189+
190+
// For the given resource, find atll the file paths for kernel specs that wewant to associate with this
191+
private async findAllResourcePossibleKernelPaths(
192+
resource: Resource,
193+
_cancelToken?: CancellationToken
194+
): Promise<string[]> {
195+
const [activePath, interpreterPaths, diskPaths] = await Promise.all([
196+
this.getActiveInterpreterPath(resource),
197+
this.getInterpreterPaths(resource),
198+
this.getDiskPaths()
199+
]);
200+
201+
return [...activePath, ...interpreterPaths, ...diskPaths];
202+
}
203+
204+
private async getActiveInterpreterPath(resource: Resource): Promise<string[]> {
205+
const activeInterpreter = await this.interpreterService.getActiveInterpreter(resource);
206+
207+
if (activeInterpreter) {
208+
return [path.join(activeInterpreter.sysPrefix, 'share', 'jupyter', 'kernels')];
209+
}
210+
211+
return [];
212+
}
213+
214+
private async getInterpreterPaths(resource: Resource): Promise<string[]> {
215+
const interpreters = await this.interpreterLocator.getInterpreters(resource, { ignoreCache: false });
216+
const interpreterPrefixPaths = interpreters.map((interpreter) => interpreter.sysPrefix);
217+
// We can get many duplicates here, so de-dupe the list
218+
const uniqueInterpreterPrefixPaths = [...new Set(interpreterPrefixPaths)];
219+
return uniqueInterpreterPrefixPaths.map((prefixPath) => path.join(prefixPath, baseKernelPath));
220+
}
221+
222+
private async getDiskPaths(): Promise<string[]> {
223+
let paths = [];
224+
225+
if (this.platformService.isWindows) {
226+
paths = [path.join(this.pathUtils.home, winJupyterPath)];
227+
228+
if (process.env.ALLUSERSPROFILE) {
229+
paths.push(path.join(process.env.ALLUSERSPROFILE, 'jupyter', 'kernels'));
230+
}
231+
} else {
232+
// Unix based
233+
const secondPart = this.platformService.isMac ? macJupyterPath : linuxJupyterPath;
234+
235+
paths = [
236+
path.join('usr', 'share', 'jupyter', 'kernels'),
237+
path.join('usr', 'local', 'share', 'jupyter', 'kernels'),
238+
path.join(this.pathUtils.home, secondPart)
239+
];
240+
}
241+
242+
return paths;
243+
}
244+
245+
// Given a set of paths, search for kernel.json files and return back the full paths of all of them that we find
246+
private async kernelGlobSearch(paths: string[]): Promise<string[]> {
247+
const promises = paths.map((kernelPath) => this.file.search(`**/kernel.json`, kernelPath));
248+
const searchResults = await Promise.all(promises);
249+
250+
// Append back on the start of each path so we have the full path in the results
251+
const fullPathResults = searchResults.map((result, index) => {
252+
return result.map((partialSpecPath) => {
253+
return path.join(paths[index], partialSpecPath);
254+
});
255+
});
256+
257+
return flatten(fullPathResults);
118258
}
119259

120260
// For the given kernelspec return back the kernelspec with ipykernel installed into it or error
@@ -143,21 +283,17 @@ export class KernelFinder implements IKernelFinder {
143283

144284
private async getKernelSpecFromActiveInterpreter(
145285
kernelName: string,
146-
activeInterpreter: PythonInterpreter
286+
resource: Resource
147287
): Promise<IJupyterKernelSpec | undefined> {
148-
return this.getKernelSpecFromDisk(
149-
[path.join(activeInterpreter.sysPrefix, 'share', 'jupyter', 'kernels')],
150-
kernelName
151-
);
288+
const activePath = await this.getActiveInterpreterPath(resource);
289+
return this.getKernelSpecFromDisk(activePath, kernelName);
152290
}
153291

154292
private async findInterpreterPath(
155293
interpreterPaths: string[],
156294
kernelName: string
157295
): Promise<IJupyterKernelSpec | undefined> {
158-
const promises = interpreterPaths.map((intPath) =>
159-
this.getKernelSpecFromDisk([path.join(intPath, kernelPaths.get('kernel')!)], kernelName)
160-
);
296+
const promises = interpreterPaths.map((intPath) => this.getKernelSpecFromDisk([intPath], kernelName));
161297

162298
const specs = await Promise.all(promises);
163299
return specs.find((sp) => sp !== undefined);
@@ -166,46 +302,23 @@ export class KernelFinder implements IKernelFinder {
166302
// Jupyter looks for kernels in these paths:
167303
// https://jupyter-client.readthedocs.io/en/stable/kernels.html#kernel-specs
168304
private async findDiskPath(kernelName: string): Promise<IJupyterKernelSpec | undefined> {
169-
let paths = [];
170-
171-
if (this.platformService.isWindows) {
172-
paths = [path.join(this.pathUtils.home, kernelPaths.get('winJupyterPath')!)];
173-
174-
if (process.env.ALLUSERSPROFILE) {
175-
paths.push(path.join(process.env.ALLUSERSPROFILE, 'jupyter', 'kernels'));
176-
}
177-
} else {
178-
// Unix based
179-
const secondPart = this.platformService.isMac
180-
? kernelPaths.get('macJupyterPath')!
181-
: kernelPaths.get('linuxJupyterPath')!;
182-
183-
paths = [
184-
path.join('usr', 'share', 'jupyter', 'kernels'),
185-
path.join('usr', 'local', 'share', 'jupyter', 'kernels'),
186-
path.join(this.pathUtils.home, secondPart)
187-
];
188-
}
305+
const paths = await this.getDiskPaths();
189306

190307
return this.getKernelSpecFromDisk(paths, kernelName);
191308
}
192309

193310
private async getKernelSpecFromDisk(paths: string[], kernelName: string): Promise<IJupyterKernelSpec | undefined> {
194-
const promises = paths.map((kernelPath) => this.file.search('**/kernel.json', kernelPath));
195-
const searchResults = await Promise.all(promises);
196-
searchResults.forEach((result, i) => {
197-
result.forEach((res) => {
198-
const specPath = path.join(paths[i], res);
199-
if (!this.cache.includes(specPath)) {
200-
this.cache.push(specPath);
201-
}
202-
});
311+
const searchResults = await this.kernelGlobSearch(paths);
312+
searchResults.forEach((specPath) => {
313+
this.updateCache(specPath);
203314
});
204315

205316
return this.searchCache(kernelName);
206317
}
207318

208-
private async getDefaultKernelSpec(activeInterpreter?: PythonInterpreter): Promise<IJupyterKernelSpec> {
319+
private async getDefaultKernelSpec(resource: Resource): Promise<IJupyterKernelSpec> {
320+
const activeInterpreter = await this.interpreterService.getActiveInterpreter(resource);
321+
209322
// This creates a default kernel spec. When launched, 'python' argument will map to using the interpreter
210323
// associated with the current resource for launching.
211324
const defaultSpec: Kernel.ISpecModel = {
@@ -231,8 +344,18 @@ export class KernelFinder implements IKernelFinder {
231344
}
232345
}
233346

347+
private updateCache(newPath: string) {
348+
if (!this.cache.includes(newPath)) {
349+
this.cache.push(newPath);
350+
this.cacheDirty = true;
351+
}
352+
}
353+
234354
private async writeCache(cache: string[]) {
235-
await this.file.writeFile(path.join(this.context.globalStoragePath, cacheFile), JSON.stringify(cache));
355+
if (this.cacheDirty) {
356+
await this.file.writeFile(path.join(this.context.globalStoragePath, cacheFile), JSON.stringify(cache));
357+
this.cacheDirty = false;
358+
}
236359
}
237360

238361
private async searchCache(kernelName: string): Promise<IJupyterKernelSpec | undefined> {
@@ -246,10 +369,12 @@ export class KernelFinder implements IKernelFinder {
246369
});
247370

248371
if (kernelJsonFile) {
249-
const kernelJson = JSON.parse(await this.file.readFile(kernelJsonFile));
250-
const spec = new JupyterKernelSpec(kernelJson, kernelJsonFile);
251-
spec.name = kernelName;
252-
return spec;
372+
const spec = await this.getKernelSpec(kernelJsonFile);
373+
374+
if (spec) {
375+
spec.name = kernelName;
376+
return spec;
377+
}
253378
}
254379

255380
return undefined;

0 commit comments

Comments
 (0)