Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Non-jupyter kernel finder #11538

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 18 commits into from
May 4, 2020
Merged

Non-jupyter kernel finder #11538

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
6f4c6c2
first working kernel finder
May 1, 2020
9ff8966
path fix
May 1, 2020
33d538b
promise all for getting paths
May 1, 2020
6eb4feb
merge path functions in
May 1, 2020
03039fb
better glob search in finder
May 1, 2020
acd249a
update comment
May 1, 2020
1f1466a
both finds use glob search
May 1, 2020
ea23557
change path constants
May 1, 2020
84fe7fb
add result to search cache
May 1, 2020
f8eca04
handle errors and have find use new load mechanism
May 1, 2020
db09c1e
remove cancellation
May 1, 2020
3a8aba2
use workspace folder not resource
May 1, 2020
1a011c0
basic test passing
May 2, 2020
3ec2a81
couple of basic unit tests for find
May 2, 2020
00c6499
remove old comments
May 2, 2020
893d8d9
de-dupe interpreters list
May 2, 2020
dcc6efa
revert glob pattern back to slash
May 4, 2020
6fcd79a
only write cache if it changes, and update cache after we do our find
May 4, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions src/client/datascience/jupyter/kernels/kernelSelections.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,10 +122,10 @@ export class InstalledJupyterKernelSelectionListProvider implements IKernelSelec
export class InstalledRawKernelSelectionListProvider implements IKernelSelectionListProvider {
constructor(private readonly kernelFinder: IKernelFinder, private readonly pathUtils: IPathUtils) {}
public async getKernelSelections(
_resource: Resource,
cancelToken?: CancellationToken
resource: Resource,
_cancelToken?: CancellationToken
): Promise<IKernelSpecQuickPickItem[]> {
const items = await this.kernelFinder.listKernelSpecs(cancelToken);
const items = await this.kernelFinder.listKernelSpecs(resource);
return items
.filter((item) => (item.language || '').toLowerCase() === PYTHON_LANGUAGE.toLowerCase())
.map((item) => getQuickPickItemForKernelSpec(item, this.pathUtils));
Expand Down
267 changes: 196 additions & 71 deletions src/client/datascience/kernel-launcher/kernelFinder.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,29 +6,26 @@ import { Kernel } from '@jupyterlab/services';
import { inject, injectable, named } from 'inversify';
import * as path from 'path';
import { CancellationToken, CancellationTokenSource } from 'vscode';
import { IWorkspaceService } from '../../common/application/types';
import { wrapCancellationTokens } from '../../common/cancellation';
import { traceInfo } from '../../common/logger';
import { traceError, traceInfo } from '../../common/logger';
import { IFileSystem, IPlatformService } from '../../common/platform/types';
import { IExtensionContext, IInstaller, InstallerResponse, IPathUtils, Product, Resource } from '../../common/types';
import {
IInterpreterLocatorService,
IInterpreterService,
KNOWN_PATH_SERVICE,
PythonInterpreter
} from '../../interpreter/contracts';
import { IInterpreterLocatorService, IInterpreterService, KNOWN_PATH_SERVICE } from '../../interpreter/contracts';
import { captureTelemetry } from '../../telemetry';
import { Telemetry } from '../constants';
import { JupyterKernelSpec } from '../jupyter/kernels/jupyterKernelSpec';
import { IJupyterKernelSpec } from '../types';
import { getKernelInterpreter } from './helpers';
import { IKernelFinder } from './types';
// tslint:disable-next-line:no-require-imports no-var-requires
const flatten = require('lodash/flatten') as typeof import('lodash/flatten');

const winJupyterPath = path.join('AppData', 'Roaming', 'jupyter', 'kernels');
const linuxJupyterPath = path.join('.local', 'share', 'jupyter', 'kernels');
const macJupyterPath = path.join('Library', 'Jupyter', 'kernels');
const baseKernelPath = path.join('share', 'jupyter', 'kernels');

const kernelPaths = new Map([
['winJupyterPath', path.join('AppData', 'Roaming', 'jupyter', 'kernels')],
['linuxJupyterPath', path.join('.local', 'share', 'jupyter', 'kernels')],
['macJupyterPath', path.join('Library', 'Jupyter', 'kernels')],
['kernel', path.join('share', 'jupyter', 'kernels')]
]);
const cacheFile = 'kernelSpecPathCache.json';
const defaultSpecName = 'python_defaultSpec_';

Expand All @@ -47,6 +44,13 @@ export function findIndexOfConnectionFile(kernelSpec: Readonly<IJupyterKernelSpe
@injectable()
export class KernelFinder implements IKernelFinder {
private cache: string[] = [];
private cacheDirty = false;

// Store our results when listing all possible kernelspecs for a resource
private workspaceToKernels = new Map<string, Promise<IJupyterKernelSpec[]>>();

// Store any json file that we have loaded from disk before
private pathToKernelSpec = new Map<string, Promise<IJupyterKernelSpec | undefined>>();

constructor(
@inject(IInterpreterService) private interpreterService: IInterpreterService,
Expand All @@ -57,7 +61,8 @@ export class KernelFinder implements IKernelFinder {
@inject(IFileSystem) private file: IFileSystem,
@inject(IPathUtils) private readonly pathUtils: IPathUtils,
@inject(IInstaller) private installer: IInstaller,
@inject(IExtensionContext) private readonly context: IExtensionContext
@inject(IExtensionContext) private readonly context: IExtensionContext,
@inject(IWorkspaceService) private readonly workspaceService: IWorkspaceService
) {}

@captureTelemetry(Telemetry.KernelFinderPerf)
Expand All @@ -68,7 +73,6 @@ export class KernelFinder implements IKernelFinder {
): Promise<IJupyterKernelSpec> {
this.cache = await this.readCache();
let foundKernel: IJupyterKernelSpec | undefined;
const activeInterpreter = await this.interpreterService.getActiveInterpreter(resource);

if (kernelName && !kernelName.includes(defaultSpecName)) {
let kernelSpec = await this.searchCache(kernelName);
Expand All @@ -78,32 +82,27 @@ export class KernelFinder implements IKernelFinder {
}

// Check in active interpreter first
if (activeInterpreter) {
kernelSpec = await this.getKernelSpecFromActiveInterpreter(kernelName, activeInterpreter);
}
kernelSpec = await this.getKernelSpecFromActiveInterpreter(kernelName, resource);

if (kernelSpec) {
this.writeCache(this.cache).ignoreErrors();
return kernelSpec;
}

const diskSearch = this.findDiskPath(kernelName);
const interpreterSearch = this.interpreterLocator
.getInterpreters(resource, { ignoreCache: false })
.then((interpreters) => {
const interpreterPaths = interpreters.map((interp) => interp.sysPrefix);
return this.findInterpreterPath(interpreterPaths, kernelName);
});
const interpreterSearch = this.getInterpreterPaths(resource).then((interpreterPaths) => {
return this.findInterpreterPath(interpreterPaths, kernelName);
});

let result = await Promise.race([diskSearch, interpreterSearch]);
if (!result) {
const both = await Promise.all([diskSearch, interpreterSearch]);
result = both[0] ? both[0] : both[1];
}

foundKernel = result ? result : await this.getDefaultKernelSpec(activeInterpreter);
foundKernel = result ? result : await this.getDefaultKernelSpec(resource);
} else {
foundKernel = await this.getDefaultKernelSpec(activeInterpreter);
foundKernel = await this.getDefaultKernelSpec(resource);
}

this.writeCache(this.cache).ignoreErrors();
Expand All @@ -113,8 +112,149 @@ export class KernelFinder implements IKernelFinder {
}

// Search all our local file system locations for installed kernel specs and return them
public async listKernelSpecs(_cancelToken?: CancellationToken): Promise<IJupyterKernelSpec[]> {
throw new Error('Not yet implmented');
public async listKernelSpecs(resource: Resource): Promise<IJupyterKernelSpec[]> {
if (!resource) {
// We need a resource to search for related kernel specs
return [];
}

// Get an id for the workspace folder, if we don't have one, use the fsPath of the resource
const workspaceFolderId = this.workspaceService.getWorkspaceFolderIdentifier(resource, resource.fsPath);

// If we have not already searched for this resource, then generate the search
if (!this.workspaceToKernels.has(workspaceFolderId)) {
this.workspaceToKernels.set(workspaceFolderId, this.findResourceKernelSpecs(resource));
}

this.writeCache(this.cache).ignoreErrors();

// ! as the has and set above verify that we have a return here
return this.workspaceToKernels.get(workspaceFolderId)!;
}

private async findResourceKernelSpecs(resource: Resource): Promise<IJupyterKernelSpec[]> {
const results: IJupyterKernelSpec[] = [];

// Find all the possible places to look for this resource
const paths = await this.findAllResourcePossibleKernelPaths(resource);

const searchResults = await this.kernelGlobSearch(paths);

await Promise.all(
searchResults.map(async (resultPath) => {
// Add these into our path cache to speed up later finds
this.updateCache(resultPath);
const kernelspec = await this.getKernelSpec(resultPath);

if (kernelspec) {
results.push(kernelspec);
}
})
);

return results;
}

// Load the IJupyterKernelSpec for a given spec path, check the ones that we have already loaded first
private async getKernelSpec(specPath: string): Promise<IJupyterKernelSpec | undefined> {
// If we have not already loaded this kernel spec, then load it
if (!this.pathToKernelSpec.has(specPath)) {
this.pathToKernelSpec.set(specPath, this.loadKernelSpec(specPath));
}

// ! as the has and set above verify that we have a return here
return this.pathToKernelSpec.get(specPath)!.then((value) => {
if (value) {
return value;
}

// If we failed to get a kernelspec pull path from our cache and loaded list
this.pathToKernelSpec.delete(specPath);
this.cache = this.cache.filter((itempath) => itempath !== specPath);
return undefined;
});
}

// Load kernelspec json from disk
private async loadKernelSpec(specPath: string): Promise<IJupyterKernelSpec | undefined> {
let kernelJson;
try {
kernelJson = JSON.parse(await this.file.readFile(specPath));
} catch {
traceError(`Failed to parse kernelspec ${specPath}`);
return undefined;
}
return new JupyterKernelSpec(kernelJson, specPath);
}

// For the given resource, find atll the file paths for kernel specs that wewant to associate with this
private async findAllResourcePossibleKernelPaths(
resource: Resource,
_cancelToken?: CancellationToken
): Promise<string[]> {
const [activePath, interpreterPaths, diskPaths] = await Promise.all([
this.getActiveInterpreterPath(resource),
this.getInterpreterPaths(resource),
this.getDiskPaths()
]);

return [...activePath, ...interpreterPaths, ...diskPaths];
}

private async getActiveInterpreterPath(resource: Resource): Promise<string[]> {
const activeInterpreter = await this.interpreterService.getActiveInterpreter(resource);

if (activeInterpreter) {
return [path.join(activeInterpreter.sysPrefix, 'share', 'jupyter', 'kernels')];
}

return [];
}

private async getInterpreterPaths(resource: Resource): Promise<string[]> {
const interpreters = await this.interpreterLocator.getInterpreters(resource, { ignoreCache: false });
const interpreterPrefixPaths = interpreters.map((interpreter) => interpreter.sysPrefix);
// We can get many duplicates here, so de-dupe the list
const uniqueInterpreterPrefixPaths = [...new Set(interpreterPrefixPaths)];
return uniqueInterpreterPrefixPaths.map((prefixPath) => path.join(prefixPath, baseKernelPath));
}

private async getDiskPaths(): Promise<string[]> {
let paths = [];

if (this.platformService.isWindows) {
paths = [path.join(this.pathUtils.home, winJupyterPath)];

if (process.env.ALLUSERSPROFILE) {
paths.push(path.join(process.env.ALLUSERSPROFILE, 'jupyter', 'kernels'));
}
} else {
// Unix based
const secondPart = this.platformService.isMac ? macJupyterPath : linuxJupyterPath;

paths = [
path.join('usr', 'share', 'jupyter', 'kernels'),
path.join('usr', 'local', 'share', 'jupyter', 'kernels'),
path.join(this.pathUtils.home, secondPart)
];
}

return paths;
}

// Given a set of paths, search for kernel.json files and return back the full paths of all of them that we find
private async kernelGlobSearch(paths: string[]): Promise<string[]> {
const promises = paths.map((kernelPath) => this.file.search(`**/kernel.json`, kernelPath));
const searchResults = await Promise.all(promises);

// Append back on the start of each path so we have the full path in the results
const fullPathResults = searchResults.map((result, index) => {
return result.map((partialSpecPath) => {
return path.join(paths[index], partialSpecPath);
});
});

return flatten(fullPathResults);
}

// For the given kernelspec return back the kernelspec with ipykernel installed into it or error
Expand Down Expand Up @@ -143,21 +283,17 @@ export class KernelFinder implements IKernelFinder {

private async getKernelSpecFromActiveInterpreter(
kernelName: string,
activeInterpreter: PythonInterpreter
resource: Resource
): Promise<IJupyterKernelSpec | undefined> {
return this.getKernelSpecFromDisk(
[path.join(activeInterpreter.sysPrefix, 'share', 'jupyter', 'kernels')],
kernelName
);
const activePath = await this.getActiveInterpreterPath(resource);
return this.getKernelSpecFromDisk(activePath, kernelName);
}

private async findInterpreterPath(
interpreterPaths: string[],
kernelName: string
): Promise<IJupyterKernelSpec | undefined> {
const promises = interpreterPaths.map((intPath) =>
this.getKernelSpecFromDisk([path.join(intPath, kernelPaths.get('kernel')!)], kernelName)
);
const promises = interpreterPaths.map((intPath) => this.getKernelSpecFromDisk([intPath], kernelName));

const specs = await Promise.all(promises);
return specs.find((sp) => sp !== undefined);
Expand All @@ -166,46 +302,23 @@ export class KernelFinder implements IKernelFinder {
// Jupyter looks for kernels in these paths:
// https://jupyter-client.readthedocs.io/en/stable/kernels.html#kernel-specs
private async findDiskPath(kernelName: string): Promise<IJupyterKernelSpec | undefined> {
let paths = [];

if (this.platformService.isWindows) {
paths = [path.join(this.pathUtils.home, kernelPaths.get('winJupyterPath')!)];

if (process.env.ALLUSERSPROFILE) {
paths.push(path.join(process.env.ALLUSERSPROFILE, 'jupyter', 'kernels'));
}
} else {
// Unix based
const secondPart = this.platformService.isMac
? kernelPaths.get('macJupyterPath')!
: kernelPaths.get('linuxJupyterPath')!;

paths = [
path.join('usr', 'share', 'jupyter', 'kernels'),
path.join('usr', 'local', 'share', 'jupyter', 'kernels'),
path.join(this.pathUtils.home, secondPart)
];
}
const paths = await this.getDiskPaths();

return this.getKernelSpecFromDisk(paths, kernelName);
}

private async getKernelSpecFromDisk(paths: string[], kernelName: string): Promise<IJupyterKernelSpec | undefined> {
const promises = paths.map((kernelPath) => this.file.search('**/kernel.json', kernelPath));
const searchResults = await Promise.all(promises);
searchResults.forEach((result, i) => {
result.forEach((res) => {
const specPath = path.join(paths[i], res);
if (!this.cache.includes(specPath)) {
this.cache.push(specPath);
}
});
const searchResults = await this.kernelGlobSearch(paths);
searchResults.forEach((specPath) => {
this.updateCache(specPath);
});

return this.searchCache(kernelName);
}

private async getDefaultKernelSpec(activeInterpreter?: PythonInterpreter): Promise<IJupyterKernelSpec> {
private async getDefaultKernelSpec(resource: Resource): Promise<IJupyterKernelSpec> {
const activeInterpreter = await this.interpreterService.getActiveInterpreter(resource);

// This creates a default kernel spec. When launched, 'python' argument will map to using the interpreter
// associated with the current resource for launching.
const defaultSpec: Kernel.ISpecModel = {
Expand All @@ -231,8 +344,18 @@ export class KernelFinder implements IKernelFinder {
}
}

private updateCache(newPath: string) {
if (!this.cache.includes(newPath)) {
this.cache.push(newPath);
this.cacheDirty = true;
}
}

private async writeCache(cache: string[]) {
await this.file.writeFile(path.join(this.context.globalStoragePath, cacheFile), JSON.stringify(cache));
if (this.cacheDirty) {
await this.file.writeFile(path.join(this.context.globalStoragePath, cacheFile), JSON.stringify(cache));
this.cacheDirty = false;
}
}

private async searchCache(kernelName: string): Promise<IJupyterKernelSpec | undefined> {
Expand All @@ -246,10 +369,12 @@ export class KernelFinder implements IKernelFinder {
});

if (kernelJsonFile) {
const kernelJson = JSON.parse(await this.file.readFile(kernelJsonFile));
const spec = new JupyterKernelSpec(kernelJson, kernelJsonFile);
spec.name = kernelName;
return spec;
const spec = await this.getKernelSpec(kernelJsonFile);

if (spec) {
spec.name = kernelName;
return spec;
}
}

return undefined;
Expand Down
Loading