Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit c84e43d

Browse files
asgerfasgerf
committed
JS: Replace yarn with manual dependency resolution
1 parent f5c3aa3 commit c84e43d

8 files changed

Lines changed: 756 additions & 129 deletions

File tree

‎javascript/extractor/src/com/semmle/js/dependencies/AsyncFetcher.java‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
package com.semmle.js.dependencies;
2+
3+
import java.io.IOException;
4+
import java.nio.file.Path;
5+
import java.util.LinkedHashMap;
6+
import java.util.Map;
7+
import java.util.concurrent.CompletableFuture;
8+
import java.util.concurrent.CompletionException;
9+
import java.util.concurrent.ExecutorService;
10+
import java.util.function.Consumer;
11+
import java.util.function.Supplier;
12+
13+
import com.semmle.js.dependencies.packument.Packument;
14+
15+
/**
16+
* Asynchronous I/O operations needed for dependency installation.
17+
* <p>
18+
* The methods in this class are non-blocking, that is, they return more or less immediately, always scheduling the work
19+
* in the provided executor service. Requests are cached where it makes sense.
20+
*/
21+
public class AsyncFetcher {
22+
private Fetcher fetcher = new Fetcher();
23+
private ExecutorService executor;
24+
private Consumer<CompletionException> errorReporter;
25+
26+
/**
27+
* @param executor thread pool to perform I/O tasks
28+
* @param errorReporter called once for each error from the underlying I/O tasks
29+
*/
30+
public AsyncFetcher(ExecutorService executor, Consumer<CompletionException> errorReporter) {
31+
this.executor = executor;
32+
this.errorReporter = errorReporter;
33+
}
34+
35+
private CompletionException makeError(String message, Exception cause) {
36+
CompletionException ex = new CompletionException(message, cause);
37+
errorReporter.accept(ex); // Handle here to ensure each exception is logged at most once, not once per consumer
38+
throw ex;
39+
}
40+
41+
class CachedOperation<K, V> {
42+
private Map<K, CompletableFuture<V>> cache = new LinkedHashMap<>();
43+
44+
public synchronized CompletableFuture<V> get(K key, Supplier<V> builder) {
45+
CompletableFuture<V> future = cache.get(key);
46+
if (future == null) {
47+
future = CompletableFuture.supplyAsync(() -> builder.get(), executor);
48+
cache.put(key, future);
49+
}
50+
return future;
51+
}
52+
}
53+
54+
private CachedOperation<String, Packument> packuments = new CachedOperation<>();
55+
56+
/**
57+
* Returns a future that completes with the packument for the given package.
58+
* <p>
59+
* At most one fetch will be performed.
60+
*/
61+
public CompletableFuture<Packument> getPackument(String packageName) {
62+
return packuments.get(packageName, () -> {
63+
try {
64+
return fetcher.getPackument(packageName);
65+
} catch (IOException e) {
66+
throw makeError("Could not fetch packument for " + packageName, e);
67+
}
68+
});
69+
}
70+
71+
/**
72+
* Extracts the relevant contents of the given tarball URL in the given folder;
73+
* the returned future completes when done.
74+
*/
75+
public CompletableFuture<Void> installFromTarballUrl(String tarballUrl, Path destDir) {
76+
return CompletableFuture.runAsync(() -> {
77+
try {
78+
fetcher.extractFromTarballUrl(tarballUrl, destDir);
79+
} catch (IOException e) {
80+
throw makeError("Could not install package from " + tarballUrl, e);
81+
}
82+
}, executor);
83+
}
84+
}

‎javascript/extractor/src/com/semmle/js/dependencies/DependencyResolver.java‎

Lines changed: 221 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,221 @@
1+
package com.semmle.js.dependencies;
2+
3+
import java.io.IOException;
4+
import java.nio.file.Files;
5+
import java.nio.file.Path;
6+
import java.nio.file.Paths;
7+
import java.util.ArrayList;
8+
import java.util.Arrays;
9+
import java.util.Collections;
10+
import java.util.LinkedHashMap;
11+
import java.util.List;
12+
import java.util.Map;
13+
import java.util.Set;
14+
import java.util.concurrent.CompletableFuture;
15+
import java.util.concurrent.CompletionException;
16+
import java.util.concurrent.ExecutorService;
17+
import java.util.concurrent.Executors;
18+
19+
import com.google.gson.Gson;
20+
import com.semmle.js.dependencies.packument.PackageJson;
21+
import com.semmle.util.data.Pair;
22+
23+
public class DependencyResolver {
24+
private AsyncFetcher fetcher;
25+
private List<Constraint> constraints = new ArrayList<>();
26+
27+
/** Packages we don't try to install because it is part of the same monorepo. */
28+
private Set<String> packagesInRepo;
29+
30+
private static class Constraint {
31+
final PackageJson targetPackage;
32+
final SemVer targetPackageVersion;
33+
final PackageJson demandingPackage;
34+
final int depth;
35+
36+
Constraint(PackageJson targetPackage, SemVer targetPackageVersion, PackageJson demandingPackage, int depth) {
37+
this.targetPackage = targetPackage;
38+
this.targetPackageVersion = targetPackageVersion;
39+
this.demandingPackage = demandingPackage;
40+
this.depth = depth;
41+
}
42+
43+
String getTargetPackageName() {
44+
return targetPackage.getName(); // Must exist as you can't depend on a package without a name
45+
}
46+
}
47+
48+
public DependencyResolver(ExecutorService threadPool, Set<String> packagesInRepo) {
49+
this.fetcher = new AsyncFetcher(threadPool, this::reportError);
50+
this.packagesInRepo = packagesInRepo;
51+
}
52+
53+
private void reportError(CompletionException ex) {
54+
System.err.println(ex);
55+
}
56+
57+
private void addConstraint(Constraint constraint) {
58+
synchronized(constraints) {
59+
constraints.add(constraint);
60+
}
61+
}
62+
63+
/**
64+
* Returns the first version number mentioned in the given constraints, excluding upper bounds such as `< 2.0.0`,
65+
* or `null` if no such version number was found.
66+
* <p>
67+
* To help ensure deterministic version resolution, we prefer the version mentioned in the constraint, rather than
68+
* the latest version satisfying the constraint (as the latter can change in time).
69+
*/
70+
private SemVer getPreferredVersionFromConstraints(List<VersionConstraint> constraints) {
71+
for (VersionConstraint constraint : constraints) {
72+
if (!constraint.getOperator().equals("<") && constraint.getVersion() != null) {
73+
return constraint.getVersion();
74+
}
75+
}
76+
return null;
77+
}
78+
79+
/**
80+
* Given a set of available versions, pick the oldest version no older than <code>preferredVersion</code>.
81+
*/
82+
private Pair<SemVer, PackageJson> getTargetVersion(Map<String, PackageJson> versions, SemVer preferredVersion) {
83+
PackageJson result = versions.get(preferredVersion.toString());
84+
if (result != null) return Pair.make(preferredVersion, result);
85+
SemVer bestVersion = null;
86+
for (Map.Entry<String, PackageJson> entry : versions.entrySet()) {
87+
SemVer version = SemVer.tryParse(entry.getKey());
88+
if (version == null) continue; // Could not parse version
89+
if (version.compareTo(preferredVersion) < 0) continue; // Version is older than preferred version, ignore
90+
if (bestVersion != null && bestVersion.compareTo(version) < 0) continue; // We already found an older version
91+
bestVersion = version;
92+
result = entry.getValue();
93+
}
94+
return Pair.make(bestVersion, result);
95+
}
96+
97+
/**
98+
* Fetches all packages and builds up the constraint system needed for resolving.
99+
*/
100+
private CompletableFuture<Void> fetchRelevantPackages(PackageJson pack, int depth) {
101+
List<CompletableFuture<Void>> futures = new ArrayList<>();
102+
List<Map<String, String>> dependencyMaps = depth == 0
103+
? Arrays.asList(pack.getDependencies(), pack.getPeerDependencies(), pack.getDevDependencies())
104+
: Arrays.asList(pack.getDependencies()); // for transitive dependencies, only follow explicit dependencies
105+
for (Map<String, String> dependencies : dependencyMaps) {
106+
if (dependencies == null) continue;
107+
dependencies.forEach((targetName, targetVersions) -> {
108+
if (packagesInRepo.contains(targetName)) {
109+
return;
110+
}
111+
List<VersionConstraint> constraints = VersionConstraint.parseVersionConstraints(targetVersions);
112+
SemVer preferredVersion = getPreferredVersionFromConstraints(constraints);
113+
if (preferredVersion == null) return;
114+
futures.add(fetcher.getPackument(targetName).exceptionally(ex -> null).thenCompose(targetPackument -> {
115+
if (targetPackument == null) {
116+
return CompletableFuture.completedFuture(null);
117+
}
118+
Map<String, PackageJson> versions = targetPackument.getVersions();
119+
if (versions == null) return CompletableFuture.completedFuture(null);
120+
121+
// Pick the matching version
122+
Pair<SemVer, PackageJson> targetVersionAndPackage = getTargetVersion(versions, preferredVersion);
123+
SemVer targetVersion = targetVersionAndPackage.fst();
124+
PackageJson targetPackage = targetVersionAndPackage.snd();
125+
if (targetPackage == null) return CompletableFuture.completedFuture(null);
126+
127+
if (targetName.startsWith("@types/")) {
128+
// Deeply install dependencies in `@types`
129+
addConstraint(new Constraint(targetPackage, targetVersion, pack, depth));
130+
return fetchRelevantPackages(targetPackage, depth + 1);
131+
} else if (dependencies != pack.getDevDependencies() && (targetPackage.getTypes() != null || targetPackage.getTypings() != null)) {
132+
// If a non-dev dependency contains its own typings, do a shallow install of that package
133+
addConstraint(new Constraint(targetPackage, targetVersion, pack, depth));
134+
}
135+
return CompletableFuture.completedFuture(null);
136+
}));
137+
});
138+
}
139+
return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0]));
140+
}
141+
142+
/**
143+
* Resolves the dependencies of the given package in a deterministic way.
144+
*/
145+
private CompletableFuture<Map<String, PackageJson>> resolvePackages(PackageJson rootPackage) {
146+
return fetchRelevantPackages(rootPackage, 0).thenApply(void_ -> {
147+
// Compute the minimum depth from which each dependency is requested.
148+
Map<String, Integer> packageDepth = new LinkedHashMap<>();
149+
for (Constraint constraint : constraints) {
150+
Integer currentDepth = packageDepth.get(constraint.getTargetPackageName());
151+
if (currentDepth == null || currentDepth > constraint.depth) {
152+
packageDepth.put(constraint.getTargetPackageName(), constraint.depth);
153+
}
154+
}
155+
156+
// We use a greedy solver: sort the constraints and then satisfy them eagerly in that order.
157+
constraints.sort((c1, c2) -> {
158+
int cmp;
159+
160+
cmp = Integer.compare(packageDepth.get(c1.getTargetPackageName()), packageDepth.get(c2.getTargetPackageName()));
161+
if (cmp != 0) return cmp;
162+
163+
cmp = c1.getTargetPackageName().compareTo(c2.getTargetPackageName());
164+
if (cmp != 0) return cmp;
165+
166+
// Pick the most recent version, so reverse-sort by package version.
167+
cmp = -c1.targetPackageVersion.compareTo(c2.targetPackageVersion);
168+
if (cmp != 0) return cmp;
169+
170+
return 0;
171+
});
172+
173+
Map<String, PackageJson> selectedPackages = new LinkedHashMap<>();
174+
for (Constraint constraint : constraints) {
175+
if (selectedPackages.containsKey(constraint.getTargetPackageName())) {
176+
// Too bad, we already picked a version for this package. Ignore the constraint.
177+
continue;
178+
}
179+
if (constraint.demandingPackage != rootPackage) {
180+
PackageJson selectedDemander = selectedPackages.get(constraint.demandingPackage.getName());
181+
if (selectedDemander != null && selectedDemander != constraint.demandingPackage) {
182+
// The constraint comes from a package version we already decided not to install (a different version was picked).
183+
// There is no need to try to satisfy this constraint, so ignore it.
184+
continue;
185+
}
186+
}
187+
System.out.println("Picked " + constraint.getTargetPackageName() + "@" + constraint.targetPackageVersion);
188+
selectedPackages.put(constraint.getTargetPackageName(), constraint.targetPackage);
189+
}
190+
191+
return selectedPackages;
192+
});
193+
}
194+
195+
public CompletableFuture<Void> installDependencies(PackageJson rootPackage, Path nodeModulesDir) {
196+
return resolvePackages(rootPackage).thenCompose(resolvedPackages -> {
197+
List<CompletableFuture<Void>> futures = new ArrayList<>();
198+
resolvedPackages.forEach((name, targetPackage) -> {
199+
Path destinationDir = nodeModulesDir.resolve(Fetcher.toSafePath(name));
200+
futures.add(fetcher.installFromTarballUrl(targetPackage.getDist().getTarball(), destinationDir));
201+
});
202+
return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0]));
203+
});
204+
}
205+
206+
/** Entry point which installs dependencies from a given `package.json`, used for testing andbenchmarking. */
207+
public static void main(String[] args) throws IOException {
208+
ExecutorService executors = Executors.newFixedThreadPool(50);
209+
try {
210+
DependencyResolver resolver = new DependencyResolver(executors, Collections.emptySet());
211+
for (String packageJsonPath : args) {
212+
Path path = Paths.get(packageJsonPath).toAbsolutePath();
213+
PackageJson packageJson = new Gson().fromJson(Files.newBufferedReader(path), PackageJson.class);
214+
resolver.installDependencies(packageJson, path.getParent().resolve("node_modules")).join();
215+
}
216+
System.out.println("Done");
217+
} finally {
218+
executors.shutdown();
219+
}
220+
}
221+
}

0 commit comments

Comments
 (0)