Skip to content

Commit

Permalink
Refine dependency detection in package lock files.
Browse files Browse the repository at this point in the history 
With this patch, we recognise locally-resolved dependencies as project
content (#284), and honour remapped names (#285).
  • Loading branch information
@waynebeaton
waynebeaton committed Nov 23, 2023
1 parent a9e6990 commit f3156b2
Show file tree
Hide file tree
Showing 3 changed files with 9,531 additions and 34 deletions.
86 changes: 57 additions & 29 deletions core/src/main/java/org/eclipse/dash/licenses/cli/PackageLockFileReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@

public class PackageLockFileReader implements IDependencyListReader {
final Logger logger = LoggerFactory.getLogger(PackageLockFileReader.class);
private static final Pattern Name_Pattern = Pattern.compile("(?:(?<scope>@[^\\/]+)\\/)?(?<name>[^\\/]+)$");

private final InputStream input;

Expand Down Expand Up @@ -73,36 +74,62 @@ class Package {
* It's relatively easy to handle all cases with a regular expression.
*/
IContentId getContentId() {
Pattern pattern = Pattern.compile("(?:(?<scope>@[^\\/]+)\\/)?(?<name>[^\\/]+)$");
Matcher matcher = pattern.matcher(key);
if (matcher.find()) {
var namespace = matcher.group("scope");
if (namespace == null)
namespace = "-";
var name = matcher.group("name");
JsonObject jsonObject = value.asJsonObject();
var version = jsonObject.getString("version", null);
var resolved = jsonObject.getString("resolved", "");
if (resolved.startsWith("file:")) {
resolved = "local";
} else if (resolved.contains("registry.npmjs.org")) {
resolved = "npmjs";
} else {
logger.debug("Unknown resolved source: {}", resolved);
resolved = "npmjs";
}

if (version != null) {
IContentId contentId = ContentId.getContentId("npm", resolved, namespace, name, version);
return contentId == null ? new InvalidContentId(key + "@" + version) : contentId;
}
var namespace = getNameSpace();
var name = getName();
var version = value.asJsonObject().getString("version", null);

if (version != null) {
IContentId contentId = ContentId.getContentId("npm", getSource(), namespace, name, version);
return contentId == null ? new InvalidContentId(key + "@" + version) : contentId;
}
return new InvalidContentId(key);
}

String getSource() {
if (isResolvedLocally())
return "local";

var resolved = value.asJsonObject().getString("resolved", "");
if (resolved.contains("registry.npmjs.org")) {
return "npmjs";
} else {
logger.debug("Unknown resolved source: {}", resolved);
return "npmjs";
}
}

String getNameSpace() {
var name = value.asJsonObject().getString("name", key);
Matcher matcher = Name_Pattern.matcher(name);
if (matcher.find()) {
var scope = matcher.group("scope");
if (scope != null)
return scope;
}

return "-";
}

String getName() {
var name = value.asJsonObject().getString("name", key);
Matcher matcher = Name_Pattern.matcher(name);
if (matcher.find()) {
return matcher.group("name");
}

return null;
}

public boolean isResolvedLocally() {
var resolved = value.asJsonObject().getString("resolved", null);
if (resolved == null)
return true;
if (resolved.startsWith("file:"))
return true;

if (key.startsWith("packages/"))
return true;

if (value.asJsonObject().getBoolean("link", false))
return true;
if (value.asJsonObject().getString("version", "").startsWith("file:"))
Expand Down Expand Up @@ -143,28 +170,29 @@ Stream<Dependency> stream() {

@Override
public Collection<IContentId> getContentIds() {
return contentIds().filter(contentId -> "local" != contentId.getSource()).collect(Collectors.toList());
}

public Stream<IContentId> contentIds() {
JsonObject json = JsonUtils.readJson(input);

switch (json.getJsonNumber("lockfileVersion").intValue()) {
case 1:
return new Dependency("", json)
.stream()
.filter(each -> !each.key.isEmpty())
.map(dependency -> dependency.getContentId())
.collect(Collectors.toList());
.map(dependency -> dependency.getContentId());
case 2:
case 3:
// @formatter:off
return json.getJsonObject("packages").entrySet().stream()
.filter(entry -> !entry.getKey().isEmpty())
.map(entry -> new Package(entry.getKey(), entry.getValue().asJsonObject()))
.filter(dependency -> !dependency.isResolvedLocally())
.map(dependency -> dependency.getContentId())
.distinct().collect(Collectors.toList());
.map(dependency -> dependency.getContentId());
// @formatter:on
}

return null;
return Stream.empty();
}

}
41 changes: 36 additions & 5 deletions core/src/test/java/org/eclipse/dash/licenses/tests/PackageLockFileReaderTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,16 @@
package org.eclipse.dash.licenses.tests;

import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.jupiter.api.Assertions.assertEquals;

import java.io.IOException;
import java.io.InputStream;
import java.util.Arrays;
import java.util.stream.Collectors;

import org.eclipse.dash.licenses.ContentId;
import org.eclipse.dash.licenses.IContentId;
import org.eclipse.dash.licenses.cli.PackageLockFileReader;
import org.junit.jupiter.api.Test;
Expand Down Expand Up @@ -57,15 +62,41 @@ void testV2Format() throws IOException {
}

@Test
void testResolvedToLocal() throws IOException {
void testV3Format() throws IOException {
try (InputStream input = this.getClass().getResourceAsStream("/test_data_package-lock-v3.json")) {
PackageLockFileReader reader = new PackageLockFileReader(input);
var ids = reader.contentIds().collect(Collectors.toList());

assertEquals(769, ids.size());

// Issue #285 Component name is remapped. Make sure that we don't see the key
// in the results. This record should manifest as langium-statemachine-dsl (see
// below)
assertFalse(ids.stream().anyMatch(each -> "statemachine".equals(each.getName())));

// Test that a handful of content ids are detected as expected.
var includes = new IContentId[] { ContentId.getContentId("npm", "npmjs", "-", "ansi-styles", "3.2.1"),
ContentId.getContentId("npm", "npmjs", "@typescript-eslint", "eslint-plugin", "6.4.1"),
ContentId.getContentId("npm", "npmjs", "@types", "minimatch", "3.0.5"),
ContentId.getContentId("npm", "local", "-", "langium-requirements-dsl", "2.1.0"),
ContentId.getContentId("npm", "local", "-", "langium-domainmodel-dsl", "2.1.0"),
ContentId.getContentId("npm", "local", "-", "langium-statemachine-dsl", "2.1.0") };

for (int index = 0; index < includes.length; index++) {
var id = includes[index];
assertTrue("Should include: " + id.toString(), ids.contains(id));
}
}
}

@Test
void testAllRecordsDetected() throws IOException {
try (InputStream input = this.getClass().getResourceAsStream("/differentResolved.json")) {
PackageLockFileReader reader = new PackageLockFileReader(input);
// This "test" is a little... abridged. At least this test proves
// that we're getting something in the right format from the reader
// without having to enumerate all 574 (I think) records).

String[] expected = { "npm/npmjs/@babel/code-frame/7.12.13", "npm/local/-/some_local_package/1.2.3", };
Arrays.sort(expected);
String[] found = reader.getContentIds().stream().map(IContentId::toString).sorted().toArray(String[]::new);
String[] found = reader.contentIds().map(IContentId::toString).sorted().toArray(String[]::new);
assertArrayEquals(expected, found);
}
}
Expand Down
Loading

0 comments on commit f3156b2

Please sign in to comment.