Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add Support for Composer 2 Metadata API #4435

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

Add Support for Composer 2 Metadata API #4435

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a0dce79
Add Support for Composer 2 Metadata API
ch8matt Dec 4, 2024
8d35647
Update ComposerMetaAnalyzer.java
ch8matt Dec 4, 2024
de42ba8
Update ComposerMetaAnalyzer.java
ch8matt Dec 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
194 changes: 122 additions & 72 deletions src/main/java/org/dependencytrack/tasks/repositories/ComposerMetaAnalyzer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,129 +20,179 @@

import alpine.common.logging.Logger;
import com.github.packageurl.PackageURL;
import org.dependencytrack.exception.MetaAnalyzerException;
import org.json.JSONObject;
import org.apache.http.HttpStatus;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.util.EntityUtils;
import org.apache.maven.artifact.versioning.ComparableVersion;
import org.dependencytrack.exception.MetaAnalyzerException;
import org.dependencytrack.model.Component;
import org.dependencytrack.model.RepositoryType;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.apache.maven.artifact.versioning.ComparableVersion;

import java.io.IOException;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;

/**
* An IMetaAnalyzer implementation that supports Composer.
*
* @author Szabolcs (Szasza) Palmer
* @since 4.1.0
*/
public class ComposerMetaAnalyzer extends AbstractMetaAnalyzer {

private static final Logger LOGGER = Logger.getLogger(ComposerMetaAnalyzer.class);
private static final String DEFAULT_BASE_URL = "https://repo.packagist.org";

/**
* @see <a href="https://packagist.org/apidoc#get-package-metadata-v1">Packagist's API doc for "Getting package data - Using the Composer v1 metadata (DEPRECATED)"</a>
*/
private static final String API_URL = "/p/%s/%s.json";
private static final String API_URL_V1 = "/p/%s/%s.json";
private static final String API_URL_V2 = "/p2/%s/%s.json";

ComposerMetaAnalyzer() {
this.baseUrl = DEFAULT_BASE_URL;
}

/**
* {@inheritDoc}
*/
@Override
public boolean isApplicable(final Component component) {
return component.getPurl() != null && PackageURL.StandardTypes.COMPOSER.equals(component.getPurl().getType());
}

/**
* {@inheritDoc}
*/
@Override
public RepositoryType supportedRepositoryType() {
return RepositoryType.COMPOSER;
}

/**
* {@inheritDoc}
*/
@Override
public MetaModel analyze(final Component component) {
final MetaModel meta = new MetaModel(component);
if (component.getPurl() == null) {
return meta;
}

final String url = String.format(baseUrl + API_URL, urlEncode(component.getPurl().getNamespace()), urlEncode(component.getPurl().getName()));
try (final CloseableHttpResponse response = processHttpRequest(url)) {
if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
handleUnexpectedHttpResponse(LOGGER, url, response.getStatusLine().getStatusCode(), response.getStatusLine().getReasonPhrase(), component);
final String urlV2 = String.format(baseUrl + API_URL_V2, urlEncode(component.getPurl().getNamespace()), urlEncode(component.getPurl().getName()));
final String urlV1 = String.format(baseUrl + API_URL_V1, urlEncode(component.getPurl().getNamespace()), urlEncode(component.getPurl().getName()));

try {
if (processRepository(urlV2, meta)) {
return meta;
}
if (response.getEntity().getContent() == null) {
if (processRepository(urlV1, meta)) {
return meta;
}
String jsonString = EntityUtils.toString(response.getEntity());
if (jsonString.equalsIgnoreCase("")) {
return meta;
LOGGER.warn("Failed to retrieve package metadata from both Composer V1 and V2 endpoints.");
} catch (IOException ex) {
handleRequestException(LOGGER, ex);
} catch (Exception ex) {
LOGGER.error("Unexpected error during analysis", ex);
throw new MetaAnalyzerException(ex);
}
return meta;
}

private boolean processRepository(String url, MetaModel meta) throws IOException {
try (final CloseableHttpResponse response = processHttpRequest(url)) {
if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK && response.getEntity() != null) {
String jsonString = EntityUtils.toString(response.getEntity());
JSONObject jsonObject = new JSONObject(jsonString);

if (jsonObject.has("packages")) {
parseComposerData(jsonObject.getJSONObject("packages"), meta);
return true;
} else {
LOGGER.warn("Unexpected JSON structure from: " + url);
}
} else {
LOGGER.warn("HTTP response status not OK for URL: " + url);
}
if (jsonString.equalsIgnoreCase("{}")) {
return meta;
} catch (JSONException e) {
LOGGER.error("Invalid JSON response from: " + url, e);
}
return false;
}

private void parseComposerData(JSONObject packages, MetaModel meta) {
for (String packageName : packages.keySet()) {
Object packageData = packages.get(packageName);
if (packageData instanceof JSONObject) {
// For Composer 1 (/p endpoint)
JSONObject packageDataObj = (JSONObject) packageData;
JSONObject versionsObj = packageDataObj.optJSONObject("versions");
if (versionsObj != null) {
parseVersions(versionsObj, meta);
}
} else if (packageData instanceof JSONArray) {
// For Composer 2 (/p2 endpoint)
JSONArray versionsArray = (JSONArray) packageData;
for (int i = 0; i < versionsArray.length(); i++) {
JSONObject versionData = versionsArray.getJSONObject(i);
parseVersionData(versionData, meta);
}
} else {
LOGGER.warn("Unexpected package data type for package: " + packageName);
}
JSONObject jsonObject = new JSONObject(jsonString);
final String expectedResponsePackage = component.getPurl().getNamespace() + "/" + component.getPurl().getName();
final JSONObject responsePackages = jsonObject
.getJSONObject("packages");
if (!responsePackages.has(expectedResponsePackage)) {
// the package no longer exists - like this one: https://repo.packagist.org/p/magento/adobe-ims.json
return meta;
}
}

private void parseVersions(JSONObject versions, MetaModel meta) {
if (versions == null) {
return;
}

for (String versionKey : versions.keySet()) {
JSONObject versionData = versions.optJSONObject(versionKey);
if (versionData != null) {
parseVersionData(versionData, meta);
}
final JSONObject composerPackage = responsePackages.getJSONObject(expectedResponsePackage);
}
}

final ComparableVersion latestVersion = new ComparableVersion(stripLeadingV(component.getPurl().getVersion()));
final DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssXXX");
private void parseVersionData(JSONObject versionData, MetaModel meta) {
String version = versionData.optString("version", null);
String normalizedVersion = normalizeVersion(versionData.optString("version_normalized", version));
String time = versionData.optString("time", null);

composerPackage.names().forEach(key_ -> {
String key = (String) key_;
if (key.startsWith("dev-") || key.endsWith("-dev")) {
// dev versions are excluded, since they are not pinned but a VCS-branch.
return;
}
if (version == null || normalizedVersion == null) {
LOGGER.warn("Version data missing required keys: " + versionData);
return;
}

final String version_normalized = composerPackage.getJSONObject(key).getString("version_normalized");
ComparableVersion currentComparableVersion = new ComparableVersion(version_normalized);
if (currentComparableVersion.compareTo(latestVersion) < 0) {
// smaller version can be skipped
return;
}
String currentLatestVersionNormalized = meta.getLatestVersion() != null
? normalizeVersion(meta.getLatestVersion())
: null;

final String version = composerPackage.getJSONObject(key).getString("version");
latestVersion.parseVersion(stripLeadingV(version_normalized));
meta.setLatestVersion(version);
try {
ComparableVersion newVersion = new ComparableVersion(normalizedVersion);
ComparableVersion currentLatestVersion = currentLatestVersionNormalized != null
? new ComparableVersion(currentLatestVersionNormalized)
: null;

final String published = composerPackage.getJSONObject(key).getString("time");
try {
meta.setPublishedTimestamp(dateFormat.parse(published));
} catch (ParseException e) {
LOGGER.warn("An error occurred while parsing upload time", e);
if (currentLatestVersion == null || newVersion.compareTo(currentLatestVersion) > 0) {
meta.setLatestVersion(version);
if (time != null) {
try {
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssXXX");
meta.setPublishedTimestamp(format.parse(time));
} catch (ParseException e) {
LOGGER.error("Failed to parse timestamp: " + time, e);
}
}
});
} catch (IOException ex) {
handleRequestException(LOGGER, ex);
} catch (Exception ex) {
throw new MetaAnalyzerException(ex);
}
} catch (IllegalArgumentException e) {
LOGGER.warn("Invalid version format: " + normalizedVersion, e);
}

return meta;
}

private static String stripLeadingV(String s) {
return s.startsWith("v")
? s.substring(1)
: s;
private static String normalizeVersion(String version) {
if (version == null) {
return null;
}
version = version.trim();

// Remove leading 'v' or 'V'
if (version.startsWith("v") || version.startsWith("V")) {
version = version.substring(1);
}

// Remove trailing ".0" components
version = version.replaceAll("(\\.0)+$", "");

return version;
}
}
41 changes: 26 additions & 15 deletions src/test/java/org/dependencytrack/tasks/repositories/ComposerMetaAnalyzerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@

import java.io.File;
import java.io.FileInputStream;
import java.net.URL;
import java.text.SimpleDateFormat;

import static org.mockserver.model.HttpRequest.request;
Expand Down Expand Up @@ -76,7 +77,7 @@ public void testAnalyzerFindsVersionWithLeadingV() throws Exception {
.when(
request()
.withMethod("GET")
.withPath("/p/typo3/class-alias-loader.json")
.withPath("/p2/typo3/class-alias-loader.json")
)
.respond(
response()
Expand All @@ -89,9 +90,9 @@ public void testAnalyzerFindsVersionWithLeadingV() throws Exception {

MetaModel metaModel = analyzer.analyze(component);

Assert.assertEquals("v1.1.3", metaModel.getLatestVersion());
Assert.assertEquals("v1.2.0", metaModel.getLatestVersion());
Assert.assertEquals(
new SimpleDateFormat("yyyy-MM-dd HH:mm:ss XXX").parse("2020-05-24 13:03:22 Z"),
new SimpleDateFormat("yyyy-MM-dd HH:mm:ss XXX").parse("2024-10-11 12:00:00 Z"),
metaModel.getPublishedTimestamp()
);
}
Expand All @@ -101,15 +102,15 @@ public void testAnalyzerGetsUnexpectedResponseContentCausingLatestVersionBeingNu
Component component = new Component();
ComposerMetaAnalyzer analyzer = new ComposerMetaAnalyzer();

component.setPurl(new PackageURL("pkg:composer/magento/[email protected]"));
final File packagistFile = getResourceFile("magento", "adobe-ims");
component.setPurl(new PackageURL("pkg:composer/mockery/[email protected]"));
final File packagistFile = getResourceFile("mockery", "mockery");

analyzer.setRepositoryBaseUrl(String.format("http://localhost:%d", mockServer.getPort()));
new MockServerClient("localhost", mockServer.getPort())
.when(
request()
.withMethod("GET")
.withPath("/p/magento/adobe-ims.json")
.withPath("/p2/mockery/mockery.json")
)
.respond(
response()
Expand All @@ -124,16 +125,26 @@ public void testAnalyzerGetsUnexpectedResponseContentCausingLatestVersionBeingNu
Assert.assertNull(metaModel.getLatestVersion());
}

private static File getResourceFile(String namespace, String name) throws Exception{
return new File(
Thread.currentThread().getContextClassLoader()
.getResource(String.format(
"unit/tasks/repositories/https---repo.packagist.org-p-%s-%s.json",
namespace,
name
))
.toURI()
private static File getResourceFile(String namespace, String name) throws Exception {
String resourcePathP = String.format(
"unit/tasks/repositories/https---repo.packagist.org-p-%s-%s.json",
namespace, name
);
String resourcePathP2 = String.format(
"unit/tasks/repositories/https---repo.packagist.org-p2-%s-%s.json",
namespace, name
);

URL resource = Thread.currentThread().getContextClassLoader().getResource(resourcePathP2);
if (resource == null) {
resource = Thread.currentThread().getContextClassLoader().getResource(resourcePathP);
}

if (resource == null) {
throw new IllegalArgumentException("Test resource not found: " + resourcePathP + " or " + resourcePathP2);
}

return new File(resource.toURI());
}

private static byte[] getTestData(File file) throws Exception {
Expand Down
1 change: 0 additions & 1 deletion ...est/resources/unit/tasks/repositories/https---repo.packagist.org-p-magento-adobe-ims.json
View file
Open in desktop

This file was deleted.

Loading
Loading