Skip to content

Commit

Permalink
Add a parameter for the repository on IP Team Review Requests #224
Browse files Browse the repository at this point in the history
  • Loading branch information
waynebeaton committed Nov 24, 2023
1 parent 9d7a6a5 commit 17e4a91
Show file tree
Hide file tree
Showing 11 changed files with 208 additions and 39 deletions.
34 changes: 18 additions & 16 deletions DEPENDENCIES
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ maven/mavencentral/com.google.inject.extensions/guice-multibindings/4.2.3, Apach
maven/mavencentral/com.google.inject/guice/5.1.0, Apache-2.0, approved, #6676
maven/mavencentral/com.google.inject/guice/7.0.0, Apache-2.0, approved, #9095
maven/mavencentral/com.google.j2objc/j2objc-annotations/1.3, Apache-2.0, approved, CQ21195
maven/mavencentral/com.googlecode.javaewah/JavaEWAH/1.1.7, Apache-2.0, approved, CQ11658
maven/mavencentral/commons-cli/commons-cli/1.5.0, Apache-2.0, approved, clearlydefined
maven/mavencentral/commons-codec/commons-codec/1.11, Apache-2.0 AND BSD-3-Clause, approved, CQ15971
maven/mavencentral/commons-logging/commons-logging/1.2, Apache-2.0, approved, CQ10162
Expand All @@ -28,24 +29,24 @@ maven/mavencentral/org.apache.commons/commons-csv/1.10.0, Apache-2.0, approved,
maven/mavencentral/org.apache.commons/commons-lang3/3.12.0, Apache-2.0, approved, clearlydefined
maven/mavencentral/org.apache.httpcomponents/httpclient/4.5.13, Apache-2.0 AND LicenseRef-Public-Domain, approved, CQ23527
maven/mavencentral/org.apache.httpcomponents/httpcore/4.4.13, Apache-2.0, approved, CQ23528
maven/mavencentral/org.apache.maven.plugin-tools/maven-plugin-annotations/3.9.0, Apache-2.0, approved, #8458
maven/mavencentral/org.apache.maven.resolver/maven-resolver-api/1.9.14, Apache-2.0, approved, #4954
maven/mavencentral/org.apache.maven.resolver/maven-resolver-impl/1.9.14, Apache-2.0, approved, #5011
maven/mavencentral/org.apache.maven.resolver/maven-resolver-named-locks/1.9.14, Apache-2.0, approved, #5012
maven/mavencentral/org.apache.maven.resolver/maven-resolver-spi/1.9.14, Apache-2.0, approved, #5013
maven/mavencentral/org.apache.maven.resolver/maven-resolver-util/1.9.14, Apache-2.0, approved, #4953
maven/mavencentral/org.apache.maven.plugin-tools/maven-plugin-annotations/3.10.2, Apache-2.0, approved, #11096
maven/mavencentral/org.apache.maven.resolver/maven-resolver-api/1.9.16, Apache-2.0, approved, #4954
maven/mavencentral/org.apache.maven.resolver/maven-resolver-impl/1.9.16, Apache-2.0, approved, #5011
maven/mavencentral/org.apache.maven.resolver/maven-resolver-named-locks/1.9.16, Apache-2.0, approved, #5012
maven/mavencentral/org.apache.maven.resolver/maven-resolver-spi/1.9.16, Apache-2.0, approved, #5013
maven/mavencentral/org.apache.maven.resolver/maven-resolver-util/1.9.16, Apache-2.0, approved, #4953
maven/mavencentral/org.apache.maven.shared/maven-common-artifact-filters/3.3.2, Apache-2.0, approved, #10055
maven/mavencentral/org.apache.maven.shared/maven-shared-utils/3.3.4, Apache-2.0, approved, clearlydefined
maven/mavencentral/org.apache.maven/maven-artifact/3.9.4, Apache-2.0, approved, #7015
maven/mavencentral/org.apache.maven/maven-builder-support/3.9.4, Apache-2.0, approved, #7027
maven/mavencentral/org.apache.maven/maven-core/3.9.4, Apache-2.0, approved, #7017
maven/mavencentral/org.apache.maven/maven-model-builder/3.9.4, Apache-2.0, approved, #7022
maven/mavencentral/org.apache.maven/maven-model/3.9.4, Apache-2.0, approved, #7014
maven/mavencentral/org.apache.maven/maven-plugin-api/3.9.4, Apache-2.0, approved, #7016
maven/mavencentral/org.apache.maven/maven-repository-metadata/3.9.4, Apache-2.0, approved, #7023
maven/mavencentral/org.apache.maven/maven-resolver-provider/3.9.4, Apache-2.0, approved, #7030
maven/mavencentral/org.apache.maven/maven-settings-builder/3.9.4, Apache-2.0, approved, #7020
maven/mavencentral/org.apache.maven/maven-settings/3.9.4, Apache-2.0, approved, #7024
maven/mavencentral/org.apache.maven/maven-artifact/3.9.5, Apache-2.0, approved, #7015
maven/mavencentral/org.apache.maven/maven-builder-support/3.9.5, Apache-2.0, approved, #7027
maven/mavencentral/org.apache.maven/maven-core/3.9.5, Apache-2.0, approved, #7017
maven/mavencentral/org.apache.maven/maven-model-builder/3.9.5, Apache-2.0, approved, #7022
maven/mavencentral/org.apache.maven/maven-model/3.9.5, Apache-2.0, approved, #7014
maven/mavencentral/org.apache.maven/maven-plugin-api/3.9.5, Apache-2.0, approved, #7016
maven/mavencentral/org.apache.maven/maven-repository-metadata/3.9.5, Apache-2.0, approved, #7023
maven/mavencentral/org.apache.maven/maven-resolver-provider/3.9.5, Apache-2.0, approved, #7030
maven/mavencentral/org.apache.maven/maven-settings-builder/3.9.5, Apache-2.0, approved, #7020
maven/mavencentral/org.apache.maven/maven-settings/3.9.5, Apache-2.0, approved, #7024
maven/mavencentral/org.checkerframework/checker-qual/3.12.0, MIT, approved, clearlydefined
maven/mavencentral/org.codehaus.plexus/plexus-cipher/2.0, Apache-2.0, approved, clearlydefined
maven/mavencentral/org.codehaus.plexus/plexus-classworlds/2.7.0, Apache-2.0, approved, clearlydefined
Expand All @@ -55,6 +56,7 @@ maven/mavencentral/org.codehaus.plexus/plexus-sec-dispatcher/2.0, Apache-2.0, ap
maven/mavencentral/org.codehaus.plexus/plexus-utils/3.5.1, Apache-2.0 AND BSD-3-Clause AND BSD-2-Clause and LicenseRef-BSD-Style AND LicenseRef-Public-Domain, approved, #4119
maven/mavencentral/org.eclipse.dash/eclipse-api-for-java/1.0.0, EPL-2.0, approved, technology.dash
maven/mavencentral/org.eclipse.dash/org.eclipse.dash.licenses.core/1.0.3-SNAPSHOT, EPL-2.0, approved, technology.dash
maven/mavencentral/org.eclipse.jgit/org.eclipse.jgit/5.11.0.202103091610-r, BSD-3-Clause, approved, technology.jgit
maven/mavencentral/org.eclipse.sisu/org.eclipse.sisu.inject/0.3.5, EPL-1.0, approved, technology.sisu
maven/mavencentral/org.eclipse.sisu/org.eclipse.sisu.plexus/0.3.5, EPL-1.0, approved, technology.sisu
maven/mavencentral/org.gitlab4j/gitlab4j-api/5.3.0, MIT, approved, #10054
Expand Down
35 changes: 23 additions & 12 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,8 @@ To use this feature, you must have committer status on at least one Eclipse proj

* Get an [authentication token](https://gitlab.eclipse.org/-/profile/personal_access_tokens) (scope: `api`) from `gitlab.eclipse.org`;
* Include the `-review` option;
* Pass the token via the `-token` option; and
* Pass the token via the `-token` option;
* Pass the Eclipse project's repository URL (e.g., `https://github.com/eclipse/dash-licenses`) via the `-repo` option; and
* Pass the Eclipse open source project id (e.g., `technology.dash`) via the `-project` option.

Note that the options are slightly different for the [Maven plugin](README.md#maven-plugin-options).
Expand Down Expand Up @@ -254,15 +255,18 @@ Add the `repo.eclipse.org` plugin repository so that the license check plugin is

#### Maven Plugin Options

The Maven Plugin has the following options that can passed either via the command-line or as configuration options in your `pom.xml` file:
The Maven Plugin has the following properties that can passed either via the command-line or as configuration parameters in your `pom.xml` file:

- `dash.skip` - Skip executing the plugin. Default: `false`.
- `dash.fail` - Force the build to fail when license issues are found. Default: `false`.
- `dash.iplab.token` - The access token for automatically creating IP Team review requests. **Do not share your access token.**
- `dash.projectId` - The Eclipse open source project id (e.g. `technology.dash`)
- `dash.summary` - The location (where) to generate the summary file.
- `dash.skip` - Skip executing the plugin. Default: `false`;
- `dash.fail` - Force the build to fail when license issues are found. Default: `false`;
- `dash.iplab.token` - The access token for automatically creating IP Team review requests **Do not share your access token.**;
- `dash.projectId` - The Eclipse open source project id (e.g. `technology.dash`);
- `dash.repo` - Specify the Eclipse Project repository that is the source of the request (e.g., `https://github.com/eclipse/dash-licenses`);
- `dash.summary` - The location (where) to generate the summary file; and
- `dash.review.summary` - The location (where) to generate the review-summary file.

All properties are optional.

Note that the Maven plugin always generates the summary file. The default location is `${project.build.directory}/dash/summary`.

To generate a summary of dependencies named `DEPENDENCIES` in the working directory:
Expand Down Expand Up @@ -512,14 +516,21 @@ Sort out the licenses and approval of dependencies.
integer percent (0-100)
-ef,--foundation-api <url> Eclipse Foundation license check API
URL.
-excludeSources <sources> Exclude values from specific sources
-help,--help Display help
-summary <file> Output a summary to a file
-lic,--licenses <url> Approved Licenses List URL
-project <shortname> Process the request in the context of
an Eclipse project (e.g.,
technology.dash)
-repo <url> The Eclipse Project repository that is
the source of the request
-review Must also specify the project and token
-summary <file> Output a summary to a file
-timeout <seconds> Timeout for HTTP calls (in seconds)
-token <token> The GitLab authentication token
<file> is the path to a file, or "-" to indicate stdin. Multiple files may
be provided
e.g.,
npm list | grep -Poh "\S+@\d+(?:\.\d+){2}" | sort | uniq | LicenseFinder -
<file> is the path to a file, or "-" to indicate stdin.
For more help and examples, see https://github.com/eclipse/dash-licenses
```

### Reusable Github workflow for automatic license check and IP Team Review Requests
Expand Down
5 changes: 5 additions & 0 deletions core/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,11 @@
<artifactId>gitlab4j-api</artifactId>
<version>5.3.0</version>
</dependency>
<dependency>
<groupId>org.eclipse.jgit</groupId>
<artifactId>org.eclipse.jgit</artifactId>
<version>5.11.0.202103091610-r</version>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-engine</artifactId>
Expand Down
6 changes: 6 additions & 0 deletions core/src/main/java/org/eclipse/dash/licenses/ISettings.java
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@

import java.io.File;

import org.eclipse.dash.licenses.util.GitUtils;

public interface ISettings {

/**
Expand Down Expand Up @@ -128,4 +130,8 @@ default String getSummaryFilePath() {
default File getSummaryFile() {
return new File(getSummaryFilePath());
}

default String getRepository() {
return null;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ public class CommandLineSettings implements ISettings {
private static final String EXCLUDE_SOURCES_OPTION = "excludeSources";
private static final String TOKEN_OPTION = "token";
private static final String PROJECT_OPTION = "project";

private static final String REPO_OPTION = "repo";

private CommandLine commandLine;

Expand Down Expand Up @@ -261,6 +263,14 @@ private static Options getOptions() {
.desc("Process the request in the context of an Eclipse project (e.g., technology.dash)")
.build());

options.addOption(Option.builder(REPO_OPTION)
.required(false)
.hasArg()
.argName("url")
.type(String.class)
.desc("The Eclipse Project repository that is the source of the request")
.build());

options.addOption(Option.builder(HELP_OPTION)
.longOpt(HELP_OPTION)
.required(false)
Expand All @@ -286,9 +296,8 @@ public static void printHelp(PrintStream out) {
final HelpFormatter formatter = new HelpFormatter();
final String syntax = String.format("%s [options] <file> ...", Main.class.getName());
final String usageHeader = "Sort out the licenses and approval of dependencies.";
final String usageFooter = "\n" + "\n<file> is the path to a file, or \"-\" to indicate stdin. "
+ "Multiple files may be provided" + "\ne.g.,"
+ "\nnpm list | grep -Poh \"\\S+@\\d+(?:\\.\\d+){2}\" | sort | uniq | LicenseFinder -";
final String usageFooter = "\n<file> is the path to a file, or \"-\" to indicate stdin. "
+ "\nFor more help and examples, see https://github.com/eclipse/dash-licenses";

formatter.printHelp(syntax, usageHeader, getOptions(), usageFooter);
}
Expand All @@ -302,6 +311,13 @@ public String getSummaryFilePath() {
public String getProjectId() {
return commandLine.getOptionValue(PROJECT_OPTION, null);
}

@Override
public String getRepository() {
var repository = commandLine.getOptionValue(REPO_OPTION, "");
if (repository.isBlank()) return null;
return repository;
}

public String getExcludedSources() {
return commandLine.getOptionValue(EXCLUDE_SOURCES_OPTION);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,11 @@
public class GitLabReview {
private String projectId;
private LicenseData licenseData;
private String repository;

public GitLabReview(String projectId, LicenseData licenseData) {
public GitLabReview(String projectId, String repository, LicenseData licenseData) {
this.projectId = projectId;
this.repository = repository;
this.licenseData = licenseData;
}

Expand All @@ -38,8 +40,12 @@ public String getDescription() {
if (projectId != null) {
builder
.append(String
.format("Project: [%s](https://projects.eclipse.org/projects/%s)\n", projectId, projectId));
.format("Project: [%s](https://projects.eclipse.org/projects/%s)\n\n", projectId, projectId));
}
if (repository != null) {
builder.append(String.format("Repository: %s\n\n", repository));
}

licenseData.contentData().forEach(data -> describeItem(data, builder));

return builder.toString();
Expand All @@ -53,8 +59,6 @@ public String getDescription() {
*/
private void describeItem(IContentData data, StringBuilder output) {
// FIXME This is clunky

output.append("\n");
String authority = data.getAuthority();
if (data.getUrl() != null)
authority = String.format("[%s](%s)", authority, data.getUrl());
Expand All @@ -66,6 +70,7 @@ private void describeItem(IContentData data, StringBuilder output) {
.discoveredLicenses()
.forEach(license -> output.append(" - Discovered: " + license).append('\n'));
};
output.append("\n");
}

private IContentId getContentId() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import org.eclipse.dash.licenses.IProxySettings;
import org.eclipse.dash.licenses.ISettings;
import org.eclipse.dash.licenses.LicenseData;
import org.eclipse.dash.licenses.util.GitUtils;
import org.gitlab4j.api.GitLabApi;
import org.gitlab4j.api.GitLabApiException;
import org.gitlab4j.api.models.Issue;
Expand All @@ -40,7 +41,7 @@ public class GitLabSupport {
/** Optional HTTP proxy settings. */
@Inject
Provider<IProxySettings> proxySettings;

public void createReviews(List<LicenseData> needsReview, BiConsumer<IContentId, String> monitor) {
execute(connection -> {
var count = 0;
Expand All @@ -67,7 +68,7 @@ public void createReviews(List<LicenseData> needsReview, BiConsumer<IContentId,
* required to prevent rare duplication.
*/
try {
GitLabReview review = new GitLabReview(settings.getProjectId(), licenseData);
GitLabReview review = new GitLabReview(settings.getProjectId(), getRepository(), licenseData);

Issue existing = connection.findIssue(review);
if (existing != null) {
Expand Down Expand Up @@ -98,6 +99,13 @@ public void createReviews(List<LicenseData> needsReview, BiConsumer<IContentId,
});
}

String getRepository() {
var repository = settings.getRepository();
if (repository != null) return repository;

return GitUtils.getEclipseRemote();
}

public void execute(Consumer<GitLabConnection> callable) {
Map<String, Object> clientConfig = null;
IProxySettings proxySettings = this.proxySettings.get();
Expand Down
89 changes: 89 additions & 0 deletions core/src/main/java/org/eclipse/dash/licenses/util/GitUtils.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
/*************************************************************************
* Copyright (c) 2023 The Eclipse Foundation and others.
*
* This program and the accompanying materials are made available under
* the terms of the Eclipse Public License 2.0 which accompanies this
* distribution, and is available at https://www.eclipse.org/legal/epl-2.0
*
* SPDX-License-Identifier: EPL-2.0
*************************************************************************/
package org.eclipse.dash.licenses.util;

import java.io.IOException;
import java.util.stream.Stream;

import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.lib.RepositoryBuilder;
import org.eclipse.jgit.transport.URIish;

public class GitUtils {

/**
* Answers the browsable (https) URL for our best guess at the URL of the remote
* Eclipse Project Git repository associated with the Git repository in the
* working directory.
*
* @return a browsable (<code>https</code>) URL or <code>null</code>.
*/
public static String getEclipseRemote() {
return findEclipseRemotes()
.map(each -> String.format("https://%s/%s", each.getHost(), clean(each.getPath())))
.findFirst().orElse(null);
}

static Stream<URIish> findEclipseRemotes() {
try {
var repository = new RepositoryBuilder().findGitDir().build();
if (repository == null) return null;

var git = Git.wrap(repository);
return git.remoteList().call().stream()
.flatMap(each -> each.getURIs().stream())
.filter(each -> each.isRemote())
.filter(each -> isEclipseRemote(each));
} catch (IOException | GitAPIException e) {
// FIXME Log this (or maybe just don't...)
}
return Stream.empty();
}

static boolean isEclipseRemote(URIish uri) {
if ("git.eclipse.org".equals(uri.getHost())) return true;
if ("gitlab.eclipse.org".equals(uri.getHost()) && isEclipseProjectGitLabGroup(uri)) return true;
if ("github.com".equals(uri.getHost()) && isEclipseProjectGitHubOrg(uri)) return true;

return false;
}

static boolean isEclipseProjectGitLabGroup(URIish uri) {
var path = clean(uri.getPath());
if (path.startsWith("eclipse/")) return true;
if (path.startsWith("eclipsefdn/")) return true;

return false;
}

static boolean isEclipseProjectGitHubOrg(URIish uri) {
var path = clean(uri.getPath());
if (path.startsWith("eclipse/")) return true;
if (path.startsWith("eclipse-")) return true;

// TODO We need to be more dynamic
if (path.startsWith("jetty/")) return true;
if (path.startsWith("deeplearning4j/")) return true;

return false;
}

/**
* The path sometimes has a leading slash and sometimes does not. I believe that
* the difference is the result of the schema (there's no slash when the
* <code>git</code> schema is used, but there is a slash when it's
* <code>https</code>. Just strip a leading slash if one exists.
*/
static private String clean(String path) {
if (path.startsWith("/")) return path.substring(1);
return path;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,13 @@ public class LicenseCheckMojo extends AbstractArtifactFilteringMojo {
*/
@Parameter(property = "dash.projectId")
private String projectId;


/**
* Optionally specify the Eclipse Project repository that is the source of the request
*/
@Parameter(property = "dash.repo")
private String repo;

/**
* Output a summary to the given file. If not specified, then a dependencies
Expand Down Expand Up @@ -182,7 +189,7 @@ public void execute() throws MojoExecutionException, MojoFailureException {
// Validate the user-given dash license tool settings
ISettings settings;
try {
settings = new MavenSettings(batch, foundationApi, clearlyDefinedApi, licenses, confidence, projectId, iplabToken);
settings = new MavenSettings(batch, foundationApi, clearlyDefinedApi, licenses, confidence, projectId, repo, iplabToken);
} catch (IllegalArgumentException e) {
throw new MojoExecutionException("Invalid setting: " + e.getMessage());
}
Expand Down
Loading

0 comments on commit 17e4a91

Please sign in to comment.