diff --git a/plugins/package-managers/nuget/src/funTest/assets/dotnet-expected-output.yml b/plugins/package-managers/nuget/src/funTest/assets/dotnet-expected-output.yml index 7b50b2ccaa6e8..ea17832e57725 100644 --- a/plugins/package-managers/nuget/src/funTest/assets/dotnet-expected-output.yml +++ b/plugins/package-managers/nuget/src/funTest/assets/dotnet-expected-output.yml @@ -18,10 +18,10 @@ project: scopes: - name: "net45" dependencies: - - id: "NuGet::System.Runtime.CompilerServices.Unsafe:4.5.3" - - id: "NuGet::System.Threading.Tasks.Extensions:4.5.4" + - id: "NuGet:System.Runtime:CompilerServices.Unsafe:4.5.3" + - id: "NuGet:System.Threading:Tasks.Extensions:4.5.4" packages: -- id: "NuGet::System.Runtime.CompilerServices.Unsafe:4.5.3" +- id: "NuGet:System.Runtime:CompilerServices.Unsafe:4.5.3" purl: "pkg:nuget/System.Runtime.CompilerServices.Unsafe@4.5.3" authors: - "Microsoft" @@ -54,7 +54,7 @@ packages: url: "" revision: "" path: "" -- id: "NuGet::System.Threading.Tasks.Extensions:4.5.4" +- id: "NuGet:System.Threading:Tasks.Extensions:4.5.4" purl: "pkg:nuget/System.Threading.Tasks.Extensions@4.5.4" authors: - "Microsoft" diff --git a/plugins/package-managers/nuget/src/funTest/assets/dotnet-license-expected-output.yml b/plugins/package-managers/nuget/src/funTest/assets/dotnet-license-expected-output.yml index c5717605538b2..1b4fea76e988e 100644 --- a/plugins/package-managers/nuget/src/funTest/assets/dotnet-license-expected-output.yml +++ b/plugins/package-managers/nuget/src/funTest/assets/dotnet-license-expected-output.yml @@ -18,9 +18,9 @@ project: scopes: - name: "netstandard2.0" dependencies: - - id: "NuGet::Newtonsoft.Json:13.0.2" + - id: "NuGet:Newtonsoft:Json:13.0.2" packages: -- id: "NuGet::Newtonsoft.Json:13.0.2" +- id: "NuGet:Newtonsoft:Json:13.0.2" purl: "pkg:nuget/Newtonsoft.Json@13.0.2" authors: - "James Newton-King" diff --git a/plugins/package-managers/nuget/src/funTest/assets/dotnet-many-deps-expected-output.yml b/plugins/package-managers/nuget/src/funTest/assets/dotnet-many-deps-expected-output.yml index 9fd501d8a9056..1995a903d0aa7 100644 --- a/plugins/package-managers/nuget/src/funTest/assets/dotnet-many-deps-expected-output.yml +++ b/plugins/package-managers/nuget/src/funTest/assets/dotnet-many-deps-expected-output.yml @@ -18,18 +18,18 @@ project: scopes: - name: "net48" dependencies: - - id: "NuGet::Microsoft.Bcl.AsyncInterfaces:7.0.0" - - id: "NuGet::System.Buffers:4.5.1" - - id: "NuGet::System.Memory:4.5.5" - - id: "NuGet::System.Net.Http.Json:7.0.0" - - id: "NuGet::System.Numerics.Vectors:4.5.0" - - id: "NuGet::System.Runtime.CompilerServices.Unsafe:6.0.0" - - id: "NuGet::System.Text.Encodings.Web:7.0.0" - - id: "NuGet::System.Text.Json:7.0.2" - - id: "NuGet::System.Threading.Tasks.Extensions:4.5.4" - - id: "NuGet::System.ValueTuple:4.5.0" + - id: "NuGet:Microsoft.Bcl:AsyncInterfaces:7.0.0" + - id: "NuGet:System:Buffers:4.5.1" + - id: "NuGet:System:Memory:4.5.5" + - id: "NuGet:System:ValueTuple:4.5.0" + - id: "NuGet:System.Net:Http.Json:7.0.0" + - id: "NuGet:System.Numerics:Vectors:4.5.0" + - id: "NuGet:System.Runtime:CompilerServices.Unsafe:6.0.0" + - id: "NuGet:System.Text:Encodings.Web:7.0.0" + - id: "NuGet:System.Text:Json:7.0.2" + - id: "NuGet:System.Threading:Tasks.Extensions:4.5.4" packages: -- id: "NuGet::Microsoft.Bcl.AsyncInterfaces:7.0.0" +- id: "NuGet:Microsoft.Bcl:AsyncInterfaces:7.0.0" purl: "pkg:nuget/Microsoft.Bcl.AsyncInterfaces@7.0.0" authors: - "Microsoft" @@ -61,7 +61,7 @@ packages: url: "https://github.com/dotnet/runtime.git" revision: "d099f075e45d2aa6007a22b71b45a08758559f80" path: "" -- id: "NuGet::System.Buffers:4.5.1" +- id: "NuGet:System:Buffers:4.5.1" purl: "pkg:nuget/System.Buffers@4.5.1" authors: - "Microsoft" @@ -94,7 +94,7 @@ packages: url: "" revision: "" path: "" -- id: "NuGet::System.Memory:4.5.5" +- id: "NuGet:System:Memory:4.5.5" purl: "pkg:nuget/System.Memory@4.5.5" authors: - "Microsoft" @@ -128,7 +128,40 @@ packages: url: "" revision: "" path: "" -- id: "NuGet::System.Net.Http.Json:7.0.0" +- id: "NuGet:System:ValueTuple:4.5.0" + purl: "pkg:nuget/System.ValueTuple@4.5.0" + authors: + - "Microsoft" + declared_licenses: + - "https://github.com/dotnet/corefx/blob/master/LICENSE.TXT" + declared_licenses_processed: + spdx_expression: "MIT" + mapped: + https://github.com/dotnet/corefx/blob/master/LICENSE.TXT: "MIT" + description: "Provides the System.ValueTuple structs, which implement the underlying\ + \ types for tuples in C# and Visual Basic." + homepage_url: "https://dot.net/" + binary_artifact: + url: "https://api.nuget.org/v3-flatcontainer/system.valuetuple/4.5.0/system.valuetuple.4.5.0.nupkg" + hash: + value: "fa00ebb5045d12c51274f64411c551981beceb1266a8606a4731063109b95ea1f15939197bf3d2ba899db61e593dc39bfce876908bba34286823525093ae3d8e" + algorithm: "SHA-512" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "" + url: "" + revision: "" + path: "" +- id: "NuGet:System.Net:Http.Json:7.0.0" purl: "pkg:nuget/System.Net.Http.Json@7.0.0" authors: - "Microsoft" @@ -159,7 +192,7 @@ packages: url: "https://github.com/dotnet/runtime.git" revision: "d099f075e45d2aa6007a22b71b45a08758559f80" path: "" -- id: "NuGet::System.Numerics.Vectors:4.5.0" +- id: "NuGet:System.Numerics:Vectors:4.5.0" purl: "pkg:nuget/System.Numerics.Vectors@4.5.0" authors: - "Microsoft" @@ -192,7 +225,7 @@ packages: url: "" revision: "" path: "" -- id: "NuGet::System.Runtime.CompilerServices.Unsafe:6.0.0" +- id: "NuGet:System.Runtime:CompilerServices.Unsafe:6.0.0" purl: "pkg:nuget/System.Runtime.CompilerServices.Unsafe@6.0.0" authors: - "Microsoft" @@ -223,7 +256,7 @@ packages: url: "https://github.com/dotnet/runtime.git" revision: "4822e3c3aa77eb82b2fb33c9321f923cf11ddde6" path: "" -- id: "NuGet::System.Text.Encodings.Web:7.0.0" +- id: "NuGet:System.Text:Encodings.Web:7.0.0" purl: "pkg:nuget/System.Text.Encodings.Web@7.0.0" authors: - "Microsoft" @@ -254,7 +287,7 @@ packages: url: "https://github.com/dotnet/runtime.git" revision: "d099f075e45d2aa6007a22b71b45a08758559f80" path: "" -- id: "NuGet::System.Text.Json:7.0.2" +- id: "NuGet:System.Text:Json:7.0.2" purl: "pkg:nuget/System.Text.Json@7.0.2" authors: - "Microsoft" @@ -289,7 +322,7 @@ packages: url: "https://github.com/dotnet/runtime.git" revision: "0a2bda10e81d901396c3cff95533529e3a93ad47" path: "" -- id: "NuGet::System.Threading.Tasks.Extensions:4.5.4" +- id: "NuGet:System.Threading:Tasks.Extensions:4.5.4" purl: "pkg:nuget/System.Threading.Tasks.Extensions@4.5.4" authors: - "Microsoft" @@ -322,36 +355,3 @@ packages: url: "" revision: "" path: "" -- id: "NuGet::System.ValueTuple:4.5.0" - purl: "pkg:nuget/System.ValueTuple@4.5.0" - authors: - - "Microsoft" - declared_licenses: - - "https://github.com/dotnet/corefx/blob/master/LICENSE.TXT" - declared_licenses_processed: - spdx_expression: "MIT" - mapped: - https://github.com/dotnet/corefx/blob/master/LICENSE.TXT: "MIT" - description: "Provides the System.ValueTuple structs, which implement the underlying\ - \ types for tuples in C# and Visual Basic." - homepage_url: "https://dot.net/" - binary_artifact: - url: "https://api.nuget.org/v3-flatcontainer/system.valuetuple/4.5.0/system.valuetuple.4.5.0.nupkg" - hash: - value: "fa00ebb5045d12c51274f64411c551981beceb1266a8606a4731063109b95ea1f15939197bf3d2ba899db61e593dc39bfce876908bba34286823525093ae3d8e" - algorithm: "SHA-512" - source_artifact: - url: "" - hash: - value: "" - algorithm: "" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "" - url: "" - revision: "" - path: "" diff --git a/plugins/package-managers/nuget/src/funTest/assets/dotnet-with-csproj-and-nuget-config-expected-output.yml b/plugins/package-managers/nuget/src/funTest/assets/dotnet-with-csproj-and-nuget-config-expected-output.yml index 3bb5dcdda4576..75305fc3bf877 100644 --- a/plugins/package-managers/nuget/src/funTest/assets/dotnet-with-csproj-and-nuget-config-expected-output.yml +++ b/plugins/package-managers/nuget/src/funTest/assets/dotnet-with-csproj-and-nuget-config-expected-output.yml @@ -18,9 +18,9 @@ project: scopes: - name: "net7.0" dependencies: - - id: "NuGet::xunit.analyzers:1.2.0-pre.7" + - id: "NuGet:xunit:analyzers:1.2.0-pre.7" packages: -- id: "NuGet::xunit.analyzers:1.2.0-pre.7" +- id: "NuGet:xunit:analyzers:1.2.0-pre.7" purl: "pkg:nuget/xunit.analyzers@1.2.0-pre.7" authors: - "jnewkirk,bradwilson,marcind" diff --git a/plugins/package-managers/nuget/src/funTest/assets/dotnet-with-nuspec-expected-output.yml b/plugins/package-managers/nuget/src/funTest/assets/dotnet-with-nuspec-expected-output.yml index 02803aa63936c..f6e72912443a1 100644 --- a/plugins/package-managers/nuget/src/funTest/assets/dotnet-with-nuspec-expected-output.yml +++ b/plugins/package-managers/nuget/src/funTest/assets/dotnet-with-nuspec-expected-output.yml @@ -18,10 +18,10 @@ project: scopes: - name: "net45" dependencies: - - id: "NuGet::System.Runtime.CompilerServices.Unsafe:4.5.3" - - id: "NuGet::System.Threading.Tasks.Extensions:4.5.4" + - id: "NuGet:System.Runtime:CompilerServices.Unsafe:4.5.3" + - id: "NuGet:System.Threading:Tasks.Extensions:4.5.4" packages: -- id: "NuGet::System.Runtime.CompilerServices.Unsafe:4.5.3" +- id: "NuGet:System.Runtime:CompilerServices.Unsafe:4.5.3" purl: "pkg:nuget/System.Runtime.CompilerServices.Unsafe@4.5.3" authors: - "Microsoft" @@ -54,7 +54,7 @@ packages: url: "" revision: "" path: "" -- id: "NuGet::System.Threading.Tasks.Extensions:4.5.4" +- id: "NuGet:System.Threading:Tasks.Extensions:4.5.4" purl: "pkg:nuget/System.Threading.Tasks.Extensions@4.5.4" authors: - "Microsoft" diff --git a/plugins/package-managers/nuget/src/funTest/assets/nuget-expected-output.yml b/plugins/package-managers/nuget/src/funTest/assets/nuget-expected-output.yml index 0e8aa46ae6bdd..c38db229ba2e4 100644 --- a/plugins/package-managers/nuget/src/funTest/assets/nuget-expected-output.yml +++ b/plugins/package-managers/nuget/src/funTest/assets/nuget-expected-output.yml @@ -18,22 +18,10 @@ project: scopes: - name: "any" dependencies: - - id: "NuGet::System.Globalization:4.3.0" - dependencies: - - id: "NuGet::Microsoft.NETCore.Platforms:1.1.0" - - id: "NuGet::Microsoft.NETCore.Targets:1.1.0" - - id: "NuGet::System.Runtime:4.3.0" - - id: "NuGet::System.Threading:4.0.11" - dependencies: - - id: "NuGet::System.Runtime:4.3.0" - - id: "NuGet::System.Threading.Tasks:4.0.11" - - id: "NuGet::System.Threading.Tasks.Extensions:4.5.4" - dependencies: - - id: "NuGet::System.Runtime.CompilerServices.Unsafe:4.5.3" - id: "NuGet::WebGrease:1.5.2" dependencies: - id: "NuGet::Antlr:3.4.1.9004" - - id: "NuGet::Newtonsoft.Json:5.0.4" + - id: "NuGet:Newtonsoft:Json:5.0.4" - id: "NuGet::foobar:1.2.3" issues: - timestamp: "1970-01-01T00:00:00Z" @@ -45,6 +33,18 @@ project: \ at NugetInspector.BasePackage.Update(NugetApi nugetApi, Boolean with_details)\ \ in ./nuget-inspector/Models.cs:line 321" severity: "WARNING" + - id: "NuGet:System:Globalization:4.3.0" + dependencies: + - id: "NuGet:Microsoft.NETCore:Platforms:1.1.0" + - id: "NuGet:Microsoft.NETCore:Targets:1.1.0" + - id: "NuGet:System:Runtime:4.3.0" + - id: "NuGet:System:Threading:4.0.11" + dependencies: + - id: "NuGet:System:Runtime:4.3.0" + - id: "NuGet:System.Threading:Tasks:4.0.11" + - id: "NuGet:System.Threading:Tasks.Extensions:4.5.4" + dependencies: + - id: "NuGet:System.Runtime:CompilerServices.Unsafe:4.5.3" packages: - id: "NuGet::Antlr:3.4.1.9004" purl: "pkg:nuget/Antlr@3.4.1.9004" @@ -76,7 +76,65 @@ packages: url: "" revision: "" path: "" -- id: "NuGet::Microsoft.NETCore.Platforms:1.1.0" +- id: "NuGet::WebGrease:1.5.2" + purl: "pkg:nuget/WebGrease@1.5.2" + authors: + - "webgrease@microsoft.com" + declared_licenses: + - "http://www.microsoft.com/web/webpi/eula/msn_webgrease_eula.htm" + declared_licenses_processed: + unmapped: + - "http://www.microsoft.com/web/webpi/eula/msn_webgrease_eula.htm" + description: "Web Grease is a suite of tools for optimizing javascript, css files\ + \ and images." + homepage_url: "" + binary_artifact: + url: "https://api.nuget.org/v3-flatcontainer/webgrease/1.5.2/webgrease.1.5.2.nupkg" + hash: + value: "453fa7a3483b424dbc27f5d5c07827e6527faa6e21c96169ef0b7423bd54c348f749f4df3a636a3ccc5f79d1da9c08ed7f7741dbfd0a8d0ddf424bd57b0b2fb0" + algorithm: "SHA-512" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "" + url: "" + revision: "" + path: "" +- id: "NuGet::foobar:1.2.3" + purl: "pkg:nuget/foobar@1.2.3" + declared_licenses: [] + declared_licenses_processed: {} + description: "" + homepage_url: "" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "" + hash: + value: "" + algorithm: "" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "" + url: "" + revision: "" + path: "" +- id: "NuGet:Microsoft.NETCore:Platforms:1.1.0" purl: "pkg:nuget/Microsoft.NETCore.Platforms@1.1.0" authors: - "Microsoft" @@ -109,7 +167,7 @@ packages: url: "" revision: "" path: "" -- id: "NuGet::Microsoft.NETCore.Targets:1.1.0" +- id: "NuGet:Microsoft.NETCore:Targets:1.1.0" purl: "pkg:nuget/Microsoft.NETCore.Targets@1.1.0" authors: - "Microsoft" @@ -143,7 +201,7 @@ packages: url: "" revision: "" path: "" -- id: "NuGet::Newtonsoft.Json:5.0.4" +- id: "NuGet:Newtonsoft:Json:5.0.4" purl: "pkg:nuget/Newtonsoft.Json@5.0.4" authors: - "James Newton-King" @@ -175,7 +233,7 @@ packages: url: "" revision: "" path: "" -- id: "NuGet::System.Globalization:4.3.0" +- id: "NuGet:System:Globalization:4.3.0" purl: "pkg:nuget/System.Globalization@4.3.0" authors: - "Microsoft" @@ -209,7 +267,7 @@ packages: url: "" revision: "" path: "" -- id: "NuGet::System.Runtime:4.3.0" +- id: "NuGet:System:Runtime:4.3.0" purl: "pkg:nuget/System.Runtime@4.3.0" authors: - "Microsoft" @@ -244,23 +302,23 @@ packages: url: "" revision: "" path: "" -- id: "NuGet::System.Runtime.CompilerServices.Unsafe:4.5.3" - purl: "pkg:nuget/System.Runtime.CompilerServices.Unsafe@4.5.3" +- id: "NuGet:System:Threading:4.0.11" + purl: "pkg:nuget/System.Threading@4.0.11" authors: - "Microsoft" declared_licenses: - - "https://github.com/dotnet/corefx/blob/master/LICENSE.TXT" + - "http://go.microsoft.com/fwlink/?LinkId=329770" declared_licenses_processed: - spdx_expression: "MIT" + spdx_expression: "LicenseRef-scancode-ms-net-library-2019-06" mapped: - https://github.com/dotnet/corefx/blob/master/LICENSE.TXT: "MIT" - description: "Provides the System.Runtime.CompilerServices.Unsafe class, which provides\ - \ generic, low-level functionality for manipulating pointers." + http://go.microsoft.com/fwlink/?LinkId=329770: "LicenseRef-scancode-ms-net-library-2019-06" + description: "Provides the fundamental synchronization primitives, including System.Threading.Monitor\ + \ and System.Threading.Mutex, that are required when writing asynchronous code." homepage_url: "https://dot.net/" binary_artifact: - url: "https://api.nuget.org/v3-flatcontainer/system.runtime.compilerservices.unsafe/4.5.3/system.runtime.compilerservices.unsafe.4.5.3.nupkg" + url: "https://api.nuget.org/v3-flatcontainer/system.threading/4.0.11/system.threading.4.0.11.nupkg" hash: - value: "765d87d36a7b7415dee5b6cbd3a08ead9762915fbfacfad8a205a78d4a187cec6677da2407f7f7c2d1b55fe9f8c0257925c9b0bc193d402972c323979678baab" + value: "05c0dd1bbcfcedb6fc6c5f311c41920a4775f8a28a61ca246b6c65ad8afd9b04881d3357880af000ac056fd121fc5c3ec0b56d6fd607e0c27e7a639157c85e3e" algorithm: "SHA-512" source_artifact: url: "" @@ -277,23 +335,23 @@ packages: url: "" revision: "" path: "" -- id: "NuGet::System.Threading:4.0.11" - purl: "pkg:nuget/System.Threading@4.0.11" +- id: "NuGet:System.Runtime:CompilerServices.Unsafe:4.5.3" + purl: "pkg:nuget/System.Runtime.CompilerServices.Unsafe@4.5.3" authors: - "Microsoft" declared_licenses: - - "http://go.microsoft.com/fwlink/?LinkId=329770" + - "https://github.com/dotnet/corefx/blob/master/LICENSE.TXT" declared_licenses_processed: - spdx_expression: "LicenseRef-scancode-ms-net-library-2019-06" + spdx_expression: "MIT" mapped: - http://go.microsoft.com/fwlink/?LinkId=329770: "LicenseRef-scancode-ms-net-library-2019-06" - description: "Provides the fundamental synchronization primitives, including System.Threading.Monitor\ - \ and System.Threading.Mutex, that are required when writing asynchronous code." + https://github.com/dotnet/corefx/blob/master/LICENSE.TXT: "MIT" + description: "Provides the System.Runtime.CompilerServices.Unsafe class, which provides\ + \ generic, low-level functionality for manipulating pointers." homepage_url: "https://dot.net/" binary_artifact: - url: "https://api.nuget.org/v3-flatcontainer/system.threading/4.0.11/system.threading.4.0.11.nupkg" + url: "https://api.nuget.org/v3-flatcontainer/system.runtime.compilerservices.unsafe/4.5.3/system.runtime.compilerservices.unsafe.4.5.3.nupkg" hash: - value: "05c0dd1bbcfcedb6fc6c5f311c41920a4775f8a28a61ca246b6c65ad8afd9b04881d3357880af000ac056fd121fc5c3ec0b56d6fd607e0c27e7a639157c85e3e" + value: "765d87d36a7b7415dee5b6cbd3a08ead9762915fbfacfad8a205a78d4a187cec6677da2407f7f7c2d1b55fe9f8c0257925c9b0bc193d402972c323979678baab" algorithm: "SHA-512" source_artifact: url: "" @@ -310,7 +368,7 @@ packages: url: "" revision: "" path: "" -- id: "NuGet::System.Threading.Tasks:4.0.11" +- id: "NuGet:System.Threading:Tasks:4.0.11" purl: "pkg:nuget/System.Threading.Tasks@4.0.11" authors: - "Microsoft" @@ -343,7 +401,7 @@ packages: url: "" revision: "" path: "" -- id: "NuGet::System.Threading.Tasks.Extensions:4.5.4" +- id: "NuGet:System.Threading:Tasks.Extensions:4.5.4" purl: "pkg:nuget/System.Threading.Tasks.Extensions@4.5.4" authors: - "Microsoft" @@ -376,61 +434,3 @@ packages: url: "" revision: "" path: "" -- id: "NuGet::WebGrease:1.5.2" - purl: "pkg:nuget/WebGrease@1.5.2" - authors: - - "webgrease@microsoft.com" - declared_licenses: - - "http://www.microsoft.com/web/webpi/eula/msn_webgrease_eula.htm" - declared_licenses_processed: - unmapped: - - "http://www.microsoft.com/web/webpi/eula/msn_webgrease_eula.htm" - description: "Web Grease is a suite of tools for optimizing javascript, css files\ - \ and images." - homepage_url: "" - binary_artifact: - url: "https://api.nuget.org/v3-flatcontainer/webgrease/1.5.2/webgrease.1.5.2.nupkg" - hash: - value: "453fa7a3483b424dbc27f5d5c07827e6527faa6e21c96169ef0b7423bd54c348f749f4df3a636a3ccc5f79d1da9c08ed7f7741dbfd0a8d0ddf424bd57b0b2fb0" - algorithm: "SHA-512" - source_artifact: - url: "" - hash: - value: "" - algorithm: "" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "" - url: "" - revision: "" - path: "" -- id: "NuGet::foobar:1.2.3" - purl: "pkg:nuget/foobar@1.2.3" - declared_licenses: [] - declared_licenses_processed: {} - description: "" - homepage_url: "" - binary_artifact: - url: "" - hash: - value: "" - algorithm: "" - source_artifact: - url: "" - hash: - value: "" - algorithm: "" - vcs: - type: "" - url: "" - revision: "" - path: "" - vcs_processed: - type: "" - url: "" - revision: "" - path: "" diff --git a/plugins/package-managers/nuget/src/main/kotlin/utils/NuGetInspector.kt b/plugins/package-managers/nuget/src/main/kotlin/utils/NuGetInspector.kt index 8570649a8cdad..ba0672a5362a7 100644 --- a/plugins/package-managers/nuget/src/main/kotlin/utils/NuGetInspector.kt +++ b/plugins/package-managers/nuget/src/main/kotlin/utils/NuGetInspector.kt @@ -26,24 +26,8 @@ import kotlinx.serialization.json.Json import kotlinx.serialization.json.JsonNamingStrategy import kotlinx.serialization.json.decodeFromStream -import org.ossreviewtoolkit.analyzer.PackageManager -import org.ossreviewtoolkit.downloader.VersionControlSystem -import org.ossreviewtoolkit.model.Hash -import org.ossreviewtoolkit.model.Identifier -import org.ossreviewtoolkit.model.Issue -import org.ossreviewtoolkit.model.Package -import org.ossreviewtoolkit.model.PackageReference -import org.ossreviewtoolkit.model.Project -import org.ossreviewtoolkit.model.RemoteArtifact -import org.ossreviewtoolkit.model.Scope -import org.ossreviewtoolkit.model.Severity -import org.ossreviewtoolkit.model.VcsInfo -import org.ossreviewtoolkit.model.VcsType -import org.ossreviewtoolkit.model.fromYaml -import org.ossreviewtoolkit.model.utils.toPurl import org.ossreviewtoolkit.utils.common.CommandLineTool import org.ossreviewtoolkit.utils.common.safeDeleteRecursively -import org.ossreviewtoolkit.utils.ort.DeclaredLicenseProcessor import org.ossreviewtoolkit.utils.ort.createOrtTempFile private val json = Json { @@ -141,120 +125,3 @@ internal object NuGetInspector : CommandLineTool { val url: String? ) } - -private const val TYPE = "NuGet" - -private fun List.toAuthors(): Set = - filter { it.role == "author" }.mapNotNullTo(mutableSetOf()) { party -> - buildString { - party.name?.let { append(it) } - party.email?.let { - append(if (party.name != null && it.isNotBlank()) " <$it>" else it) - } - }.takeIf { it.isNotBlank() } - } - -internal fun NuGetInspector.Result.toOrtProject( - managerName: String, - analysisRoot: File, - definitionFile: File -): Project { - val id = Identifier( - type = managerName, - namespace = "", - name = definitionFile.relativeTo(analysisRoot).invariantSeparatorsPath, - version = "" - ) - - val nestedPackages = mutableListOf() - - packages.forEach { pkg -> - pkg.dependencies.forEach { dep -> nestedPackages += dep } - } - - val packageReferences = nestedPackages.toPackageReferences() - val scopes = setOf(Scope(headers.first().projectFramework, packageReferences)) - - return Project( - id = id, - definitionFilePath = VersionControlSystem.getPathInfo(definitionFile).path, - vcs = VcsInfo.EMPTY, - authors = emptySet(), - vcsProcessed = PackageManager.processProjectVcs(definitionFile.parentFile), - declaredLicenses = emptySet(), - homepageUrl = "", - scopeDependencies = scopes - ) -} - -private fun List.toPackageReferences(): Set = - mapTo(mutableSetOf()) { data -> - PackageReference( - id = Identifier(type = TYPE, namespace = "", name = data.name, version = data.version.orEmpty()), - dependencies = data.dependencies.toPackageReferences(), - issues = data.errors.map { Issue(source = TYPE, message = it, severity = Severity.ERROR) } - + data.warnings.map { Issue(source = TYPE, message = it, severity = Severity.WARNING) } - ) - } - -internal fun Collection.toOrtPackages(): Set = - groupBy { "${it.name}:${it.version}" }.mapTo(mutableSetOf()) { (_, packages) -> - val pkg = packages.first() - - fun NuGetInspector.PackageData.getHash(): Hash = - Hash.create( - (sha512 ?: sha256 ?: sha1 ?: md5 ?: "").lowercase() - ) - - val id = Identifier( - type = TYPE, - namespace = pkg.namespace.orEmpty(), - name = pkg.name, - version = pkg.version.orEmpty() - ) - - val declaredLicenses = mutableSetOf() - val pkgDeclaredLicense = pkg.declaredLicense.orEmpty() - - if (pkgDeclaredLicense.isNotBlank()) { - val licenseData = pkgDeclaredLicense.fromYaml>() - val type = licenseData["LicenseType"].orEmpty() - - listOfNotNull( - licenseData["LicenseExpression"]?.takeIf { type.equals("expression", ignoreCase = true) }, - licenseData["LicenseUrl"] - ).firstOrNull()?.also { declaredLicenses += it } - } - - var vcsUrl = pkg.vcsUrl.orEmpty() - var commit = "" - - val segments = vcsUrl.split("@", limit = 2) - if (segments.size == 2) { - vcsUrl = segments[0] - commit = segments[1] - } - - Package( - id = id, - purl = pkg.purl.takeUnless { it.isEmpty() } ?: id.toPurl(), - authors = pkg.parties.toAuthors(), - declaredLicenses = declaredLicenses, - declaredLicensesProcessed = DeclaredLicenseProcessor.process(declaredLicenses), - description = pkg.description.lineSequence().firstOrNull { it.isNotBlank() }.orEmpty(), - homepageUrl = pkg.homepageUrl.orEmpty(), - binaryArtifact = RemoteArtifact( - url = pkg.downloadUrl, - hash = pkg.getHash() - ), - sourceArtifact = RemoteArtifact.EMPTY, - vcs = VcsInfo.EMPTY.copy(url = vcsUrl, revision = commit), - vcsProcessed = PackageManager.processPackageVcs( - VcsInfo(type = VcsType.UNKNOWN, url = vcsUrl, revision = commit), - fallbackUrls = listOfNotNull( - pkg.codeViewUrl, - pkg.homepageUrl - ).toTypedArray() - ) - ) - } diff --git a/plugins/package-managers/nuget/src/main/kotlin/utils/NuGetInspectorExtensions.kt b/plugins/package-managers/nuget/src/main/kotlin/utils/NuGetInspectorExtensions.kt new file mode 100644 index 0000000000000..8f5d3ef1c1450 --- /dev/null +++ b/plugins/package-managers/nuget/src/main/kotlin/utils/NuGetInspectorExtensions.kt @@ -0,0 +1,158 @@ +/* + * Copyright (C) 2023 The ORT Project Authors (see ) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.ossreviewtoolkit.plugins.packagemanagers.nuget.utils + +import java.io.File + +import org.ossreviewtoolkit.analyzer.PackageManager +import org.ossreviewtoolkit.downloader.VersionControlSystem +import org.ossreviewtoolkit.model.Hash +import org.ossreviewtoolkit.model.Identifier +import org.ossreviewtoolkit.model.Issue +import org.ossreviewtoolkit.model.Package +import org.ossreviewtoolkit.model.PackageReference +import org.ossreviewtoolkit.model.Project +import org.ossreviewtoolkit.model.RemoteArtifact +import org.ossreviewtoolkit.model.Scope +import org.ossreviewtoolkit.model.Severity +import org.ossreviewtoolkit.model.VcsInfo +import org.ossreviewtoolkit.model.VcsType +import org.ossreviewtoolkit.model.fromYaml +import org.ossreviewtoolkit.model.utils.toPurl +import org.ossreviewtoolkit.utils.ort.DeclaredLicenseProcessor + +private const val TYPE = "NuGet" + +private fun List.toAuthors(): Set = + filter { it.role == "author" }.mapNotNullTo(mutableSetOf()) { party -> + buildString { + party.name?.let { append(it) } + party.email?.let { + append(if (party.name != null && it.isNotBlank()) " <$it>" else it) + } + }.takeIf { it.isNotBlank() } + } + +internal fun NuGetInspector.Result.toOrtProject( + managerName: String, + analysisRoot: File, + definitionFile: File +): Project { + val id = Identifier( + type = managerName, + namespace = "", + name = definitionFile.relativeTo(analysisRoot).invariantSeparatorsPath, + version = "" + ) + + val nestedPackages = mutableListOf() + + packages.forEach { pkg -> + pkg.dependencies.forEach { dep -> nestedPackages += dep } + } + + val packageReferences = nestedPackages.toPackageReferences() + val scopes = setOf(Scope(headers.first().projectFramework, packageReferences)) + + return Project( + id = id, + definitionFilePath = VersionControlSystem.getPathInfo(definitionFile).path, + vcs = VcsInfo.EMPTY, + authors = emptySet(), + vcsProcessed = PackageManager.processProjectVcs(definitionFile.parentFile), + declaredLicenses = emptySet(), + homepageUrl = "", + scopeDependencies = scopes + ) +} + +private fun NuGetInspector.PackageData.getIdentifierWithNamespace(): Identifier = + if (namespace.isNullOrEmpty()) { + getIdentifierWithNamespace(TYPE, name, version.orEmpty()) + } else { + Identifier(TYPE, namespace, name, version.orEmpty()) + } + +private fun List.toPackageReferences(): Set = + mapTo(mutableSetOf()) { data -> + PackageReference( + id = data.getIdentifierWithNamespace(), + dependencies = data.dependencies.toPackageReferences(), + issues = data.errors.map { Issue(source = TYPE, message = it, severity = Severity.ERROR) } + + data.warnings.map { Issue(source = TYPE, message = it, severity = Severity.WARNING) } + ) + } + +internal fun Collection.toOrtPackages(): Set = + groupBy { "${it.name}:${it.version}" }.mapTo(mutableSetOf()) { (_, packages) -> + val pkg = packages.first() + + fun NuGetInspector.PackageData.getHash(): Hash = + Hash.create( + (sha512 ?: sha256 ?: sha1 ?: md5 ?: "").lowercase() + ) + + val id = pkg.getIdentifierWithNamespace() + + val declaredLicenses = mutableSetOf() + val pkgDeclaredLicense = pkg.declaredLicense.orEmpty() + + if (pkgDeclaredLicense.isNotBlank()) { + val licenseData = pkgDeclaredLicense.fromYaml>() + val type = licenseData["LicenseType"].orEmpty() + + listOfNotNull( + licenseData["LicenseExpression"]?.takeIf { type.equals("expression", ignoreCase = true) }, + licenseData["LicenseUrl"] + ).firstOrNull()?.also { declaredLicenses += it } + } + + var vcsUrl = pkg.vcsUrl.orEmpty() + var commit = "" + + val segments = vcsUrl.split("@", limit = 2) + if (segments.size == 2) { + vcsUrl = segments[0] + commit = segments[1] + } + + Package( + id = id, + purl = pkg.purl.takeUnless { it.isEmpty() } ?: id.toPurl(), + authors = pkg.parties.toAuthors(), + declaredLicenses = declaredLicenses, + declaredLicensesProcessed = DeclaredLicenseProcessor.process(declaredLicenses), + description = pkg.description.lineSequence().firstOrNull { it.isNotBlank() }.orEmpty(), + homepageUrl = pkg.homepageUrl.orEmpty(), + binaryArtifact = RemoteArtifact( + url = pkg.downloadUrl, + hash = pkg.getHash() + ), + sourceArtifact = RemoteArtifact.EMPTY, + vcs = VcsInfo.EMPTY.copy(url = vcsUrl, revision = commit), + vcsProcessed = PackageManager.processPackageVcs( + VcsInfo(type = VcsType.UNKNOWN, url = vcsUrl, revision = commit), + fallbackUrls = listOfNotNull( + pkg.codeViewUrl, + pkg.homepageUrl + ).toTypedArray() + ) + ) + } diff --git a/plugins/package-managers/nuget/src/main/kotlin/utils/NuGetUtils.kt b/plugins/package-managers/nuget/src/main/kotlin/utils/NuGetUtils.kt new file mode 100644 index 0000000000000..5d99d5966b24d --- /dev/null +++ b/plugins/package-managers/nuget/src/main/kotlin/utils/NuGetUtils.kt @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2023 The ORT Project Authors (see ) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.ossreviewtoolkit.plugins.packagemanagers.nuget.utils + +import org.ossreviewtoolkit.model.Identifier + +/** + * Deduce an [Identifier] that has a namespace from the conventional NuGet package [name], using the given [type] and + * [version] as-is. This function can be useful for third-party tools that use ORT as a library to be able to reproduce + * a NuGet package [Identifier], e.g. to use it as part of curations. + */ +fun getIdentifierWithNamespace(type: String, name: String, version: String): Identifier { + val namespace = name.split('.', limit = 3).toMutableList() + val nameWithoutNamespace = namespace.removeLast() + val namespaceWithoutName = namespace.joinToString(".") + return Identifier(type, namespaceWithoutName, nameWithoutNamespace, version) +} diff --git a/plugins/package-managers/nuget/src/test/kotlin/utils/NuGetUtilsTest.kt b/plugins/package-managers/nuget/src/test/kotlin/utils/NuGetUtilsTest.kt new file mode 100644 index 0000000000000..f6d133a5af786 --- /dev/null +++ b/plugins/package-managers/nuget/src/test/kotlin/utils/NuGetUtilsTest.kt @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2020 The ORT Project Authors (see ) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.ossreviewtoolkit.plugins.packagemanagers.nuget.utils + +import io.kotest.assertions.assertSoftly +import io.kotest.core.spec.style.WordSpec +import io.kotest.matchers.shouldBe + +import org.ossreviewtoolkit.model.Identifier + +class NuGetUtilsTest : WordSpec({ + "getIdentifierWithNamespace()" should { + "split the namespace from the name" { + assertSoftly { + getIdentifierWithNamespace("NuGet", "SharpCompress", "0.23.0") shouldBe + Identifier("NuGet::SharpCompress:0.23.0") + + getIdentifierWithNamespace("NuGet", "System.IO", "4.1.0") shouldBe + Identifier("NuGet:System:IO:4.1.0") + getIdentifierWithNamespace("NuGet", "System.IO.Compression", "4.3.0") shouldBe + Identifier("NuGet:System.IO:Compression:4.3.0") + getIdentifierWithNamespace("NuGet", "System.IO.Compression.ZipFile", "4.0.1") shouldBe + Identifier("NuGet:System.IO:Compression.ZipFile:4.0.1") + + getIdentifierWithNamespace( + "NuGet", + "Microsoft.Extensions.Diagnostics.HealthChecks.EntityFrameworkCore", + "2.2.1" + ) shouldBe Identifier( + "NuGet:Microsoft.Extensions:Diagnostics.HealthChecks.EntityFrameworkCore:2.2.1" + ) + + getIdentifierWithNamespace( + "NuGet", + "Microsoft.AspNetCore.Components.WebAssembly.Build.BrotliCompression", + "3.1.6" + ) shouldBe Identifier( + "NuGet:Microsoft.AspNetCore:Components.WebAssembly.Build.BrotliCompression:3.1.6" + ) + } + } + } +}) diff --git a/plugins/package-managers/python/src/main/kotlin/utils/PythonInspector.kt b/plugins/package-managers/python/src/main/kotlin/utils/PythonInspector.kt index bbfa328b807dc..0e088ffd2fd94 100644 --- a/plugins/package-managers/python/src/main/kotlin/utils/PythonInspector.kt +++ b/plugins/package-managers/python/src/main/kotlin/utils/PythonInspector.kt @@ -29,29 +29,12 @@ import kotlinx.serialization.json.decodeFromStream import org.apache.logging.log4j.kotlin.Logging -import org.ossreviewtoolkit.analyzer.PackageManager -import org.ossreviewtoolkit.downloader.VersionControlSystem -import org.ossreviewtoolkit.model.Hash -import org.ossreviewtoolkit.model.Identifier -import org.ossreviewtoolkit.model.Package -import org.ossreviewtoolkit.model.PackageReference -import org.ossreviewtoolkit.model.Project -import org.ossreviewtoolkit.model.RemoteArtifact -import org.ossreviewtoolkit.model.Scope -import org.ossreviewtoolkit.model.VcsInfo -import org.ossreviewtoolkit.model.VcsType import org.ossreviewtoolkit.utils.common.CommandLineTool -import org.ossreviewtoolkit.utils.ort.DeclaredLicenseProcessor -import org.ossreviewtoolkit.utils.ort.ProcessedDeclaredLicense import org.ossreviewtoolkit.utils.ort.createOrtTempFile -import org.ossreviewtoolkit.utils.spdx.SpdxLicenseIdExpression import org.semver4j.RangesList import org.semver4j.RangesListFactory -private const val GENERIC_BSD_LICENSE = "BSD License" -private const val SHORT_STRING_MAX_CHARS = 200 - private val json = Json { ignoreUnknownKeys = true namingStrategy = JsonNamingStrategy.SnakeCase @@ -181,192 +164,3 @@ internal object PythonInspector : CommandLineTool, Logging { val url: String? ) } - -private const val TYPE = "PyPI" - -internal fun PythonInspector.Result.toOrtProject( - managerName: String, - analysisRoot: File, - definitionFile: File -): Project { - val id = resolveIdentifier(managerName, analysisRoot, definitionFile) - - val setupProject = projects.find { it.path.endsWith("/setup.py") } - val projectData = setupProject?.packageData?.singleOrNull() - val homepageUrl = projectData?.homepageUrl.orEmpty() - - val scopes = setOf(Scope("install", resolvedDependenciesGraph.toPackageReferences())) - - return Project( - id = id, - definitionFilePath = VersionControlSystem.getPathInfo(definitionFile).path, - authors = projectData?.parties?.toAuthors() ?: emptySet(), - declaredLicenses = projectData?.declaredLicense?.getDeclaredLicenses() ?: emptySet(), - vcs = VcsInfo.EMPTY, - vcsProcessed = PackageManager.processProjectVcs(definitionFile.parentFile, VcsInfo.EMPTY, homepageUrl), - homepageUrl = homepageUrl, - scopeDependencies = scopes - ) -} - -private fun PythonInspector.Result.resolveIdentifier( - managerName: String, - analysisRoot: File, - definitionFile: File -): Identifier { - // First try to get identifier components from "setup.py" in any case, even for "requirements.txt" projects. - val (setupName, setupVersion) = projects.find { it.path.endsWith("/setup.py") } - ?.let { project -> - listOf( - project.packageData.single().name.orEmpty(), - project.packageData.single().version.orEmpty() - ) - } ?: listOf("", "") - - val (requirementsName, requirementsVersion) = projects.find { !it.path.endsWith("/setup.py") } - ?.let { - // In case of "requirements*.txt" there is no metadata at all available, so use the parent directory name - // plus what "*" expands to as the project name and the VCS revision, if any, as the project version. - val suffix = definitionFile.name.removePrefix("requirements").removeSuffix(".txt") - val name = if (definitionFile.parentFile != analysisRoot) { - definitionFile.parentFile.name + suffix - } else { - PackageManager.getFallbackProjectName(analysisRoot, definitionFile) - } - - val version = VersionControlSystem.getCloneInfo(definitionFile.parentFile).revision - - listOf(name, version) - } ?: listOf("", "") - - val hasSetupName = setupName.isNotEmpty() - val hasRequirementsName = requirementsName.isNotEmpty() - - val projectName = when { - hasSetupName && !hasRequirementsName -> setupName - !hasSetupName && hasRequirementsName -> requirementsName - hasSetupName && hasRequirementsName -> "$setupName-with-requirements-$requirementsName" - else -> PackageManager.getFallbackProjectName(analysisRoot, definitionFile) - } - - val projectVersion = setupVersion.takeIf { it.isNotEmpty() } ?: requirementsVersion - - return Identifier( - type = managerName, - namespace = "", - name = projectName, - version = projectVersion - ) -} - -private fun PythonInspector.DeclaredLicense.getDeclaredLicenses() = - buildSet { - getLicenseFromLicenseField(license)?.let { add(it) } - addAll(classifiers.mapNotNull { getLicenseFromClassifier(it) }) - } - -private fun processDeclaredLicenses(id: Identifier, declaredLicenses: Set): ProcessedDeclaredLicense { - var declaredLicensesProcessed = DeclaredLicenseProcessor.process(declaredLicenses) - - // Python's classifiers only support a coarse license declaration of "BSD License". So if there is another - // more specific declaration of a BSD license, align on that one. - if (GENERIC_BSD_LICENSE in declaredLicensesProcessed.unmapped) { - declaredLicensesProcessed.spdxExpression?.decompose()?.singleOrNull { - it is SpdxLicenseIdExpression && it.isValid() && it.toString().startsWith("BSD-") - }?.let { license -> - PythonInspector.logger.debug { "Mapping '$GENERIC_BSD_LICENSE' to '$license' for '${id.toCoordinates()}'." } - - declaredLicensesProcessed = declaredLicensesProcessed.copy( - mapped = declaredLicensesProcessed.mapped + mapOf(GENERIC_BSD_LICENSE to license), - unmapped = declaredLicensesProcessed.unmapped - GENERIC_BSD_LICENSE - ) - } - } - - return declaredLicensesProcessed -} - -internal fun List.toOrtPackages(): Set = - groupBy { "${it.name}:${it.version}" }.mapTo(mutableSetOf()) { (_, packages) -> - // The python inspector currently often contains two entries for a package where the only difference is the - // download URL. In this case, one package contains the URL of the binary artifact, the other for the source - // artifact. So take all metadata from the first package except for the artifacts. - val pkg = packages.first() - - fun PythonInspector.Package.getHash(): Hash = Hash.create(sha512 ?: sha256 ?: sha1 ?: md5 ?: "") - - fun getArtifact(fileExtension: String) = - packages.find { it.downloadUrl.endsWith(fileExtension) }?.let { - RemoteArtifact( - url = it.downloadUrl, - hash = it.getHash() - ) - } ?: RemoteArtifact.EMPTY - - val id = Identifier(type = TYPE, namespace = "", name = pkg.name, version = pkg.version) - val declaredLicenses = pkg.declaredLicense?.getDeclaredLicenses() ?: emptySet() - val declaredLicensesProcessed = processDeclaredLicenses(id, declaredLicenses) - - Package( - // The package has a namespace property which is currently always empty. Deliberately set the namespace to - // an empty string here to be consistent with the resolved packages which do not have a namespace property. - id = id, - purl = pkg.purl, - authors = pkg.parties.toAuthors(), - declaredLicenses = declaredLicenses, - declaredLicensesProcessed = declaredLicensesProcessed, - // Only use the first line of the description because the descriptions provided by python-inspector are - // currently far too long, see: https://github.com/nexB/python-inspector/issues/74 - description = pkg.description.lineSequence().firstOrNull { it.isNotBlank() }.orEmpty(), - homepageUrl = pkg.homepageUrl.orEmpty(), - binaryArtifact = getArtifact(".whl"), - sourceArtifact = getArtifact(".tar.gz"), - vcs = VcsInfo.EMPTY.copy(url = pkg.vcsUrl.orEmpty()), - vcsProcessed = PackageManager.processPackageVcs( - VcsInfo(VcsType.UNKNOWN, pkg.vcsUrl.orEmpty(), revision = ""), - fallbackUrls = listOfNotNull( - pkg.codeViewUrl, - pkg.homepageUrl - ).toTypedArray() - ) - ) - } - -private fun List.toAuthors(): Set = - filter { it.role == "author" }.mapNotNullTo(mutableSetOf()) { party -> - buildString { - party.name?.let { append(it) } - party.email?.let { - append(if (party.name != null) " <$it>" else it) - } - }.takeIf { it.isNotBlank() } - } - -private fun List.toPackageReferences(): Set = - mapTo(mutableSetOf()) { it.toPackageReference() } - -private fun PythonInspector.ResolvedDependency.toPackageReference() = - PackageReference( - id = Identifier(type = TYPE, namespace = "", name = packageName, version = installedVersion), - dependencies = dependencies.toPackageReferences() - ) - -private fun getLicenseFromClassifier(classifier: String): String? { - // Example license classifier (also see https://pypi.org/classifiers/): - // "License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)" - val classifiers = classifier.split(" :: ").map { it.trim() } - val licenseClassifiers = listOf("License", "OSI Approved") - val license = classifiers.takeIf { it.first() in licenseClassifiers }?.last() - return license?.takeUnless { it in licenseClassifiers } -} - -private fun getLicenseFromLicenseField(value: String?): String? { - if (value.isNullOrBlank() || value == "UNKNOWN") return null - - // See https://docs.python.org/3/distutils/setupscript.html#additional-meta-data for what a "short string" is. - val isShortString = value.length <= SHORT_STRING_MAX_CHARS && value.lines().size == 1 - if (!isShortString) return null - - // Apply a work-around for projects that declare licenses in classifier-syntax in the license field. - return getLicenseFromClassifier(value) ?: value -} diff --git a/plugins/package-managers/python/src/main/kotlin/utils/PythonInspectorExtensions.kt b/plugins/package-managers/python/src/main/kotlin/utils/PythonInspectorExtensions.kt new file mode 100644 index 0000000000000..2f1f62fa1e6b1 --- /dev/null +++ b/plugins/package-managers/python/src/main/kotlin/utils/PythonInspectorExtensions.kt @@ -0,0 +1,182 @@ +/* + * Copyright (C) 2022 The ORT Project Authors (see ) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.ossreviewtoolkit.plugins.packagemanagers.python.utils + +import java.io.File + +import org.ossreviewtoolkit.analyzer.PackageManager +import org.ossreviewtoolkit.downloader.VersionControlSystem +import org.ossreviewtoolkit.model.Hash +import org.ossreviewtoolkit.model.Identifier +import org.ossreviewtoolkit.model.Package +import org.ossreviewtoolkit.model.PackageReference +import org.ossreviewtoolkit.model.Project +import org.ossreviewtoolkit.model.RemoteArtifact +import org.ossreviewtoolkit.model.Scope +import org.ossreviewtoolkit.model.VcsInfo +import org.ossreviewtoolkit.model.VcsType + +private const val TYPE = "PyPI" + +internal fun PythonInspector.Result.toOrtProject( + managerName: String, + analysisRoot: File, + definitionFile: File +): Project { + val id = resolveIdentifier(managerName, analysisRoot, definitionFile) + + val setupProject = projects.find { it.path.endsWith("/setup.py") } + val projectData = setupProject?.packageData?.singleOrNull() + val homepageUrl = projectData?.homepageUrl.orEmpty() + + val scopes = setOf(Scope("install", resolvedDependenciesGraph.toPackageReferences())) + + return Project( + id = id, + definitionFilePath = VersionControlSystem.getPathInfo(definitionFile).path, + authors = projectData?.parties?.toAuthors() ?: emptySet(), + declaredLicenses = projectData?.declaredLicense?.getDeclaredLicenses() ?: emptySet(), + vcs = VcsInfo.EMPTY, + vcsProcessed = PackageManager.processProjectVcs(definitionFile.parentFile, VcsInfo.EMPTY, homepageUrl), + homepageUrl = homepageUrl, + scopeDependencies = scopes + ) +} + +private fun PythonInspector.Result.resolveIdentifier( + managerName: String, + analysisRoot: File, + definitionFile: File +): Identifier { + // First try to get identifier components from "setup.py" in any case, even for "requirements.txt" projects. + val (setupName, setupVersion) = projects.find { it.path.endsWith("/setup.py") } + ?.let { project -> + listOf( + project.packageData.single().name.orEmpty(), + project.packageData.single().version.orEmpty() + ) + } ?: listOf("", "") + + val (requirementsName, requirementsVersion) = projects.find { !it.path.endsWith("/setup.py") } + ?.let { + // In case of "requirements*.txt" there is no metadata at all available, so use the parent directory name + // plus what "*" expands to as the project name and the VCS revision, if any, as the project version. + val suffix = definitionFile.name.removePrefix("requirements").removeSuffix(".txt") + val name = if (definitionFile.parentFile != analysisRoot) { + definitionFile.parentFile.name + suffix + } else { + PackageManager.getFallbackProjectName(analysisRoot, definitionFile) + } + + val version = VersionControlSystem.getCloneInfo(definitionFile.parentFile).revision + + listOf(name, version) + } ?: listOf("", "") + + val hasSetupName = setupName.isNotEmpty() + val hasRequirementsName = requirementsName.isNotEmpty() + + val projectName = when { + hasSetupName && !hasRequirementsName -> setupName + !hasSetupName && hasRequirementsName -> requirementsName + hasSetupName && hasRequirementsName -> "$setupName-with-requirements-$requirementsName" + else -> PackageManager.getFallbackProjectName(analysisRoot, definitionFile) + } + + val projectVersion = setupVersion.takeIf { it.isNotEmpty() } ?: requirementsVersion + + return Identifier( + type = managerName, + namespace = "", + name = projectName, + version = projectVersion + ) +} + +private fun PythonInspector.DeclaredLicense.getDeclaredLicenses() = + buildSet { + getLicenseFromLicenseField(license)?.let { add(it) } + addAll(classifiers.mapNotNull { getLicenseFromClassifier(it) }) + } + +internal fun List.toOrtPackages(): Set = + groupBy { "${it.name}:${it.version}" }.mapTo(mutableSetOf()) { (_, packages) -> + // The python inspector currently often contains two entries for a package where the only difference is the + // download URL. In this case, one package contains the URL of the binary artifact, the other for the source + // artifact. So take all metadata from the first package except for the artifacts. + val pkg = packages.first() + + fun PythonInspector.Package.getHash(): Hash = Hash.create(sha512 ?: sha256 ?: sha1 ?: md5 ?: "") + + fun getArtifact(fileExtension: String) = + packages.find { it.downloadUrl.endsWith(fileExtension) }?.let { + RemoteArtifact( + url = it.downloadUrl, + hash = it.getHash() + ) + } ?: RemoteArtifact.EMPTY + + val id = Identifier(type = TYPE, namespace = "", name = pkg.name, version = pkg.version) + val declaredLicenses = pkg.declaredLicense?.getDeclaredLicenses() ?: emptySet() + val declaredLicensesProcessed = processDeclaredLicenses(id, declaredLicenses) + + Package( + // The package has a namespace property which is currently always empty. Deliberately set the namespace to + // an empty string here to be consistent with the resolved packages which do not have a namespace property. + id = id, + purl = pkg.purl, + authors = pkg.parties.toAuthors(), + declaredLicenses = declaredLicenses, + declaredLicensesProcessed = declaredLicensesProcessed, + // Only use the first line of the description because the descriptions provided by python-inspector are + // currently far too long, see: https://github.com/nexB/python-inspector/issues/74 + description = pkg.description.lineSequence().firstOrNull { it.isNotBlank() }.orEmpty(), + homepageUrl = pkg.homepageUrl.orEmpty(), + binaryArtifact = getArtifact(".whl"), + sourceArtifact = getArtifact(".tar.gz"), + vcs = VcsInfo.EMPTY.copy(url = pkg.vcsUrl.orEmpty()), + vcsProcessed = PackageManager.processPackageVcs( + VcsInfo(VcsType.UNKNOWN, pkg.vcsUrl.orEmpty(), revision = ""), + fallbackUrls = listOfNotNull( + pkg.codeViewUrl, + pkg.homepageUrl + ).toTypedArray() + ) + ) + } + +private fun List.toAuthors(): Set = + filter { it.role == "author" }.mapNotNullTo(mutableSetOf()) { party -> + buildString { + party.name?.let { append(it) } + party.email?.let { + append(if (party.name != null) " <$it>" else it) + } + }.takeIf { it.isNotBlank() } + } + +private fun List.toPackageReferences(): Set = + mapTo(mutableSetOf()) { it.toPackageReference() } + +private fun PythonInspector.ResolvedDependency.toPackageReference() = + PackageReference( + id = Identifier(type = TYPE, namespace = "", name = packageName, version = installedVersion), + dependencies = dependencies.toPackageReferences() + ) diff --git a/plugins/package-managers/python/src/main/kotlin/utils/PythonUtils.kt b/plugins/package-managers/python/src/main/kotlin/utils/PythonUtils.kt new file mode 100644 index 0000000000000..82b24fde1a52c --- /dev/null +++ b/plugins/package-managers/python/src/main/kotlin/utils/PythonUtils.kt @@ -0,0 +1,69 @@ +/* + * Copyright (C) 2022 The ORT Project Authors (see ) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.ossreviewtoolkit.plugins.packagemanagers.python.utils + +import org.ossreviewtoolkit.model.Identifier +import org.ossreviewtoolkit.utils.ort.DeclaredLicenseProcessor +import org.ossreviewtoolkit.utils.ort.ProcessedDeclaredLicense +import org.ossreviewtoolkit.utils.spdx.SpdxLicenseIdExpression + +private const val GENERIC_BSD_LICENSE = "BSD License" +private const val SHORT_STRING_MAX_CHARS = 200 + +internal fun getLicenseFromClassifier(classifier: String): String? { + // Example license classifier (also see https://pypi.org/classifiers/): + // "License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)" + val classifiers = classifier.split(" :: ").map { it.trim() } + val licenseClassifiers = listOf("License", "OSI Approved") + val license = classifiers.takeIf { it.first() in licenseClassifiers }?.last() + return license?.takeUnless { it in licenseClassifiers } +} + +internal fun getLicenseFromLicenseField(value: String?): String? { + if (value.isNullOrBlank() || value == "UNKNOWN") return null + + // See https://docs.python.org/3/distutils/setupscript.html#additional-meta-data for what a "short string" is. + val isShortString = value.length <= SHORT_STRING_MAX_CHARS && value.lines().size == 1 + if (!isShortString) return null + + // Apply a work-around for projects that declare licenses in classifier-syntax in the license field. + return getLicenseFromClassifier(value) ?: value +} + +internal fun processDeclaredLicenses(id: Identifier, declaredLicenses: Set): ProcessedDeclaredLicense { + var declaredLicensesProcessed = DeclaredLicenseProcessor.process(declaredLicenses) + + // Python's classifiers only support a coarse license declaration of "BSD License". So if there is another + // more specific declaration of a BSD license, align on that one. + if (GENERIC_BSD_LICENSE in declaredLicensesProcessed.unmapped) { + declaredLicensesProcessed.spdxExpression?.decompose()?.singleOrNull { + it is SpdxLicenseIdExpression && it.isValid() && it.toString().startsWith("BSD-") + }?.let { license -> + PythonInspector.logger.debug { "Mapping '$GENERIC_BSD_LICENSE' to '$license' for '${id.toCoordinates()}'." } + + declaredLicensesProcessed = declaredLicensesProcessed.copy( + mapped = declaredLicensesProcessed.mapped + mapOf(GENERIC_BSD_LICENSE to license), + unmapped = declaredLicensesProcessed.unmapped - GENERIC_BSD_LICENSE + ) + } + } + + return declaredLicensesProcessed +}