From dddc7b31b2bb2588c23efc6b5a43ce5acfab1329 Mon Sep 17 00:00:00 2001 From: Victor Ohashi <38299943+VictorOhashi@users.noreply.github.com> Date: Wed, 10 Jan 2024 12:16:25 -0300 Subject: [PATCH] feat: support authenticating private pub repository (#627) --- packages/melos/lib/src/commands/publish.dart | 7 +- packages/melos/lib/src/common/http.dart | 5 +- .../melos/lib/src/common/pub_credential.dart | 108 ++++++++++++ packages/melos/lib/src/common/pub_hosted.dart | 105 ++++++++++++ .../lib/src/common/pub_hosted_package.dart | 70 ++++++++ packages/melos/lib/src/package.dart | 53 +----- packages/melos/test/package_test.dart | 154 +++++++++++++----- packages/melos/test/utils.dart | 14 +- 8 files changed, 419 insertions(+), 97 deletions(-) create mode 100644 packages/melos/lib/src/common/pub_credential.dart create mode 100644 packages/melos/lib/src/common/pub_hosted.dart create mode 100644 packages/melos/lib/src/common/pub_hosted_package.dart diff --git a/packages/melos/lib/src/commands/publish.dart b/packages/melos/lib/src/commands/publish.dart index 6e405c3c..f538a677 100644 --- a/packages/melos/lib/src/commands/publish.dart +++ b/packages/melos/lib/src/commands/publish.dart @@ -107,9 +107,12 @@ mixin _PublishMixin on _ExecMixin { (package) async { if (package.isPrivate) return; - final versions = await package.getPublishedVersions(); + final pubPackage = await package.getPublishedPackage(); + final versions = pubPackage?.prioritizedVersions + .map((v) => v.version.toString()) + .toList(); - if (versions.isEmpty) { + if (versions == null || versions.isEmpty) { latestPackageVersion[package.name] = null; return; } diff --git a/packages/melos/lib/src/common/http.dart b/packages/melos/lib/src/common/http.dart index bb156878..f39b6469 100644 --- a/packages/melos/lib/src/common/http.dart +++ b/packages/melos/lib/src/common/http.dart @@ -2,7 +2,6 @@ import 'package:http/http.dart' as http; import 'package:meta/meta.dart'; @visibleForTesting -http.Client? testClient; +http.Client internalHttpClient = http.Client(); -Future get(Uri url, {Map? headers}) => - testClient?.get(url, headers: headers) ?? http.get(url, headers: headers); +http.Client get httpClient => internalHttpClient; diff --git a/packages/melos/lib/src/common/pub_credential.dart b/packages/melos/lib/src/common/pub_credential.dart new file mode 100644 index 00000000..8da310a8 --- /dev/null +++ b/packages/melos/lib/src/common/pub_credential.dart @@ -0,0 +1,108 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:cli_util/cli_util.dart'; +import 'package:collection/collection.dart'; +import 'package:meta/meta.dart'; +import 'package:path/path.dart' as path; + +import 'io.dart'; +import 'pub_hosted.dart'; + +const _pubTokenFileName = 'pub-tokens.json'; + +@visibleForTesting +PubCredentialStore internalPubCredentialStore = + PubCredentialStore.fromConfigFile(); + +PubCredentialStore get pubCredentialStore => internalPubCredentialStore; + +class PubCredentialStore { + PubCredentialStore(this.credentials); + + factory PubCredentialStore.fromConfigFile({String? configDir}) { + configDir ??= applicationConfigHome('dart'); + final tokenFilePath = path.join(configDir, _pubTokenFileName); + + if (!fileExists(tokenFilePath)) { + return PubCredentialStore([]); + } + + final content = + jsonDecode(readTextFile(tokenFilePath)) as Map?; + + final hostedCredentials = content?['hosted'] as List? ?? const []; + + final credentials = hostedCredentials + .cast>() + .map(PubCredential.fromJson) + .toList(); + + return PubCredentialStore(credentials); + } + + final List credentials; + + PubCredential? findCredential(Uri hostedUrl) { + return credentials.firstWhereOrNull( + (c) => c.url == hostedUrl && c.isValid(), + ); + } +} + +class PubCredential { + @visibleForTesting + PubCredential({ + required this.url, + required this.token, + this.env, + }); + + factory PubCredential.fromJson(Map json) { + final hostedUrl = json['url'] as String?; + + if (hostedUrl == null) { + throw const FormatException('Url is not provided for the credential'); + } + + return PubCredential( + url: normalizeHostedUrl(Uri.parse(hostedUrl)), + token: json['token'] as String?, + env: json['env'] as String?, + ); + } + + /// Server url which this token authenticates. + final Uri url; + + /// Authentication token value + final String? token; + + /// Environment variable name that stores token value + final String? env; + + bool isValid() => (token == null) ^ (env == null); + + String? get _tokenValue { + final environment = env; + if (environment != null) { + final value = Platform.environment[environment]; + + if (value == null) { + throw FormatException( + 'Saved credential for "$url" pub repository requires environment ' + 'variable named "$env" but not defined.', + ); + } + + return value; + } else { + return token; + } + } + + String? getAuthHeader() { + if (!isValid()) return null; + return 'Bearer $_tokenValue'; + } +} diff --git a/packages/melos/lib/src/common/pub_hosted.dart b/packages/melos/lib/src/common/pub_hosted.dart new file mode 100644 index 00000000..dd763f27 --- /dev/null +++ b/packages/melos/lib/src/common/pub_hosted.dart @@ -0,0 +1,105 @@ +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; + +import 'package:http/http.dart' as http; +import 'package:meta/meta.dart'; + +import 'http.dart'; +import 'platform.dart'; +import 'pub_credential.dart'; +import 'pub_hosted_package.dart'; + +/// The URL where we can find a package server. +/// +/// The default is `pub.dev`, but it can be overridden using the +/// `PUB_HOSTED_URL` environment variable. +/// https://dart.dev/tools/pub/environment-variables +Uri get defaultPubUrl => Uri.parse( + currentPlatform.environment['PUB_HOSTED_URL'] ?? 'https://pub.dev', + ); + +class PubHostedClient extends http.BaseClient { + @visibleForTesting + PubHostedClient(this.pubHosted, this._inner, this._credentialStore); + + factory PubHostedClient.fromUri({required Uri? pubHosted}) { + final store = pubCredentialStore; + final innerClient = httpClient; + final uri = normalizeHostedUrl(pubHosted ?? defaultPubUrl); + + return PubHostedClient(uri, innerClient, store); + } + + final http.Client _inner; + + final PubCredentialStore _credentialStore; + + final Uri pubHosted; + + @override + Future send(http.BaseRequest request) { + final credential = _credentialStore.findCredential(pubHosted); + + if (credential != null) { + final authToken = credential.getAuthHeader(); + if (authToken != null) { + request.headers[HttpHeaders.authorizationHeader] = authToken; + } + } + + return _inner.send(request); + } + + Future fetchPackage(String name) async { + final url = pubHosted.resolve('api/packages/$name'); + final response = await get(url); + + if (response.statusCode == 404) { + // The package was never published + return null; + } else if (response.statusCode != 200) { + throw Exception( + 'Error reading pub.dev registry for package "$name" ' + '(HTTP Status ${response.statusCode}), response: ${response.body}', + ); + } + + final data = json.decode(response.body) as Map; + return PubHostedPackage.fromJson(data); + } + + @override + void close() => _inner.close(); +} + +Uri normalizeHostedUrl(Uri uri) { + var u = uri; + + if (!u.hasScheme || (u.scheme != 'http' && u.scheme != 'https')) { + throw FormatException('url scheme must be https:// or http://', uri); + } + if (!u.hasAuthority || u.host == '') { + throw FormatException('url must have a hostname', uri); + } + if (u.userInfo != '') { + throw FormatException('user-info is not supported in url', uri); + } + if (u.hasQuery) { + throw FormatException('querystring is not supported in url', uri); + } + if (u.hasFragment) { + throw FormatException('fragment is not supported in url', uri); + } + u = u.normalizePath(); + // If we have a path of only `/` + if (u.path == '/') { + u = u.replace(path: ''); + } + // If there is a path, and it doesn't end in a slash we normalize to slash + if (u.path.isNotEmpty && !u.path.endsWith('/')) { + u = u.replace(path: '${u.path}/'); + } + + return u; +} diff --git a/packages/melos/lib/src/common/pub_hosted_package.dart b/packages/melos/lib/src/common/pub_hosted_package.dart new file mode 100644 index 00000000..e2cb3ad1 --- /dev/null +++ b/packages/melos/lib/src/common/pub_hosted_package.dart @@ -0,0 +1,70 @@ +import 'package:pub_semver/pub_semver.dart'; + +class PubHostedPackage { + PubHostedPackage({required this.name, required this.versions, this.latest}); + + factory PubHostedPackage.fromJson(Map json) { + final name = json['name'] as String?; + final latest = json['latest'] as Map?; + final versions = json['versions'] as List? ?? const []; + + if (name == null) { + throw const FormatException('Name is not provided for the package'); + } + + final packageVersions = versions + .map((v) => PubPackageVersion.fromJson(v as Map)) + .toList(); + + return PubHostedPackage( + name: name, + versions: packageVersions, + latest: latest != null ? PubPackageVersion.fromJson(latest) : null, + ); + } + + /// Returns the name of this package. + final String name; + + /// Returns the latest version of this package if available. + final PubPackageVersion? latest; + + /// Returns the versions of this package. + final List versions; + + /// Returns the sorted versions of this package. + List get prioritizedVersions { + final versions = [...this.versions]; + return versions..sort((a, b) => Version.prioritize(a.version, b.version)); + } + + bool isVersionPublished(Version version) { + if (latest != null && latest!.version == version) { + return true; + } + + return prioritizedVersions.map((v) => v.version).contains(version); + } +} + +class PubPackageVersion { + PubPackageVersion({required this.version, this.published}); + + factory PubPackageVersion.fromJson(Map json) { + final version = json['version'] as String?; + final published = json['published'] as String?; + + if (version == null) { + throw const FormatException('Version is not provided for the package'); + } + + return PubPackageVersion( + version: Version.parse(version), + published: published != null ? DateTime.tryParse(published) : null, + ); + } + + final Version version; + + final DateTime? published; +} diff --git a/packages/melos/lib/src/package.dart b/packages/melos/lib/src/package.dart index c3d0ac5a..ab776984 100644 --- a/packages/melos/lib/src/package.dart +++ b/packages/melos/lib/src/package.dart @@ -16,7 +16,6 @@ */ import 'dart:async'; -import 'dart:convert'; import 'dart:io'; import 'package:collection/collection.dart'; @@ -31,9 +30,10 @@ import 'package:pubspec/pubspec.dart'; import 'common/exception.dart'; import 'common/git.dart'; import 'common/glob.dart'; -import 'common/http.dart' as http; import 'common/io.dart'; import 'common/platform.dart'; +import 'common/pub_hosted.dart' as pub; +import 'common/pub_hosted_package.dart'; import 'common/utils.dart'; import 'common/validation.dart'; import 'logging.dart'; @@ -66,15 +66,6 @@ final List cleanablePubFilePaths = [ '.dart_tool${currentPlatform.pathSeparator}version', ]; -/// The URL where we can find a package server. -/// -/// The default is `pub.dev`, but it can be overridden using the -/// `PUB_HOSTED_URL` environment variable. -/// https://dart.dev/tools/pub/environment-variables -Uri get pubUrl => Uri.parse( - currentPlatform.environment['PUB_HOSTED_URL'] ?? 'https://pub.dev', - ); - final _isValidPubPackageNameRegExp = RegExp(r'^[a-z][a-z\d_-]*$', caseSensitive: false); @@ -667,13 +658,11 @@ extension on Iterable { final packagesFilteredWithPublishStatus = []; await pool.forEach(this, (package) async { - final packageVersion = package.version.toString(); - - final publishedVersions = await package.getPublishedVersions(); + final pubPackage = await package.getPublishedPackage(); - final isOnPubRegistry = publishedVersions.contains(packageVersion); + final isOnPubRegistry = pubPackage?.isVersionPublished(package.version); - if (published == isOnPubRegistry) { + if (published == (isOnPubRegistry ?? false)) { packagesFilteredWithPublishStatus.add(package); } }).drain(); @@ -891,37 +880,13 @@ class Package { /// Queries the pub.dev registry for published versions of this package. /// Primarily used for publish filters and versioning. - Future> getPublishedVersions() async { + Future getPublishedPackage() async { if (isPrivate) { - return []; + return null; } - final pubHosted = publishTo ?? pubUrl; - - final url = pubHosted.replace(path: '/api/packages/$name'); - final response = await http.get(url); - - if (response.statusCode == 404) { - // The package was never published - return []; - } else if (response.statusCode != 200) { - throw Exception( - 'Error reading pub.dev registry for package "$name" ' - '(HTTP Status ${response.statusCode}), response: ${response.body}', - ); - } - - final body = json.decode(response.body) as Map; - final packageVersionInfos = body['versions']! as List; - final packageVersions = packageVersionInfos - .map((info) => (info! as Map)['version']! as String) - .toList(); - - packageVersions.sort((a, b) { - return Version.prioritize(Version.parse(a), Version.parse(b)); - }); - - return packageVersions.reversed.toList(); + final pubClient = pub.PubHostedClient.fromUri(pubHosted: publishTo); + return pubClient.fetchPackage(name); } /// The example [Package] contained within this package, if any. diff --git a/packages/melos/test/package_test.dart b/packages/melos/test/package_test.dart index 62f3ee18..ae8b6fd6 100644 --- a/packages/melos/test/package_test.dart +++ b/packages/melos/test/package_test.dart @@ -1,11 +1,10 @@ import 'dart:io'; import 'package:glob/glob.dart'; -import 'package:http/http.dart' as http; import 'package:melos/melos.dart'; import 'package:melos/src/common/http.dart'; +import 'package:melos/src/common/pub_credential.dart'; import 'package:melos/src/package.dart'; -import 'package:mockito/mockito.dart'; import 'package:platform/platform.dart'; import 'package:pub_semver/pub_semver.dart'; import 'package:test/test.dart'; @@ -17,6 +16,7 @@ import 'utils.dart'; const pubPackageJson = ''' { + "name": "melos", "versions": [ { "version": "1.0.0" @@ -57,13 +57,9 @@ void main() { }); group('MelosPackage', () { - final httpClientMock = HttpClientMock(); late MelosWorkspace workspace; - setUpAll(() => testClient = httpClientMock); - setUp(() async { - reset(httpClientMock); IOOverrides.global = MockFs(); final config = await MelosWorkspaceConfig.fromWorkspaceRoot( @@ -84,40 +80,55 @@ void main() { tearDown(() => IOOverrides.global = null); - tearDownAll(() => testClient = null); + group('When requests published packages', () { + final pubCredentialStoreMock = PubCredentialStore([]); - test('requests published packages from pub.dev by default', () async { - final uri = Uri.parse('https://pub.dev/api/packages/melos'); - when(httpClientMock.get(uri)) - .thenAnswer((_) async => http.Response(pubPackageJson, 200)); + setUpAll(() { + internalPubCredentialStore = pubCredentialStoreMock; + }); - final package = workspace.allPackages.values.first; - await package.getPublishedVersions(); + tearDownAll(() { + internalPubCredentialStore = PubCredentialStore([]); + }); - verify(httpClientMock.get(uri)).called(1); - }); + test('Should fetch package from pub.dev by default', () async { + final uri = Uri.parse('https://pub.dev/api/packages/melos'); + internalHttpClient = HttpClientMock( + (request) { + expect(request.url, uri); + return HttpClientMock.parseResponse(pubPackageJson); + }, + ); - test( - 'requests published packages from PUB_HOSTED_URL if present', - withMockPlatform( - () async { - final uri = Uri.parse('http://localhost:8080/api/packages/melos'); - when(httpClientMock.get(uri)) - .thenAnswer((_) async => http.Response(pubPackageJson, 200)); + final package = workspace.allPackages.values.first; + final pubPackage = await package.getPublishedPackage(); - final package = workspace.allPackages.values.first; - await package.getPublishedVersions(); + expect(pubPackage?.name, isNotEmpty); + }); - verify(httpClientMock.get(uri)).called(1); - }, - platform: FakePlatform.fromPlatform(const LocalPlatform()) - ..environment['PUB_HOSTED_URL'] = 'http://localhost:8080', - ), - ); + test( + 'Should fetch package from PUB_HOSTED_URL if present', + withMockPlatform( + () async { + final uri = Uri.parse('http://localhost:8080/api/packages/melos'); + internalHttpClient = HttpClientMock( + (request) { + expect(request.url, uri); + return HttpClientMock.parseResponse(pubPackageJson); + }, + ); + + final package = workspace.allPackages.values.first; + final pubPackage = await package.getPublishedPackage(); + + expect(pubPackage?.name, isNotEmpty); + }, + platform: FakePlatform.fromPlatform(const LocalPlatform()) + ..environment['PUB_HOSTED_URL'] = 'http://localhost:8080', + ), + ); - test( - 'do not request published versions for private package', - () async { + test('Should not fetch versions for private package', () async { final workspaceBuilder = VirtualWorkspaceBuilder('name: test'); workspaceBuilder.addPackage(''' name: a @@ -130,17 +141,80 @@ void main() { final workspace = workspaceBuilder.build(); expect( - await workspace.allPackages['a']!.getPublishedVersions(), - isEmpty, + await workspace.allPackages['a']!.getPublishedPackage(), + isNull, ); expect( - await workspace.allPackages['b']!.getPublishedVersions(), - isEmpty, + await workspace.allPackages['b']!.getPublishedPackage(), + isNull, ); + }); + }); - verifyNever(httpClientMock.get(any)); - }, - ); + group('When requests published packages for private registries', () { + final fakeCredential = PubCredential( + url: Uri.parse('https://fake.registry'), + token: 'fake_token', + ); + + final pubCredentialStoreMock = PubCredentialStore([fakeCredential]); + + setUpAll(() { + internalPubCredentialStore = pubCredentialStoreMock; + }); + + tearDownAll(() { + internalPubCredentialStore = PubCredentialStore([]); + }); + + test( + 'Should fetch without credentials', + () async { + final uri = Uri.parse('https://pub.dev/api/packages/melos'); + internalHttpClient = HttpClientMock( + (request) { + expect(request.url, uri); + expect( + request.headers, + isNot(contains(HttpHeaders.authorizationHeader)), + ); + return HttpClientMock.parseResponse(pubPackageJson); + }, + ); + + final package = workspace.allPackages.values.first; + final pubPackage = await package.getPublishedPackage(); + + expect(pubPackage?.name, isNotEmpty); + }, + ); + + test( + 'Should fetch from private registry if present', + withMockPlatform( + () async { + final uri = fakeCredential.url.resolve('api/packages/melos'); + internalHttpClient = HttpClientMock( + (request) { + expect(request.url, uri); + expect( + request.headers[HttpHeaders.authorizationHeader], + fakeCredential.getAuthHeader(), + ); + return HttpClientMock.parseResponse(pubPackageJson); + }, + ); + + final package = workspace.allPackages.values.first; + final pubPackage = await package.getPublishedPackage(); + + expect(pubPackage?.name, isNotEmpty); + }, + platform: FakePlatform.fromPlatform(const LocalPlatform()) + ..environment['PUB_HOSTED_URL'] = fakeCredential.url.toString(), + ), + ); + }); }); group('Package', () { diff --git a/packages/melos/test/utils.dart b/packages/melos/test/utils.dart index 6d15feac..b95044db 100644 --- a/packages/melos/test/utils.dart +++ b/packages/melos/test/utils.dart @@ -3,12 +3,12 @@ import 'dart:io'; import 'package:cli_util/cli_logging.dart'; import 'package:http/http.dart' as http; +import 'package:http/testing.dart' as http_testing; import 'package:melos/melos.dart'; import 'package:melos/src/common/glob.dart'; import 'package:melos/src/common/io.dart'; import 'package:melos/src/common/platform.dart'; import 'package:melos/src/common/utils.dart'; -import 'package:mockito/mockito.dart'; import 'package:path/path.dart' as p; import 'package:pub_semver/pub_semver.dart'; import 'package:pubspec/pubspec.dart'; @@ -403,13 +403,11 @@ class _VirtualPackage { final String? path; } -class HttpClientMock extends Mock implements http.Client { - @override - Future get(Uri? url, {Map? headers}) { - return super.noSuchMethod( - Invocation.method(#get, [url], {#headers: headers}), - returnValue: Future.value(http.Response('', 200)), - ) as Future; +class HttpClientMock extends http_testing.MockClient { + HttpClientMock(super.fn); + + static Future parseResponse(String result) async { + return http.Response(result, 200); } }