diff --git a/README.md b/README.md index cf9bb04b..12899b3b 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ A base module for [drupal-helfi-platform](https://github.com/City-of-Helsinki/dr ## Features - [API user manager](documentation/api-accounts.md): Allows API users to be created/managed from an environment variable. +- [API client](documentation/api-client.md): Services for caching and mocking http responses. - [Automatic external cache invalidation](documentation/automatic-external-cache-invalidation.md): Invalidate caches from external projects using [PubSub messaging](documentation/pubsub-messaging.md) service. - [Automatic revision deletion](documentation/revisions.md): Clean up old entity revisions automatically. - [Debug collector](documentation/debug.md): A plugin to collect and show various debug information in one place. diff --git a/composer.json b/composer.json index 499ebd67..1b25a024 100644 --- a/composer.json +++ b/composer.json @@ -12,7 +12,8 @@ "php": "^8.1", "t4web/composer-lock-parser": "^1.0", "textalk/websocket": "^1.6", - "webmozart/assert": "^1.0" + "webmozart/assert": "^1.0", + "ext-curl": "*" }, "conflict": { "drupal/helfi_debug": "*" diff --git a/documentation/api-client.md b/documentation/api-client.md new file mode 100644 index 00000000..519d3134 --- /dev/null +++ b/documentation/api-client.md @@ -0,0 +1,49 @@ +# API client + +Service for HTTP JSON APIs. + +Features: + - Simple caching. + - Optional response mocking for local environment. + +## Usage + +Create your own client service from abstract service `helfi_api_base.api_client_base`. You must provide your own logger. Optionally you can provide default request parameters. + +```yaml +# my_module.services.yml +my_module.my_api: + parent: helfi_api_base.api_manager + arguments: + - '@logger.channel.my_module' + # Optional: + - { timeout: 30 } +``` + +Actual requests are usually made in the callback of `cache()` method. The callback must return `CacheValue`. + +```php +use Drupal\helfi_api_base\ApiClient\CacheValue; + +/** @var Drupal\helfi_api_base\ApiClient\ApiClient $client */ +$client = \Drupal::service('my_module.my_api'); + +$response = $client->cache($id, fn () => new CacheValue( + // Actual HTTP response. + $client->makeRequest('GET', 'https://example.com/api/v1/foo'), + // TTL. + $client->cacheMaxAge(ttl: 180), + // Custom cache tags. + ['user:1'] +)); +``` + +### Mocking + +In local environment, the `makeRequestWithFixture` method returns response from JSON file if the response fails. + +```php +$client->makeRequestWithFixture('path-to-fixture.json', 'GET', 'https://example.com/fails-in-local'), +``` + +*Warning*: The client fail any further requests to `makeRequestWithFixture` instantly after one failed requests. This is to prevent blocking the rendering process and cause the site to time-out. You should not share the client for different purposes that need fault tolerance. diff --git a/helfi_api_base.services.yml b/helfi_api_base.services.yml index 38fa3459..c9590c92 100644 --- a/helfi_api_base.services.yml +++ b/helfi_api_base.services.yml @@ -142,3 +142,12 @@ services: - '@entity_type.manager' - '@config.factory' - '@database' + + helfi_api_base.api_client_base: + class: Drupal\helfi_api_base\ApiClient\ApiClient + abstract: true + arguments: + - '@http_client' + - '@cache.default' + - '@datetime.time' + - '@helfi_api_base.environment_resolver' diff --git a/src/ApiClient/ApiClient.php b/src/ApiClient/ApiClient.php new file mode 100644 index 00000000..a50f9abc --- /dev/null +++ b/src/ApiClient/ApiClient.php @@ -0,0 +1,252 @@ +bypassCache = TRUE; + return $instance; + } + + /** + * Gets the default request options. + * + * @param string $environmentName + * Environment name. + * @param array $options + * The optional options. + * + * @return array + * The request options. + */ + protected function getRequestOptions(string $environmentName, array $options = []) : array { + // Hardcode cURL options. + // Curl options are keyed by PHP constants so there is no easy way to + // define them in yaml files yet. See: https://www.drupal.org/node/3403883 + $default = $this->defaultOptions + [ + 'curl' => [CURLOPT_TCP_KEEPALIVE => TRUE], + ]; + + if ($environmentName === 'local') { + // Disable SSL verification in local environment. + $default['verify'] = FALSE; + } + + return array_merge_recursive($options, $default); + } + + /** + * Makes HTTP request. + * + * @param string $method + * Request method. + * @param string $url + * The endpoint in the instance. + * @param array $options + * Body for requests. + * + * @return \Drupal\helfi_api_base\ApiClient\ApiResponse + * The JSON object. + * + * @throws \GuzzleHttp\Exception\GuzzleException + */ + public function makeRequest( + string $method, + string $url, + array $options = [], + ): ApiResponse { + $activeEnvironmentName = $this->environmentResolver + ->getActiveEnvironment() + ->getEnvironmentName(); + + $options = $this->getRequestOptions($activeEnvironmentName, $options); + + $response = $this->httpClient->request($method, $url, $options); + + return new ApiResponse(Utils::jsonDecode($response->getBody()->getContents())); + } + + /** + * Makes HTTP request with fixture. + * + * @param string $fixture + * File for mock data if requests fail in local environment. + * @param string $method + * Request method. + * @param string $url + * The endpoint in the instance. + * @param array $options + * Body for requests. + * + * @return \Drupal\helfi_api_base\ApiClient\ApiResponse + * The JSON object. + * + * @throws \GuzzleHttp\Exception\GuzzleException + */ + public function makeRequestWithFixture( + string $fixture, + string $method, + string $url, + array $options = [], + ): ApiResponse { + try { + if ($this->previousException instanceof \Exception) { + // Fail any further request instantly after one failed request, so we + // don't block the rendering process and cause the site to time-out. + throw $this->previousException; + } + + return $this->makeRequest($method, $url, $options); + } + catch (\Exception $e) { + if ($e instanceof GuzzleException) { + $this->previousException = $e; + } + + $activeEnvironmentName = $this->environmentResolver + ->getActiveEnvironment() + ->getEnvironmentName(); + + // Serve mock data in local environments if requests fail. + if ( + ($e instanceof ClientException || $e instanceof ConnectException) && + $activeEnvironmentName === 'local' + ) { + $this->logger->warning( + sprintf('Request failed: %s. Mock data is used instead.', $e->getMessage()) + ); + + return ApiFixture::requestFromFile($fixture); + } + + throw $e; + } + } + + /** + * Gets the cached data for given response. + * + * @param string $key + * The cache key. + * @param callable $callback + * The callback to handle requests. + * + * @return \Drupal\helfi_api_base\ApiClient\CacheValue|null + * The cache or null. + * + * @throws \GuzzleHttp\Exception\GuzzleException + */ + public function cache(string $key, callable $callback) : ?CacheValue { + $exception = new TransferException(); + $value = ($cache = $this->cache->get($key)) ? $cache->data : NULL; + + // Attempt to re-fetch the data in case cache does not exist, cache has + // expired, or bypass cache is set to true. + if ( + ($value instanceof CacheValue && $value->hasExpired($this->time->getRequestTime())) || + $this->bypassCache || + $value === NULL + ) { + try { + $value = $callback(); + $this->cache->set($key, $value, tags: $value->tags); + return $value; + } + catch (GuzzleException $e) { + // Request callback failed. Catch the exception, so we can still use + // stale cache if it exists. + $exception = $e; + } + } + + if ($value instanceof CacheValue) { + return $value; + } + + // We should only reach this if: + // 1. Cache does not exist ($value is NULL). + // 2. API request fails, and we cannot re-populate the cache (caught the + // exception). + throw $exception; + } + + /** + * Helper method for calculating cache max age. + * + * @param int $ttl + * Time to live in seconds. + * + * @return int + * Expires timestamp. + */ + public function cacheMaxAge(int $ttl): int { + return $this->time->getRequestTime() + $ttl; + } + +} diff --git a/src/ApiClient/ApiFixture.php b/src/ApiClient/ApiFixture.php new file mode 100644 index 00000000..8f845f32 --- /dev/null +++ b/src/ApiClient/ApiFixture.php @@ -0,0 +1,32 @@ + $this->expires; + } + +} diff --git a/src/Plugin/migrate/destination/TranslatableEntity.php b/src/Plugin/migrate/destination/TranslatableEntity.php index 60f71bf2..2ab6bef4 100644 --- a/src/Plugin/migrate/destination/TranslatableEntity.php +++ b/src/Plugin/migrate/destination/TranslatableEntity.php @@ -98,7 +98,7 @@ protected function populateDefaultValues(Row $row) : void { // Set default values for entity when we're creating the entity // for the first time. These are not supposed to be overridden by // migrate. - foreach ($defaultValues ?? [] as $key => $value) { + foreach ($defaultValues as $key => $value) { $row->setDestinationProperty($key, $value); } } diff --git a/tests/fixtures/response.json b/tests/fixtures/response.json new file mode 100644 index 00000000..c4bd5689 --- /dev/null +++ b/tests/fixtures/response.json @@ -0,0 +1,3 @@ +{ + "value": 1 +} diff --git a/tests/src/Unit/ApiClient/ApiClientTest.php b/tests/src/Unit/ApiClient/ApiClientTest.php new file mode 100644 index 00000000..eb712e2a --- /dev/null +++ b/tests/src/Unit/ApiClient/ApiClientTest.php @@ -0,0 +1,429 @@ +fixture = sprintf('%s/../../../fixtures/response.json', __DIR__); + $this->cache = new MemoryBackend(); + $this->environmentResolverConfiguration = [ + EnvironmentResolver::PROJECT_NAME_KEY => Project::ASUMINEN, + EnvironmentResolver::ENVIRONMENT_NAME_KEY => 'local', + ]; + } + + /** + * Create a new time mock object. + * + * @param int $expectedTime + * The expected time. + * + * @return \Prophecy\Prophecy\ObjectProphecy + * The mock. + */ + private function getTimeMock(int $expectedTime) : ObjectProphecy { + $time = $this->prophesize(TimeInterface::class); + $time->getRequestTime()->willReturn($expectedTime); + return $time; + } + + /** + * Constructs a new api client instance. + * + * @param \GuzzleHttp\ClientInterface $client + * The http client. + * @param \Drupal\Component\Datetime\TimeInterface|null $time + * The time prophecy. + * @param \Psr\Log\LoggerInterface|null $logger + * The logger. + * @param \Drupal\helfi_api_base\Environment\EnvironmentResolverInterface|null $environmentResolver + * The environment resolver. + * @param array $requestOptions + * The default request options. + * + * @return \Drupal\helfi_api_base\ApiClient\ApiClient + * The api client instance. + */ + private function getSut( + ClientInterface $client = NULL, + TimeInterface $time = NULL, + LoggerInterface $logger = NULL, + EnvironmentResolverInterface $environmentResolver = NULL, + array $requestOptions = [], + ) : ApiClient { + if (!$client) { + $client = $this->createMockHttpClient([]); + } + + if (!$time) { + $time = $this->getTimeMock(time())->reveal(); + } + + if (!$logger) { + $logger = $this->prophesize(LoggerInterface::class)->reveal(); + } + if (!$environmentResolver) { + $environmentResolver = new EnvironmentResolver('', $this->getConfigFactoryStub([ + 'helfi_api_base.environment_resolver.settings' => $this->environmentResolverConfiguration, + ])); + } + + return new ApiClient( + $client, + $this->cache, + $time, + $environmentResolver, + $logger, + $requestOptions, + ); + } + + /** + * Test makeRequest(). + * + * @covers ::__construct + * @covers ::makeRequest + * @covers ::getRequestOptions + * @covers \Drupal\helfi_api_base\ApiClient\ApiResponse::__construct + */ + public function testMakeRequest() { + $requests = []; + $client = $this->createMockHistoryMiddlewareHttpClient($requests, [ + new Response(200, body: json_encode([])), + new Response(200, body: json_encode(['key' => 'value'])), + ]); + $sut = $this->getSut($client); + + // Test empty and non-empty response. + for ($i = 0; $i < 2; $i++) { + $response = $sut->makeRequest('GET', '/foo'); + $this->assertInstanceOf(ApiResponse::class, $response); + $this->assertInstanceOf(RequestInterface::class, $requests[0]['request']); + } + } + + /** + * Tests exception when cache callback fails. + * + * @covers ::__construct + * @covers ::cache + */ + public function testCacheExceptionOnFailure() : void { + $this->expectException(GuzzleException::class); + + $this->getSut()->cache( + 'nonexistent:fi', + fn () => throw new RequestException( + 'Test', + $this->prophesize(RequestInterface::class)->reveal() + ) + ); + } + + /** + * Tests that stale cache will be returned in case callback fails. + * + * @covers ::__construct + * @covers ::cache + * @covers \Drupal\helfi_api_base\ApiClient\CacheValue::hasExpired + * @covers \Drupal\helfi_api_base\ApiClient\CacheValue::__construct + */ + public function testStaleCacheOnFailure() : void { + $time = time(); + // Expired cache object. + $cacheValue = new CacheValue( + new ApiResponse((object) ['value' => 1]), + $time - 10, + [], + ); + $this->cache->set('external_menu:main:fi', $cacheValue); + + $sut = $this->getSut( + time: $this->getTimeMock($time)->reveal(), + ); + $response = $sut->cache( + 'external_menu:main:fi', + fn () => throw new RequestException( + 'Test', + $this->prophesize(RequestInterface::class)->reveal() + ) + ); + $this->assertInstanceOf(CacheValue::class, $response); + } + + /** + * Tests that stale cache can be updated. + * + * @covers ::__construct + * @covers ::cache + * @covers ::cacheMaxAge + * @covers \Drupal\helfi_api_base\ApiClient\CacheValue::hasExpired + * @covers \Drupal\helfi_api_base\ApiClient\CacheValue::__construct + * @covers \Drupal\helfi_api_base\ApiClient\ApiResponse::__construct + */ + public function testStaleCacheUpdate() : void { + $time = time(); + + // Expired cache object. + $cacheValue = new CacheValue( + new ApiResponse((object) ['value' => 1]), + $time - 10, + [], + ); + // Populate cache with expired cache value object. + $this->cache->set('external_menu:main:en', $cacheValue); + + $sut = $this->getSut( + time: $this->getTimeMock($time)->reveal(), + ); + $value = $sut->cache('external_menu:main:en', static fn () => new CacheValue( + new ApiResponse((object) ['value' => 'value']), + $sut->cacheMaxAge(10), + [], + )); + $this->assertInstanceOf(CacheValue::class, $value); + $this->assertInstanceOf(ApiResponse::class, $value->response); + // Make sure cache was updated. + $this->assertInstanceOf(\stdClass::class, $value->response->data); + $this->assertEquals($time + 10, $value->expires); + $this->assertEquals('value', $value->response->data->value); + // Re-fetch the data to make sure we still get updated data and make sure + // no further requests are made. + $value = $sut->cache('external_menu:main:en', fn() => $this->fail('Data should be cached')); + $this->assertInstanceOf(\stdClass::class, $value->response->data); + $this->assertEquals('value', $value->response->data->value); + } + + /** + * Make sure we log the exception and then re-throw the same exception. + * + * @covers ::makeRequest + * @covers ::__construct + * @covers ::getRequestOptions + */ + public function testRequestLoggingException() : void { + $this->expectException(GuzzleException::class); + + $client = $this->createMockHttpClient([ + new RequestException('Test', $this->prophesize(RequestInterface::class)->reveal()), + ]); + $logger = $this->prophesize(LoggerInterface::class); + + $sut = $this->getSut($client, logger: $logger->reveal()); + $sut->makeRequest('GET', '/foo'); + } + + /** + * Tests that file not found exception is thrown when no mock file exists. + * + * @covers ::makeRequestWithFixture + * @covers ::__construct + * @covers ::getRequestOptions + * @covers \Drupal\helfi_api_base\ApiClient\ApiFixture::requestFromFile + */ + public function testMockFallbackException() : void { + $this->expectException(FileNotExistsException::class); + $response = $this->prophesize(ResponseInterface::class); + $response->getStatusCode()->willReturn(403); + $client = $this->createMockHttpClient([ + new ClientException( + 'Test', + $this->prophesize(RequestInterface::class)->reveal(), + $response->reveal(), + ), + ]); + $sut = $this->getSut($client); + // Test with non-existent menu to make sure no mock file exist. + $sut->makeRequestWithFixture( + sprintf('%d/should-not-exists.txt', __DIR__), + 'GET', + '/foo' + ); + } + + /** + * Tests that mock is used on local environment when GET request fails. + * + * @covers ::makeRequestWithFixture + * @covers ::__construct + * @covers ::getRequestOptions + * @covers \Drupal\helfi_api_base\ApiClient\ApiResponse::__construct + * @covers \Drupal\helfi_api_base\ApiClient\ApiFixture::requestFromFile + */ + public function testMockFallback() : void { + // Use logger to verify that mock file is actually used. + $logger = $this->prophesize(LoggerInterface::class); + $logger->warning(Argument::containingString('Mock data is used instead.')) + ->shouldBeCalled(); + $client = $this->createMockHttpClient([ + new ConnectException( + 'Test', + $this->prophesize(RequestInterface::class)->reveal(), + ), + ]); + $sut = $this->getSut( + $client, + logger: $logger->reveal(), + ); + $response = $sut->makeRequestWithFixture( + $this->fixture, + 'GET', + '/foo', + ); + $this->assertInstanceOf(ApiResponse::class, $response); + } + + /** + * Make sure subsequent requests are failed after one failed request. + * + * @covers ::makeRequestWithFixture + * @covers ::__construct + * @covers ::getRequestOptions + */ + public function testFastRequestFailure() : void { + // Override environment name so we don't fallback to mock responses. + $this->environmentResolverConfiguration[EnvironmentResolver::ENVIRONMENT_NAME_KEY] = 'test'; + + $client = $this->createMockHttpClient([ + new ConnectException( + 'Test', + $this->prophesize(RequestInterface::class)->reveal(), + ), + ]); + $sut = $this->getSut($client); + + $attempts = 0; + // Make sure only one request is sent if the first request fails. + // This should fail to \OutOfBoundsException from guzzle MockHandler + // if more than one request is sent. + for ($i = 0; $i < 50; $i++) { + try { + $sut->makeRequestWithFixture($this->fixture, 'GET', '/foo'); + } + catch (ConnectException) { + $attempts++; + } + } + $this->assertEquals(50, $attempts); + } + + /** + * Make sure cache can be bypassed when configured so. + * + * @covers ::makeRequest + * @covers ::__construct + * @covers ::cache + * @covers ::getRequestOptions + * @covers ::withBypassCache + * @covers \Drupal\helfi_api_base\ApiClient\CacheValue::hasExpired + * @covers \Drupal\helfi_api_base\ApiClient\CacheValue::__construct + * @covers \Drupal\helfi_api_base\ApiClient\ApiResponse::__construct + */ + public function testCacheBypass() : void { + $time = time(); + $requests = []; + $client = $this->createMockHistoryMiddlewareHttpClient($requests, [ + new Response(200, body: json_encode(['value' => 1])), + new Response(200, body: json_encode(['value' => 2])), + ]); + $sut = $this->getSut( + $client, + time: $this->getTimeMock($time)->reveal() + ); + // Make sure cache is used for all requests. + for ($i = 0; $i < 3; $i++) { + $response = $sut->cache('cache_key', fn () => new CacheValue( + $sut->makeRequest('GET', '/foo'), + $time + 100, + [], + ))->response; + $this->assertInstanceOf(\stdClass::class, $response->data); + $this->assertEquals(1, $response->data->value); + } + // Make sure cache is bypassed when configured so and the cached content + // is updated. + $response = $sut->withBypassCache()->cache('cache_key', fn () => new CacheValue( + $sut->makeRequest('GET', '/foo'), + $time + 100, + [] + ))->response; + $this->assertInstanceOf(\stdClass::class, $response->data); + $this->assertEquals(2, $response->data->value); + + // withBypassCache() method creates a clone of ApiManager instance to ensure + // cache is only bypassed when explicitly told so. + // We defined only two responses, so this should fail to OutOfBoundException + // if cache was bypassed here. + for ($i = 0; $i < 3; $i++) { + $response = $sut->cache('cache_key', fn () => new CacheValue( + $sut->makeRequest('GET', '/foo'), + $time + 100, + [], + ))->response; + $this->assertInstanceOf(\stdClass::class, $response->data); + $this->assertEquals(2, $response->data->value); + } + } + +} diff --git a/tests/src/Unit/ApiClient/ApiFixtureTest.php b/tests/src/Unit/ApiClient/ApiFixtureTest.php new file mode 100644 index 00000000..490237a7 --- /dev/null +++ b/tests/src/Unit/ApiClient/ApiFixtureTest.php @@ -0,0 +1,42 @@ +assertInstanceOf(ApiResponse::class, $response); + } + + /** + * Test missing file. + * + * @covers ::requestFromFile + */ + public function testException() { + $this->expectException(FileNotExistsException::class); + ApiFixture::requestFromFile(vsprintf('%s/should-not-exists', [ + __DIR__, + ])); + } + +}