diff --git a/app/Jobs/UpdateWikiSiteStatsJob.php b/app/Jobs/UpdateWikiSiteStatsJob.php index 384ba2a9..1ea77c24 100644 --- a/app/Jobs/UpdateWikiSiteStatsJob.php +++ b/app/Jobs/UpdateWikiSiteStatsJob.php @@ -4,6 +4,8 @@ use App\Wiki; use App\WikiSiteStats; +use Carbon\CarbonInterface; +use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Support\Facades\Http; use Illuminate\Support\Facades\Log; use Illuminate\Support\Facades\DB; @@ -13,6 +15,7 @@ class UpdateWikiSiteStatsJob extends Job implements ShouldBeUnique { + use Dispatchable; public $timeout = 3600; public function handle (): void { @@ -73,7 +76,7 @@ private function updateSiteStats (Wiki $wiki): void }); } - private function getFirstEditedDate (Wiki $wiki): ?\Carbon\CarbonInterface + private function getFirstEditedDate (Wiki $wiki): ?CarbonInterface { $allRevisions = Http::withHeaders(['host' => $wiki->getAttribute('domain')])->get( getenv('PLATFORM_MW_BACKEND_HOST').'/w/api.php', @@ -111,18 +114,38 @@ private function getFirstEditedDate (Wiki $wiki): ?\Carbon\CarbonInterface return Carbon::parse($result); } - private function getLastEditedDate (Wiki $wiki): ?\Carbon\CarbonInterface + private function getLastEditedDate (Wiki $wiki): ?CarbonInterface { - $recentChangesInfo = Http::withHeaders(['host' => $wiki->getAttribute('domain')])->get( + $allRevisions = Http::withHeaders(['host' => $wiki->getAttribute('domain')])->get( + getenv('PLATFORM_MW_BACKEND_HOST').'/w/api.php', + [ + 'action' => 'query', + 'format' => 'json', + 'list' => 'allrevisions', + 'formatversion' => 2, + 'arvlimit' => 1, + 'arvprop' => 'ids', + 'arvexcludeuser' => 'PlatformReservedUser', + 'arvdir' => 'older', + ], + ); + $lastRevision = data_get($allRevisions->json(), 'query.allrevisions.0.revisions.0.revid'); + if (!$lastRevision) { + return null; + } + + $revisionInfo = Http::withHeaders(['host' => $wiki->getAttribute('domain')])->get( getenv('PLATFORM_MW_BACKEND_HOST').'/w/api.php', [ 'action' => 'query', - 'list' => 'recentchanges', 'format' => 'json', - 'rcexcludeuser' => 'PlatformReservedUser', + 'prop' => 'revisions', + 'rvprop' => 'timestamp', + 'formatversion' => 2, + 'revids' => $lastRevision, ], ); - $result = data_get($recentChangesInfo->json(), 'query.recentchanges.0.timestamp'); + $result = data_get($revisionInfo->json(), 'query.pages.0.revisions.0.timestamp'); if (!$result) { return null; } diff --git a/database/migrations/2024_08_01_193038_remove_and_rebuild_wiki_lifecycle_events.php b/database/migrations/2024_08_01_193038_remove_and_rebuild_wiki_lifecycle_events.php new file mode 100644 index 00000000..6b660d84 --- /dev/null +++ b/database/migrations/2024_08_01_193038_remove_and_rebuild_wiki_lifecycle_events.php @@ -0,0 +1,25 @@ +delete(); + UpdateWikiSiteStatsJob::dispatch(); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + // + } +}; diff --git a/tests/Jobs/UpdateWikiSiteStatsJobTest.php b/tests/Jobs/UpdateWikiSiteStatsJobTest.php index 6c068840..400579dd 100644 --- a/tests/Jobs/UpdateWikiSiteStatsJobTest.php +++ b/tests/Jobs/UpdateWikiSiteStatsJobTest.php @@ -15,10 +15,14 @@ class UpdateWikiSiteStatsJobTest extends TestCase use RefreshDatabase; + private $fakeResponses; + public function setUp(): void { // Other tests leave dangling wikis around so we need to clean them up parent::setUp(); Wiki::query()->delete(); + $this->fakeResponses = []; + Http::preventStrayRequests(); } public function tearDown(): void { @@ -26,157 +30,80 @@ public function tearDown(): void { parent::tearDown(); } - public function testSuccess() - { - Wiki::factory()->create([ - 'domain' => 'this.wikibase.cloud' - ]); - Wiki::factory()->create([ - 'domain' => 'that.wikibase.cloud' - ]); + private function addFakeSiteStatsResponse($site, $response) { + $siteStatsUrl = getenv('PLATFORM_MW_BACKEND_HOST').'/w/api.php?action=query&meta=siteinfo&siprop=statistics&format=json'; + $this->fakeResponses[$siteStatsUrl][$site] = $response; + } - Http::fake(function (Request $request) { - $responses = [ - getenv('PLATFORM_MW_BACKEND_HOST').'/w/api.php?action=query&meta=siteinfo&siprop=statistics&format=json' => [ - 'this.wikibase.cloud' => Http::response(['query' => [ - 'statistics' => [ - 'pages' => 1, - 'articles' => 2, - 'edits' => 3, - 'images' => 4, - 'users' => 5, - 'activeusers' => 6, - 'admins' => 7, - 'jobs' => 8, - 'cirrussearch-article-words' => 9 - ] - ]]), - 'that.wikibase.cloud' => Http::response(['query' => [ - 'statistics' => [ - 'pages' => 19, - 'articles' => 18, - 'edits' => 17, - 'images' => 16, - 'users' => 15, - 'activeusers' => 14, - 'admins' => 13, - 'jobs' => 12, - 'cirrussearch-article-words' => 11 - ] - ]]) - ], - getenv('PLATFORM_MW_BACKEND_HOST').'/w/api.php?action=query&format=json&list=allrevisions&formatversion=2&arvlimit=1&arvprop=ids&arvexcludeuser=PlatformReservedUser&arvdir=newer' => [ - 'this.wikibase.cloud' => Http::response([ - 'query' => [ - 'allrevisions' => [ - [ - 'revisions' => [ - [ - 'revid' => 2 - ] - ] - ] - ] - ] - ]), - 'that.wikibase.cloud' => Http::response([ - 'query' => [ - 'allrevisions' => [ - [ - 'revisions' => [ - [ - 'revid' => 2 - ] - ] - ] - ] - ] - ]), - ], - getenv('PLATFORM_MW_BACKEND_HOST').'/w/api.php?action=query&format=json&prop=revisions&rvprop=timestamp&formatversion=2&revids=2' => [ - 'this.wikibase.cloud' => Http::response([ - 'query' => [ - 'pages' => [ - [ - 'revisions' => [ - [ - 'timestamp' => '2023-02-27T16:57:06Z' - ] - ] - ] + private function addFakeRevisionTimestamp($site, $revid, $timestamp) { + $revTimestampUrl = getenv('PLATFORM_MW_BACKEND_HOST').'/w/api.php?action=query&format=json&prop=revisions&rvprop=timestamp&formatversion=2&revids=' . $revid; + $this->fakeResponses[$revTimestampUrl][$site] = Http::response([ + 'query' => [ + 'pages' => [ + [ + 'revisions' => [ + [ + 'timestamp' => $timestamp ] ] - ]), - 'that.wikibase.cloud' => Http::response([ - 'query' => [ - 'pages' => [ - [ - 'revisions' => [ - [ - 'timestamp' => '2023-05-07T21:31:47Z' - ] - ] - ] - ] - ] - ]), - ], - getenv('PLATFORM_MW_BACKEND_HOST').'/w/api.php?action=query&list=recentchanges&format=json&rcexcludeuser=PlatformReservedUser' => [ - 'this.wikibase.cloud' => Http::response([ - 'query' => [ - 'recentchanges' => [ - [ - 'type' => 'new', - 'ns' => 120, - 'title' => 'Item:Q312951', - 'pageid' => 310675, - 'revid' => 830699, - 'old_revid' => 0, - 'rcid' => 829604, - 'timestamp' => '2023-09-16T17:22:33Z' - ], - [ - - 'type' => 'edit', - 'ns' => 4, - 'title' => 'Project:Home', - 'pageid' => 1, - 'revid' => 830698, - 'old_revid' => 830094, - 'rcid' => 829603, - 'timestamp' => '2023-09-16T04:50:03Z' - ] + ] + ] + ] + ]); + } + + private function addFakeEmptyRevisionList($site) { + $firstRevisionIdUrl = getenv('PLATFORM_MW_BACKEND_HOST').'/w/api.php?action=query&format=json&list=allrevisions&formatversion=2&arvlimit=1&arvprop=ids&arvexcludeuser=PlatformReservedUser&arvdir=newer'; + $this->fakeResponses[$firstRevisionIdUrl][$site] = Http::response([ + 'query' => [ + 'allrevisions' => [] + ] + ]); + $lastRevisionIdUrl = getenv('PLATFORM_MW_BACKEND_HOST').'/w/api.php?action=query&format=json&list=allrevisions&formatversion=2&arvlimit=1&arvprop=ids&arvexcludeuser=PlatformReservedUser&arvdir=older'; + $this->fakeResponses[$lastRevisionIdUrl][$site] = Http::response([ + 'query' => [ + 'allrevisions' => [] + ] + ]); + } + + private function addFakeFirstRevisionId($site, $id) { + $firstRevisionIdUrl = getenv('PLATFORM_MW_BACKEND_HOST').'/w/api.php?action=query&format=json&list=allrevisions&formatversion=2&arvlimit=1&arvprop=ids&arvexcludeuser=PlatformReservedUser&arvdir=newer'; + $this->fakeResponses[$firstRevisionIdUrl][$site] = Http::response([ + 'query' => [ + 'allrevisions' => [ + [ + 'revisions' => [ + [ + 'revid' => $id ] ] - ]), - 'that.wikibase.cloud' => Http::response([ - 'query' => [ - 'recentchanges' => [ - [ - 'type' => 'edit', - 'ns' => 120, - 'title' => 'Item:Q158700', - 'pageid' => 204252, - 'revid' => 438675, - 'old_revid' => 438674, - 'rcid' => 441539, - 'timestamp' => '2023-09-19T11:35:09Z' - ], - [ - 'type' => 'edit', - 'ns' => 120, - 'title' => 'Item:Q158700', - 'pageid' => 204252, - 'revid' => 438674, - 'old_revid' => 438673, - 'rcid' => 441538, - 'timestamp' => '2023-09-19T11:33:50Z' - ] + ] + ] + ] + ]); + } + + private function addFakeLastRevisionId($site, $id) { + $lastRevisionIdUrl = getenv('PLATFORM_MW_BACKEND_HOST').'/w/api.php?action=query&format=json&list=allrevisions&formatversion=2&arvlimit=1&arvprop=ids&arvexcludeuser=PlatformReservedUser&arvdir=older'; + $this->fakeResponses[$lastRevisionIdUrl][$site] = Http::response([ + 'query' => [ + 'allrevisions' => [ + [ + 'revisions' => [ + [ + 'revid' => $id ] ] - ]), - ], - ]; + ] + ] + ] + ]); + } + + private function fakeResponse() { + $responses = $this->fakeResponses; + $fakeFunction = function (Request $request) use ($responses) { $url = $request->url(); $hostHeader = $request->header('host')[0]; @@ -188,7 +115,100 @@ public function testSuccess() } } return Http::response('not found', 404); - }); + }; + + Http::fake( $fakeFunction ); + } + + public function testWikiSiteStatsIsSuccessfullyUpdated() { + Wiki::factory()->create([ + 'domain' => 'this.wikibase.cloud' + ]); + + $this->addFakeSiteStatsResponse( + 'this.wikibase.cloud', + Http::response(['query' => [ + 'statistics' => [ + 'pages' => 1, + 'articles' => 2, + 'edits' => 3, + 'images' => 4, + 'users' => 5, + 'activeusers' => 6, + 'admins' => 7, + 'jobs' => 8, + 'cirrussearch-article-words' => 9 + ] + ]]) + ); + $this->fakeResponse(); + + $mockJob = $this->createMock(Job::class); + $job = new UpdateWikiSiteStatsJob(); + $job->setJob($mockJob); + + $mockJob->expects($this->never())->method('fail'); + $mockJob->expects($this->never())->method('markAsFailed'); + $job->handle(); + + $stats1 = Wiki::with('wikiSiteStats')->where(['domain' => 'this.wikibase.cloud'])->first()->wikiSiteStats()->first(); + $this->assertEquals($stats1['admins'], 7); + + } + + public function testSuccessOfMultipleWikisTogether() + { + + Wiki::factory()->create([ + 'domain' => 'that.wikibase.cloud' + ]); + Wiki::factory()->create([ + 'domain' => 'this.wikibase.cloud' + ]); + + $this->addFakeSiteStatsResponse( + 'this.wikibase.cloud', + Http::response(['query' => [ + 'statistics' => [ + 'pages' => 1, + 'articles' => 2, + 'edits' => 3, + 'images' => 4, + 'users' => 5, + 'activeusers' => 6, + 'admins' => 7, + 'jobs' => 8, + 'cirrussearch-article-words' => 9 + ] + ]]) + ); + + $this->addFakeSiteStatsResponse( + 'that.wikibase.cloud', + Http::response(['query' => [ + 'statistics' => [ + 'pages' => 19, + 'articles' => 18, + 'edits' => 17, + 'images' => 16, + 'users' => 15, + 'activeusers' => 14, + 'admins' => 13, + 'jobs' => 12, + 'cirrussearch-article-words' => 11 + ] + ]]) + ); + + $this->addFakeFirstRevisionId('this.wikibase.cloud', 2); + $this->addFakeFirstRevisionId('that.wikibase.cloud', 2); + $this->addFakeLastRevisionId('this.wikibase.cloud', 1); + $this->addFakeLastRevisionId('that.wikibase.cloud', 1); + $this->addFakeRevisionTimestamp('that.wikibase.cloud', 2, '2023-05-07T21:31:47Z'); + $this->addFakeRevisionTimestamp('this.wikibase.cloud', 2, '2023-02-27T16:57:06Z'); + $this->addFakeRevisionTimestamp('this.wikibase.cloud', 1, '2023-09-16T17:22:33Z'); + $this->addFakeRevisionTimestamp('that.wikibase.cloud', 1, '2023-09-19T11:35:09Z'); + $this->fakeResponse(); $mockJob = $this->createMock(Job::class); $job = new UpdateWikiSiteStatsJob(); @@ -211,114 +231,82 @@ public function testSuccess() $this->assertEquals($events2['last_edited']->toIso8601String(), '2023-09-19T11:35:09+00:00'); } - public function testFailure() - { + public function testJobFailsIfSiteStatsLookupFails() { Wiki::factory()->create([ 'domain' => 'fail.wikibase.cloud' ]); - Wiki::factory()->create([ - 'domain' => 'that.wikibase.cloud' - ]); + + $this->addFakeSiteStatsResponse( + 'fail.wikibase.cloud', + Http::response('DINOSAUR OUTBREAK!', 500) + ); + + $this->fakeResponse(); + + $mockJob = $this->createMock(Job::class); + $job = new UpdateWikiSiteStatsJob(); + $job->setJob($mockJob); + + $mockJob->expects($this->never())->method('fail'); + $mockJob->expects($this->once())->method('markAsFailed'); + $job->handle(); + } + + public function testIncompleteSiteStatsDoesNotCauseFailure() { Wiki::factory()->create([ 'domain' => 'incomplete.wikibase.cloud' ]); - Http::fake(function (Request $request) { - $responses = [ - getenv('PLATFORM_MW_BACKEND_HOST').'/w/api.php?action=query&meta=siteinfo&siprop=statistics&format=json' => [ - 'fail.wikibase.cloud' => Http::response('DINOSAUR OUTBREAK!', 500), - 'incomplete.wikibase.cloud' => Http::response(['query' => [ - 'statistics' => [ - 'articles' => 99, - 'not' => 129, - 'sure' => 11, - 'what' => 102, - 'happened' => 20 - ] - ]]), - 'that.wikibase.cloud' => Http::response(['query' => [ - 'statistics' => [ - 'pages' => 1, - 'articles' => 2, - 'edits' => 3, - 'images' => 4, - 'users' => 5, - 'activeusers' => 6, - 'admins' => 7, - 'jobs' => 8, - 'cirrussearch-article-words' => 9 - ] - ]]) - ], - getenv('PLATFORM_MW_BACKEND_HOST').'/w/api.php?action=query&format=json&list=allrevisions&formatversion=2&arvlimit=1&arvprop=ids&arvexcludeuser=PlatformReservedUser&arvdir=newer' => [ - 'fail.wikibase.cloud' => Http::response([]), - 'incomplete.wikibase.cloud' => Http::response([ - 'query' => [ - 'allrevisions' => [ - [ - 'revisions' => [ - [ - 'revid' => 1 - ] - ] - ] - ] - ] - ]), - 'that.wikibase.cloud' => Http::response([]), - ], - getenv('PLATFORM_MW_BACKEND_HOST').'/w/api.php?action=query&format=json&prop=revisions&rvprop=timestamp&formatversion=2&revids=1' => [ - 'fail.wikibase.cloud' => Http::response([]), - 'incomplete.wikibase.cloud' => Http::response([ - 'query' => [ - 'pages' => [ - [ - 'revisions' => [ - [ - 'timestamp' => '2023-05-07T21:31:47Z' - ] - ] - ] - ] - ] - ]), - 'that.wikibase.cloud' => Http::response([]), - ], - getenv('PLATFORM_MW_BACKEND_HOST').'/w/api.php?action=query&list=recentchanges&format=json&rcexcludeuser=PlatformReservedUser' => [ - 'fail.wikibase.cloud' => Http::response([]), - 'incomplete.wikibase.cloud' => Http::response('haha whoops', 500), - 'that.wikibase.cloud' => Http::response([]), - ], - ]; + $this->addFakeSiteStatsResponse( + 'incomplete.wikibase.cloud', + Http::response(['query' => [ + 'statistics' => [ + 'articles' => 99, + 'not' => 129, + 'sure' => 11, + 'what' => 102, + 'happened' => 20 + ] + ]]) + ); + + $this->fakeResponse(); - $url = $request->url(); - $hostHeader = $request->header('host')[0]; - // N.B.: using `data_get` is not feasible here as the array keys - // contain dots - if (array_key_exists($url, $responses)) { - if (array_key_exists($hostHeader, $responses[$url])) { - return $responses[$url][$hostHeader]; - } - } - return Http::response('not found', 404); - }); $mockJob = $this->createMock(Job::class); $job = new UpdateWikiSiteStatsJob(); $job->setJob($mockJob); $mockJob->expects($this->never())->method('fail'); - $mockJob->expects($this->once())->method('markAsFailed'); + $mockJob->expects($this->never())->method('markAsFailed'); $job->handle(); - $stats1 = Wiki::with('wikiSiteStats')->where(['domain' => 'that.wikibase.cloud'])->first()->wikiSiteStats()->first(); - $this->assertEquals($stats1['admins'], 7); - $stats2 = Wiki::with('wikiSiteStats')->where(['domain' => 'incomplete.wikibase.cloud'])->first()->wikiSiteStats()->first(); $this->assertEquals($stats2['articles'], 99); $this->assertEquals($stats2['images'], 0); - $events2 = Wiki::with('wikiLifecycleEvents')->where(['domain' => 'incomplete.wikibase.cloud'])->first()->wikiLifecycleEvents()->first(); - $this->assertEquals($events2['first_edited']->toIso8601String(), '2023-05-07T21:31:47+00:00'); - $this->assertEquals($events2['last_edited'], null); } + + public function testNeverEditedWikiCreatesEmptyLifecycleEvents() { + Wiki::factory()->create([ + 'domain' => 'this.wikibase.cloud' + ]); + + $this->addFakeSiteStatsResponse('this.wikibase.cloud', Http::response()); + $this->addFakeEmptyRevisionList('this.wikibase.cloud'); + $this->fakeResponse(); + + + $mockJob = $this->createMock(Job::class); + $job = new UpdateWikiSiteStatsJob(); + $job->setJob($mockJob); + + $mockJob->expects($this->never())->method('fail'); + $mockJob->expects($this->never())->method('markAsFailed'); + $job->handle(); + + $events = Wiki::with('wikiLifecycleEvents')->where(['domain' => 'this.wikibase.cloud'])->first()->wikiLifecycleEvents()->first(); + $this->assertNull($events['first_edited']); + $this->assertNull($events['last_edited']); + } + }