From 34145736d0ba78b7f9142fe2bb0ec89ab09670d8 Mon Sep 17 00:00:00 2001 From: Andrew Newton Date: Thu, 3 Oct 2024 12:21:57 +0100 Subject: [PATCH] feat: Update DataGovUkExport.php to upload to S3 bucket. (#357) --- app/api/config/autoload/config.global.php | 8 +- app/api/config/autoload/local.php.dist | 7 + app/api/module/Api/config/module.config.php | 2 + .../Api/src/Service/S3/S3ClientFactory.php | 23 ++++ .../CommandHandler/AbstractDataExport.php | 121 +++++++++++++----- .../Domain/CommandHandler/DataDvaNiExport.php | 30 +++-- .../Domain/CommandHandler/DataGovUkExport.php | 25 ++-- .../CommandHandler/DataDvaNiExportTest.php | 66 +++++----- .../CommandHandler/DataGovUkExportTest.php | 106 ++++++--------- 9 files changed, 231 insertions(+), 157 deletions(-) create mode 100644 app/api/module/Api/src/Service/S3/S3ClientFactory.php diff --git a/app/api/config/autoload/config.global.php b/app/api/config/autoload/config.global.php index 27a9323f2f..f30bf42e40 100644 --- a/app/api/config/autoload/config.global.php +++ b/app/api/config/autoload/config.global.php @@ -408,9 +408,13 @@ 'export_path' => '/tmp/ViExtract' ], - // Path to export CSV data for data.gov.uk + // s3 bucket URI to export CSV data for data.gov.uk 'data-gov-uk-export' => [ - 'path' => '/tmp/dataGovUk', + 's3_uri' => '%data-gov-uk-export-s3uri%' + ], + + 'data-dva-ni-export' => [ + 's3_uri' => '%data-dva-ni-export-s3uri%' ], // Path to export CSV data for Companies House differences diff --git a/app/api/config/autoload/local.php.dist b/app/api/config/autoload/local.php.dist index 7fd5882d84..b629171e12 100644 --- a/app/api/config/autoload/local.php.dist +++ b/app/api/config/autoload/local.php.dist @@ -373,4 +373,11 @@ return [ 'proxy' => new \Laminas\Stdlib\ArrayUtils\MergeRemoveKey(), ], ], + 'data-gov-uk-export' => [ + 's3_uri' => 's3://devapp-vol-content/olcs.local.nonprod.dvsa.aws/data-gov-uk-export/' + ], + + 'data-dva-ni-export' => [ + 's3_uri' => 's3://devapp-olcs-pri-integration-dva-s3/ + ], ]; diff --git a/app/api/module/Api/config/module.config.php b/app/api/module/Api/config/module.config.php index 3904e9fbba..90ac791df9 100644 --- a/app/api/module/Api/config/module.config.php +++ b/app/api/module/Api/config/module.config.php @@ -522,6 +522,8 @@ ApiSrv\GovUkAccount\GovUkAccountService::class => ApiSrv\GovUkAccount\GovUkAccountServiceFactory::class, ApiSrv\AddressHelper\AddressHelperService::class => ApiSrv\AddressHelper\AddressHelperServiceFactory::class, + + Aws\S3\S3Client::class => Dvsa\Olcs\Api\Service\S3\S3ClientFactory::class, ], ], 'view_manager' => [ diff --git a/app/api/module/Api/src/Service/S3/S3ClientFactory.php b/app/api/module/Api/src/Service/S3/S3ClientFactory.php new file mode 100644 index 0000000000..2e8c275af7 --- /dev/null +++ b/app/api/module/Api/src/Service/S3/S3ClientFactory.php @@ -0,0 +1,23 @@ +get('config'); + $awsOptions = $config['awsOptions']; + + $s3Client = new S3Client([ + 'version' => $awsOptions['version'], + 'region' => $awsOptions['region'], + 'use_path_style_endpoint' => $awsOptions['s3']['use_path_style_endpoint'], + ]); + + return $s3Client; + } +} diff --git a/app/api/module/Cli/src/Domain/CommandHandler/AbstractDataExport.php b/app/api/module/Cli/src/Domain/CommandHandler/AbstractDataExport.php index e609af6cbc..597d478a30 100644 --- a/app/api/module/Cli/src/Domain/CommandHandler/AbstractDataExport.php +++ b/app/api/module/Cli/src/Domain/CommandHandler/AbstractDataExport.php @@ -4,11 +4,11 @@ namespace Dvsa\Olcs\Cli\Domain\CommandHandler; +use Aws\S3\S3Client; use Doctrine\DBAL\Result; use Dvsa\Olcs\Api\Domain\Util\DateTime\DateTime; use Dvsa\Olcs\Api\Entity\TrafficArea\TrafficArea as TrafficAreaEntity; use Dvsa\Olcs\Api\Domain\CommandHandler\AbstractCommandHandler; -use Dvsa\Olcs\Cli\Service\Utils\ExportToCsv; use Dvsa\Olcs\Api\Domain\QueueAwareTrait; use Dvsa\Olcs\Api\Domain\Repository; use Dvsa\Olcs\Api\Service\Exception; @@ -36,15 +36,13 @@ abstract class AbstractDataExport extends AbstractCommandHandler 'Licence' ]; - /** - * @var string - */ - protected $path; + protected string $path; - /** - * @var array - */ - private $csvPool = []; + private array $csvPool = []; + + protected S3Client $s3Client; + + protected string $s3Bucket; /** * Fill a CSV with the result of a doctrine statement @@ -58,18 +56,16 @@ abstract class AbstractDataExport extends AbstractCommandHandler protected function singleCsvFromDbalResult(Result $dbalResult, $fileName, $fileNameSeparator = '_') { $date = new DateTime('now'); + $fileBaseName = $fileName . $fileNameSeparator . $date->format(static::FILE_DATETIME_FORMAT) . '.csv'; - $filePath = $this->path . '/' . $fileName . $fileNameSeparator . $date->format(static::FILE_DATETIME_FORMAT) . '.csv'; + $tempCsvPath = sys_get_temp_dir() . '/' . $fileBaseName; + $this->result->addMessage('Creating CSV file: ' . $tempCsvPath); + $fh = fopen($tempCsvPath, 'w'); - // create csv file - $this->result->addMessage('create csv file: ' . $filePath); - $fh = ExportToCsv::createFile($filePath); $firstRow = false; - // add rows while (($row = $dbalResult->fetchAssociative()) !== false) { if (!$firstRow) { - //add title fputcsv($fh, array_keys($row)); $firstRow = true; } @@ -79,7 +75,7 @@ protected function singleCsvFromDbalResult(Result $dbalResult, $fileName, $fileN fclose($fh); - return file_get_contents($filePath); + return $tempCsvPath; } /** @@ -93,35 +89,38 @@ protected function singleCsvFromDbalResult(Result $dbalResult, $fileName, $fileN */ protected function makeCsvsFromDbalResult(Result $dbalResult, $keyFld, $fileName) { - // add rows + $filePaths = []; + $fileHandles = []; + + // add rows while (($row = $dbalResult->fetchAssociative()) !== false) { $key = $row[$keyFld]; - if (!isset($this->csvPool[$key])) { - // create csv file - $filePath = $this->path . '/' . $fileName . '_' . $key . '.csv'; + if (!isset($fileHandles[$key])) { + $fileBaseName = $fileName . '_' . $key . '.csv'; + $filePath = sys_get_temp_dir() . '/' . $fileBaseName; - $this->result->addMessage('create csv file: ' . $filePath); - $fh = ExportToCsv::createFile($filePath); + $this->result->addMessage('Creating CSV file: ' . $filePath); + $fh = fopen($filePath, 'w'); - // add title & first row fputcsv($fh, array_keys($row)); fputcsv($fh, $row); - $this->csvPool[$key] = $fh; + $fileHandles[$key] = $fh; + $filePaths[$key] = $filePath; continue; } - // add rows to csv from pool - $fh = $this->csvPool[$key]; - + $fh = $fileHandles[$key]; fputcsv($fh, $row); } - // close files - foreach ($this->csvPool as $fh) { + foreach ($fileHandles as $key => $fh) { fclose($fh); + $filePath = $filePaths[$key]; + $this->uploadToS3($filePath); + unlink($filePath); } } @@ -179,4 +178,68 @@ protected function getTrafficAreas() return $items; } + + protected function createManifest(array $filePaths) + { + $manifestLines = []; + + foreach ($filePaths as $filePath) { + $hash = hash_file('sha256', $filePath); + $fileName = basename($filePath); + $manifestLines[] = $hash . ' ' . $fileName; + } + + $manifestContent = implode("\n", $manifestLines); + + $manifestPath = sys_get_temp_dir() . '/dvaoplic-manifest.txt'; + file_put_contents($manifestPath, $manifestContent); + + return $manifestPath; + } + + protected function createTarGzArchive(array $filePaths, $manifestPath) + { + $date = new DateTime('now'); + $archiveBaseName = 'dvaoplic-' . $date->format(static::FILE_DATETIME_FORMAT) . '.tar'; + $archiveGzBaseName = $archiveBaseName . '.gz'; + $archivePath = sys_get_temp_dir() . '/' . $archiveBaseName; + $archiveGzPath = sys_get_temp_dir() . '/' . $archiveGzBaseName; + + $tar = new \PharData($archivePath); + + foreach ($filePaths as $filePath) { + $tar->addFile($filePath, basename($filePath)); + } + + $tar->addFile($manifestPath, basename($manifestPath)); + $tar->compress(\Phar::GZ); + unlink($archivePath); + + return $archiveGzPath; + } + + protected function uploadToS3($filePath) + { + $fileName = basename($filePath); + $fileResource = fopen($filePath, 'r'); + + $this->s3Client->putObject([ + 'Bucket' => $this->s3Bucket, + 'Key' => $this->path . '/' . $fileName, + 'Body' => $fileResource, + ]); + + fclose($fileResource); + + $this->result->addMessage('Uploaded file to S3: ' . $fileName); + } + + protected function cleanUpFiles(array $filePaths) + { + foreach ($filePaths as $filePath) { + if (file_exists($filePath)) { + unlink($filePath); + } + } + } } diff --git a/app/api/module/Cli/src/Domain/CommandHandler/DataDvaNiExport.php b/app/api/module/Cli/src/Domain/CommandHandler/DataDvaNiExport.php index 1363ff332d..25f57be729 100644 --- a/app/api/module/Cli/src/Domain/CommandHandler/DataDvaNiExport.php +++ b/app/api/module/Cli/src/Domain/CommandHandler/DataDvaNiExport.php @@ -2,6 +2,7 @@ namespace Dvsa\Olcs\Cli\Domain\CommandHandler; +use Aws\S3\S3Client; use Dvsa\Olcs\Transfer\Command\CommandInterface; use Psr\Container\ContainerInterface; use Dvsa\Olcs\Api\Domain\QueueAwareTrait; @@ -29,11 +30,6 @@ final class DataDvaNiExport extends AbstractDataExport */ private $reportName; - /** - * @var string - */ - protected $path; - /** * @var Repository\DataDvaNi */ @@ -50,7 +46,6 @@ final class DataDvaNiExport extends AbstractDataExport */ public function handleCommand(CommandInterface $command) { - $this->path = (trim($command->getPath()) ?: $this->path); $this->reportName = $command->getReportName(); $this->dataDvaNiRepo = $this->getRepo(); @@ -71,20 +66,33 @@ public function handleCommand(CommandInterface $command) */ private function processNiOperatorLicences() { - $this->result->addMessage('Fetching data from DB for NI Operator Licences'); $dbalResult = $this->dataDvaNiRepo->fetchNiOperatorLicences(); - $this->singleCsvFromDbalResult($dbalResult, 'NiGvLicences', '-'); + $csvFilePath = $this->singleCsvFromDbalResult($dbalResult, 'NiGvLicences', '-'); + + $manifestPath = $this->createManifest([$csvFilePath]); + + $archivePath = $this->createTarGzArchive([$csvFilePath], $manifestPath); + + $this->uploadToS3($archivePath); + + $this->cleanUpFiles([$csvFilePath, $manifestPath, $archivePath]); } public function __invoke(ContainerInterface $container, $requestedName, array $options = null) { $config = $container->get('config'); - $exportCfg = (!empty($config['data-dva-ni-export']) ? $config['data-dva-ni-export'] : []); - if (isset($exportCfg['path'])) { - $this->path = $exportCfg['path']; + $exportCfg = $config['data-dva-ni-export'] ?? []; + + if (isset($exportCfg['s3_uri'])) { + $parsedUrl = parse_url(rtrim($exportCfg['s3_uri'], '/')); + $this->s3Bucket = $parsedUrl['host']; + $this->path = ltrim($parsedUrl['path'], '/'); } + + $this->s3Client = $container->get(S3Client::class); + return parent::__invoke($container, $requestedName, $options); } } diff --git a/app/api/module/Cli/src/Domain/CommandHandler/DataGovUkExport.php b/app/api/module/Cli/src/Domain/CommandHandler/DataGovUkExport.php index 5939ea5fff..b824ef7756 100644 --- a/app/api/module/Cli/src/Domain/CommandHandler/DataGovUkExport.php +++ b/app/api/module/Cli/src/Domain/CommandHandler/DataGovUkExport.php @@ -2,6 +2,7 @@ namespace Dvsa\Olcs\Cli\Domain\CommandHandler; +use Aws\S3\S3Client; use Dvsa\Olcs\Api\Entity\TrafficArea\TrafficArea as TrafficAreaEntity; use Dvsa\Olcs\Api\Domain\Command\Email\SendPsvOperatorListReport; use Dvsa\Olcs\Api\Domain\Command\Email\SendInternationalGoods as SendIntlGoodsEmailCmd; @@ -45,11 +46,6 @@ final class DataGovUkExport extends AbstractDataExport */ private $reportName; - /** - * @var string - */ - protected $path; - /** * @var Repository\DataGovUk */ @@ -66,9 +62,7 @@ final class DataGovUkExport extends AbstractDataExport */ public function handleCommand(CommandInterface $command) { - $this->path = trim($command->getPath() ?? '') ?: $this->path; $this->reportName = $command->getReportName(); - $this->dataGovUkRepo = $this->getRepo(); if ($this->reportName === self::OPERATOR_LICENCE) { @@ -110,7 +104,6 @@ private function processPsvOperatorList() ['id' => $document->getId('document')], $document->getId('document') ); - $email = $this->handleSideEffect($emailQueue); $this->result->merge($email); @@ -129,7 +122,6 @@ private function processInternationalGoodsList() /** @var Repository\Licence $repo */ $repo = $this->getRepo('Licence'); $dbalResult = $repo->internationalGoodsReport(); - $csvContent = $this->singleCsvFromDbalResult($dbalResult, 'international_goods'); $document = $this->handleSideEffect( @@ -205,7 +197,6 @@ private function processOperatorLicences() $this->result->addMessage('Fetching data from DB for Operator Licences'); $dbalResult = $this->dataGovUkRepo->fetchOperatorLicences($areas); - $this->makeCsvsFromDbalResult($dbalResult, 'GeographicRegion', 'OLBSLicenceReport'); } @@ -223,7 +214,6 @@ private function processBusRegOnly() $this->result->addMessage('Fetching data from DB for Bus Registered Only'); $dbalResult = $this->dataGovUkRepo->fetchBusRegisteredOnly($areas); - $this->makeCsvsFromDbalResult($dbalResult, 'Current Traffic Area', 'Bus_RegisteredOnly'); } @@ -241,17 +231,22 @@ private function processBusVariation() $this->result->addMessage('Fetching data from DB for Bus Variation'); $dbalResult = $this->dataGovUkRepo->fetchBusVariation($areas); - $this->makeCsvsFromDbalResult($dbalResult, 'Current Traffic Area', 'Bus_Variation'); } public function __invoke(ContainerInterface $container, $requestedName, array $options = null) { $config = $container->get('config'); - $exportCfg = (!empty($config['data-gov-uk-export']) ? $config['data-gov-uk-export'] : []); - if (isset($exportCfg['path'])) { - $this->path = $exportCfg['path']; + $exportCfg = $config['data-gov-uk-export'] ?? []; + + if (isset($exportCfg['s3_uri'])) { + $parsedUrl = parse_url($exportCfg['s3_uri']); + $this->s3Bucket = $parsedUrl['host']; + $this->path = ltrim($parsedUrl['path'], '/'); } + + $this->s3Client = $container->get(S3Client::class); + return parent::__invoke($container, $requestedName, $options); } } diff --git a/app/api/test/module/Cli/src/Domain/CommandHandler/DataDvaNiExportTest.php b/app/api/test/module/Cli/src/Domain/CommandHandler/DataDvaNiExportTest.php index 4a51d98ff9..3e55ab7ef3 100644 --- a/app/api/test/module/Cli/src/Domain/CommandHandler/DataDvaNiExportTest.php +++ b/app/api/test/module/Cli/src/Domain/CommandHandler/DataDvaNiExportTest.php @@ -2,12 +2,12 @@ namespace Dvsa\OlcsTest\Cli\Domain\CommandHandler; +use Aws\S3\S3Client; use Doctrine\DBAL\Result; use Dvsa\OlcsTest\Api\Domain\CommandHandler\AbstractCommandHandlerTestCase; use Dvsa\Olcs\Cli\Domain\Command\DataDvaNiExport as Cmd; use Dvsa\Olcs\Cli\Domain\CommandHandler\DataDvaNiExport; use Dvsa\Olcs\Api\Domain\Repository; -use org\bovigo\vfs\vfsStream; use Mockery as m; use Dvsa\Olcs\Api\Domain\Util\DateTime\DateTime; @@ -22,20 +22,7 @@ class DataDvaNiExportTest extends AbstractCommandHandlerTestCase */ protected $sut; - /** - * @var string - */ - private $tmpPath; - - /** - * @var vfsStream - */ - private $vfsStream; - - /** - * @var m\MockInterface - */ - private $mockStmt; + private $tempDir; public function setUp(): void { @@ -49,19 +36,40 @@ public function setUp(): void $this->mockDbalResult = m::mock(Result::class); - // mock config $this->mockedSmServices['config'] = [ 'data-dva-ni-export' => [ - 'path' => 'unit_CfgPath', + 's3_uri' => 's3://testbucket/testprefix/', ], ]; + $this->mockS3client = m::mock(S3Client::class); + $this->mockedSmServices[S3Client::class] = $this->mockS3client; + $this->sut = new DataDvaNiExport(); parent::setUp(); + } - $this->vfsStream = vfsStream::setup('root'); - $this->tmpPath = $this->vfsStream->url() . '/unit'; + private function createTempDir() + { + $this->tempDir = sys_get_temp_dir() . '/phpunit_' . uniqid(); + mkdir($this->tempDir); + return $this->tempDir; + } + + private function cleanupRealTempDir() + { + if ($this->tempDir && is_dir($this->tempDir)) { + $files = new \RecursiveIteratorIterator( + new \RecursiveDirectoryIterator($this->tempDir, \RecursiveDirectoryIterator::SKIP_DOTS), + \RecursiveIteratorIterator::CHILD_FIRST + ); + foreach ($files as $fileinfo) { + $todo = ($fileinfo->isDir() ? 'rmdir' : 'unlink'); + $todo($fileinfo->getRealPath()); + } + rmdir($this->tempDir); + } } public function testInvalidReportException() @@ -82,10 +90,12 @@ public function testInvalidReportException() public function testNiOperatorLicence() { + $tempDir = $this->createTempDir(); + $cmd = Cmd::create( [ 'reportName' => DataDvaNiExport::NI_OPERATOR_LICENCE, - 'path' => $this->tmpPath, + 'path' => $tempDir, ] ); @@ -112,28 +122,24 @@ public function testNiOperatorLicence() ->once() ->andReturn($this->mockDbalResult); + $this->mockS3client->shouldReceive('putObject')->once()->andReturn([]); + // call & check $actual = $this->sut->handleCommand($cmd); $date = new DateTime('now'); - $expectFile = $this->tmpPath . '/NiGvLicences-' . $date->format(DataDvaNiExport::FILE_DATETIME_FORMAT) . '.csv'; + $expectCsvFile = '/tmp/NiGvLicences-' . $date->format(DataDvaNiExport::FILE_DATETIME_FORMAT) . '.csv'; + $expectTgzFile = 'dvaoplic-' . $date->format(DataDvaNiExport::FILE_DATETIME_FORMAT) . '.tar.gz'; $expectMsg = 'Fetching data from DB for NI Operator Licences' . - 'create csv file: ' . $expectFile; + 'Creating CSV file: ' . $expectCsvFile . + 'Uploaded file to S3: ' . $expectTgzFile; static::assertEquals( $expectMsg, implode('', $actual->toArray()['messages']) ); - - static::assertSame( - 'LicenceNumber,LicenceType' . "\n" . - '123455,test_type' . "\n" . - '123456,test_type' . "\n" . - '123457,test_type' . "\n", - file_get_contents($expectFile) - ); } } diff --git a/app/api/test/module/Cli/src/Domain/CommandHandler/DataGovUkExportTest.php b/app/api/test/module/Cli/src/Domain/CommandHandler/DataGovUkExportTest.php index defdc1951b..387ed89f79 100644 --- a/app/api/test/module/Cli/src/Domain/CommandHandler/DataGovUkExportTest.php +++ b/app/api/test/module/Cli/src/Domain/CommandHandler/DataGovUkExportTest.php @@ -2,6 +2,7 @@ namespace Dvsa\OlcsTest\Cli\Domain\CommandHandler; +use Aws\S3\S3Client; use Dvsa\Olcs\Api\Rbac\IdentityProviderInterface; use Dvsa\OlcsTest\Api\Domain\CommandHandler\AbstractCommandHandlerTestCase; use Dvsa\Olcs\Api\Domain\Command\Email\SendPsvOperatorListReport; @@ -18,10 +19,8 @@ use Dvsa\Olcs\Api\Domain\Command\Result; use Dvsa\Olcs\Api\Domain\Repository; use Dvsa\Olcs\Email\Service\Email; -use org\bovigo\vfs\vfsStream; +use Psr\Container\ContainerInterface; use Mockery as m; -use org\bovigo\vfs\vfsStreamDirectory; -use org\bovigo\vfs\vfsStreamFile; use Dvsa\Olcs\Api\Domain\Util\DateTime\DateTime; /** @@ -39,18 +38,18 @@ class DataGovUkExportTest extends AbstractCommandHandlerTestCase */ private $tmpPath; - /** - * @var vfsStream - */ - private $vfsStream; - /** * @var m\MockInterface */ private $mockDbalResult; + protected $mockS3client; + + public function setUp(): void { + $this->tmpPath = sys_get_temp_dir(); + // mock repos $this->mockRepo('DataGovUk', Repository\DataGovUk::class); $this->mockRepo('TrafficArea', Repository\TrafficArea::class); @@ -61,10 +60,9 @@ public function setUp(): void $this->mockDbalResult = m::mock(\Doctrine\DBAL\Result::class); - // mock config $this->mockedSmServices['config'] = [ 'data-gov-uk-export' => [ - 'path' => 'unit_CfgPath', + 's3_uri' => 's3://testbucket/testprefix/', ], ]; @@ -81,12 +79,14 @@ public function setUp(): void $this->mockedSmServices['FileUploader'] = $mockFileUploader; $this->mockedSmServices['DocumentNamingService'] = $mockDocumentNaming; + $this->mockS3client = m::mock(S3Client::class); + $this->mockedSmServices[S3Client::class] = $this->mockS3client; + $sm = m::mock(ContainerInterface::class); + $sm->shouldReceive('get')->with(S3Client::class)->andReturn($this->mockS3client); + $this->sut = new DataGovUkExport(); parent::setUp(); - - $this->vfsStream = vfsStream::setup('root'); - $this->tmpPath = $this->vfsStream->url() . '/unit'; } protected function initReferences() @@ -123,13 +123,6 @@ public function testInternationalGoods() { $fileName = 'international_goods'; - $vfsFile = new vfsStreamFile($fileName, 0777); - $tmpFolder = new vfsStreamDirectory('unit'); - $tmpFolder->addChild($vfsFile); - - $directory = new vfsStreamDirectory('root'); - $directory->addChild($tmpFolder); - $row1 = [ 'Licence number' => 'LicNo1', 'col1' => 'val11', @@ -179,7 +172,7 @@ public function testInternationalGoods() new Result() ); - // call & check + // Call & check $cmd = Cmd::create( [ 'reportName' => DataGovUkExport::INTERNATIONAL_GOODS, @@ -195,16 +188,7 @@ public function testInternationalGoods() $expectMsg = 'Fetching data for international goods list' . - 'create csv file: ' . $expectedFile; - - $this->assertSame( - '"Licence number",col1,col2' . "\n" . - 'LicNo1,val11,"v""\'-/\\,"' . "\n" . - 'LicNo2,val21,val22' . "\n" . - 'LicNo3,val31,val32' . "\n" . - '', - file_get_contents($expectedFile) - ); + 'Creating CSV file: ' . $expectedFile; $this->assertEquals( $expectMsg, @@ -216,12 +200,6 @@ public function testPsvOperatorListOk() { $fileName = 'psv-operator-list.csv'; - $vfsFile = new vfsStreamFile($fileName, 0777); - $tmpFolder = new vfsStreamDirectory('unit'); - $tmpFolder->addChild($vfsFile); - - $directory = new vfsStreamDirectory('root'); - $directory->addChild($tmpFolder); $row1 = [ 'Licence number' => 'areaName1', @@ -318,8 +296,6 @@ public function testPsvOperatorListOk() $actual = $this->sut->handleCommand($cmd); - $expectFile = $this->tmpPath . '/' . $fileName; - $expectMsg = 'Fetching data from DB for PSV Operators' . 'create csv file content'; @@ -368,32 +344,27 @@ public function testOperatorLicenceOk() ->once() ->andReturn($this->mockDbalResult); + $this->mockS3client->shouldAllowMockingMethod('putObject'); + $this->mockS3client->shouldReceive('putObject') + ->twice() + ->andReturn([]); + // call & check $actual = $this->sut->handleCommand($cmd); - $expectFile1 = $this->tmpPath . '/OLBSLicenceReport_areaName1.csv'; - $expectFile2 = $this->tmpPath . '/OLBSLicenceReport_areaName2.csv'; + $expectFile1 = $this->tmpPath. '/OLBSLicenceReport_areaName1.csv'; + $expectFile2 = $this->tmpPath. '/OLBSLicenceReport_areaName2.csv'; $expectMsg = 'Fetching data from DB for Operator Licences' . - 'create csv file: ' . $expectFile1 . - 'create csv file: ' . $expectFile2; + 'Creating CSV file: ' . $expectFile1 . + 'Creating CSV file: ' . $expectFile2 . + 'Uploaded file to S3: OLBSLicenceReport_areaName1.csvUploaded file to S3: OLBSLicenceReport_areaName2.csv'; static::assertEquals( $expectMsg, implode('', $actual->toArray()['messages']) ); - - static::assertSame( - 'GeographicRegion,col1,col2' . "\n" . - 'areaName1,val11,"v""\'-/\,"' . "\n" . - 'areaName1,val31,val32' . "\n", - file_get_contents($expectFile1) - ); - static::assertSame( - 'GeographicRegion,col1,col2' . "\n" . 'areaName2,val21,val22' . "\n", - file_get_contents($expectFile2) - ); } public function testBugRegOnlyOk() @@ -428,26 +399,23 @@ public function testBugRegOnlyOk() ->once() ->andReturn($this->mockDbalResult); + $this->mockS3client->shouldReceive('putObject') + ->once() + ->andReturn([]); + // call & check $actual = $this->sut->handleCommand($cmd); - $expectFile1 = $this->tmpPath . '/Bus_RegisteredOnly_areaId1.csv'; + $expectFile1 = $this->tmpPath. '/Bus_RegisteredOnly_areaId1.csv'; $expectMsg = 'Fetching data from DB for Bus Registered Only' . - 'create csv file: ' . $expectFile1; + 'Creating CSV file: ' . $expectFile1.'Uploaded file to S3: Bus_RegisteredOnly_areaId1.csv'; static::assertEquals( $expectMsg, implode('', $actual->toArray()['messages']) ); - - static::assertSame( - '"Current Traffic Area",col1,col2' . "\n" . - 'areaId1,val11,"v""\'-/\,"' . "\n" . - 'areaId1,val21,val22' . "\n", - file_get_contents($expectFile1) - ); } public function testBugVariationOk() @@ -476,6 +444,10 @@ public function testBugVariationOk() ->once() ->andReturn($this->mockDbalResult); + $this->mockS3client->shouldReceive('putObject') + ->once() + ->andReturn([]); + // call & check $actual = $this->sut->handleCommand($cmd); @@ -483,18 +455,12 @@ public function testBugVariationOk() $expectMsg = 'Fetching data from DB for Bus Variation' . - 'create csv file: ' . $expectFile1; + 'Creating CSV file: ' . $expectFile1.'Uploaded file to S3: Bus_Variation_areaId1.csv'; static::assertEquals( $expectMsg, implode('', $actual->toArray()['messages']) ); - - static::assertSame( - '"Current Traffic Area",col1,col2' . "\n" . - 'areaId1,val11,"v""\'-/\,"' . "\n", - file_get_contents($expectFile1) - ); } public function testTrafficAreaNotFound()