Skip to content

Commit

Permalink
Merge pull request #1092 from pnax/new-table-result-entries
Browse files Browse the repository at this point in the history 
New result entries table

Store the `test_results.results` field into a dedicated `result_entries`
table.
  • Loading branch information
Alexandre Pion committed Nov 21, 2023
2 parents 8ebe174 + 33e32e4 commit dfd197d
Show file tree
Hide file tree
Showing 9 changed files with 676 additions and 60 deletions.
1 change: 1 addition & 0 deletions MANIFEST
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ share/locale/nb/LC_MESSAGES/Zonemaster-Backend.mo
share/locale/sv/LC_MESSAGES/Zonemaster-Backend.mo
share/Makefile
share/patch/patch_db_zonemaster_backend_ver_9.0.0.pl
share/patch/patch_db_zonemaster_backend_ver_11.0.3.pl
share/patch/patch_mysql_db_zonemaster_backend_ver_1.0.3.pl
share/patch/patch_mysql_db_zonemaster_backend_ver_5.0.0.pl
share/patch/patch_mysql_db_zonemaster_backend_ver_5.0.2.pl
Expand Down
193 changes: 137 additions & 56 deletions lib/Zonemaster/Backend/DB.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ use Readonly;
use Try::Tiny;

use Zonemaster::Backend::Errors;
use Zonemaster::Engine::Logger::Entry;

requires qw(
add_batch_job
Expand Down Expand Up @@ -371,6 +372,34 @@ sub test_state {
}
}

sub set_test_completed {
my ( $self, $test_id ) = @_;

my $current_state = $self->test_state( $test_id );

if ( $current_state ne $TEST_RUNNING ) {
die Zonemaster::Backend::Error::Internal->new( reason => 'illegal transition to COMPLETED' );
}

my $rows_affected = $self->dbh->do(
q[
UPDATE test_results
SET progress = 100,
ended_at = ?
WHERE hash_id = ?
AND 0 < progress
AND progress < 100
],
undef,
$self->format_time( time() ),
$test_id,
);

if ( $rows_affected == 0 ) {
die Zonemaster::Backend::Error::Internal->new( reason => "job not found or illegal transition" );
}
}

sub select_test_results {
my ( $self, $test_id ) = @_;

Expand All @@ -379,8 +408,7 @@ sub select_test_results {
SELECT
hash_id,
created_at,
params,
results
params
FROM test_results
WHERE hash_id = ?
],
Expand Down Expand Up @@ -431,14 +459,35 @@ sub test_results {

my $result = $self->select_test_results( $test_id );

my @result_entries = $self->dbh->selectall_array(
q[
SELECT
l.level,
r.module,
r.testcase,
r.tag,
r.timestamp,
r.args
FROM result_entries r
INNER JOIN log_level l
ON r.level = l.value
WHERE hash_id = ?
],
{ Slice => {} },
$test_id
);

eval {
$result->{params} = decode_json( $result->{params} );

if (defined $result->{results}) {
$result->{results} = decode_json( $result->{results} );
} else {
$result->{results} = [];
}
@result_entries = map {
{
%$_,
args => decode_json( $_->{args} ),
}
} @result_entries;

$result->{results} = \@result_entries;
};

die Zonemaster::Backend::Error::JsonError->new( reason => "$@", data => { test_id => $test_id } )
Expand All @@ -462,11 +511,13 @@ sub get_test_history {
my @results;
my $query = q[
SELECT
(SELECT count(*) FROM result_entries WHERE result_entries.hash_id = test_results.hash_id AND level = ?) AS nb_critical,
(SELECT count(*) FROM result_entries WHERE result_entries.hash_id = test_results.hash_id AND level = ?) AS nb_error,
(SELECT count(*) FROM result_entries WHERE result_entries.hash_id = test_results.hash_id AND level = ?) AS nb_warning,
id,
hash_id,
created_at,
undelegated,
results
undelegated
FROM test_results
WHERE progress = 100 AND domain = ? AND ( ? IS NULL OR undelegated = ? )
ORDER BY id DESC
Expand All @@ -475,33 +526,37 @@ sub get_test_history {

my $sth = $dbh->prepare( $query );

$sth->bind_param( 1, _normalize_domain( $p->{frontend_params}{domain} ) );
$sth->bind_param( 2, $undelegated, SQL_INTEGER );
$sth->bind_param( 3, $undelegated, SQL_INTEGER );
$sth->bind_param( 4, $p->{limit} );
$sth->bind_param( 5, $p->{offset} );
my %levels = Zonemaster::Engine::Logger::Entry->levels();
$sth->bind_param( 1, $levels{CRITICAL} );
$sth->bind_param( 2, $levels{ERROR} );
$sth->bind_param( 3, $levels{WARNING} );
$sth->bind_param( 4, _normalize_domain( $p->{frontend_params}{domain} ) );
$sth->bind_param( 5, $undelegated, SQL_INTEGER );
$sth->bind_param( 6, $undelegated, SQL_INTEGER );
$sth->bind_param( 7, $p->{limit} );
$sth->bind_param( 8, $p->{offset} );

$sth->execute();

while ( my $h = $sth->fetchrow_hashref ) {
$h->{results} = decode_json($h->{results}) if $h->{results};
my $critical = ( grep { $_->{level} eq 'CRITICAL' } @{ $h->{results} } );
my $error = ( grep { $_->{level} eq 'ERROR' } @{ $h->{results} } );
my $warning = ( grep { $_->{level} eq 'WARNING' } @{ $h->{results} } );

# More important overwrites
my $overall = 'ok';
$overall = 'warning' if $warning;
$overall = 'error' if $error;
$overall = 'critical' if $critical;
my $overall_result = 'ok';
if ( $h->{nb_critical} ) {
$overall_result = 'critical';
}
elsif ( $h->{nb_error} ) {
$overall_result = 'error';
}
elsif ( $h->{nb_warning} ) {
$overall_result = 'warning';
}

push(
@results,
{
id => $h->{hash_id},
created_at => $self->to_iso8601( $h->{created_at} ),
undelegated => $h->{undelegated},
overall_result => $overall,
overall_result => $overall_result,
}
);
}
Expand Down Expand Up @@ -725,15 +780,18 @@ sub process_unfinished_tests {
$test_run_timeout,
);

my $msg = {
"level" => "CRITICAL",
"module" => "BACKEND_TEST_AGENT",
"tag" => "UNABLE_TO_FINISH_TEST",
"args" => { max_execution_time => $test_run_timeout },
"timestamp" => $test_run_timeout
};
my $msg = Zonemaster::Engine::Logger::Entry->new(
{
level => "CRITICAL",
module => "BACKEND_TEST_AGENT",
testcase => "",
tag => "UNABLE_TO_FINISH_TEST",
args => { max_execution_time => $test_run_timeout },
timestamp => $test_run_timeout
}
);
while ( my $h = $sth1->fetchrow_hashref ) {
$self->force_end_test($h->{hash_id}, $h->{results}, $msg);
$self->force_end_test($h->{hash_id}, $msg);
}
}

Expand Down Expand Up @@ -775,44 +833,40 @@ sub select_unfinished_tests {
}
}

=head2 force_end_test($hash_id, $results, $msg)
=head2 force_end_test($hash_id, $msg)
Append the $msg log entry to the $results arrayref and store the results into
the database.
Store the L<Zonemaster::Engine::Logger::Entry> $msg log entry into the database
and mark test with $hash_id as COMPLETED.
=cut

sub force_end_test {
my ( $self, $hash_id, $results, $msg ) = @_;
my $result;
if ( defined $results && $results =~ /^\[/ ) {
$result = decode_json( $results );
}
else {
$result = [];
}
push @$result, $msg;
my ( $self, $hash_id, $msg ) = @_;

$self->store_results( $hash_id, encode_json($result) );
$self->add_result_entries( $hash_id, $msg );
$self->set_test_completed( $hash_id );
}

=head2 process_dead_test($hash_id)
Append a new log entry C<BACKEND_TEST_AGENT:TEST_DIED> to the test with $hash_id.
Then store the results in database.
Store a new log entry C<BACKEND_TEST_AGENT:TEST_DIED> in database for the test
with $hash_id.
=cut

sub process_dead_test {
my ( $self, $hash_id ) = @_;
my ( $results ) = $self->dbh->selectrow_array("SELECT results FROM test_results WHERE hash_id = ?", undef, $hash_id);
my $msg = {
"level" => "CRITICAL",
"module" => "BACKEND_TEST_AGENT",
"tag" => "TEST_DIED",
"timestamp" => $self->get_relative_start_time($hash_id)
};
$self->force_end_test($hash_id, $results, $msg);
my $msg = Zonemaster::Engine::Logger::Entry->new(
{
level => "CRITICAL",
module => "BACKEND_TEST_AGENT",
testcase => "",
tag => "TEST_DIED",
args => {},
timestamp => $self->get_relative_start_time($hash_id)
}
);
$self->force_end_test($hash_id, $msg);
}

# Converts the domain to lowercase and if the domain is not the root ('.')
Expand Down Expand Up @@ -945,6 +999,33 @@ sub to_iso8601 {
return $time;
}

sub add_result_entries {
my ( $self, $hash_id, @entries ) = @_;
my @records;

my $json = JSON::PP->new->allow_blessed->convert_blessed->canonical;

my %levels = Zonemaster::Engine::Logger::Entry->levels();

foreach my $e ( @entries ) {
my $r = [
$hash_id,
$levels{ $e->level },
$e->module,
$e->testcase,
$e->tag,
$e->timestamp,
$json->encode( $e->args // {} ),
];

push @records, $r;
}
my $query_values = join ", ", ("(?, ?, ?, ?, ?, ?, ?)") x @records;
my $query = "INSERT INTO result_entries (hash_id, level, module, testcase, tag, timestamp, args) VALUES $query_values";
my $sth = $self->dbh->prepare($query);
$sth = $sth->execute(map { @$_ } @records);
}

no Moose::Role;

1;
81 changes: 81 additions & 0 deletions lib/Zonemaster/Backend/DB/MySQL.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,68 @@ sub create_schema {
);
}

####################################################################
# LOG LEVEL
####################################################################
$dbh->do(
"CREATE TABLE IF NOT EXISTS log_level (
value INT,
level VARCHAR(15),
UNIQUE (value)
) ENGINE=InnoDB
"
) or die Zonemaster::Backend::Error::Internal->new( reason => "MySQL error, could not create 'log_level' table", data => $dbh->errstr() );

my ( $c ) = $dbh->selectrow_array( "SELECT count(*) FROM log_level" );
if ( $c == 0 ) {
$dbh->do(
"INSERT INTO log_level (value, level)
VALUES
(-2, 'DEBUG3'),
(-1, 'DEBUG2'),
( 0, 'DEBUG'),
( 1, 'INFO'),
( 2, 'NOTICE'),
( 3, 'WARNING'),
( 4, 'ERROR'),
( 5, 'CRITICAL')
"
);
}

####################################################################
# RESULT ENTRIES
####################################################################
$dbh->do(
"CREATE TABLE IF NOT EXISTS result_entries (
hash_id VARCHAR(16) NOT NULL,
level INT NOT NULL,
module VARCHAR(255) NOT NULL,
testcase VARCHAR(255) NOT NULL,
tag VARCHAR(255) NOT NULL,
timestamp REAL NOT NULL,
args BLOB NOT NULL,
CONSTRAINT fk_hash_id FOREIGN KEY (hash_id) REFERENCES test_results(hash_id),
CONSTRAINT fk_level FOREIGN KEY (level) REFERENCES log_level(value)
) ENGINE=InnoDB
"
) or die Zonemaster::Backend::Error::Internal->new( reason => "MySQL error, could not create 'result_entries' table", data => $dbh->errstr() );

$indexes = $dbh->selectall_hashref( 'SHOW INDEXES FROM result_entries', 'Key_name' );
if ( not exists($indexes->{result_entries__hash_id}) ) {
$dbh->do(
'CREATE INDEX result_entries__hash_id ON result_entries (hash_id)'
);
}

if ( not exists($indexes->{result_entries__level}) ) {
$dbh->do(
'CREATE INDEX result_entries__level ON result_entries (level)'
);
}


####################################################################
# BATCH JOBS
Expand Down Expand Up @@ -155,7 +217,26 @@ Drop all the tables if they exist.
sub drop_tables {
my ( $self ) = @_;

# remove any FOREIGN KEY before droping the table
# MariaDB <10.4 and MySQL do not support the IF EXISTS syntax
# on ALTER TABLE and DROP FOREIGN KEY
# MariaDB 10.3 is used on Ubuntu 20.04 LTS (eol 2023-04)
# MySQL is used on FreeBSD
my $tables = $self->dbh->selectall_hashref( 'SHOW TABLE STATUS', 'Name' );
if ( exists $tables->{result_entries} ) {
my @fk = $self->dbh->selectall_array( 'SELECT constraint_name FROM information_schema.referential_constraints' );
@fk = map { ref eq 'ARRAY' ? @$_ : $_ } @fk;
if ( grep( /^fk_hash_id$/, @fk ) ) {
$self->dbh->do( "ALTER TABLE result_entries DROP FOREIGN KEY fk_hash_id" );
}
if ( grep( /^fk_level$/, @fk ) ) {
$self->dbh->do( "ALTER TABLE result_entries DROP FOREIGN KEY fk_level" );
}
}

$self->dbh->do( "DROP TABLE IF EXISTS test_results" );
$self->dbh->do( "DROP TABLE IF EXISTS result_entries" );
$self->dbh->do( "DROP TABLE IF EXISTS log_level" );
$self->dbh->do( "DROP TABLE IF EXISTS users" );
$self->dbh->do( "DROP TABLE IF EXISTS batch_jobs" );

Expand Down
Loading

0 comments on commit dfd197d

Please sign in to comment.