From f90b4e79a6e7f049511d3f60afb752adb34729e0 Mon Sep 17 00:00:00 2001 From: Ed Santiago Date: Mon, 18 Dec 2023 06:12:10 -0700 Subject: [PATCH] Tests for xref-helpmsgs-manpages In the process of adding new functionality to the xref script, I realized it is much too fragile. It's too easy to make some minor change that could break the crossrefs, giving us the illusion of testing. Solution: add a test suite for the script. Still incomplete, but an important step toward building confidence. Requires minor surgery to the script itself Signed-off-by: Ed Santiago --- hack/xref-helpmsgs-manpages | 34 ++++--- hack/xref-helpmsgs-manpages.t | 181 ++++++++++++++++++++++++++++++++++ 2 files changed, 199 insertions(+), 16 deletions(-) create mode 100644 hack/xref-helpmsgs-manpages.t diff --git a/hack/xref-helpmsgs-manpages b/hack/xref-helpmsgs-manpages index 4b925737ea..575f3daecc 100755 --- a/hack/xref-helpmsgs-manpages +++ b/hack/xref-helpmsgs-manpages @@ -9,6 +9,7 @@ use utf8; use strict; use warnings; +use FindBin; (our $ME = $0) =~ s|.*/||; our $VERSION = '0.1'; @@ -23,17 +24,17 @@ $| = 1; # BEGIN user-customizable section # Path to podman executable -my $Default_Podman = './bin/podman'; +my $Default_Podman = "$FindBin::Bin/../bin/podman"; my $PODMAN = $ENV{PODMAN} || $Default_Podman; # Path to all doc files, including .rst and (down one level) markdown -my $Docs_Path = 'docs/source'; +our $Docs_Path = 'docs/source'; # Path to podman markdown source files (of the form podman-*.1.md) -my $Markdown_Path = "$Docs_Path/markdown"; +our $Markdown_Path = "$Docs_Path/markdown"; # Global error count -my $Errs = 0; +our $Errs = 0; # Table of exceptions for documenting fields in '--format {{.Foo}}' # @@ -111,10 +112,6 @@ my %Skip_Subcommand = map { $_ => 1 } ( ); # END user-customizable section -############################################################################### - -use FindBin; - ############################################################################### # BEGIN boilerplate args checking, usage messages @@ -180,6 +177,9 @@ sub main { # Fetch command-line arguments. Barf if too many. die "$ME: Too many arguments; try $ME --help\n" if @ARGV; + chdir "$FindBin::Bin/.." + or die "$ME: FATAL: Cannot cd $FindBin::Bin/..: $!"; + my $help = podman_help(); my $man = podman_man('podman'); my $rst = podman_rst(); @@ -216,9 +216,12 @@ sub xref_by_help { # autocompletion that looks like a Go template, but those # template options aren't documented in the man pages. if ($k eq '--format' && ! ref($man->{$k})) { - warn "$ME: 'podman @subcommand': --format options are available through autocomplete, but are not documented in $man->{_path}\n"; - - ++$Errs unless "@subcommand" eq "inspect"; + # "podman inspect" tries to autodetect if it's being run + # on an image or container. It cannot sanely be documented. + unless ("@subcommand" eq "inspect") { + warn "$ME: 'podman @subcommand': --format options are available through autocomplete, but are not documented in $man->{_path}\n"; + ++$Errs; + } next OPTION; } @@ -243,7 +246,7 @@ sub xref_by_help { # The usual case is "podman ... --help"... my $what = '--help'; # ...but for *options* (e.g. --filter), we're checking command completion - $what = '' if $subcommand[-1] =~ /^--/; + $what = '' if @subcommand && $subcommand[-1] =~ /^--/; warn "$ME: 'podman @subcommand $what' lists '$k', which is not in $man\n"; ++$Errs; } @@ -309,7 +312,7 @@ sub xref_by_man { # It's not always --help, sometimes we check completion my $what = '--help'; - $what = 'command completion' if $subcommand[-1] =~ /^--/; + $what = 'command completion' if @subcommand && $subcommand[-1] =~ /^--/; warn "$ME: 'podman @subcommand': '$k' in $man, but not in $what\n"; ++$Errs; } @@ -460,12 +463,11 @@ sub podman_help { sub podman_man { my $command = shift; my $subpath = "$Markdown_Path/$command.1.md"; - my $manpath = "$FindBin::Bin/../$subpath"; print "** $subpath \n" if $debug; my %man = (_path => $subpath); - open my $fh, '<', $manpath - or die "$ME: Cannot read $manpath: $!\n"; + open my $fh, '<', $subpath + or die "$ME: Cannot read $subpath: $!\n"; my $section = ''; my @most_recent_flags; my $previous_subcmd = ''; diff --git a/hack/xref-helpmsgs-manpages.t b/hack/xref-helpmsgs-manpages.t new file mode 100644 index 0000000000..e85d598316 --- /dev/null +++ b/hack/xref-helpmsgs-manpages.t @@ -0,0 +1,181 @@ +# -*- perl -*- +# +# tests for xref-helpmsgs-manpages +# + +use v5.20; +use strict; +use warnings; + +use Clone qw(clone); +use File::Basename; +use File::Path qw(make_path); +use File::Temp qw(tempdir); +use FindBin; +use Test::More; +use Test::Differences; + +plan tests => 9; + +require_ok "$FindBin::Bin/xref-helpmsgs-manpages"; + + +my $workdir = tempdir(basename($0) . ".XXXXXXXX", TMPDIR => 1, CLEANUP => 1); + +# Copy man pages to tmpdir, so we can fiddle with them +my $doc_path = do { + no warnings 'once'; + "$LibPod::CI::XrefHelpmsgsManpages::Markdown_Path"; +}; + +make_path "$workdir/$doc_path" + or die "Internal error: could not make_path $workdir/$doc_path"; +system('rsync', '-a', "$doc_path/." => "$workdir/$doc_path/.") == 0 + or die "Internal error: could not rsync $doc_path to $workdir"; + +chdir $workdir + or die "Internal error: could not cd $workdir: $!"; + +my @warnings_seen; +$SIG{__WARN__} = sub { + my $msg = shift; + chomp $msg; + $msg =~ s/^xref-\S+?:\s+//; + $msg =~ s|\s+$doc_path/| |g; + push @warnings_seen, $msg; +}; + +# When we get errors (hopefully only when adding new functionality +# to this test!), this format is MUCH easier to read and copy-paste. +unified_diff; + +# Helper function for running xref tests. +sub test_xref { + my $name = shift; + my $h = shift; + my $m = shift; + my $expect_by_help = shift; + my $expect_by_man = shift; + + @warnings_seen = (); + LibPod::CI::XrefHelpmsgsManpages::xref_by_help($h, $m); + eq_or_diff_text \@warnings_seen, $expect_by_help, "$name: xref_by_help()"; + + @warnings_seen = (); + LibPod::CI::XrefHelpmsgsManpages::xref_by_man($h, $m); + eq_or_diff_text \@warnings_seen, $expect_by_man, "$name: xref_by_man()"; +} + +############################################################################### +# BEGIN Baseline tests +# +# Confirm that everything passes in the current tree + +my $help = LibPod::CI::XrefHelpmsgsManpages::podman_help(); +eq_or_diff_text \@warnings_seen, [], "podman_help() runs cleanly, no warnings"; + +@warnings_seen = (); +my $man = LibPod::CI::XrefHelpmsgsManpages::podman_man('podman'); +eq_or_diff_text \@warnings_seen, [], "podman_man() runs cleanly, no warnings"; + +# If this doesn't pass, we've got big problems. +test_xref("baseline", $help, $man, [], []); + +#use Data::Dump; dd $man; exit 0; + +# END Baseline tests +########################################################################## +# BEGIN fault injection tests on xref_by_man() +# +# These are really simple: only two different warnings. + +my $hclone = clone($help); +my $mclone = clone($man); + +delete $hclone->{network}{ls}{"--format"}; +delete $hclone->{save}; +$mclone->{"command-in-man"} = {}; +$mclone->{"system"}{"subcommand-in-man"} = {}; + +# --format field documented in man page but not in autocomplete +delete $hclone->{events}{"--format"}{".HealthStatus"}; + +test_xref("xref_by_man injection", $hclone, $mclone, + [], + [ + "'podman ': 'command-in-man' in podman.1.md, but not in --help", + "'podman events --format': '.HealthStatus' in podman-events.1.md, but not in command completion", + "'podman network ls': --format options documented in man page, but not available via autocomplete", + "'podman ': 'save' in podman.1.md, but not in --help", + "'podman system': 'subcommand-in-man' in podman-system.1.md, but not in --help", + ], + ); + +# END fault injection tests on xref_by_man() +############################################################################### +# BEGIN fault injection tests on xref_by_help() +# +# These are much more complicated. + +$hclone = clone($help); +$mclone = clone($man); + +# --format is not documented in man page +delete $mclone->{"auto-update"}{"--format"}; +# --format is documented, but without a table +$mclone->{container}{list}{"--format"} = 1; +# --format is documented, with a table, but one entry missing +delete $mclone->{events}{"--format"}{".HealthStatus"}; + +# -l option is not documented +delete $mclone->{pod}{inspect}{"-l"}; + +# Command and subcommand in podman --help, but not in man pages +$hclone->{"new-command-in-help"} = {}; +$hclone->{"secret"}{"subcommand-in-help"} = {}; + +# Can happen if podman-partlydocumented exists in --help, and is listed +# in podman.1.md, but does not have its own actual man page. +$hclone->{partlydocumented} = { "something" => 1 }; +$mclone->{partlydocumented} = undef; + +test_xref("xref_by_help() injection", $hclone, $mclone, + [ + "'podman auto-update --help' lists '--format', which is not in podman-auto-update.1.md", + "'podman container list': --format options are available through autocomplete, but are not documented in podman-ps.1.md", + "'podman events --format ' lists '.HealthStatus', which is not in podman-events.1.md", + "'podman --help' lists 'new-command-in-help', which is not in podman.1.md", + "'podman partlydocumented' is not documented in man pages!", + "'podman pod inspect --help' lists '-l', which is not in podman-pod-inspect.1.md", + "'podman secret --help' lists 'subcommand-in-help', which is not in podman-secret.1.md", + ], + [], + ); + +# END fault injection tests on xref_by_help() +############################################################################### +# BEGIN fault injection tests on podman_man() +# +# This function has a ton of sanity checks. To test them we need to +# perform minor surgery on lots of .md files. +# +# FIXME: TBD. This PR has grown big enough as it is. + + +# END fault injection tests on podman_man() +############################################################################### +# BEGIN fault injection tests on podman_help() +# +# Nope, this is not likely to happen. In order to do this we'd need to: +# +# * instrument podman and cobra to emit fake output; or +# * write a podman wrapper that selectively munges output; or +# * write a dummy podman that generates the right form of (broken) output. +# +# podman_help() has few sanity checks, and those are unlikely, so doing this +# is way more effort than it's worth. +# +# END fault injection tests on podman_help() +############################################################################### + +1;