diff --git a/.gitignore b/.gitignore index 630bb6f237f..c81e19c3c12 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ # no backup files *~ *.vscode +*.vstags **.kate-swp # Go binaries common/Go/yabi/yabi @@ -22,4 +23,5 @@ venv infrastructure-tooling # solus image symlink or clone -solus-image-* \ No newline at end of file +solus-image-* + diff --git a/common/.gitignore b/common/.gitignore index 7e99e367f84..73fdec5956a 100644 --- a/common/.gitignore +++ b/common/.gitignore @@ -1 +1,3 @@ -*.pyc \ No newline at end of file +*.pyc +*.vstags +*.bak diff --git a/common/perl/README.md b/common/perl/README.md new file mode 100644 index 00000000000..a6e03077773 --- /dev/null +++ b/common/perl/README.md @@ -0,0 +1,25 @@ +# README for perl scripts + +## Documentation and help output + +For each script, there should be documentation written in pod and help output. + +Docs are read with perldoc. For example: + +```bash +perldoc get_unique_updates/get_unique_updates.pl +``` + +Each script *should* have help output. For example: + +```bash +get_unique_updates.pl --help +``` + +## Running tests + +Tests are run with `prove`. To run all tests for a script, run prove on all the test files in the test directory: + +```bash +prove get_unique_updates/t/*.t +``` \ No newline at end of file diff --git a/common/perl/get_unique_updates/get_unique_updates.pl b/common/perl/get_unique_updates/get_unique_updates.pl new file mode 100755 index 00000000000..ee45fe49d92 --- /dev/null +++ b/common/perl/get_unique_updates/get_unique_updates.pl @@ -0,0 +1,115 @@ +#!/usr/bin/env perl + +# ABSTRACT: Find and return all unique package names in input file with highest version string +# PODNAME: get_unique_updates.pl + +use FindBin; +use lib "$FindBin::RealBin/lib"; + +BEGIN { + # First, check if all the required modules have + # been installed inthe system this script will run on. + my @import_modules = ( + 'Modern::Perl', + 'Getopt::Long', + 'Pod::Usage', + 'ExtUtils::Installed' + ); + + for my $module ( @import_modules ) { + eval 'require ' . $module; + if ($@) { + # problems finding module + # say "Cant find $module"; + print qq{ +Perl module $module does not seem to be installed in this system. Please install it and try again.\n +First install cpanm with + curl -L https://cpanmin.us | perl - --sudo App::cpanminus\n +Then install the missing module like so: + sudo cpanm $module +}; + exit 1; + } + } +} + +use Modern::Perl; +use GetUniqueLines; +use Getopt::Long; +use Pod::Usage; + +my $successful_parse = GetOptions( + 'help|?' => \my $help, + man => \my $man, + 'input-file=s' => \my $input_file, + 'output-dir=s' => \( my $output_dir = "$FindBin::RealBin" ), + 'output-file=s' => \my $output_file + ) + or pod2usage(2); + +if ( !$successful_parse || ( @ARGV > 2 ) ) { + die qq{Usage: $0 [ --input-file input_filename --output-dir output_directory]\n}; +} + +if ( !defined $input_file ) { + die qq{ An input file is required\n + Usage: $0 [ --input-file input_filename --output-dir output_directory --output-file output_filename]\n}; +} + +pod2usage(1) if $help; +pod2usage( -exitval => 0, -verbose => 2 ) if $man; + +unless ( -e -f -r $input_file ) { + die("Unable to read '$input_file': $!\n"); +} + +unless ( GetUniqueLines::check_file_format($input_file) ) { + die("File '$input_file' is not in the expected markdown format\n"); +} + +my %packages = GetUniqueLines::get_unique_lines($input_file); +my $pkg_href = \%packages; + +# Write to STDOUT, unless output file provided +if ($output_file) { + my $file_path = $output_dir . '/' . $output_file; + GetUniqueLines::write_to_file( $file_path, $pkg_href ); +} else { + GetUniqueLines::write_to_stdout($pkg_href); +} + +__END__ +=pod + +=encoding UTF-8 + +=head1 NAME + +get_unique_updates.pl + +=head1 SYNOPSIS + +get_unique_updates.pl --input-file ~/temp/ --output-dir ~/temp + + Options: + -help Brief help message + -man Full documentation + --input-file The file to parse, generated by worklog.py (required) + --output-dir Directory to store output file (defaults to the directory this script is in, only used if --output-file is provided) + --output-file The name of the file to save the output to. If this is not provided, the list of packages will be printed to STDOUT. + +=head1 DESCRIPTION + +Solus uses worklog.py to generate a list of packages that were updated in a given timeframe. The file it generates includes all updates to every package. + +This script takes that file as input and returns a list to STDOUT or a file that has only one line per package. That line will be the latest version built for the package. + +You must use the markdown output type with worklog.py for use with this script. e.g. + + ./worklog.py --format md builds 2024-02-04 now >| ~/temp/solus_builds_2024-02-04.md + +Examples: + + get_unique_updates.pl --input-file ~/temp/solus_builds_2024-02-04.md + + get_unique_updates.pl --input-file ~/temp/solus_builds_2024-02-04.md --output-dir ~/temp --output-file solus_unique_builds_2024-02-04.md diff --git a/common/perl/get_unique_updates/lib/GetUniqueLines.pm b/common/perl/get_unique_updates/lib/GetUniqueLines.pm new file mode 100644 index 00000000000..b16d0568518 --- /dev/null +++ b/common/perl/get_unique_updates/lib/GetUniqueLines.pm @@ -0,0 +1,99 @@ +package GetUniqueLines; + +use Modern::Perl; +use List::Util; + +use Data::Dumper; + +use Pod::Usage; + +sub check_file_format { + my $file = shift; + my $is_markdown; + + # Make sure file is in the right format + # This is a very simple check to avoid needing to have more perl modules installed + + open( my $fh, "<", $file ) + or die "Can't open '$file': $!"; + my @rows = <$fh>; + chomp @rows; + + # Line one should be a number of builds + unless ( $rows[0] =~ /\d+\sbuilds/i ) { + die( "First line of the filed does not appear to show the number of builds. Got:\n'" . $rows[0] . "'\n" ); + } + + # Line two should be an unordered list item which is a package name and link + unless ( $rows[1] =~ /^-\s\[[^\s]+\s\d+[^\s]+\]\(https:\/\/[^\s]+\)/ ) { + die( "Second line of the file is not in expected markdown format. Got:\n'" . $rows[1] . "'\n" ); + } + $is_markdown = 1; + + close($fh) or die $!; + + return $is_markdown; +} + +sub get_unique_lines { + + # Take input file + my $input_file = shift; + my %packages; + + open( my $fh, "<", $input_file ) + or die "Can't open file '$input_file': $!"; + my @lines = <$fh>; + chomp @lines; + + # iterate through lines and parse out package names + # Create a hash of packages, package names are the keys + # Each update of each key will write in the line with the highest version + # Skip first line since that is the package count + for my $i ( 1 .. $#lines ) { + my $line = $lines[$i]; + + # Get package name + if ( $line =~ /^-\s\[([^\s]+)\s/ ) { + my $pkg_name = $1; + + # say "Got package name $1"; + $packages{$pkg_name} = $line; + } else { + next; + } + } + + # return the hash of unique package lines + return %packages; +} + +sub write_to_stdout { + my $package_href = shift; + my %package_list = %{$package_href}; + + foreach my $name ( sort( keys %package_list ) ) { + print $package_list{$name} . "\n"; + } + + return; +} + +sub write_to_file { + my ( $file_path, $package_href ) = @_; + my %package_list = %{$package_href}; + + open( my $fh, ">", $file_path ) + or die "Can't open '$file_path': $!"; + + foreach my $name ( sort( keys %package_list ) ) { + print $fh $package_list{$name} . "\n"; + } + + close($fh) or die $!; + + return; + +} + +1; diff --git a/common/perl/get_unique_updates/t/data/markdown_text.md b/common/perl/get_unique_updates/t/data/markdown_text.md new file mode 100644 index 00000000000..584aa0ccc41 --- /dev/null +++ b/common/perl/get_unique_updates/t/data/markdown_text.md @@ -0,0 +1,20 @@ +123 builds: +- [0ad 0.0.26-53](https://github.com/getsolus/packages/commit/bfef63e3245209ab3b32ddb6b11d93681a904e12) +- [abseil-cpp 20230802.1-5](https://github.com/getsolus/packages/commit/354b2c458704bb639e6437b44dd668dace98c63b) +- [abseil-cpp 20230802.1-6](https://github.com/getsolus/packages/commit/91739a1a49c48e68279370ba5400059a7dcef840) +- [aegisub 3.3.3-39](https://github.com/getsolus/packages/commit/d86decc52fcd44ea02a7f999ad5870bab781d977) +- [alsa-lib 1.2.11-37](https://github.com/getsolus/packages/commit/83989aad2c1d565c92ba6c2a9b4b459a869495ab) +- [alsa-tools 1.2.11-14](https://github.com/getsolus/packages/commit/0f355093daca07dbed0adc630d154db8868ad4de) +- [alsa-utils 1.2.11-27](https://github.com/getsolus/packages/commit/33cef9e6346e8cd7ce96731a9cf3842a36a5287f) +- [amdgpu_top 0.7.0-10](https://github.com/getsolus/packages/commit/bdcfc6f42d356ce5c08812a535e35b3054c71b4d) +- [anki 23.12.1-33](https://github.com/getsolus/packages/commit/113bae3c5a250823065b5f09f0da9821164579d5) +- [appstream-data 32-35](https://github.com/getsolus/packages/commit/d8c7882c390d10e82748d9ddd04b306bb3415c0d) +- [apr-util 1.6.3-18](https://github.com/getsolus/packages/commit/ef1b3da340dd02c5df6665b1a87d46102cc4a30b) +- [attica 5.115.0-74](https://github.com/getsolus/packages/commit/40d198c4c875d288646358fdf202f4ecddeda389) +- [autofs 5.1.9-12](https://github.com/getsolus/packages/commit/b6edcf792be13222ac8a8a3e13545f0fad1122ba) +- [baloo 5.115.0-75](https://github.com/getsolus/packages/commit/630558119e3dec18ce36aeed0f6d51146e855b1d) +- [broadcom-sta 6.30.223.271-384](https://github.com/getsolus/packages/commit/57ba69fbadfde4170037f1af1fca46213154c01e) +- [broadcom-sta 6.30.223.271-385](https://github.com/getsolus/packages/commit/d633bea9ffa45f50d0aff82d2c943ceea96a15f6) +- [darktable 4.6.0-75](https://github.com/getsolus/packages/commit/acecb5d8c41d18c3a2b81a7436e6ba5525c32ae7) +- [darktable 4.6.0-76](https://github.com/getsolus/packages/commit/1f8be1a336c6c5ed6966ae32846b7a6b31c13286) +- [darktable 4.6.0-77](https://github.com/getsolus/packages/commit/3e12b4295851f3325da4d4f43ff1cd4e093d65eb) \ No newline at end of file diff --git a/common/perl/get_unique_updates/t/data/random_text b/common/perl/get_unique_updates/t/data/random_text new file mode 100644 index 00000000000..08e00ed2916 --- /dev/null +++ b/common/perl/get_unique_updates/t/data/random_text @@ -0,0 +1 @@ +Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. \ No newline at end of file diff --git a/common/perl/get_unique_updates/t/get_unique_updates.t b/common/perl/get_unique_updates/t/get_unique_updates.t new file mode 100644 index 00000000000..e286735adf0 --- /dev/null +++ b/common/perl/get_unique_updates/t/get_unique_updates.t @@ -0,0 +1,68 @@ +use Test2::V0; +use Test2::Tools::Exception qw/dies lives/; +use Test2::Plugin::DieOnFail; +use Test2::Plugin::ExitSummary; +use Test::File; + +use FindBin qw($Bin); +use lib "$Bin/../lib"; + +my $text_file = "$Bin/../t/data/random_text"; +my $md_file = "$Bin/../t/data/markdown_text.md"; +my $output_dir = "$Bin/../t/data"; +my $output_file = "markdown_output.md"; +my $file_path = $output_dir . '/' . $output_file; + +my $expected_broadcom_ver = 'broadcom-sta 6.30.223.271-385'; +my $expected_darktable_ver = 'darktable 4.6.0-77'; + +BEGIN { + plan(6); + use GetUniqueLines; + pass('Module loads correctly.'); +} + +ok( GetUniqueLines::check_file_format($md_file), "Markdown file passes file check" ); + +like( dies { GetUniqueLines::check_file_format($text_file) }, qr/First line/, "Text file fails file check" ); + +# Make sure we get highest package versions from the result +my %packages = GetUniqueLines::get_unique_lines($md_file); +my $pkg_href = \%packages; + +# Capture output of write_to_stdout +my $stdout_result; +open( my $outputFH, '>', \$stdout_result ) or die; +my $oldFH = select $outputFH; +GetUniqueLines::write_to_stdout($pkg_href); +select $oldFH; +close $outputFH; + +my ( $broadcom_result, $darktable_result ); + +if ( $stdout_result =~ /\[(broadcom-sta \S+)\]/g ) { + $broadcom_result = $1; + like( "$broadcom_result", qr/$expected_broadcom_ver/, "Entry for package with 2 versions has highest value" ); +} else { + fail('Broadcom entry found in result'); +} + +if ( $stdout_result =~ /\[(darktable \S+)\]/g ) { + $darktable_result = $1; + like( "$darktable_result", qr/$expected_darktable_ver/, "Entry for package with 3 versions has highest version" ); +} else { + fail('Darktable entry found in result'); +} + +# Ensure module can write a file + +GetUniqueLines::write_to_file( $file_path, $pkg_href ); + +print "test looking for $file_path\n"; +# file_exists_ok( $file_path, 'Output file is written'); +file_line_count_is( $file_path, 15, 'Output file is written and has expected number of packages'); + +END { + # Remove test file + unlink($file_path) or die "Can't delete $file_path: $!\n"; +}