office-gobmx/bin/lo-commit-stat
Xisco Fauli df30a4515b lo-commit-stat: Only search for bug ids in the summary
Sometimes, developers mention other bugs in the body as
related issues.
This might create mislead information in the release notes
E.g: https://wiki.documentfoundation.org/Releases/6.1.1/RC1
* Caolán is mentioned as fixer of i#66963 because the id is in
the body of 9c3d8b1c64
which fixes tdf#98558
* M. Stahl is mentioned as fixer of tdf#112447 because the id is
in the body of 31e66bd07c
This bug was fixed by Bjoern Michaelsen in
ce2fce9a41

Change-Id: I8d0ee038741dbb27364a56a7fa9285f32839c37b
Reviewed-on: https://gerrit.libreoffice.org/69750
Tested-by: Jenkins
Reviewed-by: Andras Timar <andras.timar@collabora.com>
2019-04-05 12:41:27 +02:00

584 lines
19 KiB
Perl
Executable file
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/perl
eval 'exec /usr/bin/perl -S $0 ${1+"$@"}'
if $running_under_some_shell;
#!/usr/bin/perl
use strict;
use warnings;
use LWP::UserAgent;
use utf8;
use File::Temp;
use Encode;
use open ':encoding(utf8)';
use open ':std' => ':encoding(utf8)';
my %module_dirname = (
"core" => "",
"dictionaries" => "dictionaries",
"help" => "helpcontent2",
"translations" => "translations"
);
my %bugzillas = (
fdo => "https://bugs.documentfoundation.org/show_bug.cgi?id=",
tdf => "https://bugs.documentfoundation.org/show_bug.cgi?id=",
bnc => "https://bugzilla.novell.com/show_bug.cgi?id=",
rhbz => "https://bugzilla.redhat.com/show_bug.cgi?id=",
i => "https://bz.apache.org/ooo/show_bug.cgi?id=",
fate => "https://features.opensuse.org/",
);
sub search_bugs($$$$)
{
my ($pdata, $module, $commit_id, $line) = @_;
my $bug = "";
my $bug_orig;
while (defined $bug) {
# match fdo#123, rhz#123, i#123, #123
# but match only bug number with >= 4 digits
if ( $line =~ m/(\w+\#+\d{4,})/ ) {
$bug_orig = $1;
$bug = $1;
# default to issuezilla for the #123 variant
# but match only bug number with >= 4 digits
} elsif ( $line =~ m/(\#)(\d{4,})/ ) {
$bug_orig = $1 . $2;
$bug = "i#$2";
# match #i123#
} elsif ( $line =~ m/(\#i)(\d+)(\#)/ ) {
$bug_orig = $1 . $2 . $3;
$bug = "i#$2";
} else {
$bug = undef;
next;
}
# print " found $bug\n";
# remove bug number from the comment; it will be added later a standardized way
$bug_orig =~ s/\#/\\#/;
$line =~ s/(,\s)*[Rr](elated|esolve[ds]):?\s*$bug_orig\s?:?\s*//;
$line =~ s/\s*-\s*$bug_orig\s*//;
$line =~ s/\(?$bug_orig\)?\s*[:,-]?\s*//;
# bnc# is preferred over n# for novell bugs
$bug =~ s/^n\#/bnc#/;
# deb# is preferred over debian# for debian bugs
$bug =~ s/^debian\#/deb#/;
# easyhack# is sometimes used for fdo# - based easy hacks
$bug =~ s/^easyhack\#/fdo#/;
# someone mistyped fdo as fd0
$bug =~ s/^fd0\#/fdo#/;
# save the bug number
$pdata->{$module}{$commit_id}{'bugs'}{$bug} = 1;
}
return $line;
}
sub standardize_summary($)
{
my $line = shift;
$line =~ s/^\s*//;
$line =~ s/\s*$//;
# lower first letter if the word contains only lowercase letter
if ( $line =~ m/(^.[a-z]+\b)/ ) {
$line =~ m/(^.)/;
my $first_char = lc($1);
$line =~ s/^./$first_char/;
}
# FIXME: remove do at the end of line
# remove bug numbers
return $line;
}
sub generate_git_cherry_ids_log($$$$$)
{
my ($pdata, $repo_dir, $module, $branch_name, $git_args) = @_;
my $commit_ids_log;
my $commit_ids_log_fh;
$commit_ids_log_fh = File::Temp->new(TEMPLATE => 'lo-commit-stat-ids-XXXXXX',
DIR => '/tmp',
UNLINK => 0);
$commit_ids_log = $commit_ids_log_fh->filename;
print STDERR "Filtering cherry-picked commits in the git repo: $module...\n";
my $cmd = "cd $repo_dir; git cherry $git_args";
open (GIT, "$cmd 2>&1|") || die "Can't run $cmd: $!";
while (my $line = <GIT>) {
# skip cherry-picked commits
next if ( $line =~ m/^\-/ );
if ( $line =~ m/^\+ / ) {
$line =~ s/^\+ //;
print $commit_ids_log_fh $line;
}
}
close GIT;
close $commit_ids_log_fh;
return $commit_ids_log;
}
sub load_git_log($$$$$$$)
{
my ($pdata, $repo_dir, $module, $branch_name, $git_command, $git_cherry, $git_args) = @_;
my $cmd = "cd $repo_dir;";
my $commit_ids_log;
if ($git_cherry) {
$commit_ids_log = generate_git_cherry_ids_log($pdata, $repo_dir, $module, $branch_name, $git_args);
$cmd .= " cat $commit_ids_log | xargs -n 1 $git_command -1";
} else {
$cmd .= " $git_command $git_args";
}
my $commit_id;
my $summary;
print STDERR "Analyzing log from the git repo: $module...\n";
# FIXME: ./g pull move submodules in unnamed branches
# my $repo_branch_name = get_branch_name($repo_dir);
# if ( $branch_name ne $repo_branch_name ) {
# die "Error: mismatch of branches:\n" .
# " main repo is on the branch: $branch_name\n" .
# " $module repo is on the branch: $repo_branch_name\n";
# }
open (GIT, "$cmd 2>&1|") || die "Can't run $cmd: $!";
while (my $line = <GIT>) {
chomp $line;
if ( $line =~ m/^commit ([0-9a-z]{20})/ ) {
$commit_id = $1;
$summary=undef;
next;
}
if ( $line =~ /^Author:\s*([^\<]*)\<([^\>]*)>/ ) {
# get rid of extra empty spaces;
my $name = $1;
my $email = $2;
$name =~ s/\s+$//;
die "Error: Author already defined for the commit {$commit_id}\n" if defined ($pdata->{$module}{$commit_id}{'author'});
$pdata->{$module}{$commit_id}{'author'}{'name'} = $name;
$pdata->{$module}{$commit_id}{'author'}{'email'} = $email;
next;
}
if ( $line =~ /^Date:\s+/ ) {
# ignore date line
next;
}
if ( $line =~ /^\s*$/ ) {
# ignore empty line
next;
}
unless (defined $pdata->{$module}{$commit_id}{'summary'}) {
$line = search_bugs($pdata, $module, $commit_id, $line);
# FIXME: need to be implemented
# search_keywords($pdata, $line);
$summary = standardize_summary($line);
$pdata->{$module}{$commit_id}{'summary'} = $summary;
}
}
close GIT;
unlink $commit_ids_log if ($git_cherry);
}
sub get_repo_name($)
{
my $repo_dir = shift;
open (GIT_CONFIG, "$repo_dir/.git/config") ||
die "can't open \"$$repo_dir/.git/config\" for reading: $!\n";
while (my $line = <GIT_CONFIG>) {
chomp $line;
if ( $line =~ /^\s*url\s*=\s*(\S+)$/ ) {
my $repo_name = "$1";
$repo_name = s/.*\///g;
return "$repo_name";
}
}
die "Error: can't find repo name in \"$$repo_dir/.git/config\"\n";
}
sub load_data($$$$$$$)
{
my ($pdata, $top_dir, $p_module_dirname, $branch_name, $git_command, $git_cherry, $git_args) = @_;
foreach my $module (sort { $a cmp $b } keys %{$p_module_dirname}) {
load_git_log($pdata, "$top_dir/$p_module_dirname->{$module}", $module, $branch_name, $git_command, $git_cherry, $git_args);
}
}
sub get_branch_name($)
{
my ($top_dir) = @_;
my $branch;
my $cmd = "cd $top_dir && git branch";
open (GIT, "$cmd 2>&1|") || die "Can't run $cmd: $!";
while (my $line = <GIT>) {
chomp $line;
if ( $line =~ m/^\*\s*(\S+)/ ) {
$branch = "$1";
}
}
close GIT;
die "Error: did not detect git branch name in $top_dir\n" unless defined ($branch);
return $branch;
}
sub get_bug_list($$$)
{
my ($pdata, $pbugs, $check_bugzilla) = @_;
# associate bugs with their summaries and fixers
foreach my $module ( keys %{$pdata}) {
foreach my $id ( keys %{$pdata->{$module}}) {
foreach my $bug (keys %{$pdata->{$module}{$id}{'bugs'}}) {
my $author = $pdata->{$module}{$id}{'author'}{'name'};
my $summary = $pdata->{$module}{$id}{'summary'};
$pbugs->{$bug}{'summary'} = $summary;
$pbugs->{$bug}{'author'}{$author} = 1;
}
}
}
# try to replace summaries with bug names from bugzilla
if ($check_bugzilla) {
print "Getting bug titles:\n";
foreach my $bug ( sort { $a cmp $b } keys %{$pbugs}) {
$pbugs->{$bug}{'summary'} = get_bug_name($bug, $pbugs->{$bug}{'summary'});
}
}
}
sub open_log_file($$$$$$)
{
my ($log_dir, $log_prefix, $log_suffix, $top_dir, $branch_name, $wiki) = @_;
my $logfilename = "$log_prefix-$branch_name-$log_suffix";
$logfilename = "$log_dir/$logfilename" if (defined $log_dir);
if ($wiki) {
$logfilename .= ".wiki";
} else {
$logfilename .= ".log";
}
if (-f $logfilename) {
print "WARNING: The log file already exists: $logfilename\n";
print "Do you want to overwrite it? (Y/n)?\n";
my $answer = <STDIN>;
chomp $answer;
$answer = "y" unless ($answer);
die "Please, rename the file or choose another log suffix\n" if ( lc($answer) ne "y" );
}
my $log;
open($log, '>', $logfilename) || die "Can't open \"$logfilename\" for writing: $!\n";
return $log;
}
sub print_commit_summary($$$$$$)
{
my ($summary, $pmodule_title, $pbugs, $pauthors, $prefix, $log) = @_;
return if ( $summary eq "" );
# print module title if not done yet
if ( defined ${$pmodule_title} ) {
print $log "${$pmodule_title}\n";
${$pmodule_title} = undef;
}
# finally print the summary line
my $bugs = "";
if ( %{$pbugs} ) {
$bugs = " (" . join (", ", keys %{$pbugs}) . ")";
}
my $authors = "";
if ( %{$pauthors} ) {
$authors = " [" . join (", ", keys %{$pauthors}) . "]";
}
print $log $prefix, $summary, $bugs, $authors, "\n";
}
sub print_commits($$$)
{
my ($pdata, $log, $wiki) = @_;
foreach my $module ( sort { $a cmp $b } keys %{$pdata}) {
# check if this module has any entries at all
my $module_title = "+ $module";
if ( %{$pdata->{$module}} ) {
my $old_summary="";
my %authors = ();
my %bugs = ();
foreach my $id ( sort { lc $pdata->{$module}{$a}{'summary'} cmp lc $pdata->{$module}{$b}{'summary'} } keys %{$pdata->{$module}}) {
my $summary = $pdata->{$module}{$id}{'summary'};
if ($summary ne $old_summary) {
print_commit_summary($old_summary, \$module_title, \%bugs, \%authors, " + ", $log);
$old_summary = $summary;
%authors = ();
%bugs = ();
}
# collect bug numbers
if (defined $pdata->{$module}{$id}{'bugs'}) {
foreach my $bug (keys %{$pdata->{$module}{$id}{'bugs'}}) {
$bugs{$bug} = 1;
}
}
# collect author names
my $author = $pdata->{$module}{$id}{'author'}{'name'};
$authors{$author} = 1;
}
print_commit_summary($old_summary, \$module_title, \%bugs, \%authors, " + ", $log);
}
}
}
sub get_bug_name($$)
{
my ($bug, $summary) = @_;
print "$bug: ";
$bug =~ m/(?:(\w*)\#+(\d+))/; # fdo#12345
my $bugzilla = $1; # fdo
my $bug_number = $2; # 12345
if ( $bugzillas{$bugzilla} ) {
my $url = $bugzillas{$bugzilla} . $bug_number;
my $ua = LWP::UserAgent->new;
$ua->timeout(10);
$ua->env_proxy;
my $response = $ua->get($url);
if ($response->is_success) {
my $title = decode('utf8', $response->title);
if ( $title =~ s/^(?:Bug $bug_number \S+|$bug_number ) // ) {
print "$title\n";
return $title;
} else {
print "warning: not found; using commit message (only got $title)";
}
}
}
print "\n";
return $summary;
}
sub print_bugs($$$$)
{
my ($pbugs, $log, $wiki) = @_;
# sort alphabetically by bugzilla-type, but within that numerically
foreach my $bug ( sort { ($a =~ /(\D+)/)[0] cmp ($b =~ /(\D+)/)[0] ||
($a =~ /(\d+)/)[0] <=> ($b =~ /(\d+)/)[0] } keys %{$pbugs}) {
my $summary = $pbugs->{$bug}{'summary'};
my $authors = "";
if ( %{$pbugs->{$bug}{'author'}} ) {
$authors = " [" . join (", ", keys %{$pbugs->{$bug}{'author'}}) . "]";
}
$bug =~ s/(.*)\#(.*)/# {{$1|$2}}/ if ($wiki);
print $log $bug, " ", $summary, $authors, "\n";
}
}
sub print_bugs_changelog($$$$)
{
my ($pbugs, $log, $wiki) = @_;
foreach my $bug ( sort { $a cmp $b } keys %{$pbugs}) {
my $summary = $pbugs->{$bug}{'summary'};
my $authors = "";
if ( %{$pbugs->{$bug}{'author'}} ) {
$authors = " [" . join (", ", keys %{$pbugs->{$bug}{'author'}}) . "]";
}
print $log " + $summary ($bug)$authors\n";
}
}
sub print_bugnumbers($$$$)
{
my ($pbugs, $log, $wiki) = @_;
print $log join ("\n", sort { $a cmp $b } keys %{$pbugs}), "\n";
}
sub generate_log($$$$$$$$)
{
my ($pused_data, $print_func, $log_dir, $log_prefix, $log_suffix, $top_dir, $branch_name, $wiki) = @_;
my $log = open_log_file($log_dir, $log_prefix, $log_suffix, $top_dir, $branch_name, $wiki);
& {$print_func} ($pused_data, $log, $wiki);
close $log;
}
########################################################################
# help
sub usage()
{
print "This script generates LO git commit summary\n\n" .
"Usage: lo-commit-stat [--help] [--no-submodules] [--module=<module>] --log-dir=<dir> --log-suffix=<string> topdir [git_arg...]\n\n" .
"Options:\n" .
" --help print this help\n" .
" --no-submodule read changes just from the main repository, ignore submodules\n" .
" --module=<module> summarize just changes from the given module, use \"core\"\n" .
" for the main module\n" .
" --log-dir=<dir> directory where to put the generated log\n" .
" --log-suffix=<string> suffix of the log file name; the result will be\n" .
" commit-log-<branch>-<log-name-suffix>.log; the branch name\n" .
" is detected automatically\n" .
" --commits generate log with all commits (default)\n" .
" --bugs generate log with bugzilla entries\n" .
" --bugs-changelog generate log with bugzilla entries, use changelog style\n" .
" --bugs-wiki generate log with bugzilla entries, use wiki markup\n" .
" --bugs-numbers generate log with bugzilla numbers\n" .
" --rev-list use \"git rev-list\" instead of \"git log\"; useful to check\n" .
" differences between branches\n" .
" --cherry use \"git cherry\" instead of \"git log\"; detects cherry-picked\n" .
" commits between branches\n" .
" topdir directory with the libreoffice/core clone\n" .
" git_arg extra parameters passed to the git command to define\n" .
" the area of interest; The default command is \"git log\" and\n" .
" parameters might be, for example, --after=\"2010-09-27\" or\n" .
" TAG..HEAD; with the option --rev-list, useful might be, for\n" .
" example origin/master ^origin/libreoffice-3-3; with the option\n" .
" --rev-list, useful might be, for example libreoffice-3.6.3.2\n" .
" libreoffice-3.6.4.1\n";
}
#######################################################################
#######################################################################
# MAIN
#######################################################################
#######################################################################
my $module;
my %generate_log = ();
my $top_dir;
my $log_dir;
my $log_suffix;
my $log;
my $list_bugs = 0;
my $check_bugzilla = 0;
my $branch_name;
my $git_command = "git log";
my $git_cherry;
my $git_args = "";
my %data;
my %bugs = ();
foreach my $arg (@ARGV) {
if ($arg eq '--help') {
usage();
exit;
} elsif ($arg eq '--no-submodule') {
$module = "core";
} elsif ($arg =~ m/--module=(.*)/) {
$module = $1;
} elsif ($arg =~ m/--log-suffix=(.*)/) {
$log_suffix = "$1";
} elsif ($arg =~ m/--log-dir=(.*)/) {
$log_dir = "$1";
} elsif ($arg eq '--commits') {
$generate_log{"commits"} = 1;
} elsif ($arg eq '--bugs') {
$generate_log{"bugs"} = 1;
$check_bugzilla = 1;
$list_bugs = 1;
} elsif ($arg eq '--bugs-changelog') {
$generate_log{"bugs-changelog"} = 1;
$check_bugzilla = 1;
$list_bugs = 1;
} elsif ($arg eq '--bugs-wiki' || $arg eq '--wikibugs') {
$generate_log{"bugs-wiki"} = 1;
$check_bugzilla = 1;
$list_bugs = 1;
} elsif ($arg eq '--bugs-numbers' || $arg eq '--bug-numbers') {
$generate_log{"bugs-numbers"} = 1;
$list_bugs = 1;
} elsif ($arg eq '--rev-list') {
$git_command = "git rev-list --pretty=medium"
} elsif ($arg eq '--cherry') {
$git_command = "git log";
$git_cherry = 1;
} else {
if (! defined $top_dir) {
$top_dir=$arg;
} else {
$git_args .= " $arg";
}
}
}
# default log
unless (%generate_log) {
$generate_log{"commits"} = 1;
}
# we want only one module
if ($module) {
my $name = $module_dirname{$module};
%module_dirname = ();
$module_dirname{$module} = $name;
}
(defined $top_dir) || die "Error: top directory is not defined\n";
(-d "$top_dir") || die "Error: not a directory: $top_dir\n";
(-f "$top_dir/.git/config") || die "Error: can't find $top_dir/.git/config\n";
(!defined $log_dir) || (-d $log_dir) || die "Error: directory does no exist: $log_dir\n";
(defined $log_suffix) || die "Error: define log suffix using --log-suffix=<string>\n";
$branch_name = get_branch_name($top_dir);
load_data(\%data, $top_dir, \%module_dirname, $branch_name, $git_command, $git_cherry, $git_args);
get_bug_list(\%data, \%bugs, $check_bugzilla) if ($list_bugs);
generate_log(\%data, \&print_commits, $log_dir, "commits", $log_suffix, $top_dir, $branch_name, 0) if (defined $generate_log{"commits"});
generate_log(\%bugs, \&print_bugs, $log_dir, "bugs", $log_suffix, $top_dir, $branch_name, 0) if (defined $generate_log{"bugs"});
generate_log(\%bugs, \&print_bugs, $log_dir, "bugs", $log_suffix, $top_dir, $branch_name, 1) if (defined $generate_log{"bugs-wiki"});
generate_log(\%bugs, \&print_bugs_changelog, $log_dir, "bugs-changelog", $log_suffix, $top_dir, $branch_name, 0) if (defined $generate_log{"bugs-changelog"});
generate_log(\%bugs, \&print_bugnumbers, $log_dir, "bug-numbers", $log_suffix, $top_dir, $branch_name, 0) if (defined $generate_log{"bugs-numbers"});