From cb9883b1e9fb620f280e950de2bb50f9ec3a4c5e Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Tue, 24 Jan 2023 23:00:44 +0100 Subject: [PATCH] lo-pack-sources: Use threaded compression for xz. The xz utility can compress in parallel by using more CPU cores. The more CPUs are available the quicker it gets. The option -T0 automatically detects the number of available CPUs and uses all of them. Additional benefit is that xz starting with v5.4 can decompress the archive using multiple CPUs as well. The parallel compression works by splitting the input in multiple blocks of equal size which are compressed in parallel. Since the state is not preserved / reuse across blocks, the compression gets a little worse: -6 255M (current default) -T0 261M -eT0 259M -7T0 251M -7eT0 249M -7 uses larger blocks and dictionary which requires more memory on the compressing and decompressing side vs the current default -6. The -e option spends some extra cycles and improves the compression a bit. Use parallel compression and spend some extra cycles. Signed-off-by: Sebastian Andrzej Siewior Change-Id: Ib69c4bc1996ddd52153b8ac27b653c21e3998a68 Reviewed-on: https://gerrit.libreoffice.org/c/core/+/146793 Tested-by: Jenkins Reviewed-by: Thorsten Behrens --- bin/lo-pack-sources | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/bin/lo-pack-sources b/bin/lo-pack-sources index b4150ab87ad7..289a595b06e0 100755 --- a/bin/lo-pack-sources +++ b/bin/lo-pack-sources @@ -283,10 +283,22 @@ sub pack_module_sources($$$$) sub generate_module_tarball($$$$$$$$) { my ($source_dir, $release_version, $module, $md5, $bzip2, $xz, $lo_topdir_name, $module_tarball_name) = @_; + my $PARALLELISM = $ENV{'PARALLELISM'}; + my $CPUS_XZ = 0; + + # Set CPUS_ to the number of CPUs that should be used. If PARALLELISM + # is set then this is used otherwise autodetect is used. + if (defined $PARALLELISM) { + if ($PARALLELISM > 0) { + $CPUS_XZ = $PARALLELISM; + } else { + $CPUS_XZ = 1; + } + } my $temp_dir = prepare_module_sources($source_dir, $release_version, $module, $lo_topdir_name); pack_module_sources($temp_dir, $md5, "$module_tarball_name.tar.bz2", "bzip2 -z --best > ") if (defined $bzip2); - pack_module_sources($temp_dir, $md5, "$module_tarball_name.tar.xz", "xz -z > ") if (defined $xz); + pack_module_sources($temp_dir, $md5, "$module_tarball_name.tar.xz", "xz -z -T$CPUS_XZ -e > ") if (defined $xz); remove_tempdir($temp_dir); }