From c469bffa3be4b2d3ca7f59448dbcc7b9bfdd7bf5 Mon Sep 17 00:00:00 2001 From: Stephan Bergmann Date: Fri, 12 Jan 2024 13:51:57 +0100 Subject: [PATCH] Further adapting of create-partial-info ...along the lines of what had already been done for create-update-info, passing files to the mar executable with an -f files.txt instead of on the command line (which could grow too long for Windows), and taking the archive metadata into account. For the latter, `make create-partial-info` now expects a second make variable, ONLINEUPDATE_MAR_OLDMETADATA, in addition to ONLINEUPDATE_MAR_OLDARCHIVE. And there is a new TODO in update/create_partial_update.py that the old and new metadata should be compared for problematic changes. (For now, any such changes would just get lost by an update.) Change-Id: Ide45adedb72958fc32c5b26aa7ebb01530ccf3d7 Reviewed-on: https://gerrit.libreoffice.org/c/core/+/161972 Tested-by: Jenkins Reviewed-by: Stephan Bergmann --- Makefile.gbuild | 2 +- bin/update/create_partial_update.py | 25 +++++++- external/onlineupdate/cygpath.patch | 2 +- external/onlineupdate/lo.patch | 88 ++++++++++++++++++++++++++++- 4 files changed, 113 insertions(+), 4 deletions(-) diff --git a/Makefile.gbuild b/Makefile.gbuild index 211066becd37..3dce54520b27 100644 --- a/Makefile.gbuild +++ b/Makefile.gbuild @@ -60,7 +60,7 @@ create-partial-info: $(eval VERSION := $(LIBO_VERSION_MAJOR).$(LIBO_VERSION_MINOR).$(LIBO_VERSION_MICRO).$(LIBO_VERSION_PATCH)$(LIBO_VERSION_SUFFIX)$(LIBO_VERSION_SUFFIX_SUFFIX)) $(eval PLATFORM := $(RTL_OS)_$(RTL_ARCH)) $(eval MAR_NAME_PREFIX := $(PRODUCTNAME)_$(VERSION)_$(PLATFORM)_$(BUILDID)) - MBSDIFF=$(WORKDIR)/LinkTarget/Executable/mbsdiff MAR=$(INSTDIR)/program/mar $(if $(filter WNT,$(OS)),$(shell cygpath -u $(SRCDIR)/bin/update/create_partial_update.py),$(SRCDIR)/bin/update/create_partial_update.py) "$(WORKDIR)" "$(MAR_NAME_PREFIX)" LOOnlineUpdater "$(ONLINEUPDATE_MAR_CERTIFICATEPATH)" "$(ONLINEUPDATE_MAR_CERTIFICATENAME)" "$(ONLINEUPDATE_MAR_BASEURL)" "$(PRODUCTNAME)" '$(LIBO_VERSION_MAJOR).$(LIBO_VERSION_MINOR).$(LIBO_VERSION_MICRO).$(LIBO_VERSION_PATCH)' "$(ONLINEUPDATE_MAR_OLDARCHIVE)" + MBSDIFF=$(WORKDIR)/LinkTarget/Executable/mbsdiff MAR=$(INSTDIR)/program/mar $(if $(filter WNT,$(OS)),$(shell cygpath -u $(SRCDIR)/bin/update/create_partial_update.py),$(SRCDIR)/bin/update/create_partial_update.py) "$(WORKDIR)" "$(MAR_NAME_PREFIX)" LOOnlineUpdater "$(ONLINEUPDATE_MAR_CERTIFICATEPATH)" "$(ONLINEUPDATE_MAR_CERTIFICATENAME)" "$(ONLINEUPDATE_MAR_BASEURL)" "$(PRODUCTNAME)" '$(LIBO_VERSION_MAJOR).$(LIBO_VERSION_MINOR).$(LIBO_VERSION_MICRO).$(LIBO_VERSION_PATCH)' "$(ONLINEUPDATE_MAR_OLDARCHIVE)" "$(ONLINEUPDATE_MAR_OLDMETADATA)" # also possible to bypass the dependencies/the gbuild processing by just running # LD_LIBRARY_PATH=instdir/program make cmd cmd='ALL_LANGS="$(ALL_LANGS)" workdir/LinkTarget/Executable/pocheck' diff --git a/bin/update/create_partial_update.py b/bin/update/create_partial_update.py index cc9bb745852b..88a4553d6a36 100755 --- a/bin/update/create_partial_update.py +++ b/bin/update/create_partial_update.py @@ -30,6 +30,7 @@ def main(): product_name = sys.argv[7] version = sys.argv[8] old_archive = sys.argv[9] + old_metadata = sys.argv[10] old_uncompress_dir = uncompress_file_to_dir(old_archive, updater_path.get_previous_build_dir()) versionini = os.path.join(old_uncompress_dir, 'program', 'version.ini') #TODO: Linux, macOS @@ -43,6 +44,14 @@ def main(): if old_build_id is None: raise Exception(f'Cannot find buildid in {versionini}') + #TODO: check for problematic changes between old and new metadata + with open(old_metadata) as meta: + for l in meta: + m = re.fullmatch('skip (.*)', l.rstrip()) + if m and m.group(1).startswith(f'{product_name}/'): + path = m.group(1)[len(f'{product_name}/'):] + os.remove(os.path.join(old_uncompress_dir, path)) + new_tar_file_glob = os.path.join(updater_path.get_workdir(), "installation", product_name, "archive", "install", "*", f'{product_name}_*_archive*') new_tar_files = glob.glob(new_tar_file_glob) if len(new_tar_files) != 1: @@ -50,6 +59,19 @@ def main(): new_tar_file = new_tar_files[0] new_uncompress_dir = uncompress_file_to_dir(new_tar_file, updater_path.get_current_build_dir()) + new_metadata = os.path.join( + updater_path.get_workdir(), 'installation', product_name, 'archive', 'install', 'metadata') + ifsfile = os.path.join(updater_path.get_mar_dir(), 'ifs') + with open(new_metadata) as meta, open(ifsfile, 'w') as ifs: + for l in meta: + m = re.fullmatch('(skip|cond) (.*)', l.rstrip()) + if m and m.group(2).startswith(f'{product_name}/'): + path = m.group(2)[len(f'{product_name}/'):] + if m.group(1) == 'skip': + os.remove(os.path.join(new_uncompress_dir, path)) + else: + ifs.write(f'"{path}" "{path}"\n') + update_dir = updater_path.get_update_dir() file_name = generate_file_name(old_build_id, mar_name_prefix) @@ -58,7 +80,8 @@ def main(): os.putenv('MOZ_PRODUCT_VERSION', version) os.putenv('MAR_CHANNEL_ID', 'LOOnlineUpdater') subprocess.call([os.path.join(workdir, 'UnpackedTarball/onlineupdate/tools/update-packaging/make_incremental_update.sh'), convert_to_native(mar_file), - convert_to_native(old_uncompress_dir), convert_to_native(new_uncompress_dir)]) + convert_to_native(old_uncompress_dir), convert_to_native(new_uncompress_dir), + convert_to_native(ifsfile)]) sign_mar_file(update_dir, certificate_path, certificate_name, mar_file, mar_name_prefix) diff --git a/external/onlineupdate/cygpath.patch b/external/onlineupdate/cygpath.patch index e5ca1d0bcf8c..81003a2e7dae 100644 --- a/external/onlineupdate/cygpath.patch +++ b/external/onlineupdate/cygpath.patch @@ -7,5 +7,5 @@ -workdir="$(mktemp -d)" +workdir=$(cygpath -m "$(mktemp -d)") updatemanifestv3="$workdir/updatev3.manifest" - archivefiles="updatev3.manifest" + mkdir -p "$workdir" diff --git a/external/onlineupdate/lo.patch b/external/onlineupdate/lo.patch index 6935afef1558..ce4f2e4c77ec 100644 --- a/external/onlineupdate/lo.patch +++ b/external/onlineupdate/lo.patch @@ -237,6 +237,22 @@ + echo "add-if \"$testfile\" \"$f\"" >> "$filev3" + return + fi ++ fi + is_extension=$(echo "$f" | grep -c 'distribution/extensions/.*/') + if [ $is_extension = "1" ]; then + # Use the subdirectory of the extensions folder as the file to test +@@ -113,6 +122,15 @@ + f="$1" + filev3="$2" + ++ if [ -n "$IFSFILE" ]; then ++ ifsline=$(grep -F " \"$f\"" "$IFSFILE") ++ if [ -n "$ifsline" ]; then ++ testfile=$(printf '%s' "$ifsline" | cut -f 2 -d '"') ++ verbose_notice " patch-if \"$testfile\" \"$f.patch\" \"$f\"" ++ echo "patch-if \"$testfile\" \"$f.patch\" \"$f\"" >> "$filev3" ++ return ++ fi + fi is_extension=$(echo "$f" | grep -c 'distribution/extensions/.*/') if [ $is_extension = "1" ]; then @@ -291,7 +307,27 @@ # cleanup --- tools/update-packaging/make_incremental_update.sh +++ tools/update-packaging/make_incremental_update.sh -@@ -135,7 +135,6 @@ +@@ -104,6 +104,7 @@ + archive="$1" + olddir="$2" + newdir="$3" ++IFSFILE=$4 + # Prevent the workdir from being inside the targetdir so it isn't included in + # the update mar. + if [ $(echo "$newdir" | grep -c '\/$') = 1 ]; then +@@ -112,9 +113,10 @@ + fi + workdir="$(mktemp -d)" + updatemanifestv3="$workdir/updatev3.manifest" +-archivefiles="updatev3.manifest" + + mkdir -p "$workdir" ++ ++printf 'updatev3.manifest\n' >"$workdir/files.txt" + + # Generate a list of all files in the target directory. + pushd "$olddir" +@@ -135,7 +137,6 @@ if [ ! -f "precomplete" ]; then if [ ! -f "Contents/Resources/precomplete" ]; then notice "precomplete file is missing!" @@ -299,3 +335,53 @@ fi fi +@@ -170,7 +171,7 @@ + $XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f" + copy_perm "$newdir/$f" "$workdir/$f" + make_add_if_not_instruction "$f" "$updatemanifestv3" +- archivefiles="$archivefiles \"$f\"" ++ printf '%s\n' "$f" >>"$workdir/files.txt" + continue 1 + fi + +@@ -180,7 +181,7 @@ + $XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f" + copy_perm "$newdir/$f" "$workdir/$f" + make_add_instruction "$f" "$updatemanifestv3" 1 +- archivefiles="$archivefiles \"$f\"" ++ printf '%s\n' "$f" >>"$workdir/files.txt" + continue 1 + fi + +@@ -227,11 +228,11 @@ + make_patch_instruction "$f" "$updatemanifestv3" + mv -f "$patchfile" "$workdir/$f.patch" + rm -f "$workdir/$f" +- archivefiles="$archivefiles \"$f.patch\"" ++ printf '%s\n' "$f.patch" >>"$workdir/files.txt" + else + make_add_instruction "$f" "$updatemanifestv3" + rm -f "$patchfile" +- archivefiles="$archivefiles \"$f\"" ++ printf '%s\n' "$f" >>"$workdir/files.txt" + fi + fi + else +@@ -270,7 +271,7 @@ + fi + + +- archivefiles="$archivefiles \"$f\"" ++ printf '%s\n' "$f" >>"$workdir/files.txt" + done + + notice "" +@@ -302,7 +303,7 @@ + $XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force "$updatemanifestv3" && mv -f "$updatemanifestv3.xz" "$updatemanifestv3" + + mar_command="$mar_command -C \"$workdir\" -c output.mar" +-eval "$mar_command $archivefiles" ++eval "$mar_command -f $workdir/files.txt" + mv -f "$workdir/output.mar" "$archive" + + # cleanup