Fix make create-partial-info (for Windows, at least)

I got lost trying to figure out how the original
bin/update/create_partial_update.py code was meant to obtain old and new
installation trees to diff, so I simplified that down to the create-partial-info
make target now expecting an ONLINEUPDATE_MAR_OLDARCHIVE make variable that
points at the old archive install set.  (And the
--with-online-update-mar-serverurl configure option is gone for good again.)
The remaining changes are similar to what was needed in
28bad382fa "Fix `make create-update-info` (for
Windows, at least)".  (And the mbsdiff and mar tools expect Windows-style
pathnames, but mktemp returns a Unix-style pathname in cygwin shell scripts, so
this needed an additional Windows-only external/onlineupdate/cygpath.patch.)

Change-Id: I40690210d62e3f26fb2d574914a0dd4323e6cd62
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/161924
Tested-by: Jenkins
Reviewed-by: Stephan Bergmann <stephan.bergmann@allotropia.de>

Makefile.gbuild

@@ -60,7 +60,7 @@ create-partial-info:
 	$(eval VERSION := $(LIBO_VERSION_MAJOR).$(LIBO_VERSION_MINOR).$(LIBO_VERSION_MICRO).$(LIBO_VERSION_PATCH)$(LIBO_VERSION_SUFFIX)$(LIBO_VERSION_SUFFIX_SUFFIX))
 	$(eval PLATFORM := $(RTL_OS)_$(RTL_ARCH))
 	$(eval MAR_NAME_PREFIX := $(PRODUCTNAME)_$(VERSION)_$(PLATFORM)_$(BUILDID))
-	MBSDIFF=$(WORKDIR)/LinkTarget/Executable/mbsdiff MAR=$(INSTDIR)/program/mar $(SRCDIR)/bin/update/create_partial_update.py "$(WORKDIR)" "$(MAR_NAME_PREFIX)" "$(ONLINEUPDATE_MAR_SERVERURL)" LOOnlineUpdater "$(ONLINEUPDATE_MAR_CERTIFICATEPATH)" "$(ONLINEUPDATE_MAR_CERTIFICATENAME)" "$(ONLINEUPDATE_MAR_BASEURL)" "$(PLATFORM)" "$(BUILDID)"
+	MBSDIFF=$(WORKDIR)/LinkTarget/Executable/mbsdiff MAR=$(INSTDIR)/program/mar $(if $(filter WNT,$(OS)),$(shell cygpath -u $(SRCDIR)/bin/update/create_partial_update.py),$(SRCDIR)/bin/update/create_partial_update.py) "$(WORKDIR)" "$(MAR_NAME_PREFIX)" LOOnlineUpdater "$(ONLINEUPDATE_MAR_CERTIFICATEPATH)" "$(ONLINEUPDATE_MAR_CERTIFICATENAME)" "$(ONLINEUPDATE_MAR_BASEURL)" "$(PRODUCTNAME)" '$(LIBO_VERSION_MAJOR).$(LIBO_VERSION_MINOR).$(LIBO_VERSION_MICRO).$(LIBO_VERSION_PATCH)' "$(ONLINEUPDATE_MAR_OLDARCHIVE)"
 
 # also possible to bypass the dependencies/the gbuild processing by just running
 # LD_LIBRARY_PATH=instdir/program make cmd cmd='ALL_LANGS="$(ALL_LANGS)" workdir/LinkTarget/Executable/pocheck'

bin/update/create_partial_update.py

@@ -1,90 +1,14 @@
 #!/usr/bin/env python3
+import glob
 import json
 import os
+import re
 import subprocess
 import sys
 
-import requests
-
-from path import UpdaterPath, mkdir_p, convert_to_unix, convert_to_native
+from path import UpdaterPath, convert_to_native
 from signing import sign_mar_file
-from tools import get_file_info, get_hash
-from uncompress_mar import extract_mar
-
-BUF_SIZE = 1024
-current_dir_path = os.path.dirname(os.path.realpath(convert_to_unix(__file__)))
-
-
-class InvalidFileException(Exception):
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(self, *args, **kwargs)
-
-
-def download_file(filepath, url, hash_string):
-    with open(filepath, "wb") as f:
-        response = requests.get(url, stream=True)
-
-        if not response.ok:
-            return
-
-        for block in response.iter_content(1024):
-            f.write(block)
-
-    file_hash = get_hash(filepath)
-
-    if file_hash != hash_string:
-        raise InvalidFileException(
-            "file hash does not match for file %s: Expected %s, Got: %s" % (url, hash_string, file_hash))
-
-
-def handle_language(lang_entries, filedir):
-    langs = {}
-    for lang, data in lang_entries.items():
-        lang_dir = os.path.join(filedir, lang)
-        lang_file = os.path.join(lang_dir, "lang.mar")
-        mkdir_p(lang_dir)
-        download_file(lang_file, data["url"], data["hash"])
-        dir_path = os.path.join(lang_dir, "lang")
-        mkdir_p(dir_path)
-        extract_mar(lang_file, dir_path)
-        langs[lang] = dir_path
-
-    return langs
-
-
-def download_mar_for_update_channel_and_platform(server_url, channel, platform, temp_dir):
-    base_url = server_url + "update/partial-targets/1/"
-    url = base_url + platform + "/" + channel
-    r = requests.get(url)
-    if r.status_code != 200:
-        print(r.content)
-        raise Exception("download failed")
-
-    update_info = json.loads(r.content.decode("utf-8"))
-    update_files = update_info['updates']
-    downloaded_updates = {}
-    for update_file in update_files:
-        build = update_file["build"]
-        filedir = os.path.join(temp_dir, build)
-
-        mkdir_p(filedir)
-
-        filepath = filedir + "/complete.mar"
-        url = update_file["update"]["url"]
-        expected_hash = update_file["update"]["hash"]
-        download_file(filepath, url, expected_hash)
-
-        dir_path = os.path.join(filedir, "complete")
-        mkdir_p(dir_path)
-        extract_mar(filepath, dir_path)
-
-        downloaded_updates[build] = {"complete": dir_path}
-
-        langs = handle_language(update_file["languages"], filedir)
-        downloaded_updates[build]["languages"] = langs
-
-    return downloaded_updates
+from tools import get_file_info, uncompress_file_to_dir
 
 
 def generate_file_name(old_build_id, mar_name_prefix):
@@ -92,16 +16,6 @@ def generate_file_name(old_build_id, mar_name_prefix):
     return name
 
 
-def generate_lang_file_name(old_build_id, mar_name_prefix, lang):
-    name = "%s_%s_from_%s_partial.mar" % (mar_name_prefix, lang, old_build_id)
-    return name
-
-
-def add_single_dir(path):
-    dir_name = [os.path.join(path, name) for name in os.listdir(path) if os.path.isdir(os.path.join(path, name))]
-    return dir_name[0]
-
-
 def main():
     workdir = sys.argv[1]
 
@@ -109,60 +23,53 @@ def main():
     updater_path.ensure_dir_exist()
 
     mar_name_prefix = sys.argv[2]
-    server_url = sys.argv[3]
-    channel = sys.argv[4]
-    certificate_path = sys.argv[5]
-    certificate_name = sys.argv[6]
-    base_url = sys.argv[7]
-    platform = sys.argv[8]
-    build_id = sys.argv[9]
+    channel = sys.argv[3]
+    certificate_path = sys.argv[4]
+    certificate_name = sys.argv[5]
+    base_url = sys.argv[6]
+    product_name = sys.argv[7]
+    version = sys.argv[8]
+    old_archive = sys.argv[9]
+
+    old_uncompress_dir = uncompress_file_to_dir(old_archive, updater_path.get_previous_build_dir())
+    versionini = os.path.join(old_uncompress_dir, 'program', 'version.ini') #TODO: Linux, macOS
+    old_build_id = None
+    with open(versionini) as f:
+        for l in f:
+            m = re.fullmatch('buildid=(.*)', l.rstrip())
+            if m:
+                old_build_id = m.group(1)
+                break
+    if old_build_id is None:
+        raise Exception(f'Cannot find buildid in {versionini}')
+
+    new_tar_file_glob = os.path.join(updater_path.get_workdir(), "installation", product_name, "archive", "install", "*", f'{product_name}_*_archive*')
+    new_tar_files = glob.glob(new_tar_file_glob)
+    if len(new_tar_files) != 1:
+        raise Exception(f'`{new_tar_file_glob}` does not match exactly one file')
+    new_tar_file = new_tar_files[0]
+    new_uncompress_dir = uncompress_file_to_dir(new_tar_file, updater_path.get_current_build_dir())
 
-    current_build_path = updater_path.get_current_build_dir()
-    mar_dir = updater_path.get_mar_dir()
-    temp_dir = updater_path.get_previous_build_dir()
     update_dir = updater_path.get_update_dir()
 
-    current_build_path = add_single_dir(current_build_path)
-    if sys.platform == "cygwin":
-        current_build_path = add_single_dir(current_build_path)
+    file_name = generate_file_name(old_build_id, mar_name_prefix)
+    mar_file = os.path.join(update_dir, file_name)
 
-    updates = download_mar_for_update_channel_and_platform(server_url, channel, platform, temp_dir)
+    os.putenv('MOZ_PRODUCT_VERSION', version)
+    os.putenv('MAR_CHANNEL_ID', 'LOOnlineUpdater')
+    subprocess.call([os.path.join(workdir, 'UnpackedTarball/onlineupdate/tools/update-packaging/make_incremental_update.sh'), convert_to_native(mar_file),
+                     convert_to_native(old_uncompress_dir), convert_to_native(new_uncompress_dir)])
 
-    data = {"partials": []}
+    sign_mar_file(update_dir, certificate_path, certificate_name, mar_file, mar_name_prefix)
 
-    for build, update in updates.items():
-        file_name = generate_file_name(build, mar_name_prefix)
-        mar_file = os.path.join(update_dir, file_name)
-        subprocess.call([os.path.join(current_dir_path, 'make_incremental_update.sh'), convert_to_native(mar_file),
-                         convert_to_native(update["complete"]), convert_to_native(current_build_path)])
-        sign_mar_file(update_dir, certificate_path, certificate_name, mar_file, mar_name_prefix)
-
-        partial_info = {"file": get_file_info(mar_file, base_url), "from": build, "to": build_id,
-                        "languages": {}}
-
-        # on Windows we don't use language packs
-        if sys.platform != "cygwin":
-            for lang, lang_info in update["languages"].items():
-                lang_name = generate_lang_file_name(build, mar_name_prefix, lang)
-
-                # write the file into the final directory
-                lang_mar_file = os.path.join(update_dir, lang_name)
-
-                # the directory of the old language file is of the form
-                # workdir/mar/language/en-US/LibreOffice_<version>_<os>_archive_langpack_<lang>/
-                language_dir = add_single_dir(os.path.join(mar_dir, "language", lang))
-                subprocess.call(
-                    [os.path.join(current_dir_path, 'make_incremental_update.sh'), convert_to_native(lang_mar_file),
-                     convert_to_native(lang_info), convert_to_native(language_dir)])
-                sign_mar_file(update_dir, certificate_path, certificate_name, lang_mar_file, mar_name_prefix)
-
-                # add the partial language info
-                partial_info["languages"][lang] = get_file_info(lang_mar_file, base_url)
-
-        data["partials"].append(partial_info)
-
-    with open(os.path.join(update_dir, "partial_update_info.json"), "w") as f:
-        json.dump(data, f)
+    data = {
+        'from': old_build_id,
+        'see also': '',
+        'update': get_file_info(mar_file, base_url),
+        'languages': {}
+    };
+    with open(os.path.join(update_dir, channel), "w") as f:
+        json.dump(data, f, indent=4)
 
 
 if __name__ == '__main__':

bin/update/path.py

@@ -8,19 +8,9 @@
 #
 
 import os
-import errno
 import subprocess
 from sys import platform
 
-def mkdir_p(path):
-    try:
-        os.makedirs(path)
-    except OSError as exc:  # Python >2.5
-        if exc.errno == errno.EEXIST and os.path.isdir(path):
-            pass
-        else:
-            raise
-
 def convert_to_unix(path):
     if platform == "cygwin":
         return subprocess.check_output(["cygpath", "-u", path]).decode("utf-8", "strict").rstrip()

config_host.mk.in

@@ -514,7 +514,6 @@ export ONLINEUPDATE_MAR_BASEURL=@ONLINEUPDATE_MAR_BASEURL@
 export ONLINEUPDATE_MAR_CERTIFICATEDER=@ONLINEUPDATE_MAR_CERTIFICATEDER@
 export ONLINEUPDATE_MAR_CERTIFICATENAME=@ONLINEUPDATE_MAR_CERTIFICATENAME@
 export ONLINEUPDATE_MAR_CERTIFICATEPATH=@ONLINEUPDATE_MAR_CERTIFICATEPATH@
-export ONLINEUPDATE_MAR_SERVERURL=@ONLINEUPDATE_MAR_SERVERURL@
 export ONLINEUPDATE_MAR_UPLOADURL=@ONLINEUPDATE_MAR_UPLOADURL@
 export OOO_JUNIT_JAR=@OOO_JUNIT_JAR@
 export OOO_VENDOR=@OOO_VENDOR@

configure.ac

@@ -1893,13 +1893,6 @@ libo_FUZZ_ARG_WITH(online-update-mar-certificatepath,
          non-functional.)]),
 ,)
 
-libo_FUZZ_ARG_WITH(online-update-mar-serverurl,
-    AS_HELP_STRING([--with-online-update-mar-serverurl=...],
-        [Set the server URL value for --enable-online-update-mar.
-         (Can be left off for debug purposes, even if that may render the feature
-         non-functional.)]),
-,)
-
 libo_FUZZ_ARG_WITH(online-update-mar-uploadurl,
     AS_HELP_STRING([--with-online-update-mar-uploadurl=...],
         [Set the upload URL value for --enable-online-update-mar.
@@ -13686,15 +13679,6 @@ else
 fi
 AC_SUBST(ONLINEUPDATE_MAR_CERTIFICATEPATH)
 
-AC_MSG_CHECKING([for mar online update serverurl])
-ONLINEUPDATE_MAR_SERVERURL=$with_online_update_mar_serverurl
-if test -n "$ONLINEUPDATE_MAR_SERVERURL"; then
-    AC_MSG_RESULT([yes])
-else
-    AC_MSG_RESULT([no])
-fi
-AC_SUBST(ONLINEUPDATE_MAR_SERVERURL)
-
 AC_MSG_CHECKING([for mar online update uploadurl])
 ONLINEUPDATE_MAR_UPLOADURL=$with_online_update_mar_uploadurl
 if test -n "$ONLINEUPDATE_MAR_UPLOADURL"; then

distro-configs/Jenkins/LibreOfficeLinuxUpdater.conf

@@ -36,7 +36,6 @@
 --with-online-update-mar-certificateder=TODO
 --with-online-update-mar-certificatename=TODO
 --with-online-update-mar-certificatepath=TODO
---with-online-update-mar-serverurl=TODO
 --with-online-update-mar-uploadurl=TODO
 --with-jdk-home=/etc/alternatives/java_sdk_17
 --enable-odk

external/onlineupdate/UnpackedTarball_onlineupdate.mk

@@ -18,6 +18,12 @@ $(eval $(call gb_UnpackedTarball_add_patches,onlineupdate, \
     external/onlineupdate/lo.patch \
 ))
 
+ifeq ($(OS),WNT)
+$(eval $(call gb_UnpackedTarball_add_patches,onlineupdate, \
+    external/onlineupdate/cygpath.patch \
+))
+endif
+
 # The update maintenance service that is used on Windows has a couple of checks that files in the
 # to-be-updated installation set are signed, which would fail for --disable-windows-build-signing;
 # so, as a HACK for debugging purposes, silence those problematic checks for --enable-dbgutil:

external/onlineupdate/cygpath.patch

@@ -0,0 +1,11 @@
+--- tools/update-packaging/make_incremental_update.sh
++++ tools/update-packaging/make_incremental_update.sh
+@@ -110,7 +110,7 @@
+   # Remove the /
+   newdir=$(echo "$newdir" | sed -e 's:\/$::')
+ fi
+-workdir="$(mktemp -d)"
++workdir=$(cygpath -m "$(mktemp -d)")
+ updatemanifestv3="$workdir/updatev3.manifest"
+ archivefiles="updatev3.manifest"
+ 

external/onlineupdate/lo.patch

@@ -289,3 +289,13 @@
  mv -f "$workdir/output.mar" "$archive"
  
  # cleanup
+--- tools/update-packaging/make_incremental_update.sh
++++ tools/update-packaging/make_incremental_update.sh
+@@ -135,7 +135,6 @@
+ if [ ! -f "precomplete" ]; then
+   if [ ! -f "Contents/Resources/precomplete" ]; then
+     notice "precomplete file is missing!"
+-    exit 1
+   fi
+ fi
+