mar: allow parallel creation of diff packages

also include the version number in the base-URL for the mar files in the
json-file

Change-Id: I613ac7c1cf3aeb1eff856c58f00a67151267606c
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/168100
Tested-by: Jenkins
Reviewed-by: Christian Lohmaier <lohmaier+LibreOffice@googlemail.com>

Makefile.gbuild

@@ -43,12 +43,26 @@ create-update-info:
 	mkdir -p $(MAR_DIR)/language
 	MAR=$(INSTDIR)/program/mar $(if $(filter WNT,$(OS)),$(shell cygpath -u $(SRCDIR)/bin/update/create_full_mar.py),$(SRCDIR)/bin/update/create_full_mar.py) "$(PRODUCTNAME)" "$(WORKDIR)" "$(MAR_NAME_PREFIX)" "$(ONLINEUPDATE_MAR_CERTIFICATEPATH)" "$(ONLINEUPDATE_MAR_CERTIFICATENAME)" "$(ONLINEUPDATE_MAR_BASEURL)" '$(LIBO_VERSION_MAJOR).$(LIBO_VERSION_MINOR).$(LIBO_VERSION_MICRO).$(LIBO_VERSION_PATCH)'
 
-create-partial-info:
+# target-variables also will apply to the target's prerequisites (i.e. the individual platform targets)
-	$(eval BUILDID := $(shell git -C $(SRCDIR) log -1 --format=%H))
+# arch is taken from the stem of the rule
-	$(eval VERSION := $(LIBO_VERSION_MAJOR).$(LIBO_VERSION_MINOR).$(LIBO_VERSION_MICRO).$(LIBO_VERSION_PATCH)$(LIBO_VERSION_SUFFIX)$(LIBO_VERSION_SUFFIX_SUFFIX))
+# MSIDIR and OLD_VER are expected to be specified on the make commandline but others like VERSION,
-	$(eval PLATFORM := $(RTL_OS)_$(RTL_ARCH))
+# BUILDID, ONLINEUPDATE_MAR_NEWMSI and ONLINEUPDATE_MAR_OLDMSI can be overridden as well if needed.
-	$(eval MAR_NAME_PREFIX := $(PRODUCTNAME)_$(VERSION)_$(PLATFORM)_$(BUILDID))
+# If VERSION is specified on the commandline, BUILDID defaults to the corresponding libreoffice tag
-	MBSDIFF=$(WORKDIR)/LinkTarget/Executable/mbsdiff MAR=$(INSTDIR)/program/mar $(if $(filter WNT,$(OS)),$(shell cygpath -u $(SRCDIR)/bin/update/create_partial_update.py),$(SRCDIR)/bin/update/create_partial_update.py) "$(WORKDIR)" "$(MAR_NAME_PREFIX)" LOOnlineUpdater "$(ONLINEUPDATE_MAR_CERTIFICATEPATH)" "$(ONLINEUPDATE_MAR_CERTIFICATENAME)" "$(ONLINEUPDATE_MAR_BASEURL)" "$(PRODUCTNAME)" '$(LIBO_VERSION_MAJOR).$(LIBO_VERSION_MINOR).$(LIBO_VERSION_MICRO).$(LIBO_VERSION_PATCH)' "$(ONLINEUPDATE_MAR_OLDMSI)"
+create-partial-info: export ARCH = $*
+create-partial-info: PLATFORM = $(RTL_OS)_$(ARCH)
+create-partial-info: VERSION := $(LIBO_VERSION_MAJOR).$(LIBO_VERSION_MINOR).$(LIBO_VERSION_MICRO).$(LIBO_VERSION_PATCH)
+create-partial-info: VERSION_FULL := $(VERSION)$(LIBO_VERSION_SUFFIX)$(LIBO_VERSION_SUFFIX_SUFFIX)
+create-partial-info: BUILDID := $(shell git -C $(SRCDIR) log -1 --format=%H$(if $(filter command line,$(origin VERSION)), libreoffice-$(VERSION)))
+create-partial-info: MAR_NAME_PREFIX = $(PRODUCTNAME)_$(VERSION_FULL)_$(PLATFORM)_$(BUILDID)
+create-partial-info: ONLINEUPDATE_MAR_OLDMSI = $(MSIDIR)$(PRODUCTNAME)_$(OLD_VER)_Win_$(subst AARCH64,aarch64,$(subst X86_64,x86-64,$(ARCH))).msi
+create-partial-info: ONLINEUPDATE_MAR_NEWMSI = $(MSIDIR)$(PRODUCTNAME)_$(VERSION)_Win_$(subst AARCH64,aarch64,$(subst X86_64,x86-64,$(ARCH))).msi
+create-partial-info: export MBSDIFF := $(WORKDIR)/LinkTarget/Executable/mbsdiff
+create-partial-info: export MAR := $(INSTDIR)/program/mar
+
+create-partial-info: create-partial-info-AARCH64 create-partial-info-X86_64 create-partial-info-x86
+
+create-partial-info-%:
+	$(if $(filter WNT,$(OS)),$(shell cygpath -u $(SRCDIR)/bin/update/create_partial_update.py),$(SRCDIR)/bin/update/create_partial_update.py) "$(WORKDIR)" "$(MAR_NAME_PREFIX)" LOOnlineUpdater "$(ONLINEUPDATE_MAR_CERTIFICATEPATH)" "$(ONLINEUPDATE_MAR_CERTIFICATENAME)" "$(ONLINEUPDATE_MAR_BASEURL)$(VERSION)/" "$(PRODUCTNAME)" '$(VERSION)' "$(ONLINEUPDATE_MAR_OLDMSI)" "$(ONLINEUPDATE_MAR_NEWMSI)"
 
 # also possible to bypass the dependencies/the gbuild processing by just running
 # LD_LIBRARY_PATH=instdir/program make cmd cmd='ALL_LANGS="$(ALL_LANGS)" workdir/LinkTarget/Executable/pocheck'

bin/update/create_partial_update.py

@@ -1,10 +1,10 @@
 #!/usr/bin/env python3
-import glob
 import json
 import os
 import re
 import subprocess
 import sys
+import time
 
 from path import UpdaterPath, convert_to_native
 from signing import sign_mar_file
@@ -15,11 +15,23 @@ def generate_file_name(old_build_id, mar_name_prefix):
     name = "%s_from_%s_partial.mar" % (mar_name_prefix, old_build_id)
     return name
 
+def waitforlock(lockfile):
+    while True:
+        try:
+            os.close(os.open(lockfile, os.O_CREAT | os.O_EXCL))
+            break
+        except OSError:
+            print("waiting for lockfile/msiexec already running, sleeping 10s")
+            time.sleep(10)
+
+def releaselock(lockfile):
+    os.remove(lockfile)
 
 def main():
     workdir = sys.argv[1]
+    lockfile = os.path.join(workdir,"msiexeclock")
 
-    updater_path = UpdaterPath(workdir)
+    updater_path = UpdaterPath(os.path.join(workdir,os.environ.get('ARCH','unknown')))
     updater_path.ensure_dir_exist()
 
     mar_name_prefix = sys.argv[2]
@@ -30,8 +42,12 @@ def main():
     product_name = sys.argv[7]
     version = sys.argv[8]
     old_msi = sys.argv[9]
+    new_msi_file = sys.argv[10]
 
+    waitforlock(lockfile)
     old_uncompress_dir = uncompress_file_to_dir(old_msi, updater_path.get_previous_build_dir())
+    new_uncompress_dir = uncompress_file_to_dir(new_msi_file, updater_path.get_current_build_dir())
+    releaselock(lockfile)
     versionini = os.path.join(old_uncompress_dir, 'program', 'version.ini') #TODO: Linux, macOS
     old_build_id = None
     with open(versionini) as f:
@@ -43,13 +59,6 @@ def main():
     if old_build_id is None:
         raise Exception(f'Cannot find buildid in {versionini}')
 
-    new_msi_file_glob = os.path.join(updater_path.get_workdir(), "installation", product_name, "msi", "install", "*", f'{product_name}_*.msi')
-    new_msi_files = glob.glob(new_msi_file_glob)
-    if len(new_msi_files) != 1:
-        raise Exception(f'`{new_msi_file_glob}` does not match exactly one file')
-    new_msi_file = new_msi_files[0]
-    new_uncompress_dir = uncompress_file_to_dir(new_msi_file, updater_path.get_current_build_dir())
-
     update_dir = updater_path.get_update_dir()
 
     file_name = generate_file_name(old_build_id, mar_name_prefix)