Publish the libraries a few at a time

Once the tests are done, the run publishes one library at a time, and
what that costs is not the copying but waiting for the far side: each
library asks rsync four times over ssh, and the one that matters lists
a remote directory of tens of thousands of files to honour
`--delete-excluded`.

Three things were paid for per library and are now paid for once:

| What                               | Before       | After        |
| ---------------------------------- | ------------ | ------------ |
| git log of this repository         | once per lib | once         |
| git log of the reference files     | once per lib | once per dir |
| rsync connections to make the dirs | 3            | 1            |

The directories are made in one connection by telling rsync -R where
the relative part of the source path starts, so `emptydir/./<lib>/files`
creates the branch, the library and its files directory at once instead
of one rsync per level.

The libraries are then published four at a time. Not more: each one is
an ssh connection of its own, and sshd allows ten unauthenticated
connections by default.

The loop body became a function, which also stops it shadowing the
`stats` dictionary of the run with a list of its own.

Assisted-by: Claude Opus 5

Author: sjoelund

test.py

@@ -1254,6 +1254,79 @@ def artifactSuffix(simulator):
   publishes the workspace itself, so only the others have one."""
   return "_%s" % simulator if simulator else ""
 
+try:
+  githuburltesting = "https://github.com/OpenModelica/OpenModelicaLibraryTesting/commit/"
+  gitloglibrarytesting = check_output_log(["git", "log", '--pretty=<table><tr><th>Commit</th><th>Date</th><th>Author</th><th>Summary</th></tr><tr><td><a href="%s/%%h">%%h</a></td><td>%%ai</td><td>%%an</td><td>%%s</td></tr></table>' % (githuburltesting), "-1"], cwd="./").decode("utf-8")
+except subprocess.CalledProcessError as e:
+  print(str(e))
+  gitloglibrarytesting = "<table><tr><td>could not get the git log for OpenModelicaLibraryTesting</td></tr></table>"
+
+def readReferenceFilesVersion(c):
+  # adrpo: attempt to get the revision of the reference files if possible
+  if DEBUG:
+    print("referenceFiles git ... attempting to retrieve info from directory: %s" % c)
+    sys.stdout.flush()
+  gitReferenceFiles = c
+  # see if we have a commit file
+  f = os.path.join(c, "commit")
+  if os.path.exists(f):
+    with open(f) as fin:
+      print("referenceFiles git ... read from file %s" % f)
+      sys.stdout.flush()
+      return fin.read()
+  try:
+    if isinstance(c, (str, bytes)):
+      if DEBUG:
+        print("referenceFiles git ... see if directory has an evironment variable")
+        sys.stdout.flush()
+      m = re.search("^[$][A-Z_]+", c)
+      if m:
+        k = m.group(0)[1:]
+        if k not in os.environ:
+          if DEBUG:
+            print("referenceFiles git ... environment variable used in the directory cannot be found in the environment: %s" % k)
+            sys.stdout.flush()
+          raise Exception("Environment variable %s not defined, but used in JSON config for reference files" % k)
+        gitReferenceFiles = c.replace(m.group(0), os.environ[k])
+        if DEBUG:
+          print("referenceFiles git ... directory after replacing the environment variable: %s" % gitReferenceFiles)
+          sys.stdout.flush()
+      sys.stdout.flush()
+    try:
+      gitReferenceFilesURL = check_output_log(["git", "config", "--get", "remote.origin.url"], cwd=gitReferenceFiles).decode("utf-8")
+    except subprocess.CalledProcessError as e:
+      print(e)
+      gitReferenceFilesURL = gitReferenceFiles
+    gitReferenceFilesVersion = check_output_log(["git", "log", '--pretty=<table><tr><th>Commit</th><th>Date</th><th>Author</th><th>Summary</th></tr><tr><td><a href="%s/%%h">%%h</a></td><td>%%ai</td><td>%%an</td><td>%%s</td></tr></table>' % (gitReferenceFilesURL), "-1"], cwd=gitReferenceFiles).decode("utf-8")
+    print("referenceFiles git ... got version information: %s" % gitReferenceFilesVersion)
+    sys.stdout.flush()
+    return gitReferenceFilesVersion
+  except subprocess.CalledProcessError as e:
+    print("referenceFiles git ... something went wrong with getting the git info for directory: %s" % c)
+    print(str(e))
+    sys.stdout.flush()
+    return ""
+
+referenceFilesVersions = {}
+
+def referenceFilesVersion(c):
+  """The git log of a directory of reference files.
+
+  Nearly every library shares the directory with another, and the git log of it
+  is the same for all of them, so it is asked for once per directory instead of
+  once per library.
+  """
+  if not c:
+    return ""
+  if c not in referenceFilesVersions:
+    referenceFilesVersions[c] = readReferenceFilesVersion(c)
+  return referenceFilesVersions[c]
+
+# Publishing one library is rsync waiting for the far side to enumerate a
+# directory of tens of thousands of files, so the libraries are published a few
+# at a time. Not many more: each one is an ssh connection of its own.
+PUBLISH_JOBS = 4
+
 # Every runner publishes its own results - .sim and diff files included - to the
 # directory of its own branch; the .err of the build is shared, so each of them
 # gets a copy of it.
@@ -1268,9 +1341,9 @@ def artifactSuffix(simulator):
     os.makedirs(resRootPath)
 
   htmltpl=open("library.html.tpl").read()
-  for libname in stats_by_libname.keys():
+  def publishLibrary(libname):
     if libname in skipped_libs or not ranRunner(libname, runner):
-      continue
+      return
     s = None # Make sure I don't use this
     simulator = simulatorKey(libname, runner)
     suffix = artifactSuffix(simulator)
@@ -1332,61 +1405,7 @@ def artifactSuffix(simulator):
       for s in natsorted(stats, key=lambda s: s[1])])
     numSucceeded = [len(stats)] + [sum(1 if s[3]["phase"]>=i else 0 for s in stats) for i in range(1,8)]
 
-    try:
-      githuburltesting = "https://github.com/OpenModelica/OpenModelicaLibraryTesting/commit/"
-      gitloglibrarytesting = check_output_log(["git", "log", '--pretty=<table><tr><th>Commit</th><th>Date</th><th>Author</th><th>Summary</th></tr><tr><td><a href="%s/%%h">%%h</a></td><td>%%ai</td><td>%%an</td><td>%%s</td></tr></table>' % (githuburltesting), "-1"], cwd="./").decode("utf-8")
-    except subprocess.CalledProcessError as e:
-      print(str(e))
-      gitloglibrarytesting = "<table><tr><td>could not get the git log for OpenModelicaLibraryTesting</td></tr></table>"
-
-    # adrpo: attempt to get the revision of the reference files if possible
-    if conf.get("referenceFiles"):
-      c = conf.get("referenceFiles")
-      if DEBUG:
-        print("referenceFiles git ... attempting to retrieve info from directory: %s" % c)
-        sys.stdout.flush()
-      gitReferenceFiles = c
-      # see if we have a commit file
-      f = os.path.join(c, "commit")
-      if os.path.exists(f):
-        with open(f) as fin:
-          gitReferenceFilesVersion = fin.read()
-          print("referenceFiles git ... read from file %s" % f)
-          sys.stdout.flush()
-      else:
-        try:
-          if isinstance(c, (str, bytes)):
-            if DEBUG:
-              print("referenceFiles git ... see if directory has an evironment variable")
-              sys.stdout.flush()
-            m = re.search("^[$][A-Z_]+", c)
-            if m:
-              k = m.group(0)[1:]
-              if k not in os.environ:
-                if DEBUG:
-                  print("referenceFiles git ... environment variable used in the directory cannot be found in the environment: %s" % k)
-                  sys.stdout.flush()
-                raise Exception("Environment variable %s not defined, but used in JSON config for reference files" % k)
-              gitReferenceFiles = c.replace(m.group(0), os.environ[k])
-              if DEBUG:
-                print("referenceFiles git ... directory after replacing the environment variable: %s" % gitReferenceFiles)
-                sys.stdout.flush()
-            sys.stdout.flush()
-          try:
-            gitReferenceFilesURL = check_output_log(["git", "config", "--get", "remote.origin.url"], cwd=gitReferenceFiles).decode("utf-8")
-          except subprocess.CalledProcessError as e:
-            print(e)
-            gitReferenceFilesURL = gitReferenceFiles
-          gitReferenceFilesVersion = check_output_log(["git", "log", '--pretty=<table><tr><th>Commit</th><th>Date</th><th>Author</th><th>Summary</th></tr><tr><td><a href="%s/%%h">%%h</a></td><td>%%ai</td><td>%%an</td><td>%%s</td></tr></table>' % (gitReferenceFilesURL), "-1"], cwd=gitReferenceFiles).decode("utf-8")
-          print("referenceFiles git ... got version information: %s" % gitReferenceFilesVersion)
-          sys.stdout.flush()
-        except subprocess.CalledProcessError as e:
-          print("referenceFiles git ... something went wrong with getting the git info for directory: %s" % c)
-          print(str(e))
-          sys.stdout.flush()
-          gitReferenceFilesVersion = ""
-    else:
-      gitReferenceFilesVersion = ""
+    gitReferenceFilesVersion = referenceFilesVersion(conf.get("referenceFiles"))
 
     replacements = (
       (u"#sysInfo#", html.escape(sysInfo)),
@@ -1433,19 +1452,16 @@ def artifactSuffix(simulator):
     # move results by sync operations (not available under win)
     if result_location != "" and not isWin and not noSync:
       result_location_libname = "%s/%s" % (result_location, libname)
-      try:
-        os.mkdir(os.path.join(stageRoot, "emptydir"))
-      except:
-        pass
-      check_output_log(["rsync", "-aR", "emptydir/", result_location], cwd=stageRoot)
-      check_output_log(["rsync", "-aR", "emptydir/", result_location_libname], cwd=stageRoot)
-      check_output_log(["rsync", "-aR", "emptydir/", result_location_libname+"/files"], cwd=stageRoot)
+      def makeRemoteDirs():
+        # The /./ tells rsync -R where the relative part starts, so the branch,
+        # the library and its files directory are created in one connection.
+        os.makedirs(os.path.join(stageRoot, "emptydir", libname, "files"), exist_ok=True)
+        check_output_log(["rsync", "-aR", "emptydir/./%s/files" % libname, result_location], cwd=stageRoot)
+      makeRemoteDirs()
       try:
         check_output_log(["rsync", "-aR", "--delete-excluded", "--include-from=%s.files" % libname, "--exclude=*", "./", result_location_libname], cwd=stageRoot)
       except:
-        check_output_log(["rsync", "-aR", "emptydir/", result_location], cwd=stageRoot)
-        check_output_log(["rsync", "-aR", "emptydir/", result_location_libname], cwd=stageRoot)
-        check_output_log(["rsync", "-aR", "emptydir/", result_location_libname+"/files"], cwd=stageRoot)
+        makeRemoteDirs()
         check_output_log(["rsync", "-aR", "--delete-excluded", "--include-from=%s.files" % libname, "--exclude=*", "./", result_location_libname], cwd=stageRoot)
       if (conf.get("referenceFiles") or "") != "" and dygraphs:
         check_output_log(["rsync", "-a", dygraphs, result_location_libname+"/files"])
@@ -1482,6 +1498,8 @@ def artifactSuffix(simulator):
           print("-- problem during file copy... maybe the file is still hooked by a process... :" + file)
           pass
 
+  Parallel(n_jobs=PUBLISH_JOBS, backend="threading")(delayed(publishLibrary)(libname) for libname in stats_by_libname.keys())
+
 if clean:
   for g in ["*.o","*.so","*.h","*.c","*.cpp","*.simsuccess","*.conf.json","*.tmpfiles","*.log","*.libs","OMCpp*","*.fmu*","temp_*", "*.exe", "HelloWorld.bat", "*.makefile", "*.mat","*.xml", "*.bin", "*.json"]:
     for f in glob.glob(g):