From 23c93821ba3bd6726018e00f702bb2892b342aba Mon Sep 17 00:00:00 2001 From: Johannes Meusburger Date: Mon, 28 Sep 2026 22:53:29 +0000 Subject: [PATCH 1/2] Avoid launching git & file subprocesses in G2Project.save() G2Project.save() is called by every G2Project.refine() and launched three external processes per call: 'git' twice (via GitPython) and 'file' once (via platform.architecture()). For scripted use this serializes on process creation; with many parallel workers on a shared filesystem (HPC) it dominates the run time. Three small, behavior-preserving changes: * GSASIIpath.HowIsG2Installed(): the two 'github'+rev returns did not set G2_installed_result, so the documented cache was never filled for GitHub installs and every call reopened the repo and ran 'git remote get-url'. Set the cache before returning. * GSASIIpath.getG2CommitHash() (new): returns the HEAD commit hexsha, cached, since the checked-out commit cannot change while GSAS-II is running. G2Project.save() now uses it instead of creating a new Repo and reading head.commit on every save. * GSASIIfiles.get_python_versions(): use struct.calcsize('P') instead of platform.architecture(), which runs 'file ' on POSIX on every call. Produces the identical '32bit'/'64bit' string. Saved .gpx metadata (PythonVersions, LastSavedUsing) is unchanged. Child processes per save() drop from 3 to 0 after the first call. --- GSASII/GSASIIfiles.py | 7 ++++++- GSASII/GSASIIpath.py | 25 +++++++++++++++++++++++-- GSASII/GSASIIscriptable.py | 5 ++--- 3 files changed, 31 insertions(+), 6 deletions(-) diff --git a/GSASII/GSASIIfiles.py b/GSASII/GSASIIfiles.py index 59dd11e4..dc7db58e 100644 --- a/GSASII/GSASIIfiles.py +++ b/GSASII/GSASIIfiles.py @@ -11,6 +11,7 @@ from __future__ import division, print_function import platform import os +import struct import sys import glob import copy @@ -119,8 +120,12 @@ def get_python_versions(packagelist): versions.append([pack.__name__, pack.__version__]) except: pass + # struct.calcsize('P') gives the pointer size, producing the same + # '32bit'/'64bit' string as platform.architecture()[0] without + # launching an external process (platform.architecture runs + # 'file ' on POSIX systems every time it is called) versions.append(['Platform', - sys.platform + ' ' + platform.architecture()[0] + + sys.platform + ' ' + f"{struct.calcsize('P')*8}bit" + ' ' + platform.machine()]) return versions diff --git a/GSASII/GSASIIpath.py b/GSASII/GSASIIpath.py index c858f73b..7e288826 100644 --- a/GSASII/GSASIIpath.py +++ b/GSASII/GSASIIpath.py @@ -205,10 +205,12 @@ def HowIsG2Installed(): else: rev = '-rev' if g2URL in g2repo.remote().urls: - return 'github'+rev + G2_installed_result = 'github'+rev + return G2_installed_result elif g2URL.replace('https://github.com/', 'git@github.com:') in g2repo.remote().urls: - return 'github'+rev + G2_installed_result = 'github'+rev + return G2_installed_result G2_installed_result = 'git'+rev return G2_installed_result except: @@ -216,6 +218,25 @@ def HowIsG2Installed(): G2_installed_result = 'noVCS' return G2_installed_result +G2_commit_hash_result = None +def getG2CommitHash(): + '''Returns the hex sha for the current GSAS-II commit (HEAD) for a + git-based install. The result is cached to avoid the cost of + starting a git process for repeated calls, since the checked-out + commit will not change while GSAS-II is running. + + :returns: the hexsha string for the current commit, or None if it + cannot be determined (e.g. not installed via git). + ''' + global G2_commit_hash_result + if G2_commit_hash_result is not None: return G2_commit_hash_result + try: + g2repo = openGitRepo(path2GSAS2) + G2_commit_hash_result = g2repo.head.commit.hexsha + except: + pass + return G2_commit_hash_result + def getSavedVersionInfo(): '''Get version number information from a file written by install routines. This is faster than getting the information from git. Also, diff --git a/GSASII/GSASIIscriptable.py b/GSASII/GSASIIscriptable.py index 0e96df96..de915722 100644 --- a/GSASII/GSASIIscriptable.py +++ b/GSASII/GSASIIscriptable.py @@ -1023,9 +1023,8 @@ def save(self, filename=None): controls_data['LastSavedUsing'] = str(GSASIIpath.GetVersionNumber()) try: if GSASIIpath.HowIsG2Installed().startswith('git'): - g2repo = GSASIIpath.openGitRepo(GSASIIpath.path2GSAS2) - commit = g2repo.head.commit - controls_data['LastSavedUsing'] += f" git {commit.hexsha[:8]} script" + commit = GSASIIpath.getG2CommitHash() + controls_data['LastSavedUsing'] += f" git {commit[:8]} script" else: gv = GSASIIpath.getSavedVersionInfo() if gv is not None: From f66c665c3d0e863054e55c5832a5a865431bceeb Mon Sep 17 00:00:00 2001 From: BHT Date: Thu, 22 Oct 2026 12:03:19 -0500 Subject: [PATCH 2/2] Add fastsave option to G2Project to speed save; Remove use of platform.architecture() throughout; minor string formatting --- GSASII/GSASIIdataGUI.py | 6 +++--- GSASII/GSASIIfiles.py | 8 +------- GSASII/GSASIIpath.py | 29 ++++++++++++++++++++--------- GSASII/GSASIIpwd.py | 2 +- GSASII/GSASIIscriptable.py | 15 +++++++++++---- 5 files changed, 36 insertions(+), 24 deletions(-) diff --git a/GSASII/GSASIIdataGUI.py b/GSASII/GSASIIdataGUI.py index 81bfcea3..cfd0af74 100644 --- a/GSASII/GSASIIdataGUI.py +++ b/GSASII/GSASIIdataGUI.py @@ -596,11 +596,11 @@ def ShowVersions(): break except: pass - print (" Image: %s (PIL or Pillow)"%version) - print (" Platform: %s %s %s"%(sys.platform,platform.architecture()[0],platform.machine())) + print (f" Image: {version} (PIL or Pillow)") + print (f" Platform: {sys.platform} {GSASIIpath.GetPlatformArchitecture()} {platform.machine()}") try: import mkl - print (" Max threads:%s"%mkl.get_max_threads()) + print (f" Max threads:{mkl.get_max_threads()}") except: pass diff --git a/GSASII/GSASIIfiles.py b/GSASII/GSASIIfiles.py index dc7db58e..b639ce97 100644 --- a/GSASII/GSASIIfiles.py +++ b/GSASII/GSASIIfiles.py @@ -11,7 +11,6 @@ from __future__ import division, print_function import platform import os -import struct import sys import glob import copy @@ -120,13 +119,8 @@ def get_python_versions(packagelist): versions.append([pack.__name__, pack.__version__]) except: pass - # struct.calcsize('P') gives the pointer size, producing the same - # '32bit'/'64bit' string as platform.architecture()[0] without - # launching an external process (platform.architecture runs - # 'file ' on POSIX systems every time it is called) versions.append(['Platform', - sys.platform + ' ' + f"{struct.calcsize('P')*8}bit" + - ' ' + platform.machine()]) + sys.platform + ' ' + GSASIIpath.GetPlatformArchitecture() + ' ' + platform.machine()]) return versions ImportErrors = [] diff --git a/GSASII/GSASIIpath.py b/GSASII/GSASIIpath.py index 7e288826..2a83a0b1 100644 --- a/GSASII/GSASIIpath.py +++ b/GSASII/GSASIIpath.py @@ -146,6 +146,17 @@ def LoadConfigFile(filename): fp.close() return info +def GetPlatformArchitecture(): + '''Produces same result as platform.architecture()[0], but runs much + faster because struct.calcsize('P') gives the pointer size while + platform.architecture()[0] launches an external process + ('file ') on POSIX systems every time it is called. + + :returns: '32bit' or '64bit' + ''' + import struct + return f"{struct.calcsize('P')*8}bit" + def GetBinaryPrefix(pyver=None): '''Creates the first part of the binary directory name such as linux_64_p3.9 (where the full name will be @@ -165,11 +176,11 @@ def GetBinaryPrefix(pyver=None): raise Exception('Unknown platform') if 'arm' in platform.machine() and sys.platform == "darwin": bits = 'arm' - elif 'aarch' in platform.machine() and '64' in platform.architecture()[0]: + elif 'aarch' in platform.machine() and '64' in GSASIIpath.GetPlatformArchitecture(): bits = 'arm64' elif 'arm' in platform.machine(): bits = 'arm32' - elif '64' in platform.architecture()[0]: + elif '64' in GSASIIpath.GetPlatformArchitecture(): bits = '64' else: bits = '32' @@ -205,11 +216,11 @@ def HowIsG2Installed(): else: rev = '-rev' if g2URL in g2repo.remote().urls: - G2_installed_result = 'github'+rev + G2_installed_result = f'github{rev}' return G2_installed_result elif g2URL.replace('https://github.com/', 'git@github.com:') in g2repo.remote().urls: - G2_installed_result = 'github'+rev + G2_installed_result = f'github{rev}' return G2_installed_result G2_installed_result = 'git'+rev return G2_installed_result @@ -2154,7 +2165,7 @@ def testLLamaModel(): Returns the name of the most recently installed model or None if the model needs to be installed. ''' - modelDir = os.path.expanduser("~/.GSASII/llama_models") + modelDir = os.path.join(LocalG2Dir(),"llama_models") if not os.path.exists(modelDir): return None fileList = glob.glob(os.path.join(modelDir,'*.gguf')) @@ -2168,7 +2179,7 @@ def testLLamaModel(): def installLLamaModel(): '''Download the Qwen2.5-3B-Instruct llama model''' - modelDir = os.path.expanduser("~/.GSASII/llama_models") + modelDir = os.path.join(LocalG2Dir(),"llama_models") os.makedirs(modelDir, exist_ok=True) print('Downloading a model...') from huggingface_hub import hf_hub_download @@ -2200,7 +2211,7 @@ def ageLLMindex(): ''' import datetime f = 'chroma_db' - dbdir = os.path.expanduser('~/.GSASII/query_gsas2') + dbdir = os.path.join(LocalG2Dir(),"query_gsas2") db = os.path.join(dbdir,f) if os.path.exists(db): m_time_timestamp = os.path.getmtime(db) @@ -2249,9 +2260,9 @@ def getLLMindex(): #print("Saved to:", tmp.name) # create the files in a new location and then move them to the final location f = 'chroma_db' - finaldir = os.path.expanduser('~/.GSASII/query_gsas2') + finaldir = os.path.join(LocalG2Dir(),"query_gsas2") finaldb = os.path.join(finaldir,f) - newdir = os.path.expanduser('~/.GSASII/new_query_gsas2') + newdir = os.path.join(LocalG2Dir(),"new_query_gsas2") newdb = os.path.join(newdir,f) os.makedirs(newdir, exist_ok=True) os.makedirs(finaldir, exist_ok=True) diff --git a/GSASII/GSASIIpwd.py b/GSASII/GSASIIpwd.py index 95d922b9..7ef5033b 100644 --- a/GSASII/GSASIIpwd.py +++ b/GSASII/GSASIIpwd.py @@ -3800,7 +3800,7 @@ def fullrmcDownload(): import os import requests import platform - if platform.architecture()[0] != '64bit': + if GSASIIpath.GetPlatformArchitecture() != '64bit': return "fullrmc is only available for 64 bit machines. This is 32 bit" setXbit = True if sys.platform == "darwin": diff --git a/GSASII/GSASIIscriptable.py b/GSASII/GSASIIscriptable.py index de915722..c7e44dc1 100644 --- a/GSASII/GSASIIscriptable.py +++ b/GSASII/GSASIIscriptable.py @@ -919,6 +919,9 @@ class variables: :param str filename: To be deprecated. Serves the same function as newgpx, which has a somewhat more clear name. (Do not specify both newgpx and filename). + :param bool fastsave: When specified as True, the version number + is taken from the git_verinfo.py file rather than using git. + This saves a bit of time. There are two ways to initialize this object: @@ -956,7 +959,9 @@ class variables: :meth:`~G2Project.set_refinement`, :meth:`~G2Project.iter_refinements` and :meth:`~G2Project.do_refinements`. """ - def __init__(self, gpxfile=None, author=None, filename=None, newgpx=None): + def __init__(self, gpxfile=None, author=None, filename=None, newgpx=None, + fastsave=False): + self.fastsave = fastsave if filename is not None and newgpx is not None: raise G2ScriptException('Do not use filename and newgpx together') elif filename is not None: @@ -1005,7 +1010,9 @@ def from_dict_and_names(cls, gpxdict, names, filename=None): def save(self, filename=None): """Saves the project, either to the current filename, or to a new file. - Updates self.filename if a new filename provided""" + :param str filename: a file name to be used to save the .gpx file. + Updates self.filename if this is provided + """ controls_data = self.data['Controls']['data'] # place info in .gpx about GSAS-II # report versions of modules; ignore what has not been used @@ -1022,7 +1029,7 @@ def save(self, filename=None): # G2 version info controls_data['LastSavedUsing'] = str(GSASIIpath.GetVersionNumber()) try: - if GSASIIpath.HowIsG2Installed().startswith('git'): + if GSASIIpath.HowIsG2Installed().startswith('git') and not self.fastsave: commit = GSASIIpath.getG2CommitHash() controls_data['LastSavedUsing'] += f" git {commit[:8]} script" else: @@ -2794,7 +2801,7 @@ def add_image(self, imagefile, fmthint=None, defaultImage=None, Data['DetDepthRef'] = False else: Data['type'] = 'PWDR' - Data['color'] = GSASIIpath.GetConfigValue('Contour_color','Paired') + Data['color'] = GSASIIpath.GetConfigValue('Contour_color','GSPaired') if 'tilt' not in Data: #defaults if not preset in e.g. Bruker importer Data['tilt'] = 0.0 Data['rotation'] = 0.0