Prepare release 1.3.0

docs/user_guide.md

@@ -143,7 +143,7 @@ schema = load_schema_version(["score_2.1.0", "lang_1.1.0"])
 ```
 
 - All libraries in a merge group must be partnered with the **same** standard schema version. Listing that standard version as well is allowed and adds nothing; listing a different one fails.
-- Duplicate versions in the list are ignored, but two *different* versions of the same schema cannot be combined.
+- A schema may appear only once in a merge group: listing the same version twice, or two *different* versions of the same schema, makes the load raise a `HedFileError` with code `SCHEMA_LOAD_FAILED`.
 - An element (tag, unit class, unit, value class, or schema attribute) declared by more than one library must be declared identically - same attributes, description, position in the hierarchy, and placeholder (`#`) child. Any conflict makes the load raise a `HedFileError` with code `SCHEMA_LOAD_FAILED` whose `issues` list names each conflicting element.
 
 Schemas that cannot be merged can still be used side by side by giving them *namespace prefixes*. Each prefix forms its own merge group, resolved independently, and tags from a prefixed schema are written with the prefix in annotations:

hed/errors/error_types.py

@@ -198,7 +198,8 @@ class SchemaAttributeErrors:
 
     # Extras section errors
     SCHEMA_MISSING_EXTRA = "SCHEMA_MISSING_EXTRA"
-    # Deprecated alias for the pre-1.3.0 name; same code string. Remove in 2.0.0.
+    # Deprecated alias for the pre-1.3.0 name. Its value is now "SCHEMA_MISSING_EXTRA", so code that compared issue
+    # codes to the old string "SCHEMA_MISSING_EXTRA_VALUE" must change. Remove in 2.0.0.
     SCHEMA_MISSING_EXTRA_VALUE = SCHEMA_MISSING_EXTRA
 
     # Annotation attribute errors

hed/models/base_input.py

@@ -826,7 +826,8 @@ def _load_text_file(self, file, pandas_header):
             - Converts all data to string type for consistency
             - Replaces NaN values with "n/a" for consistent handling
             - Skips blank lines during parsing
-            - Uses specific na_values configuration ("", "null")
+            - Treats only an empty cell and "n/a" as missing (na_values ("", "n/a"), keep_default_na=False), so
+              "NaN", "NULL" and the other pandas default NA strings are values
             - Handles pandas.errors.EmptyDataError for files with no data
             - With a header, the header record is read first so that blank and repeated names are known before
               pandas renames them (``blank_columns``, ``duplicate_columns``); the column names are the header

hed/schema/hed_cache.py

@@ -115,7 +115,11 @@ def get_hed_versions(local_hed_directory=None, library_name=None, check_prerelea
             hed_files += os.listdir(hed_dir)
         except FileNotFoundError:
             pass
-    if not any(version_pattern.match(f) for f in hed_files):
+    # The default cache always holds every bundled schema, so schemas bundled by a newer hedtools reach a cache
+    # populated by an older one; any other folder is seeded only when it holds no schema at all.
+    if (local_hed_directory == HED_CACHE_DIRECTORY and _bundled_schemas_missing(local_hed_directory)) or not any(
+        version_pattern.match(f) for f in hed_files
+    ):
         cache_local_versions(local_hed_directory)
         hed_files = []
         for hed_dir in local_directories:
@@ -145,10 +149,11 @@ def get_hed_versions(local_hed_directory=None, library_name=None, check_prerelea
 def get_hed_version_path(xml_version, library_name=None, local_hed_directory=None) -> str | None:
     """Get the HED XML file path for a given version.
 
-    Searches the local cache first (including the bundled schemas that are always present).
-    If the version is not found and local_hed_directory is the default HED cache, downloads
-    only the single requested file from GitHub — never the entire catalog.
-    No network call is made for custom directories.
+    In the default HED cache, a version bundled with hedtools always resolves to the bundled file: a released
+    schema never changes, so a stale or prerelease copy left in the cache cannot shadow it. Otherwise the
+    local cache is searched. If the version is not found and local_hed_directory is the default HED cache,
+    only the single requested file is downloaded from GitHub, never the entire catalog.
+    No network call is made for custom directories, and their files are used as they are.
 
     Parameters:
         xml_version (str): The version string to look up.
@@ -163,6 +168,11 @@ def get_hed_version_path(xml_version, library_name=None, local_hed_directory=Non
     if not local_hed_directory:
         local_hed_directory = HED_CACHE_DIRECTORY
 
+    if xml_version and local_hed_directory == HED_CACHE_DIRECTORY:
+        bundled_path = _create_xml_filename(xml_version, library_name, INSTALLED_CACHE_LOCATION, False)
+        if os.path.isfile(bundled_path):
+            return bundled_path
+
     result = _find_hed_version_path(xml_version, library_name, local_hed_directory)
     if result:
         return result
@@ -793,6 +803,22 @@ def _write_library_data_file(filename, library_data):
         pass
 
 
+def _bundled_schemas_missing(cache_folder) -> bool:
+    """Return True if a schema file bundled with hedtools is not in the root of the cache folder.
+
+    Parameters:
+        cache_folder (str): The folder holding the cache.
+
+    Returns:
+        bool: True if at least one bundled schema file is absent from the cache folder.
+    """
+    try:
+        cached = set(os.listdir(cache_folder))
+    except FileNotFoundError:
+        return True
+    return any(version_pattern.match(name) and name not in cached for name in os.listdir(INSTALLED_CACHE_LOCATION))
+
+
 def _copy_installed_folder_to_cache(cache_folder, sub_folder=""):
     """Copies the schemas from the install folder to the cache"""
     source_folder = INSTALLED_CACHE_LOCATION

tests/schema/test_hed_schema_io.py

@@ -616,6 +616,65 @@ def test_load_schema_version_libraries(self):
             load_schema_version("sc1:")
 
 
+class TestBundledSchemasInCache(unittest.TestCase):
+    """A schema bundled with hedtools wins over any copy in the default cache, and reaches an older cache."""
+
+    STALE_MARKER = "STALE-COPY-MARKER"
+
+    def setUp(self):
+        self.saved_cache_folder = hed_cache.HED_CACHE_DIRECTORY
+        self.cache = tempfile.mkdtemp()
+        schema.set_cache_directory(self.cache)
+        _load_schema_version.cache_clear()
+
+    def tearDown(self):
+        schema.set_cache_directory(self.saved_cache_folder)
+        _load_schema_version.cache_clear()
+        shutil.rmtree(self.cache, ignore_errors=True)
+
+    def _bundled(self, name):
+        return os.path.join(hed_cache.INSTALLED_CACHE_LOCATION, name)
+
+    def _write_stale_850(self, folder):
+        os.makedirs(folder, exist_ok=True)
+        with open(self._bundled("HED8.5.0.xml"), encoding="utf-8") as f:
+            text = f.read().replace("</prologue>", self.STALE_MARKER + "</prologue>", 1)
+        with open(os.path.join(folder, "HED8.5.0.xml"), "w", encoding="utf-8") as f:
+            f.write(text)
+
+    def _fill_as_older_release(self):
+        # A cache an older hedtools populated: the released standard schemas up to 8.4.0, nothing newer.
+        for version in ("8.0.0", "8.1.0", "8.2.0", "8.3.0", "8.4.0"):
+            shutil.copy(self._bundled(f"HED{version}.xml"), self.cache)
+
+    def test_stale_prerelease_copy_does_not_shadow_the_release(self):
+        self._fill_as_older_release()
+        self._write_stale_850(os.path.join(self.cache, "prerelease"))
+        loaded = load_schema_version("8.5.0")
+        self.assertNotIn(self.STALE_MARKER, loaded.prologue)
+
+    def test_stale_copy_in_cache_root_does_not_shadow_the_release(self):
+        self._fill_as_older_release()
+        self._write_stale_850(self.cache)
+        self.assertEqual(hed_cache.get_hed_version_path("8.5.0"), self._bundled("HED8.5.0.xml"))
+        self.assertNotIn(self.STALE_MARKER, load_schema_version("8.5.0").prologue)
+
+    def test_newly_bundled_schemas_reach_an_older_cache(self):
+        self._fill_as_older_release()
+        self.assertIn("8.5.0", hed_cache.get_hed_versions())
+        self.assertIn("2.2.0", hed_cache.get_hed_versions(library_name="score"))
+        self.assertEqual(load_schema_version().version_number, hed_cache.get_hed_versions()[0])
+
+    def test_custom_folder_keeps_its_own_file(self):
+        folder = tempfile.mkdtemp()
+        try:
+            self._write_stale_850(folder)
+            path = hed_cache.get_hed_version_path("8.5.0", local_hed_directory=folder)
+            self.assertEqual(path, os.path.join(folder, "HED8.5.0.xml"))
+        finally:
+            shutil.rmtree(folder, ignore_errors=True)
+
+
 class TestHedSchemaUnmerged(unittest.TestCase):
     # Verify the HED cache can handle loading unmerged with_standard schemas in case they are ever used
     @classmethod

tests/test_deprecations.py

@@ -17,6 +17,8 @@
 
 import hed
 from hed.errors.error_types import SchemaAttributeErrors
+from hed.models.base_input import BaseInput
+from hed.models.column_mapper import ColumnMapper
 from hed.schema import load_schema
 from hed.schema.hed_schema_io import from_string
 from hed.schema.schema_io.base2schema import SchemaLoader
@@ -61,6 +63,21 @@ def _has_parameter(func, name):
         lambda: hasattr(data_util, "delete_rows_by_column"),
         2,
     ),
+    (
+        "allow_blank_names= parameter of BaseInput.__init__ (ignored; blank names are TABULAR_INVALID)",
+        lambda: _has_parameter(BaseInput.__init__, "allow_blank_names"),
+        2,
+    ),
+    (
+        "allow_blank_names= parameter of ColumnMapper.check_for_mapping_issues (ignored)",
+        lambda: _has_parameter(ColumnMapper.check_for_mapping_issues, "allow_blank_names"),
+        2,
+    ),
+    (
+        "ColumnMapper.check_for_blank_names (validation reports blank names as TABULAR_INVALID)",
+        lambda: hasattr(ColumnMapper, "check_for_blank_names"),
+        2,
+    ),
 ]
 
 

tests/validator/test_tag_validator.py

@@ -5,6 +5,7 @@
 from hed.errors.error_types import DefinitionErrors, ValidationErrors
 from hed.models.hed_string import HedString
 from hed.schema.hed_schema_io import load_schema_version
+from hed.validator import HedValidator
 from hed.validator.util.char_util import COLUMN_BRACES_RULE
 from hed.validator.util.character_sets import CharacterSets
 from tests.validator.test_tag_validator_base import TestValidatorBase
@@ -1245,5 +1246,37 @@ def test_special_units(self):
         self.validator_semantic(test_strings, expected_results, expected_issues, True)
 
 
+class TestReleased850Units(unittest.TestCase):
+    """Unit rules against the released, bundled HED 8.5.0 (Quantity/# takes every unit class through anyUnits)."""
+
+    @classmethod
+    def setUpClass(cls):
+        cls.hed_schema = load_schema_version("8.5.0")
+        cls.validator = HedValidator(cls.hed_schema)
+
+    def codes(self, text):
+        issues = self.validator.validate(HedString(text, self.hed_schema), allow_placeholders=False)
+        return sorted({issue["code"] for issue in issues})
+
+    def test_derived_si_unit_symbol_is_valid(self):
+        self.assertEqual(self.codes("Quantity/3 uV"), [])
+
+    def test_unit_symbols_are_case_sensitive(self):
+        self.assertEqual(self.codes("Quantity/3 UV"), [ValidationErrors.UNITS_INVALID])
+
+    def test_derived_default_unit_resolves(self):
+        # currentUnits has defaultUnits=mA, a form derived from A by an SI modifier.
+        self.assertEqual(self.codes("Quantity/3 mA"), [])
+
+    def test_unit_without_symbol_takes_no_modifier_symbol(self):
+        # calorie has no unit symbol, so kcal is not a unit (kilocalorie is).
+        self.assertEqual(self.codes("Quantity/3 kcal"), [ValidationErrors.UNITS_INVALID])
+        self.assertEqual(self.codes("Quantity/3 kilocalorie"), [])
+
+    def test_deprecated_unit_prefix_is_not_accepted(self):
+        self.assertEqual(self.codes("Quantity/$5"), [ValidationErrors.VALUE_INVALID])
+        self.assertEqual(self.codes("Quantity/5 dollars"), [])
+
+
 if __name__ == "__main__":
     unittest.main()