[CoreAI] Prepare source bundles

backends/apple/coreai/CMakeLists.txt

@@ -34,7 +34,9 @@ if(EXECUTORCH_BUILD_TESTS AND CMAKE_SYSTEM_NAME STREQUAL "Darwin")
     runtime/test/coreai_manifest_test.mm
     runtime/test/coreai_storage_test.mm
     runtime/test/coreai_filesystem_fixture.mm
+    runtime/test/coreai_source_test.mm
     runtime/test/coreai_bookmark_test.mm
+    runtime/test/coreai_source_fixture.mm
     runtime/test/coreai_bookmark_fixture.mm
     ${_coreai_runtime_sources}
   )

backends/apple/coreai/README.md

@@ -50,11 +50,43 @@ The assets root must be an absolute path. The application chooses it, should
 reserve it for this backend, and is trusted not to rename, replace or modify its
 contents while the backend uses it; the backend does not defend against other
 processes of the same user rebinding paths. Written files and changed directories
-are synced with `fsync`. Atomic publication uses `F_FULLFSYNC` on the temporary
-file before rename and on the directory afterward. A failure before rename
-preserves the old contents; a failure of the final flush reports an error even
-though the new complete contents are already visible. Use a local APFS volume;
-unsupported full-device flushes are reported as errors, not downgraded to `fsync`.
+are synced with `fsync`. Bookmark publication uses `F_FULLFSYNC` on the temporary
+file before rename and on the directory afterward. Source-bundle publication
+fully flushes the staging directory after syncing its files and subdirectories,
+then fully flushes the destination directory after rename. A failure before
+rename preserves the old contents; a failure of the final flush reports an error
+even though the new complete contents are already visible. Use a local APFS
+volume; unsupported full-device flushes are reported as errors, not downgraded
+to `fsync`.
+
+Inline bundles are read through `NamedDataMap` and materialized as unchanged
+files under the assets root. Named data can be supplied externally; inline
+packaging does not require all bytes to reside in one physical PTE. The exporter
+computes each bundle digest from all delivered relative filenames and file bytes,
+not just the SDK's bytecode-only `main.hash`. Each AOT architecture has an
+independent digest; only the selected architecture is materialized. NDS key names
+and raw asset bytes are unchanged.
+
+When source recovery is needed, existing bundles are checked for the expected
+file set, sizes, entry types and directory structure without reading asset
+contents or requesting NDS payloads. Unicode paths are compared in canonical
+decomposed form; NDS keys and bytes remain unchanged. Reuse fully flushes the
+source root, including after a previous publication's final flush failed.
+Missing bundles are written to a staging directory and published with an
+exclusive rename; each selected NDS payload is fetched once and checked for size
+before writing. If another loader publishes first, its bundle is validated and
+used. Incomplete, size-mismatched or otherwise malformed existing bundles fail
+loading without replacement, since an SDK model may still be using them.
+Only interrupted system calls are retried; other failures are returned to the
+caller without automatic retries or repair. Temporary bundles are removed on
+ordinary failures before publication; a caller can explicitly retry after
+addressing the failure.
+
+These are completeness checks, not full content-integrity verification. Same-size
+content changes are not detected, and a complete stored bundle does not cause its
+NDS source to be reread. The export-time digest is a trusted artifact identifier;
+the runtime does not rehash source or stored bytes. Core AI may reject invalid
+contents, but SDK load failures do not automatically evict the extracted copy.
 
 Preparing the assets root sets `NSURLIsExcludedFromBackupKey` on it, which covers
 everything beneath it. Ancestors are not modified. This is backup exclusion, not

backends/apple/coreai/runtime/coreai_assets.h

@@ -9,6 +9,7 @@
 #pragma once
 
 #import <Foundation/Foundation.h>
+#include <executorch/runtime/core/named_data_map.h>
 #include <executorch/runtime/core/result.h>
 
 namespace executorch::backends::coreai {
@@ -33,5 +34,15 @@ runtime::Result<Manifest> select_assets(
     const Manifest& manifest,
     NSString* device_architecture,
     NSString* platform);
+// Caller holds the key's disk lock and has prepared source_root.
+// The key is 64 lowercase hex characters identifying the selected export digest
+// and bundle name, optionally including platform or SDK context.
+// Bundles are published at source_root/key/bundle. Reuse validates structure,
+// not contents, and fully flushes source_root before returning.
+runtime::Result<NSURL*> prepare_source_bundle(
+    const Manifest& manifest,
+    const runtime::NamedDataMap* named_data,
+    NSString* source_root,
+    NSString* key);
 
 } // namespace executorch::backends::coreai

backends/apple/coreai/runtime/coreai_assets.mm

@@ -7,14 +7,29 @@
  */
 
 #import "coreai_assets.h"
+#include "coreai_file.h"
+
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <algorithm>
+#include <cerrno>
 #include <cstdint>
+#include <cstdio>
 #include <limits>
+#import "coreai_storage.h"
 
 namespace executorch::backends::coreai {
 namespace {
 using runtime::Error;
 using runtime::Result;
 
+Error path_error(int error) {
+  return error == ENOENT || error == ENOTDIR || error == ELOOP
+      ? Error::InvalidExternalData
+      : Error::AccessFailed;
+}
+
 bool nonempty_string(id value) {
   return [value isKindOfClass:NSString.class] && [value length] > 0 &&
       [value rangeOfString:@"\0"].location == NSNotFound;
@@ -135,6 +150,129 @@ bool parse_version(id value, NSOperatingSystemVersion& version) {
   return numbers[0] > 0;
 }
 
+Error validate_directory(
+    int fd,
+    NSString* relative,
+    NSDictionary<NSString*, NSNumber*>* files,
+    NSSet<NSString*>* expected_directories,
+    NSMutableSet<NSString*>* actual_files) {
+  auto children = storage_children(fd);
+  if (!children.ok()) {
+    return children.error();
+  }
+  for (NSString* name in children.get()) {
+    @autoreleasepool {
+      NSString* path = [relative stringByAppendingPathComponent:name]
+                           .decomposedStringWithCanonicalMapping;
+      struct stat info;
+      if (retry_eintr([&] {
+            return fstatat(
+                fd, name.fileSystemRepresentation, &info, AT_SYMLINK_NOFOLLOW);
+          }) != 0) {
+        return path_error(errno);
+      }
+      const bool directory = S_ISDIR(info.st_mode);
+      ET_CHECK_OR_RETURN_ERROR(
+          (directory && [expected_directories containsObject:path]) ||
+              (S_ISREG(info.st_mode) && files[path] != nil),
+          InvalidExternalData,
+          "Unexpected Core AI bundle entry: %s",
+          path.UTF8String);
+      if (directory) {
+        FileDescriptor child(retry_eintr([&] {
+          return openat(
+              fd,
+              name.fileSystemRepresentation,
+              O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
+        }));
+        if (child.get() < 0) {
+          return path_error(errno);
+        }
+        ET_CHECK_OK_OR_RETURN_ERROR(validate_directory(
+            child.get(), path, files, expected_directories, actual_files));
+      } else {
+        ET_CHECK_OR_RETURN_ERROR(
+            static_cast<uint64_t>(info.st_size) ==
+                files[path].unsignedLongLongValue,
+            InvalidExternalData,
+            "Core AI asset size does not match manifest");
+        [actual_files addObject:path];
+      }
+    }
+  }
+  return Error::Ok;
+}
+
+// Checks the exact file set, entry types and sizes. Contents are not hashed.
+Error validate_tree(int fd, NSDictionary<NSString*, NSNumber*>* files) {
+  NSMutableDictionary<NSString*, NSNumber*>* normalized =
+      [NSMutableDictionary dictionary];
+  NSMutableSet<NSString*>* expected_directories = [NSMutableSet set];
+  for (NSString* file in files) {
+    // Filesystem path conversion decomposes Unicode without changing NDS keys.
+    NSString* path = file.decomposedStringWithCanonicalMapping;
+    normalized[path] = files[file];
+    NSString* directory = path.stringByDeletingLastPathComponent;
+    while (directory.length > 0) {
+      [expected_directories addObject:directory];
+      directory = directory.stringByDeletingLastPathComponent;
+    }
+  }
+  NSMutableSet<NSString*>* actual_files = [NSMutableSet set];
+  ET_CHECK_OK_OR_RETURN_ERROR(validate_directory(
+      fd, @"", normalized, expected_directories, actual_files));
+  ET_CHECK_OR_RETURN_ERROR(
+      [actual_files isEqualToSet:[NSSet setWithArray:normalized.allKeys]],
+      InvalidExternalData,
+      "Core AI bundle file set does not match manifest");
+  return Error::Ok;
+}
+
+enum class EntryState { Missing, Valid, Corrupt };
+
+Result<EntryState> inspect_entry(
+    int root_fd,
+    NSString* name,
+    NSDictionary<NSString*, NSNumber*>* files) {
+  FileDescriptor entry(retry_eintr([&] {
+    return openat(
+        root_fd,
+        name.fileSystemRepresentation,
+        O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
+  }));
+  if (entry.get() < 0) {
+    if (errno == ENOENT)
+      return EntryState::Missing;
+    ET_CHECK_OR_RETURN_ERROR(
+        errno == ENOTDIR || errno == ELOOP,
+        AccessFailed,
+        "Cannot inspect Core AI stored bundle");
+    return EntryState::Corrupt;
+  }
+  const auto error = validate_tree(entry.get(), files);
+  if (error == Error::Ok) {
+    return EntryState::Valid;
+  }
+  if (error == Error::InvalidExternalData) {
+    return EntryState::Corrupt;
+  }
+  return error;
+}
+
+struct StagingDirectory {
+  NSURL* url;
+  ~StagingDirectory() {
+    if (url != nil) {
+      NSError* error = nil;
+      if (![NSFileManager.defaultManager removeItemAtURL:url error:&error]) {
+        ET_LOG(
+            Error,
+            "Cannot remove Core AI staging directory: %s",
+            error.localizedDescription.UTF8String);
+      }
+    }
+  }
+};
 } // namespace
 
 Result<Manifest> parse_manifest(NSData* data) {
@@ -303,4 +441,132 @@ bool parse_version(id value, NSOperatingSystemVersion& version) {
   return selected;
 }
 
+Result<NSURL*> prepare_source_bundle(
+    const Manifest& manifest,
+    const runtime::NamedDataMap* named_data,
+    NSString* source_root,
+    NSString* key) {
+  ET_CHECK_OR_RETURN_ERROR(
+      relative_path(manifest.path),
+      InvalidProgram,
+      "Core AI bundle must be selected before preparing its source");
+  ET_CHECK_OR_RETURN_ERROR(
+      nonempty_string(key) && key.length == 64 &&
+          [key rangeOfCharacterFromSet:
+                   [[NSCharacterSet
+                       characterSetWithCharactersInString:@"0123456789abcdef"]
+                       invertedSet]]
+                  .location == NSNotFound,
+      InvalidArgument,
+      "Core AI staging key must be a 64-character lowercase hex digest");
+  auto prepared = prepare_storage_root(source_root, false);
+  if (!prepared.ok()) {
+    return prepared.error();
+  }
+  NSURL* root = [NSURL fileURLWithPath:prepared.get() isDirectory:YES];
+  ET_CHECK_OK_OR_RETURN_ERROR(validate_files(
+      manifest.files, [NSSet setWithObject:manifest.path.lastPathComponent]));
+  NSDictionary<NSString*, NSNumber*>* files = manifest.files;
+  NSURL* model_url = [[root URLByAppendingPathComponent:key isDirectory:YES]
+      URLByAppendingPathComponent:manifest.path.lastPathComponent
+                      isDirectory:YES];
+  // Inspection and publication stay bound to this root descriptor.
+  FileDescriptor root_fd(retry_eintr([&] {
+    return open(
+        root.fileSystemRepresentation, O_RDONLY | O_DIRECTORY | O_CLOEXEC);
+  }));
+  ET_CHECK_OR_RETURN_ERROR(
+      root_fd.get() >= 0, AccessFailed, "Cannot open Core AI storage root");
+  auto state = inspect_entry(root_fd.get(), key, files);
+  if (!state.ok()) {
+    return state.error();
+  }
+  if (*state == EntryState::Valid) {
+    ET_CHECK_OK_OR_RETURN_ERROR(sync_storage_directory(root_fd.get(), true));
+    return model_url;
+  }
+  // Recovery sources may still be in use by a previously acquired SDK model.
+  ET_CHECK_OR_RETURN_ERROR(
+      *state == EntryState::Missing,
+      InvalidExternalData,
+      "Cannot replace a damaged Core AI stored bundle");
+  ET_CHECK_OR_RETURN_ERROR(
+      named_data != nullptr,
+      InvalidProgram,
+      "Missing NamedDataMap for Core AI source recovery");
+  NSString* staging_name =
+      [@".staging-" stringByAppendingString:NSUUID.UUID.UUIDString];
+  ET_CHECK_OR_RETURN_ERROR(
+      storage_call(
+          StorageOperation::CreateStagingDirectory,
+          [&] {
+            return mkdirat(
+                root_fd.get(), staging_name.fileSystemRepresentation, 0777);
+          }) == 0,
+      AccessFailed,
+      "Cannot create Core AI staging directory");
+  StagingDirectory staging{[root URLByAppendingPathComponent:staging_name
+                                                 isDirectory:YES]};
+  FileDescriptor staging_fd(
+      storage_call(StorageOperation::OpenStagingDirectory, [&] {
+        return openat(
+            root_fd.get(),
+            staging_name.fileSystemRepresentation,
+            O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
+      }));
+  ET_CHECK_OR_RETURN_ERROR(
+      staging_fd.get() >= 0,
+      AccessFailed,
+      "Cannot open Core AI staging directory");
+  for (NSString* file in files) {
+    @autoreleasepool {
+      NSString* data_key =
+          [NSString stringWithFormat:@"coreai/%@/%@", manifest.hash, file];
+      auto buffer = named_data->get_data(data_key.UTF8String);
+      if (!buffer.ok()) {
+        ET_CHECK_OR_RETURN_ERROR(
+            buffer.error() != Error::NotFound,
+            InvalidExternalData,
+            "Missing selected Core AI named data: %s",
+            data_key.UTF8String);
+        return buffer.error();
+      }
+      ET_CHECK_OR_RETURN_ERROR(
+          buffer->size() == files[file].unsignedLongLongValue &&
+              (buffer->size() == 0 || buffer->data() != nullptr),
+          InvalidExternalData,
+          "Invalid Core AI named data buffer");
+      ET_CHECK_OK_OR_RETURN_ERROR(write_storage_file(
+          staging_fd.get(), file, buffer->data(), buffer->size()));
+    }
+  }
+  ET_CHECK_OK_OR_RETURN_ERROR(validate_tree(staging_fd.get(), files));
+  ET_CHECK_OK_OR_RETURN_ERROR(sync_storage_directory(staging_fd.get(), true));
+  if (storage_call(StorageOperation::Rename, [&] {
+        return renameatx_np(
+            root_fd.get(),
+            staging_name.fileSystemRepresentation,
+            root_fd.get(),
+            key.fileSystemRepresentation,
+            RENAME_EXCL);
+      }) == 0) {
+    staging.url = nil;
+    ET_CHECK_OK_OR_RETURN_ERROR(sync_storage_directory(root_fd.get(), true));
+    return model_url;
+  }
+  ET_CHECK_OR_RETURN_ERROR(
+      errno == EEXIST, AccessFailed, "Cannot publish Core AI stored bundle");
+  // Another loader published first; use its source only if it is complete.
+  auto winner = inspect_entry(root_fd.get(), key, files);
+  if (!winner.ok()) {
+    return winner.error();
+  }
+  ET_CHECK_OR_RETURN_ERROR(
+      *winner == EntryState::Valid,
+      InvalidExternalData,
+      "Concurrently published Core AI source is incomplete");
+  ET_CHECK_OK_OR_RETURN_ERROR(sync_storage_directory(root_fd.get(), true));
+  return model_url;
+}
+
 } // namespace executorch::backends::coreai