From fe4a9d68b29f98807b6795f589ed541cbf221a00 Mon Sep 17 00:00:00 2001 From: Alexander Ratajczak Date: Sat, 15 Aug 2026 23:23:28 +0200 Subject: [PATCH] Added a Setting to set the Model Download Folder Signed-off-by: Alexander Ratajczak --- appinfo/info.xml | 1 + lib/Classifiers/Audio/MusicnnClassifier.php | 5 +- lib/Classifiers/Classifier.php | 6 +- .../Images/ClusteringFaceClassifier.php | 5 +- lib/Classifiers/Images/ImagenetClassifier.php | 5 +- .../Images/LandmarksClassifier.php | 5 +- lib/Classifiers/Video/MovinetClassifier.php | 5 +- lib/Migration/MoveDefaultModelFolder.php | 70 +++++++++++++++++++ lib/Service/DownloadModelsService.php | 23 ++++-- lib/Service/SettingsService.php | 39 ++++++++++- lib/Settings/AdminSettings.php | 6 +- src/classifier_imagenet.js | 4 +- src/classifier_landmarks.js | 3 +- src/classifier_movinet.js | 3 +- src/classifier_musicnn.js | 3 +- src/components/ViewAdmin.vue | 31 ++++++-- 16 files changed, 183 insertions(+), 31 deletions(-) create mode 100644 lib/Migration/MoveDefaultModelFolder.php diff --git a/appinfo/info.xml b/appinfo/info.xml index e7e201b50..b53ce2526 100644 --- a/appinfo/info.xml +++ b/appinfo/info.xml @@ -101,6 +101,7 @@ The app does not send any sensitive data to cloud providers or similar services. OCA\Recognize\Migration\InstallDeps + OCA\Recognize\Migration\MoveDefaultModelFolder OCA\Recognize\Migration\RemoveDuplicateFaceDetections diff --git a/lib/Classifiers/Audio/MusicnnClassifier.php b/lib/Classifiers/Audio/MusicnnClassifier.php index de5dba13f..3bd142bde 100644 --- a/lib/Classifiers/Audio/MusicnnClassifier.php +++ b/lib/Classifiers/Audio/MusicnnClassifier.php @@ -10,6 +10,7 @@ use OCA\Recognize\Classifiers\Classifier; use OCA\Recognize\Service\Logger; use OCA\Recognize\Service\QueueService; +use OCA\Recognize\Service\SettingsService; use OCA\Recognize\Service\TagManager; use OCP\AppFramework\Services\IAppConfig; use OCP\Files\IRootFolder; @@ -24,8 +25,8 @@ final class MusicnnClassifier extends Classifier { private TagManager $tagManager; - public function __construct(Logger $logger, IAppConfig $config, TagManager $tagManager, QueueService $queue, IRootFolder $rootFolder, ITempManager $tempManager, IPreview $previewProvider) { - parent::__construct($logger, $config, $rootFolder, $queue, $tempManager, $previewProvider); + public function __construct(Logger $logger, IAppConfig $config, TagManager $tagManager, QueueService $queue, IRootFolder $rootFolder, ITempManager $tempManager, IPreview $previewProvider, SettingsService $settingsService) { + parent::__construct($logger, $config, $rootFolder, $queue, $tempManager, $previewProvider, $settingsService); $this->tagManager = $tagManager; } diff --git a/lib/Classifiers/Classifier.php b/lib/Classifiers/Classifier.php index a1c88994c..6c7355951 100644 --- a/lib/Classifiers/Classifier.php +++ b/lib/Classifiers/Classifier.php @@ -13,6 +13,7 @@ use OCA\Recognize\Constants; use OCA\Recognize\Db\QueueFile; use OCA\Recognize\Service\QueueService; +use OCA\Recognize\Service\SettingsService; use OCP\AppFramework\Services\IAppConfig; use OCP\DB\Exception; use OCP\Encryption\Exceptions\GenericEncryptionException; @@ -41,14 +42,16 @@ abstract class Classifier { private ITempManager $tempManager; private IPreview $previewProvider; private int $maxExecutionTime = self::MAX_EXECUTION_TIME; + private SettingsService $settingsService; - public function __construct(LoggerInterface $logger, IAppConfig $config, IRootFolder $rootFolder, QueueService $queue, ITempManager $tempManager, IPreview $previewProvider) { + public function __construct(LoggerInterface $logger, IAppConfig $config, IRootFolder $rootFolder, QueueService $queue, ITempManager $tempManager, IPreview $previewProvider, SettingsService $settingsService) { $this->logger = $logger; $this->config = $config; $this->rootFolder = $rootFolder; $this->queue = $queue; $this->tempManager = $tempManager; $this->previewProvider = $previewProvider; + $this->settingsService = $settingsService; } public function setMaxExecutionTime(int $time): void { @@ -187,6 +190,7 @@ public function classifyFiles(string $model, array $queueFiles, int $timeout): \ if ($cores !== '0') { $env['RECOGNIZE_CORES'] = $cores; } + $env['MODEL_DIR'] = $this->settingsService->getSetting('models_target_path').'/models'; $proc->setEnv($env); $proc->setTimeout(count($paths) * $timeout); $proc->setInput(implode("\n", $paths)); diff --git a/lib/Classifiers/Images/ClusteringFaceClassifier.php b/lib/Classifiers/Images/ClusteringFaceClassifier.php index 81eff74a6..cd2cf962a 100644 --- a/lib/Classifiers/Images/ClusteringFaceClassifier.php +++ b/lib/Classifiers/Images/ClusteringFaceClassifier.php @@ -13,6 +13,7 @@ use OCA\Recognize\Db\FaceDetectionMapper; use OCA\Recognize\Service\Logger; use OCA\Recognize\Service\QueueService; +use OCA\Recognize\Service\SettingsService; use OCP\AppFramework\Services\IAppConfig; use OCP\BackgroundJob\IJobList; use OCP\DB\Exception; @@ -43,8 +44,8 @@ public function __construct( private FaceDetectionMapper $faceDetections, QueueService $queue, IRootFolder $rootFolder, private IUserMountCache $userMountCache, private IJobList $jobList, ITempManager $tempManager, IPreview $previewProvider, - private IManager $shareManager) { - parent::__construct($logger, $config, $rootFolder, $queue, $tempManager, $previewProvider); + private IManager $shareManager, SettingsService $settingsService) { + parent::__construct($logger, $config, $rootFolder, $queue, $tempManager, $previewProvider, $settingsService); } /** diff --git a/lib/Classifiers/Images/ImagenetClassifier.php b/lib/Classifiers/Images/ImagenetClassifier.php index 576c0546d..4d226a40b 100644 --- a/lib/Classifiers/Images/ImagenetClassifier.php +++ b/lib/Classifiers/Images/ImagenetClassifier.php @@ -10,6 +10,7 @@ use OCA\Recognize\Classifiers\Classifier; use OCA\Recognize\Service\Logger; use OCA\Recognize\Service\QueueService; +use OCA\Recognize\Service\SettingsService; use OCA\Recognize\Service\TagManager; use OCP\AppFramework\Services\IAppConfig; use OCP\Files\IRootFolder; @@ -25,8 +26,8 @@ final class ImagenetClassifier extends Classifier { private TagManager $tagManager; protected QueueService $queue; - public function __construct(Logger $logger, IAppConfig $config, TagManager $tagManager, QueueService $queue, IRootFolder $rootFolder, ITempManager $tempManager, IPreview $previewProvider) { - parent::__construct($logger, $config, $rootFolder, $queue, $tempManager, $previewProvider); + public function __construct(Logger $logger, IAppConfig $config, TagManager $tagManager, QueueService $queue, IRootFolder $rootFolder, ITempManager $tempManager, IPreview $previewProvider, SettingsService $settingsService) { + parent::__construct($logger, $config, $rootFolder, $queue, $tempManager, $previewProvider, $settingsService); $this->tagManager = $tagManager; $this->queue = $queue; } diff --git a/lib/Classifiers/Images/LandmarksClassifier.php b/lib/Classifiers/Images/LandmarksClassifier.php index adcf60ad8..f3f502cc8 100644 --- a/lib/Classifiers/Images/LandmarksClassifier.php +++ b/lib/Classifiers/Images/LandmarksClassifier.php @@ -11,6 +11,7 @@ use OCA\Recognize\Db\QueueFile; use OCA\Recognize\Service\Logger; use OCA\Recognize\Service\QueueService; +use OCA\Recognize\Service\SettingsService; use OCA\Recognize\Service\TagManager; use OCP\AppFramework\Services\IAppConfig; use OCP\Files\IRootFolder; @@ -26,8 +27,8 @@ final class LandmarksClassifier extends Classifier { private TagManager $tagManager; - public function __construct(Logger $logger, IAppConfig $config, TagManager $tagManager, QueueService $queue, IRootFolder $rootFolder, ITempManager $tempManager, IPreview $previewProvider) { - parent::__construct($logger, $config, $rootFolder, $queue, $tempManager, $previewProvider); + public function __construct(Logger $logger, IAppConfig $config, TagManager $tagManager, QueueService $queue, IRootFolder $rootFolder, ITempManager $tempManager, IPreview $previewProvider, SettingsService $settingsService) { + parent::__construct($logger, $config, $rootFolder, $queue, $tempManager, $previewProvider, $settingsService); $this->tagManager = $tagManager; } diff --git a/lib/Classifiers/Video/MovinetClassifier.php b/lib/Classifiers/Video/MovinetClassifier.php index 5f9b6a602..2e682b421 100644 --- a/lib/Classifiers/Video/MovinetClassifier.php +++ b/lib/Classifiers/Video/MovinetClassifier.php @@ -11,6 +11,7 @@ use OCA\Recognize\Exception\Exception; use OCA\Recognize\Service\Logger; use OCA\Recognize\Service\QueueService; +use OCA\Recognize\Service\SettingsService; use OCA\Recognize\Service\TagManager; use OCP\AppFramework\Services\IAppConfig; use OCP\Files\IRootFolder; @@ -24,8 +25,8 @@ final class MovinetClassifier extends Classifier { private TagManager $tagManager; - public function __construct(Logger $logger, IAppConfig $config, TagManager $tagManager, QueueService $queue, IRootFolder $rootFolder, ITempManager $tempManager, IPreview $previewProvider) { - parent::__construct($logger, $config, $rootFolder, $queue, $tempManager, $previewProvider); + public function __construct(Logger $logger, IAppConfig $config, TagManager $tagManager, QueueService $queue, IRootFolder $rootFolder, ITempManager $tempManager, IPreview $previewProvider, SettingsService $settingsService) { + parent::__construct($logger, $config, $rootFolder, $queue, $tempManager, $previewProvider, $settingsService); $this->tagManager = $tagManager; } diff --git a/lib/Migration/MoveDefaultModelFolder.php b/lib/Migration/MoveDefaultModelFolder.php new file mode 100644 index 000000000..fd77f170f --- /dev/null +++ b/lib/Migration/MoveDefaultModelFolder.php @@ -0,0 +1,70 @@ + + * @copyright Copyright (c) 2021, Marcel Klehr + * + * @author Joas Schilling + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ +namespace OCA\Recognize\Migration; + +use OCA\Recognize\Service\SettingsService; +use OCP\Migration\IOutput; +use OCP\Migration\IRepairStep; +use Psr\Log\LoggerInterface; + +final class MoveDefaultModelFolder implements IRepairStep { + + public function __construct( + private SettingsService $settingsService, + private LoggerInterface $logger, + ) { + } + + public function getName(): string { + return 'Try to move the default Model Folder'; + } + + public function run(IOutput $output): void { + $oldModelTargetPath = __DIR__ . '/../../models'; + $oldModelArchivePath = __DIR__ . '/../../models.tar.gz'; + $newPath = $this->settingsService->getSetting('models_target_path'); + $newModelTargetPath = $newPath . '/models'; + $newModelArchivePath = $newPath . '/models.tar.gz'; + + if (is_dir($oldModelTargetPath)) { + /** @var array $filesToMove */ + $filesToMove = scandir($oldModelTargetPath); + $filesToMove = array_filter($filesToMove, fn (string $value) => $value !== '.' && $value !== '..'); + $filesToMove = array_map(fn ($value) => $oldModelTargetPath.'/'.$value, $filesToMove); + mkdir($newModelTargetPath, recursive: true); + foreach ($filesToMove as $file) { + // Moving files across Partitions will fail, so instead we copy and delete see https://www.php.net/manual/en/function.rename.php#113943 + copy($file, $newModelTargetPath.'/'.basename($file)); + unlink($file); + + } + } + + if (is_file($oldModelArchivePath)) { + rename($oldModelArchivePath, $newModelArchivePath); + } + } +} diff --git a/lib/Service/DownloadModelsService.php b/lib/Service/DownloadModelsService.php index 17dd8a815..5efb412be 100644 --- a/lib/Service/DownloadModelsService.php +++ b/lib/Service/DownloadModelsService.php @@ -12,14 +12,17 @@ use OCP\Http\Client\IClientService; use RecursiveDirectoryIterator; use RecursiveIteratorIterator; +use function Safe\mkdir; final class DownloadModelsService { private IClientService $clientService; private bool $isCLI; + private SettingsService $settingsService; - public function __construct(IClientService $clientService, bool $isCLI) { + public function __construct(IClientService $clientService, bool $isCLI, SettingsService $settingsService) { $this->clientService = $clientService; $this->isCLI = $isCLI; + $this->settingsService = $settingsService; } /** @@ -27,14 +30,19 @@ public function __construct(IClientService $clientService, bool $isCLI) { * @return void * @throws \Exception */ - public function download(?callable $log = null) : void { + + public function download() : void { $log ??= static function (string $message): void { }; - $targetPath = __DIR__ . '/../../models'; - if (file_exists($targetPath)) { + $targetPath = $this->settingsService->getSetting('models_target_path'); + if (!file_exists($targetPath)) { + mkdir($targetPath, recursive: true); + } + $modelPath = $targetPath . '/models'; + if (file_exists($modelPath)) { $log('Removing existing models directory at ' . $targetPath); // remove models directory - $it = new RecursiveDirectoryIterator($targetPath, FilesystemIterator::SKIP_DOTS); + $it = new RecursiveDirectoryIterator($modelPath, FilesystemIterator::SKIP_DOTS); $files = new RecursiveIteratorIterator($it, RecursiveIteratorIterator::CHILD_FIRST); foreach ($files as $file) { @@ -44,11 +52,12 @@ public function download(?callable $log = null) : void { unlink($file->getRealPath()); } } - rmdir($targetPath); + rmdir($modelPath); } $archiveUrl = $this->getArchiveUrl($this->getNeededArchiveRef()); - $archivePath = __DIR__ . '/../../models.tar.gz'; + + $archivePath = $targetPath . '/models.tar.gz'; $log('Downloading models archive from ' . $archiveUrl); $log('Saving archive to ' . $archivePath); $timeout = $this->isCLI ? 0 : 480; diff --git a/lib/Service/SettingsService.php b/lib/Service/SettingsService.php index 8ea9d217b..395041ad0 100644 --- a/lib/Service/SettingsService.php +++ b/lib/Service/SettingsService.php @@ -61,6 +61,7 @@ final class SettingsService { 'nice_value' => '0', 'concurrency.enabled' => 'false', 'ffmpeg_binary' => '', + 'models_target_path' => '../../models_cache', ]; /** @var array */ @@ -94,7 +95,13 @@ final class SettingsService { 'landmarks.batchSize', 'movinet.batchSize', 'musicnn.batchSize', - 'concurrency.enabled' + 'concurrency.enabled', + 'models_target_path', + 'models_archive_file', + ]; + + private const PATH_SETTINGS = [ + 'models_target_path', ]; private IAppConfig $config; @@ -121,6 +128,14 @@ public function getSetting(string $key): string { if (in_array($key, self::LAZY_SETTINGS, true)) { $lazy = true; } + + if (in_array($key, self::PATH_SETTINGS, true)) { + $path = $this->config->getAppValueString($key, self::DEFAULTS[$key], lazy: $lazy); + if (!$this->isPathAbsolute($path)) { + $path = __DIR__ .'/'. $path; + } + return $path; + } return $this->config->getAppValueString($key, self::DEFAULTS[$key], lazy: $lazy); } @@ -182,6 +197,9 @@ public function setSetting(string $key, string $value): void { if (in_array($key, self::LAZY_SETTINGS, true)) { $lazy = true; } + if (in_array($key, self::PATH_SETTINGS) && $value === '') { + $value = self::DEFAULTS[$key]; + } $this->config->setAppValueString($key, $value, lazy: $lazy); } @@ -195,4 +213,23 @@ public function getAll(): array { } return $settings; } + + private function isPathAbsolute(string $path): bool { + if ($path === '') { + return false; + } + if ($path[0] === '/') { + return true; + } + + if (ctype_alpha($path[0]) && $path[1] === ':') { + return true; + } + + if ($path[0] === '\\' && $path[1] === '\\') { + return true; + } + + return false; + } } diff --git a/lib/Settings/AdminSettings.php b/lib/Settings/AdminSettings.php index 31f133706..e35ab4a3f 100644 --- a/lib/Settings/AdminSettings.php +++ b/lib/Settings/AdminSettings.php @@ -28,9 +28,11 @@ public function getForm(): TemplateResponse { $settings = $this->settingsService->getAll(); $this->initialState->provideInitialState('settings', $settings); - $modelsPath = __DIR__ . '/../../models'; - $modelsDownloaded = file_exists($modelsPath); + $targetPath = $this->settingsService->getSetting('models_target_path'); + $modelsDownloaded = file_exists($targetPath .'/models'); + $modelsTargetPathWritable = is_writable($targetPath); $this->initialState->provideInitialState('modelsDownloaded', $modelsDownloaded); + $this->initialState->provideInitialState('modelsTargetPathWritable', $modelsTargetPathWritable); $tagsEnabled = $this->appManager->isEnabledForAnyone('systemtags'); $this->initialState->provideInitialState('tagsEnabled', $tagsEnabled); diff --git a/src/classifier_imagenet.js b/src/classifier_imagenet.js index f4cb788be..5014525b4 100644 --- a/src/classifier_imagenet.js +++ b/src/classifier_imagenet.js @@ -4,9 +4,11 @@ const YAML = require('yaml') const _ = require('lodash') const rules = YAML.parse(fsSync.readFileSync(path.join(__dirname, 'rules.yml')).toString('utf8')) const { IMAGENET_CLASSES } = require('./efficientnet/classes') +const MODEL_DIR = process.env.MODEL_DIR let tf, getPort, StaticServer let PUREJS = false + if (process.env.RECOGNIZE_PUREJS === 'true') { tf = require('@tensorflow/tfjs') require('@tensorflow/tfjs-backend-wasm') @@ -58,7 +60,7 @@ if (process.argv.length < 3) throw new Error('Incorrect arguments: node classify * @param minInput */ async function main(modelName, imgSize, minInput) { - const modelPath = path.resolve(__dirname, '..', 'models', modelName) + const modelPath = path.resolve(MODEL_DIR, modelName) const modelFileName = 'model.json' let modelUrl diff --git a/src/classifier_landmarks.js b/src/classifier_landmarks.js index 63ec3a070..31f2b8036 100644 --- a/src/classifier_landmarks.js +++ b/src/classifier_landmarks.js @@ -9,6 +9,7 @@ const LABELS = { landmarks_south_america: require('./landmarks/south_america.json').name, landmarks_oceania: require('./landmarks/oceania.json').name, } +const MODEL_DIR = process.env.MODEL_DIR let tf, getPort, StaticServer let PUREJS = false @@ -48,7 +49,7 @@ if (process.argv.length < 3) throw new Error('Incorrect arguments: node classify * @param paths */ async function main(modelName, imgSize, minInput, paths) { - const modelPath = path.resolve(__dirname, '..', 'models', modelName) + const modelPath = path.resolve(MODEL_DIR, modelName) const modelFileName = 'model.json' let modelUrl diff --git a/src/classifier_movinet.js b/src/classifier_movinet.js index 499ea6be9..5a98185a4 100644 --- a/src/classifier_movinet.js +++ b/src/classifier_movinet.js @@ -1,6 +1,7 @@ const path = require('path') const fsSync = require('fs') const _ = require('lodash') +const MODEL_DIR = process.env.MODEL_DIR let tf, getPort, StaticServer let PUREJS = false @@ -33,7 +34,7 @@ if (process.argv.length < 3) throw new Error('Incorrect arguments: node classify * */ async function main() { - const modelPath = path.resolve(__dirname, '..', 'models', 'movinet-a3') + const modelPath = path.resolve(MODEL_DIR, 'movinet-a3') const modelFileName = 'model.json' let modelUrl diff --git a/src/classifier_musicnn.js b/src/classifier_musicnn.js index 5b35bcc68..4b9826b9f 100644 --- a/src/classifier_musicnn.js +++ b/src/classifier_musicnn.js @@ -3,6 +3,7 @@ const fsSync = require('fs') const YAML = require('yaml') const _ = require('lodash') const rules = YAML.parse(fsSync.readFileSync(path.join(__dirname, 'musicnn_rules.yml')).toString('utf8')) +const MODEL_DIR = process.env.MODEL_DIR let tf, getPort, StaticServer let PUREJS = false @@ -51,7 +52,7 @@ if (process.argv.length < 3) throw new Error('Incorrect arguments: node classify * */ async function main() { - const modelPath = path.resolve(__dirname, '..', 'models', 'musicnn') + const modelPath = path.resolve(MODEL_DIR, 'musicnn') const modelFileName = 'model.json' let modelUrl diff --git a/src/components/ViewAdmin.vue b/src/components/ViewAdmin.vue index 21e1d894d..7aa2e0bb3 100644 --- a/src/components/ViewAdmin.vue +++ b/src/components/ViewAdmin.vue @@ -38,6 +38,23 @@

+ + + {{ t('recognize', 'The Model Target Path is Writable') }} + + + {{ t('recognize', 'Model Target Path is not Writable') }} + +

+ +

+

+ {{ t('recognize', 'Changing this Value will require you to redownload the Models. Also the models in the old Directory need to be deleted manually') }} +

+ {{ t('recognize', 'The recognize_backend ExApp is installed; TaskProcessing mode is recommended.') }} @@ -81,8 +98,8 @@ {{ t('recognize', 'Enable face recognition (groups photos by faces that appear in them; UI is in the photos app)') }}