diff --git a/runtime/datamate-python/app/core/config.py b/runtime/datamate-python/app/core/config.py index 77139ab62..5f6d3ae55 100644 --- a/runtime/datamate-python/app/core/config.py +++ b/runtime/datamate-python/app/core/config.py @@ -84,5 +84,29 @@ def build_database_url(self): # 文件存储配置（共享文件系统） file_storage_path: str = "/data/files" + # ==================== 配比任务并行复制配置 ==================== + # 动态并发计算参数（全闪存储高性能场景默认值） + + # 并发下限（最少并发数） + ratio_copy_min_concurrent: int = 8 + + # 并发上限（最多并发数，防止资源耗尽） + ratio_copy_max_concurrent: int = 128 + + # CPU核心系数（每个核心贡献的并发数，全闪存储建议4.0） + ratio_copy_cpu_factor: float = 4.0 + + # 每并发任务预估内存占用（MB） + ratio_copy_memory_per_task_mb: int = 32 + + # 内存安全保留比例（保留给其他进程） + ratio_copy_memory_reserve_ratio: float = 0.2 + + # 是否启用动态计算（False则使用固定值） + ratio_copy_dynamic_concurrent: bool = True + + # 固定并发数（当 dynamic_concurrent=False 时使用） + ratio_copy_fixed_concurrent: int = 10 + # 全局设置实例 settings = Settings() diff --git a/runtime/datamate-python/app/module/ratio/service/ratio_task.py b/runtime/datamate-python/app/module/ratio/service/ratio_task.py index 9a865f7e6..b39602eb0 100644 --- a/runtime/datamate-python/app/module/ratio/service/ratio_task.py +++ b/runtime/datamate-python/app/module/ratio/service/ratio_task.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import List, Optional, Dict, Any +from typing import List, Optional, Dict, Any, Tuple import random import json import os @@ -10,6 +10,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.core.logging import get_logger +from app.core.config import settings from app.db.models.base_entity import LineageNode, LineageEdge from app.db.models.ratio_task import RatioInstance, RatioRelation from app.db.models import Dataset, DatasetFiles @@ -18,6 +19,7 @@ from app.module.shared.common.lineage import LineageService from app.module.shared.schema import TaskStatus, NodeType, EdgeType from app.module.ratio.schema.ratio_task import FilterCondition +from app.module.shared.util.resource_utils import get_concurrent_for_ratio_copy, get_system_resource_info logger = get_logger(__name__) @@ -78,37 +80,24 @@ async def create_task( @staticmethod async def execute_dataset_ratio_task(instance_id: str) -> None: - """Execute a ratio task in background. - - Supported ratio_method: - - DATASET: randomly select counts files from each source dataset - - TAG: randomly select counts files matching relation.filter_conditions tags - - Steps: - - Mark instance RUNNING - - For each relation: fetch ACTIVE files, optionally filter by tags - - Copy selected files into target dataset - - Update dataset statistics and mark instance SUCCESS/FAILED - """ - async with AsyncSessionLocal() as session: # type: AsyncSession + async with AsyncSessionLocal() as session: try: - # Load instance and relations inst_res = await session.execute(select(RatioInstance).where(RatioInstance.id == instance_id)) instance: Optional[RatioInstance] = inst_res.scalar_one_or_none() if not instance: logger.error(f"Ratio instance not found: {instance_id}") return - logger.info(f"start execute ratio task: {instance_id}") + + logger.info(f"Starting ratio task {instance_id}") + logger.info(f"System resources: {get_system_resource_info()}") rel_res = await session.execute( select(RatioRelation).where(RatioRelation.ratio_instance_id == instance_id) ) relations: List[RatioRelation] = list(rel_res.scalars().all()) - # Mark running instance.status = TaskStatus.RUNNING.name - # Load target dataset ds_res = await session.execute(select(Dataset).where(Dataset.id == instance.target_dataset_id)) target_ds: Optional[Dataset] = ds_res.scalar_one_or_none() if not target_ds: @@ -116,18 +105,21 @@ async def execute_dataset_ratio_task(instance_id: str) -> None: instance.status = TaskStatus.FAILED.name return - added_count, added_size = await RatioTaskService.handle_ratio_relations(relations,session, target_ds) + max_concurrent = get_concurrent_for_ratio_copy(settings) + logger.info(f"Using {max_concurrent} concurrent workers for ratio task {instance_id}") - # Update target dataset statistics - target_ds.file_count = (target_ds.file_count or 0) + added_count # type: ignore - target_ds.size_bytes = (target_ds.size_bytes or 0) + added_size # type: ignore - # If target dataset has files, mark it ACTIVE - if (target_ds.file_count or 0) > 0: # type: ignore + added_count, added_size = await RatioTaskService.handle_ratio_relations_parallel( + relations, session, target_ds, max_concurrent + ) + + target_ds.file_count = (target_ds.file_count or 0) + added_count + target_ds.size_bytes = (target_ds.size_bytes or 0) + added_size + if (target_ds.file_count or 0) > 0: target_ds.status = "ACTIVE" - # Done instance.status = TaskStatus.COMPLETED.name - logger.info(f"Dataset ratio execution completed: instance={instance_id}, files={added_count}, size={added_size}, {instance.status}") + logger.info(f"Ratio task completed: {instance_id}, files={added_count}, size={added_size}") + await RatioTaskService._add_task_to_graph( session=session, src_relations=relations, @@ -135,9 +127,8 @@ async def execute_dataset_ratio_task(instance_id: str) -> None: dst_dataset=target_ds, ) except Exception as e: - logger.exception(f"Dataset ratio execution failed for {instance_id}: {e}") + logger.exception(f"Ratio task failed for {instance_id}: {e}") try: - # Try mark failed inst_res = await session.execute(select(RatioInstance).where(RatioInstance.id == instance_id)) instance = inst_res.scalar_one_or_none() if instance: @@ -147,6 +138,116 @@ async def execute_dataset_ratio_task(instance_id: str) -> None: finally: await session.commit() + @staticmethod + async def handle_ratio_relations_parallel( + relations: list[RatioRelation], + session: AsyncSession, + target_ds: Dataset, + max_concurrent: int = 10 + ) -> Tuple[int, int]: + existing_path_rows = await session.execute( + select(DatasetFiles.file_path).where(DatasetFiles.dataset_id == target_ds.id) + ) + existing_paths = set(p for p in existing_path_rows.scalars().all() if p) + source_paths = set() + + all_copy_tasks: List[Tuple[DatasetFiles, str, str]] = [] + + for rel in relations: + if not rel.source_dataset_id or not rel.counts or rel.counts <= 0: + continue + + files = await RatioTaskService.get_files(rel, session) + if not files: + continue + + pick_n = min(rel.counts or 0, len(files)) + chosen = random.sample(files, pick_n) if pick_n < len(files) else files + + for file in chosen: + if file.file_path in source_paths: + continue + + dst_prefix = f"/dataset/{target_ds.id}/" + file_name = RatioTaskService.get_new_file_name(dst_prefix, existing_paths, file) + new_path = dst_prefix + file_name + + file_record = DatasetFiles( + dataset_id=target_ds.id, + file_name=file_name, + file_path=new_path, + file_type=file.file_type, + file_size=file.file_size, + check_sum=file.check_sum, + tags=file.tags, + tags_updated_at=datetime.now(), + dataset_filemetadata=file.dataset_filemetadata, + status="ACTIVE", + ) + + all_copy_tasks.append((file_record, file.file_path, new_path)) + existing_paths.add(new_path) + source_paths.add(file.file_path) + + if not all_copy_tasks: + return 0, 0 + + dst_dir = f"/dataset/{target_ds.id}/" + await asyncio.to_thread(os.makedirs, dst_dir, exist_ok=True) + + semaphore = asyncio.Semaphore(max_concurrent) + successful_records: List[DatasetFiles] = [] + added_count = 0 + added_size = 0 + + async def copy_with_semaphore( + file_record: DatasetFiles, + src_path: str, + dst_path: str + ) -> Tuple[bool, DatasetFiles]: + async with semaphore: + try: + file_dst_dir = os.path.dirname(dst_path) + if file_dst_dir != dst_dir: + await asyncio.to_thread(os.makedirs, file_dst_dir, exist_ok=True) + + try: + await asyncio.to_thread(os.link, src_path, dst_path) + except OSError: + try: + await asyncio.to_thread(os.symlink, src_path, dst_path) + except OSError: + await asyncio.to_thread(shutil.copy2, src_path, dst_path) + + return True, file_record + except Exception as e: + logger.error(f"Copy failed: {src_path} -> {dst_path}: {e}") + return False, file_record + + tasks = [copy_with_semaphore(rec, src, dst) for rec, src, dst in all_copy_tasks] + results = await asyncio.gather(*tasks, return_exceptions=True) + + for i, result in enumerate(results): + if isinstance(result, Exception): + logger.error(f"Copy task {i} raised exception: {result}") + continue + success, file_record = result + if success: + added_count += 1 + added_size += int(file_record.file_size or 0) + successful_records.append(file_record) + + if successful_records: + session.add_all(successful_records) + await session.flush() + + logger.info( + f"Parallel copy completed: {added_count}/{len(all_copy_tasks)} files, " + f"{added_size} bytes, {len(results) - added_count} failures" + ) + + return added_count, added_size + @staticmethod async def handle_ratio_relations(relations: list[RatioRelation], session, target_ds: Dataset) -> tuple[int, int]: # Preload existing target file paths for deduplication diff --git a/runtime/datamate-python/app/module/shared/util/resource_utils.py b/runtime/datamate-python/app/module/shared/util/resource_utils.py new file mode 100644 index 000000000..afc953364 --- /dev/null +++ b/runtime/datamate-python/app/module/shared/util/resource_utils.py @@ -0,0 +1,159 @@ +""" +资源感知并发计算工具 + +根据系统 CPU 和内存资源动态计算最优并发数 +适配全闪存储等高性能存储场景 +""" +import os +import logging +from typing import Optional + +logger = logging.getLogger(__name__) + +# 尝试导入 psutil，失败则使用 os.cpu_count +try: + import psutil + HAS_PSUTIL = True +except ImportError: + HAS_PSUTIL = False + logger.warning("psutil not available, falling back to os.cpu_count() for CPU detection") + + +def get_cpu_count(logical: bool = True) -> int: + """获取 CPU 核心数 + + Args: + logical: 是否返回逻辑核心数（超线程），False返回物理核心数 + + Returns: + CPU 核心数，无法检测时返回默认值 4 + """ + if HAS_PSUTIL: + count = psutil.cpu_count(logical=logical) + if count: + return count + + # 回退到 os.cpu_count() + count = os.cpu_count() + return count if count else 4 + + +def get_available_memory_gb() -> float: + """获取可用内存（GB） + + Returns: + 可用内存大小（GB），无法检测时返回默认值 8.0 + """ + if HAS_PSUTIL: + mem = psutil.virtual_memory() + return mem.available / (1024 ** 3) + + # 无法检测，返回保守默认值 + logger.warning("psutil not available, using default memory estimate") + return 8.0 + + +def calculate_optimal_concurrent( + min_concurrent: int = 8, + max_concurrent: int = 128, + cpu_factor: float = 4.0, + memory_per_task_mb: int = 32, + memory_reserve_ratio: float = 0.2, +) -> int: + """根据系统资源计算最优并发数 + + 计算逻辑： + 1. 基于 CPU 核心数：并发数 = CPU核心数 × cpu_factor + 2. 基于内存限制：可用内存 × (1-reserve_ratio) ÷ memory_per_task + 3. 最终取两者最小值，并限制在 [min_concurrent, max_concurrent] 范围内 + + Args: + min_concurrent: 并发下限 + max_concurrent: 并发上限 + cpu_factor: CPU核心系数（全闪存储建议4.0） + memory_per_task_mb: 每任务预估内存占用（MB） + memory_reserve_ratio: 内存安全保留比例 + + Returns: + 计算得出的最优并发数 + """ + # 获取 CPU 核心数 + cpu_count = get_cpu_count(logical=True) + + # 基于 CPU 计算并发数 + cpu_based_concurrent = int(cpu_count * cpu_factor) + + # 获取可用内存 + available_memory_gb = get_available_memory_gb() + + # 计算可用于并发任务的内存（扣除保留部分） + usable_memory_gb = available_memory_gb * (1 - memory_reserve_ratio) + usable_memory_mb = usable_memory_gb * 1024 + + # 基于内存计算最大并发数 + memory_based_concurrent = int(usable_memory_mb / memory_per_task_mb) + + # 取两者最小值（避免内存耗尽） + calculated_concurrent = min(cpu_based_concurrent, memory_based_concurrent) + + # 限制在范围内 + final_concurrent = max(min_concurrent, min(max_concurrent, calculated_concurrent)) + + # 记录计算过程 + logger.info( + f"Concurrency calculation: " + f"CPU cores={cpu_count}, CPU-based={cpu_based_concurrent}, " + f"Available memory={available_memory_gb:.1f}GB, Memory-based={memory_based_concurrent}, " + f"Final concurrent={final_concurrent} (range=[{min_concurrent}, {max_concurrent}])" + ) + + return final_concurrent + + +def get_concurrent_for_ratio_copy(settings) -> int: + """获取配比文件复制的并发数 + + Args: + settings: 配置实例 + + Returns: + 并发数（动态计算或固定值） + """ + if not settings.ratio_copy_dynamic_concurrent: + return settings.ratio_copy_fixed_concurrent + + return calculate_optimal_concurrent( + min_concurrent=settings.ratio_copy_min_concurrent, + max_concurrent=settings.ratio_copy_max_concurrent, + cpu_factor=settings.ratio_copy_cpu_factor, + memory_per_task_mb=settings.ratio_copy_memory_per_task_mb, + memory_reserve_ratio=settings.ratio_copy_memory_reserve_ratio, + ) + + +def get_system_resource_info() -> dict: + """获取系统资源信息（用于日志和调试） + + Returns: + 系统资源信息字典 + """ + cpu_logical = get_cpu_count(logical=True) + cpu_physical = get_cpu_count(logical=False) if HAS_PSUTIL else cpu_logical + + if HAS_PSUTIL: + mem = psutil.virtual_memory() + return { + "cpu_logical_cores": cpu_logical, + "cpu_physical_cores": cpu_physical, + "memory_total_gb": round(mem.total / (1024 ** 3), 2), + "memory_available_gb": round(mem.available / (1024 ** 3), 2), + "memory_used_percent": round(mem.percent, 1), + "psutil_available": True, + } + + return { + "cpu_logical_cores": cpu_logical, + "cpu_physical_cores": cpu_physical, + "memory_available_gb": get_available_memory_gb(), + "psutil_available": False, + } \ No newline at end of file diff --git a/runtime/datamate-python/poetry.lock b/runtime/datamate-python/poetry.lock index 066340b5d..4583b2ddf 100644 --- a/runtime/datamate-python/poetry.lock +++ b/runtime/datamate-python/poetry.lock @@ -3363,38 +3363,34 @@ files = [ [[package]] name = "psutil" -version = "7.2.2" -description = "Cross-platform lib for process and system monitoring." -optional = false -python-versions = ">=3.6" -groups = ["main"] -files = [ - {file = "psutil-7.2.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:2edccc433cbfa046b980b0df0171cd25bcaeb3a68fe9022db0979e7aa74a826b"}, - {file = "psutil-7.2.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78c8603dcd9a04c7364f1a3e670cea95d51ee865e4efb3556a3a63adef958ea"}, - {file = "psutil-7.2.2-cp313-cp313t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a571f2330c966c62aeda00dd24620425d4b0cc86881c89861fbc04549e5dc63"}, - {file = "psutil-7.2.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:917e891983ca3c1887b4ef36447b1e0873e70c933afc831c6b6da078ba474312"}, - {file = "psutil-7.2.2-cp313-cp313t-win_amd64.whl", hash = "sha256:ab486563df44c17f5173621c7b198955bd6b613fb87c71c161f827d3fb149a9b"}, - {file = "psutil-7.2.2-cp313-cp313t-win_arm64.whl", hash = "sha256:ae0aefdd8796a7737eccea863f80f81e468a1e4cf14d926bd9b6f5f2d5f90ca9"}, - {file = "psutil-7.2.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:eed63d3b4d62449571547b60578c5b2c4bcccc5387148db46e0c2313dad0ee00"}, - {file = "psutil-7.2.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7b6d09433a10592ce39b13d7be5a54fbac1d1228ed29abc880fb23df7cb694c9"}, - {file = "psutil-7.2.2-cp314-cp314t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1fa4ecf83bcdf6e6c8f4449aff98eefb5d0604bf88cb883d7da3d8d2d909546a"}, - {file = "psutil-7.2.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e452c464a02e7dc7822a05d25db4cde564444a67e58539a00f929c51eddda0cf"}, - {file = "psutil-7.2.2-cp314-cp314t-win_amd64.whl", hash = "sha256:c7663d4e37f13e884d13994247449e9f8f574bc4655d509c3b95e9ec9e2b9dc1"}, - {file = "psutil-7.2.2-cp314-cp314t-win_arm64.whl", hash = "sha256:11fe5a4f613759764e79c65cf11ebdf26e33d6dd34336f8a337aa2996d71c841"}, - {file = "psutil-7.2.2-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:ed0cace939114f62738d808fdcecd4c869222507e266e574799e9c0faa17d486"}, - {file = "psutil-7.2.2-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:1a7b04c10f32cc88ab39cbf606e117fd74721c831c98a27dc04578deb0c16979"}, - {file = "psutil-7.2.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:076a2d2f923fd4821644f5ba89f059523da90dc9014e85f8e45a5774ca5bc6f9"}, - {file = "psutil-7.2.2-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b0726cecd84f9474419d67252add4ac0cd9811b04d61123054b9fb6f57df6e9e"}, - {file = "psutil-7.2.2-cp36-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:fd04ef36b4a6d599bbdb225dd1d3f51e00105f6d48a28f006da7f9822f2606d8"}, - {file = "psutil-7.2.2-cp36-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b58fabe35e80b264a4e3bb23e6b96f9e45a3df7fb7eed419ac0e5947c61e47cc"}, - {file = "psutil-7.2.2-cp37-abi3-win_amd64.whl", hash = "sha256:eb7e81434c8d223ec4a219b5fc1c47d0417b12be7ea866e24fb5ad6e84b3d988"}, - {file = "psutil-7.2.2-cp37-abi3-win_arm64.whl", hash = "sha256:8c233660f575a5a89e6d4cb65d9f938126312bca76d8fe087b947b3a1aaac9ee"}, - {file = "psutil-7.2.2.tar.gz", hash = "sha256:0746f5f8d406af344fd547f1c8daa5f5c33dbc293bb8d6a16d80b4bb88f59372"}, +version = "6.1.1" +description = "Cross-platform lib for process and system monitoring in Python." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" +groups = ["main"] +files = [ + {file = "psutil-6.1.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:9ccc4316f24409159897799b83004cb1e24f9819b0dcf9c0b68bdcb6cefee6a8"}, + {file = "psutil-6.1.1-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:ca9609c77ea3b8481ab005da74ed894035936223422dc591d6772b147421f777"}, + {file = "psutil-6.1.1-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:8df0178ba8a9e5bc84fed9cfa61d54601b371fbec5c8eebad27575f1e105c0d4"}, + {file = "psutil-6.1.1-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:1924e659d6c19c647e763e78670a05dbb7feaf44a0e9c94bf9e14dfc6ba50468"}, + {file = "psutil-6.1.1-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:018aeae2af92d943fdf1da6b58665124897cfc94faa2ca92098838f83e1b1bca"}, + {file = "psutil-6.1.1-cp27-none-win32.whl", hash = "sha256:6d4281f5bbca041e2292be3380ec56a9413b790579b8e593b1784499d0005dac"}, + {file = "psutil-6.1.1-cp27-none-win_amd64.whl", hash = "sha256:c777eb75bb33c47377c9af68f30e9f11bc78e0f07fbf907be4a5d70b2fe5f030"}, + {file = "psutil-6.1.1-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:fc0ed7fe2231a444fc219b9c42d0376e0a9a1a72f16c5cfa0f68d19f1a0663e8"}, + {file = "psutil-6.1.1-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:0bdd4eab935276290ad3cb718e9809412895ca6b5b334f5a9111ee6d9aff9377"}, + {file = "psutil-6.1.1-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b6e06c20c05fe95a3d7302d74e7097756d4ba1247975ad6905441ae1b5b66003"}, + {file = "psutil-6.1.1-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:97f7cb9921fbec4904f522d972f0c0e1f4fabbdd4e0287813b21215074a0f160"}, + {file = "psutil-6.1.1-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:33431e84fee02bc84ea36d9e2c4a6d395d479c9dd9bba2376c1f6ee8f3a4e0b3"}, + {file = "psutil-6.1.1-cp36-cp36m-win32.whl", hash = "sha256:384636b1a64b47814437d1173be1427a7c83681b17a450bfc309a1953e329603"}, + {file = "psutil-6.1.1-cp36-cp36m-win_amd64.whl", hash = "sha256:8be07491f6ebe1a693f17d4f11e69d0dc1811fa082736500f649f79df7735303"}, + {file = "psutil-6.1.1-cp37-abi3-win32.whl", hash = "sha256:eaa912e0b11848c4d9279a93d7e2783df352b082f40111e078388701fd479e53"}, + {file = "psutil-6.1.1-cp37-abi3-win_amd64.whl", hash = "sha256:f35cfccb065fff93529d2afb4a2e89e363fe63ca1e4a5da22b603a85833c2649"}, + {file = "psutil-6.1.1.tar.gz", hash = "sha256:cf8496728c18f2d0b45198f06895be52f36611711746b7f30c464b422b50e2f5"}, ] [package.extras] -dev = ["abi3audit", "black", "check-manifest", "colorama ; os_name == \"nt\"", "coverage", "packaging", "psleak", "pylint", "pyperf", "pypinfo", "pyreadline3 ; os_name == \"nt\"", "pytest", "pytest-cov", "pytest-instafail", "pytest-xdist", "pywin32 ; os_name == \"nt\" and implementation_name != \"pypy\"", "requests", "rstcheck", "ruff", "setuptools", "sphinx", "sphinx_rtd_theme", "toml-sort", "twine", "validate-pyproject[all]", "virtualenv", "vulture", "wheel", "wheel ; os_name == \"nt\" and implementation_name != \"pypy\"", "wmi ; os_name == \"nt\" and implementation_name != \"pypy\""] -test = ["psleak", "pytest", "pytest-instafail", "pytest-xdist", "pywin32 ; os_name == \"nt\" and implementation_name != \"pypy\"", "setuptools", "wheel ; os_name == \"nt\" and implementation_name != \"pypy\"", "wmi ; os_name == \"nt\" and implementation_name != \"pypy\""] +dev = ["abi3audit", "black", "check-manifest", "coverage", "packaging", "pylint", "pyperf", "pypinfo", "pytest-cov", "requests", "rstcheck", "ruff", "sphinx", "sphinx_rtd_theme", "toml-sort", "twine", "virtualenv", "vulture", "wheel"] +test = ["pytest", "pytest-xdist", "setuptools"] [[package]] name = "pyasn1" @@ -5860,4 +5856,4 @@ cffi = ["cffi (>=1.17,<2.0) ; platform_python_implementation != \"PyPy\" and pyt [metadata] lock-version = "2.1" python-versions = ">=3.12,<4.0.0" -content-hash = "c279476b859b8dcd95060c06fd3115da67ccfa52e0cd07934b56a892b08da837" +content-hash = "55bd6e72737ca11461f971674fdc8bc25ee52ff2d768315083c47b89fec113d9" diff --git a/runtime/datamate-python/pyproject.toml b/runtime/datamate-python/pyproject.toml index 1ca9b4723..4a2de4e1f 100644 --- a/runtime/datamate-python/pyproject.toml +++ b/runtime/datamate-python/pyproject.toml @@ -48,6 +48,7 @@ dependencies = [ "apscheduler (>=3.11.2,<4.0.0)", "msoffcrypto-tool (>=6.0.0,<7.0.0)", "tzlocal (>=5.2,<6.0)", + "psutil (>=6.0.0,<8.0.0)", ]