Fix performance issues caused by inefficient DB usage

server/mergin/sync/interfaces.py

@@ -110,6 +110,12 @@ def get_by_name(self, name):
         """
         pass
 
+    def get_by_names(self, names):
+        """
+        Return list of workspaces whose names are in the given collection.
+        """
+        pass
+
     @abstractmethod
     def get_by_project(self, project):
         """

server/mergin/sync/models.py

@@ -6,6 +6,7 @@
 import json
 import logging
 import os
+import re
 import threading
 import time
 import uuid
@@ -18,7 +19,9 @@
 from blinker import signal
 from flask_login import current_user
 from pygeodiff import GeoDiff
+from functools import cached_property
 from sqlalchemy import text, null, desc, nullslast, tuple_
+from sqlalchemy.orm import contains_eager, joinedload, load_only
 from sqlalchemy.dialects.postgresql import ARRAY, BIGINT, UUID, JSONB, ENUM, insert
 from sqlalchemy.types import String
 from sqlalchemy.ext.hybrid import hybrid_property
@@ -136,6 +139,12 @@ def workspace(self):
         project_workspace = current_app.ws_handler.get(self.workspace_id)
         return project_workspace
 
+    @cached_property
+    def _has_conflict(self) -> bool:
+        """True if any current project file matches a known conflict-copy pattern."""
+        pattern = r"(\.gpkg|\.qgs|.qgz)(.*conflict.*)|( \(.*conflict.*)"
+        return any(re.search(pattern, f.path) for f in self.files)
+
     def get_latest_files_cache(self) -> List[int]:
         """Get latest file history ids either from cached table or calculate them on the fly"""
         if self.latest_project_files.file_history_ids is not None:
@@ -658,7 +667,7 @@ def __init__(
     def path(self) -> str:
         return self.file.path
 
-    @property
+    @cached_property
     def diff(self) -> Optional[FileDiff]:
         """Diff file pushed with UPDATE_DIFF change type.
 
@@ -713,9 +722,37 @@ def changes(
         if not (is_versioned_file(file) and since is not None and to is not None):
             return []
 
-        history = []
+        # when since=1 the range spans the entire project history; narrow it to
+        # the most recent CREATE/DELETE so we don't load records from previous
+        # file lifecycles that the Python break would discard anyway
+        if since == 1:
+            boundary = (
+                FileHistory.query.join(ProjectFilePath)
+                .filter(
+                    ProjectFilePath.project_id == project_id,
+                    ProjectFilePath.path == file,
+                    FileHistory.project_version_name <= to,
+                    FileHistory.change.in_(
+                        [PushChangeType.CREATE.value, PushChangeType.DELETE.value]
+                    ),
+                )
+                .order_by(desc(FileHistory.project_version_name))
+                .with_entities(FileHistory.project_version_name)
+                .first()
+            )
+            since = boundary[0] if boundary else since
+
         full_history = (
             FileHistory.query.join(ProjectFilePath)
+            .join(FileHistory.version)
+            .join(ProjectVersion.project)
+            .options(
+                contains_eager(FileHistory.file).load_only(ProjectFilePath.path),
+                contains_eager(FileHistory.version)
+                .load_only(ProjectVersion.name, ProjectVersion.project_id)
+                .contains_eager(ProjectVersion.project)
+                .load_only(Project.storage_params),
+            )
             .filter(
                 ProjectFilePath.project_id == project_id,
                 FileHistory.project_version_name <= to,
@@ -726,6 +763,7 @@ def changes(
             .all()
         )
 
+        history = []
         for item in full_history:
             history.append(item)
 
@@ -1781,11 +1819,15 @@ def diff_summary(self):
 
     def changes_count(self) -> Dict:
         """Return number of changes by type"""
-        query = f"SELECT change, COUNT(change) FROM file_history WHERE version_id = :version_id GROUP BY change;"
+        query = "SELECT change, COUNT(change) FROM file_history WHERE version_id = :version_id GROUP BY change;"
         params = {"version_id": self.id}
         result = db.session.execute(text(query), params).fetchall()
         return {row[0]: row[1] for row in result}
 
+    @cached_property
+    def _changes_count(self) -> Dict:
+        return self.changes_count()
+
     @property
     def zip_path(self):
         return os.path.join(

server/mergin/sync/public_api.yaml

@@ -1124,11 +1124,6 @@ components:
                           - added
                           - updated
                           - removed
-                      expiration:
-                        nullable: true
-                        type: string
-                        format: date-time
-                        example: 2019-02-26T08:47:58.636074Z
     UploadFileInfo:
       allOf:
         - $ref: "#/components/schemas/FileInfo"