From 8e742c7be18e325a50772c9d89d2934c30ebb3d5 Mon Sep 17 00:00:00 2001 From: harminius Date: Tue, 11 Aug 2026 14:22:06 +0200 Subject: [PATCH 01/13] add safeguard for too long path --- mergin/common.py | 3 +++ mergin/merginproject.py | 15 ++++++++++++++- mergin/utils.py | 17 ++++++++++++++++- 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/mergin/common.py b/mergin/common.py index 3d3f7e8..c3d11ca 100644 --- a/mergin/common.py +++ b/mergin/common.py @@ -33,6 +33,9 @@ # Maximum changes uploading to server MAX_UPLOAD_CHANGES = 100 +# maximum length of a path supported by Windows without long paths enabled (MAX_PATH) +WINDOWS_MAX_PATH = 260 + # default URL for submitting logs MERGIN_DEFAULT_LOGS_URL = "https://g4pfq226j0.execute-api.eu-west-1.amazonaws.com/mergin_client_log_submit" diff --git a/mergin/merginproject.py b/mergin/merginproject.py index 12d798f..2237084 100644 --- a/mergin/merginproject.py +++ b/mergin/merginproject.py @@ -19,6 +19,7 @@ from .utils import ( generate_checksum, is_versioned_file, + is_path_too_long, int_version, do_sqlite_checkpoint, unique_path_name, @@ -623,8 +624,14 @@ def get_local_delta(self, diff_directory: str) -> List[ProjectDeltaChange]: delta_item.size = checkpoint_size delta_item.checksum = checkpoint_checksum + diff_location = self.fpath(diff_file, diff_directory) + if is_path_too_long(diff_location): + raise ClientError( + f"Cannot create changeset for '{path}': diff file path is too long " + f"({len(diff_location)} characters) for this OS: {diff_location}\n" + "Move the project to a directory with a shorter path and try again." + ) try: - diff_location = self.fpath(diff_file, diff_directory) self.geodiff.create_changeset(origin_file, current_file, diff_location) if not self.geodiff.has_changes(diff_location): os.remove(diff_location) @@ -677,6 +684,12 @@ def get_push_changes(self): diff_id = str(uuid.uuid4()) diff_name = path + "-diff-" + diff_id diff_file = self.fpath_meta(diff_name) + if is_path_too_long(diff_file): + raise ClientError( + f"Cannot create changeset for '{path}': diff file path is too long " + f"({len(diff_file)} characters) for this OS: {diff_file}\n" + "Move the project to a directory with a shorter path and try again." + ) try: self.geodiff.create_changeset(origin_file, current_file, diff_file) if self.geodiff.has_changes(diff_file): diff --git a/mergin/utils.py b/mergin/utils.py index 91796f3..6de05b3 100644 --- a/mergin/utils.py +++ b/mergin/utils.py @@ -9,7 +9,7 @@ import tempfile from enum import Enum from typing import Optional, Type, Union, ByteString -from .common import ClientError +from .common import ClientError, WINDOWS_MAX_PATH def generate_checksum(file, chunk_size=4096): @@ -266,6 +266,21 @@ def is_versioned_file(path: str) -> bool: return f_extension.lower() in diff_extensions +def is_path_too_long(path: str) -> bool: + """ + Check whether an absolute path is too long to be reliably created/opened on this OS. + + Windows limits paths to WINDOWS_MAX_PATH (260) characters unless long paths have been + explicitly enabled (which we cannot rely on being the case), so we treat that as the limit. + + :param path: absolute path to check + :type path: str + :returns: whether the path is likely to be rejected by the OS + :rtype: bool + """ + return os.name == "nt" and len(path) >= WINDOWS_MAX_PATH + + def is_qgis_file(path: str) -> bool: """ Check if file is a QGIS project file. From 77ae14ba1bbc455c66e57c8ecf9d05d09a49ad4e Mon Sep 17 00:00:00 2001 From: harminius Date: Tue, 11 Aug 2026 14:30:09 +0200 Subject: [PATCH 02/13] docstring --- mergin/utils.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/mergin/utils.py b/mergin/utils.py index 6de05b3..02bee94 100644 --- a/mergin/utils.py +++ b/mergin/utils.py @@ -270,9 +270,6 @@ def is_path_too_long(path: str) -> bool: """ Check whether an absolute path is too long to be reliably created/opened on this OS. - Windows limits paths to WINDOWS_MAX_PATH (260) characters unless long paths have been - explicitly enabled (which we cannot rely on being the case), so we treat that as the limit. - :param path: absolute path to check :type path: str :returns: whether the path is likely to be rejected by the OS From 6cc7fc6dd7fa7811a2fac639ec606fabaca90559 Mon Sep 17 00:00:00 2001 From: harminius Date: Fri, 14 Aug 2026 11:21:08 +0200 Subject: [PATCH 03/13] Add path length safeguards to pull and local create changeset --- mergin/client_pull.py | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/mergin/client_pull.py b/mergin/client_pull.py index 5210089..d9cce37 100644 --- a/mergin/client_pull.py +++ b/mergin/client_pull.py @@ -25,7 +25,7 @@ from .common import CHUNK_SIZE, ClientError, DeltaChangeType, PullActionType from .models import ProjectDelta, ProjectDeltaChange, PullAction from .merginproject import MerginProject -from .utils import cleanup_tmp_dir, save_to_file +from .utils import cleanup_tmp_dir, save_to_file, is_path_too_long from typing import List, Optional # status = download_project_async(...) @@ -480,6 +480,12 @@ def get_download_diff_files(delta_item: ProjectDeltaChange, target_dir: str) -> for diff in delta_item.diffs: dest_file_path = os.path.normpath(os.path.join(target_dir, diff.id)) + if is_path_too_long(dest_file_path): + raise ClientError( + f"Cannot download diff for '{delta_item.path}': diff file path is too long " + f"({len(dest_file_path)} characters) for this OS: {dest_file_path}\n" + "Move the project to a directory with a shorter path and try again." + ) download_items = get_download_items(delta_item.path, diff.size, diff.version, target_dir, diff.id, True) result.append(DownloadFile(dest_file_path, download_items)) return result @@ -574,12 +580,15 @@ def pull_project_async(mc, directory) -> Optional[PullJob]: # if we have conflict and diff update, download the diff files if v2_pull_enabled: # using v2 endpoint to download diff files, without chunks. Then we are creating DownloadDiffQueueItem instances for each diff file. - diff_files.extend( - [ - DownloadDiffQueueItem(diff_item.id, os.path.join(tmp_dir.name, diff_item.id)) - for diff_item in change.diffs - ] - ) + for diff_item in change.diffs: + diff_path = os.path.join(tmp_dir.name, diff_item.id) + if is_path_too_long(diff_path): + raise ClientError( + f"Cannot download diff for '{change.path}': diff file path is too long " + f"({len(diff_path)} characters) for this OS: {diff_path}\n" + "Move the project to a directory with a shorter path and try again." + ) + diff_files.append(DownloadDiffQueueItem(diff_item.id, diff_path)) basefiles_to_patch.append((change.path, [diff.id for diff in change.diffs])) else: @@ -830,6 +839,12 @@ def download_diffs_async(mc, project_directory, file_path, versions): diff_only=True, ) dest_file_path = mp.fpath_cache(diff["path"], version=file["version"]) + if is_path_too_long(dest_file_path): + raise ClientError( + f"Cannot download diff for '{file.get('path')}': diff file path is too long " + f"({len(dest_file_path)} characters) for this OS: {dest_file_path}\n" + "Move the project to a directory with a shorter path and try again." + ) if os.path.exists(dest_file_path): continue download_files.append(DownloadFile(dest_file_path, items)) From 2d38d242820ec7b0aebc8d77c5f84af5b9cda22f Mon Sep 17 00:00:00 2001 From: harminius Date: Mon, 17 Aug 2026 22:53:32 +0200 Subject: [PATCH 04/13] Escape for windows long path --- mergin/client.py | 9 +- mergin/client_pull.py | 53 ++++------- mergin/client_push.py | 6 +- mergin/merginproject.py | 164 +++++++++++++++++--------------- mergin/report.py | 6 +- mergin/test/test_client_pull.py | 13 +-- mergin/utils.py | 29 +++++- 7 files changed, 150 insertions(+), 130 deletions(-) diff --git a/mergin/client.py b/mergin/client.py index 1555051..8467a1c 100644 --- a/mergin/client.py +++ b/mergin/client.py @@ -67,6 +67,7 @@ int_version, is_version_acceptable, normalize_role, + long_path, ) from .version import __version__ @@ -1237,7 +1238,7 @@ def get_file_diff(self, project_dir, file_path, output_diff, version_from, versi # collect required versions from the cache diffs = [] for v in versions_to_fetch[1:]: - diffs.append(mp.fpath_cache(file_history["history"][v]["diff"]["path"], v)) + diffs.append(long_path(mp.fpath_cache(file_history["history"][v]["diff"]["path"], v))) # concatenate diffs, if needed output_dir = os.path.dirname(output_diff) @@ -1377,13 +1378,15 @@ def reset_local_changes(self, directory: str, files_to_reset: typing.List[str] = # remove all added files for file in push_changes["added"]: if all_files or file["path"] in files_to_reset: - os.remove(mp.fpath(file["path"])) + os.remove(long_path(mp.fpath(file["path"]))) # update files get override with previous version for file in push_changes["updated"]: if all_files or file["path"] in files_to_reset: if mp.is_versioned_file(file["path"]): - mp.geodiff.make_copy_sqlite(mp.fpath_meta(file["path"]), mp.fpath(file["path"])) + mp.geodiff.make_copy_sqlite( + long_path(mp.fpath_meta(file["path"])), long_path(mp.fpath(file["path"])) + ) else: files_download.append(file["path"]) diff --git a/mergin/client_pull.py b/mergin/client_pull.py index d9cce37..2030457 100644 --- a/mergin/client_pull.py +++ b/mergin/client_pull.py @@ -25,7 +25,7 @@ from .common import CHUNK_SIZE, ClientError, DeltaChangeType, PullActionType from .models import ProjectDelta, ProjectDeltaChange, PullAction from .merginproject import MerginProject -from .utils import cleanup_tmp_dir, save_to_file, is_path_too_long +from .utils import cleanup_tmp_dir, save_to_file, long_path from typing import List, Optional # status = download_project_async(...) @@ -93,7 +93,9 @@ def __init__(self, file_path, size, version, diff_only, part_index, download_fil self.version = version # version of the file ("v123") self.diff_only = diff_only # whether downloading diff or full version self.part_index = part_index # index of the chunk - self.download_file_path = download_file_path # full path to a temporary file which will receive the content + self.download_file_path = long_path( + download_file_path + ) # full path to a temporary file which will receive the content def __repr__(self): return "".format( @@ -128,7 +130,9 @@ class DownloadDiffQueueItem: def __init__(self, diff_id, download_file_path): self.diff_id = diff_id # relative path to the file within project - self.download_file_path = download_file_path # full path to a temporary file which will receive the content + self.download_file_path = long_path( + download_file_path + ) # full path to a temporary file which will receive the content self.size = 0 # size of the item in bytes def __repr__(self): @@ -157,7 +161,7 @@ class DownloadFile: """ def __init__(self, dest_file, downloaded_items: typing.List[DownloadQueueItem], size_check=True): - self.dest_file = dest_file # full path to the destination file to be created + self.dest_file = long_path(dest_file) # full path to the destination file to be created self.downloaded_items = downloaded_items # list of pieces of the destination file to be merged self.size_check = size_check # whether we want to do merged file size check @@ -196,7 +200,7 @@ def get_download_items( items = [] for part_index in range(chunks): - download_file_path = os.path.join(file_dir, basename + ".{}".format(part_index)) + download_file_path = long_path(os.path.join(file_dir, basename + ".{}".format(part_index))) size = min(CHUNK_SIZE, file_size - part_index * CHUNK_SIZE) items.append(DownloadQueueItem(file_path, size, file_version, diff_only, part_index, download_file_path)) @@ -419,7 +423,7 @@ def apply(self, directory, mp): # Make a copy of the file to meta dir only if there is no user-specified path for the file. # destination_file is None for full project download and takes a meaningful value for a single file download. if mp.is_versioned_file(self.file_path) and self.destination_file is None: - mp.geodiff.make_copy_sqlite(mp.fpath(self.file_path), mp.fpath_meta(self.file_path)) + mp.geodiff.make_copy_sqlite(long_path(mp.fpath(self.file_path)), long_path(mp.fpath_meta(self.file_path))) class PullJob: @@ -479,13 +483,7 @@ def get_download_diff_files(delta_item: ProjectDeltaChange, target_dir: str) -> result = [] for diff in delta_item.diffs: - dest_file_path = os.path.normpath(os.path.join(target_dir, diff.id)) - if is_path_too_long(dest_file_path): - raise ClientError( - f"Cannot download diff for '{delta_item.path}': diff file path is too long " - f"({len(dest_file_path)} characters) for this OS: {dest_file_path}\n" - "Move the project to a directory with a shorter path and try again." - ) + dest_file_path = long_path(os.path.normpath(os.path.join(target_dir, diff.id))) download_items = get_download_items(delta_item.path, diff.size, diff.version, target_dir, diff.id, True) result.append(DownloadFile(dest_file_path, download_items)) return result @@ -561,7 +559,7 @@ def pull_project_async(mc, directory) -> Optional[PullJob]: pull_action_type == PullActionType.COPY_CONFLICT and change.type == DeltaChangeType.UPDATE_DIFF ): basefile = mp.fpath_meta(change.path) - if not os.path.exists(basefile): + if not os.path.exists(long_path(basefile)): # The basefile does not exist for some reason. This should not happen normally (maybe user removed the file # or we removed it within previous pull because we failed to apply patch the older version for some reason). # But it's not a problem - we will download the newest version and we're sorted. @@ -580,15 +578,12 @@ def pull_project_async(mc, directory) -> Optional[PullJob]: # if we have conflict and diff update, download the diff files if v2_pull_enabled: # using v2 endpoint to download diff files, without chunks. Then we are creating DownloadDiffQueueItem instances for each diff file. - for diff_item in change.diffs: - diff_path = os.path.join(tmp_dir.name, diff_item.id) - if is_path_too_long(diff_path): - raise ClientError( - f"Cannot download diff for '{change.path}': diff file path is too long " - f"({len(diff_path)} characters) for this OS: {diff_path}\n" - "Move the project to a directory with a shorter path and try again." - ) - diff_files.append(DownloadDiffQueueItem(diff_item.id, diff_path)) + diff_files.extend( + [ + DownloadDiffQueueItem(diff_item.id, os.path.join(tmp_dir.name, diff_item.id)) + for diff_item in change.diffs + ] + ) basefiles_to_patch.append((change.path, [diff.id for diff in change.diffs])) else: @@ -731,7 +726,7 @@ def pull_project_finalize(job: PullJob): basefile = job.mp.fpath_meta(file_path) server_file = job.mp.fpath(file_path, job.tmp_dir.name) - shutil.copy(basefile, server_file) + shutil.copy(long_path(basefile), long_path(server_file)) diffs = [job.mp.fpath(f, job.tmp_dir.name) for f in file_diffs] patch_error = job.mp.apply_diffs(server_file, diffs) if patch_error: @@ -744,7 +739,7 @@ def pull_project_finalize(job: PullJob): job.mp.log.error("Diffs we were applying: " + str(diffs)) job.mp.log.error("Removing basefile because it would be corrupted anyway...") job.mp.log.info("--- pull aborted") - os.remove(basefile) + os.remove(long_path(basefile)) raise ClientError("Cannot patch basefile {}! Please try syncing again.".format(basefile)) conflicts = [] job.mp.log.info(f"--- applying pull actions {job.pull_actions}") @@ -838,13 +833,7 @@ def download_diffs_async(mc, project_directory, file_path, versions): download_path=diff.get("path"), diff_only=True, ) - dest_file_path = mp.fpath_cache(diff["path"], version=file["version"]) - if is_path_too_long(dest_file_path): - raise ClientError( - f"Cannot download diff for '{file.get('path')}': diff file path is too long " - f"({len(dest_file_path)} characters) for this OS: {dest_file_path}\n" - "Move the project to a directory with a shorter path and try again." - ) + dest_file_path = long_path(mp.fpath_cache(diff["path"], version=file["version"])) if os.path.exists(dest_file_path): continue download_files.append(DownloadFile(dest_file_path, items)) diff --git a/mergin/client_push.py b/mergin/client_push.py index 831b59b..bdcf0d1 100644 --- a/mergin/client_push.py +++ b/mergin/client_push.py @@ -34,7 +34,7 @@ ) from .merginproject import MerginProject, pygeodiff from .editor import filter_changes -from .utils import get_data_checksum, cleanup_tmp_dir +from .utils import get_data_checksum, cleanup_tmp_dir, long_path POST_JSON_HEADERS = {"Content-Type": "application/json"} @@ -114,7 +114,7 @@ def upload_chunk_v2_api(self, data: ByteString, checksum: str): self.mc.upload_chunks_cache.add(checksum, self.server_chunk_id) def upload_blocking(self): - with open(self.file_path, "rb") as file_handle: + with open(long_path(self.file_path), "rb") as file_handle: file_handle.seek(self.chunk_index * UPLOAD_CHUNK_SIZE) data = file_handle.read(UPLOAD_CHUNK_SIZE) checksum_str = get_data_checksum(data) @@ -507,7 +507,7 @@ def remove_diff_files(job: UploadJob) -> None: for change in job.changes.updated: diff = change.get_diff() if diff: - diff_file = job.mp.fpath_meta(diff.path) + diff_file = long_path(job.mp.fpath_meta(diff.path)) if os.path.exists(diff_file): os.remove(diff_file) diff --git a/mergin/merginproject.py b/mergin/merginproject.py index 2237084..d49870b 100644 --- a/mergin/merginproject.py +++ b/mergin/merginproject.py @@ -19,7 +19,7 @@ from .utils import ( generate_checksum, is_versioned_file, - is_path_too_long, + long_path, int_version, do_sqlite_checkpoint, unique_path_name, @@ -280,7 +280,7 @@ def is_gpkg_open(self, path): f_extension = os.path.splitext(path)[1] if f_extension != ".gpkg": return False - if os.path.exists(f"{path}-wal"): + if os.path.exists(f"{long_path(path)}-wal"): return True return False @@ -625,21 +625,16 @@ def get_local_delta(self, diff_directory: str) -> List[ProjectDeltaChange]: delta_item.checksum = checkpoint_checksum diff_location = self.fpath(diff_file, diff_directory) - if is_path_too_long(diff_location): - raise ClientError( - f"Cannot create changeset for '{path}': diff file path is too long " - f"({len(diff_location)} characters) for this OS: {diff_location}\n" - "Move the project to a directory with a shorter path and try again." - ) + diff_location_lp = long_path(diff_location) try: - self.geodiff.create_changeset(origin_file, current_file, diff_location) - if not self.geodiff.has_changes(diff_location): - os.remove(diff_location) + self.geodiff.create_changeset(long_path(origin_file), long_path(current_file), diff_location_lp) + if not self.geodiff.has_changes(diff_location_lp): + os.remove(diff_location_lp) continue delta_item.checksum = change.get("origin_checksum") delta_item.type = DeltaChangeType.UPDATE_DIFF - os.remove(diff_location) + os.remove(diff_location_lp) except (pygeodiff.GeoDiffLibError, pygeodiff.GeoDiffLibConflictError) as e: self.log.warning("failed to create changeset for " + path) # probably the database schema has been modified if geodiff cannot create changeset. @@ -684,28 +679,23 @@ def get_push_changes(self): diff_id = str(uuid.uuid4()) diff_name = path + "-diff-" + diff_id diff_file = self.fpath_meta(diff_name) - if is_path_too_long(diff_file): - raise ClientError( - f"Cannot create changeset for '{path}': diff file path is too long " - f"({len(diff_file)} characters) for this OS: {diff_file}\n" - "Move the project to a directory with a shorter path and try again." - ) + diff_file_lp = long_path(diff_file) try: - self.geodiff.create_changeset(origin_file, current_file, diff_file) - if self.geodiff.has_changes(diff_file): - diff_size = os.path.getsize(diff_file) + self.geodiff.create_changeset(long_path(origin_file), long_path(current_file), diff_file_lp) + if self.geodiff.has_changes(diff_file_lp): + diff_size = os.path.getsize(diff_file_lp) file["checksum"] = file["origin_checksum"] # need to match basefile on server file["chunks"] = [str(uuid.uuid4()) for i in range(math.ceil(diff_size / UPLOAD_CHUNK_SIZE))] - file["mtime"] = datetime.fromtimestamp(os.path.getmtime(current_file), tzlocal()) + file["mtime"] = datetime.fromtimestamp(os.path.getmtime(long_path(current_file)), tzlocal()) file["diff"] = { "path": diff_name, - "checksum": generate_checksum(diff_file), + "checksum": generate_checksum(diff_file_lp), "size": diff_size, - "mtime": datetime.fromtimestamp(os.path.getmtime(diff_file), tzlocal()), + "mtime": datetime.fromtimestamp(os.path.getmtime(diff_file_lp), tzlocal()), } else: - if os.path.exists(diff_file): - os.remove(diff_file) + if os.path.exists(diff_file_lp): + os.remove(diff_file_lp) not_updated.append(file) except (pygeodiff.GeoDiffLibError, pygeodiff.GeoDiffLibConflictError) as e: self.log.warning("failed to create changeset for " + path) @@ -725,9 +715,10 @@ def copy_versioned_file_for_upload(self, f: FileChange, tmp_dir: str) -> str: self.log.info("Making a temporary copy (full upload): " + path) tmp_file = os.path.join(tmp_dir, path) os.makedirs(os.path.dirname(tmp_file), exist_ok=True) - self.geodiff.make_copy_sqlite(self.fpath(path), tmp_file) - f.size = os.path.getsize(tmp_file) - f.checksum = generate_checksum(tmp_file) + tmp_file_lp = long_path(tmp_file) + self.geodiff.make_copy_sqlite(long_path(self.fpath(path)), tmp_file_lp) + f.size = os.path.getsize(tmp_file_lp) + f.checksum = generate_checksum(tmp_file_lp) f.chunks = [str(uuid.uuid4()) for i in range(math.ceil(f.size / UPLOAD_CHUNK_SIZE))] f.upload_file = tmp_file return tmp_file @@ -740,7 +731,7 @@ def get_list_of_push_changes(self, push_changes): changeset = self.fpath_meta(changeset_path) result_file = self.fpath("change_list" + str(idx), self.meta_dir) try: - self.geodiff.list_changes_summary(changeset, result_file) + self.geodiff.list_changes_summary(long_path(changeset), result_file) with open(result_file, "r") as f: change = f.read() changes[file["path"]] = json.loads(change) @@ -774,14 +765,17 @@ def apply_pull_actions(self, actions: List[PullAction], download_dir: str, serve server_file = self.fpath(path, download_dir) live_file = self.fpath(path) basefile = self.fpath_meta(path) + server_file_lp = long_path(server_file) + live_file_lp = long_path(live_file) + basefile_lp = long_path(basefile) action_type = action.type if action_type == PullActionType.COPY: # simply copy the file from server if is_versioned_file(path): - self.geodiff.make_copy_sqlite(server_file, live_file) - self.geodiff.make_copy_sqlite(server_file, basefile) + self.geodiff.make_copy_sqlite(server_file_lp, live_file_lp) + self.geodiff.make_copy_sqlite(server_file_lp, basefile_lp) else: - shutil.copy(server_file, live_file) + shutil.copy(server_file_lp, live_file_lp) elif action_type == PullActionType.APPLY_DIFF_NO_REBASE: # simply apply the diff without rebase (no local changes or non-conflicting local changes) self.update_without_rebase(path, server_file, live_file, basefile, download_dir) @@ -799,22 +793,22 @@ def apply_pull_actions(self, actions: List[PullAction], download_dir: str, serve conflicts.append(conflict) if self.is_versioned_file(path): try: - self.geodiff.make_copy_sqlite(server_file, live_file) - self.geodiff.make_copy_sqlite(server_file, basefile) + self.geodiff.make_copy_sqlite(server_file_lp, live_file_lp) + self.geodiff.make_copy_sqlite(server_file_lp, basefile_lp) except pygeodiff.GeoDiffLibError: self.log.info("failed to create SQLite copy for file: " + path) # create unfinished pull copy instead - f_server_unfinished = self.fpath_unfinished_pull(path) - self.geodiff.make_copy_sqlite(server_file, f_server_unfinished) + f_server_unfinished = long_path(self.fpath_unfinished_pull(path)) + self.geodiff.make_copy_sqlite(server_file_lp, f_server_unfinished) else: - shutil.copy(server_file, live_file) + shutil.copy(server_file_lp, live_file_lp) elif action_type == PullActionType.DELETE: # remove local file - if os.path.exists(live_file): - os.remove(live_file) - if self.is_versioned_file(path) and os.path.exists(basefile): - os.remove(basefile) + if os.path.exists(live_file_lp): + os.remove(live_file_lp) + if self.is_versioned_file(path) and os.path.exists(basefile_lp): + os.remove(basefile_lp) return conflicts @@ -841,51 +835,55 @@ def update_with_rebase(self, path, src, dest, basefile, temp_dir, user_name): """ self.log.info("updating file with rebase: " + path) - server_diff = self.fpath(f"{path}-server_diff", temp_dir) # diff between server file and local basefile - local_diff = self.fpath(f"{path}-local_diff", temp_dir) + src_lp = long_path(src) + dest_lp = long_path(dest) + basefile_lp = long_path(basefile) + + server_diff = long_path(self.fpath(f"{path}-server_diff", temp_dir)) # diff between server file and local basefile + local_diff = long_path(self.fpath(f"{path}-local_diff", temp_dir)) # temporary backup of file pulled from server for recovery - f_server_backup = self.fpath(f"{path}-server_backup", temp_dir) - self.geodiff.make_copy_sqlite(src, f_server_backup) + f_server_backup = long_path(self.fpath(f"{path}-server_backup", temp_dir)) + self.geodiff.make_copy_sqlite(src_lp, f_server_backup) # create temp backup (ideally with geodiff) of locally modified file if needed later - f_conflict_file = self.fpath(f"{path}-local_backup", temp_dir) + f_conflict_file = long_path(self.fpath(f"{path}-local_backup", temp_dir)) try: - self.geodiff.create_changeset(basefile, dest, local_diff) - self.geodiff.make_copy_sqlite(basefile, f_conflict_file) + self.geodiff.create_changeset(basefile_lp, dest_lp, local_diff) + self.geodiff.make_copy_sqlite(basefile_lp, f_conflict_file) self.geodiff.apply_changeset(f_conflict_file, local_diff) except (pygeodiff.GeoDiffLibError, pygeodiff.GeoDiffLibConflictError): self.log.info("backup of local file with geodiff failed - need to do hard copy") - self.geodiff.make_copy_sqlite(dest, f_conflict_file) + self.geodiff.make_copy_sqlite(dest_lp, f_conflict_file) # in case there will be any conflicting operations found during rebase, # they will be stored in a JSON file - if there are no conflicts, the file # won't even be created - rebase_conflicts = unique_path_name( - edit_conflict_file_name(self.fpath(path), user_name, int_version(self.version())) + rebase_conflicts = long_path( + unique_path_name(edit_conflict_file_name(self.fpath(path), user_name, int_version(self.version()))) ) # try to do rebase magic try: - self.geodiff.create_changeset(basefile, src, server_diff) - self.geodiff.rebase(basefile, src, dest, rebase_conflicts) + self.geodiff.create_changeset(basefile_lp, src_lp, server_diff) + self.geodiff.rebase(basefile_lp, src_lp, dest_lp, rebase_conflicts) # make sure basefile is in the same state as remote server file (for calc of push changes) - self.geodiff.apply_changeset(basefile, server_diff) + self.geodiff.apply_changeset(basefile_lp, server_diff) self.log.info("rebase successful!") except (pygeodiff.GeoDiffLibError, pygeodiff.GeoDiffLibConflictError) as err: self.log.warning("rebase failed! going to create conflict file") try: # it would not be possible to commit local changes, they need to end up in new conflict file - self.geodiff.make_copy_sqlite(f_conflict_file, dest) + self.geodiff.make_copy_sqlite(f_conflict_file, dest_lp) conflict = self.create_conflicted_copy(path, user_name) # original file synced with server - self.geodiff.make_copy_sqlite(f_server_backup, basefile) - self.geodiff.make_copy_sqlite(f_server_backup, dest) + self.geodiff.make_copy_sqlite(f_server_backup, basefile_lp) + self.geodiff.make_copy_sqlite(f_server_backup, dest_lp) return conflict except pygeodiff.GeoDiffLibError as err: self.log.warning("creation of conflicted copy failed! going to create an unfinished pull") - f_server_unfinished = self.fpath_unfinished_pull(path) + f_server_unfinished = long_path(self.fpath_unfinished_pull(path)) self.geodiff.make_copy_sqlite(f_server_backup, f_server_unfinished) return "" @@ -911,22 +909,27 @@ def update_without_rebase(self, path, src, dest, basefile, temp_dir): :type temp_dir: str """ self.log.info("updating file without rebase: " + path) + src_lp = long_path(src) + dest_lp = long_path(dest) + basefile_lp = long_path(basefile) try: - server_diff = self.fpath(f"{path}-server_diff", temp_dir) # diff between server file and local basefile + server_diff = long_path( + self.fpath(f"{path}-server_diff", temp_dir) + ) # diff between server file and local basefile # TODO: it could happen that basefile does not exist. # It was either never created (e.g. when pushing without geodiff) # or it was deleted by mistake(?) by the user. We should detect that # when starting pull and download it as well - self.geodiff.create_changeset(basefile, src, server_diff) - self.geodiff.apply_changeset(dest, server_diff) - self.geodiff.apply_changeset(basefile, server_diff) + self.geodiff.create_changeset(basefile_lp, src_lp, server_diff) + self.geodiff.apply_changeset(dest_lp, server_diff) + self.geodiff.apply_changeset(basefile_lp, server_diff) self.log.info("update successful") except (pygeodiff.GeoDiffLibError, pygeodiff.GeoDiffLibConflictError): self.log.warning("update failed! going to copy file") # something bad happened and we have failed to patch our local files - this should not happen if there # wasn't a schema change or something similar that geodiff can't handle. - self.geodiff.make_copy_sqlite(src, dest) - self.geodiff.make_copy_sqlite(src, basefile) + self.geodiff.make_copy_sqlite(src_lp, dest_lp) + self.geodiff.make_copy_sqlite(src_lp, basefile_lp) def apply_push_changes(self, changes): """ @@ -942,16 +945,17 @@ def apply_push_changes(self, changes): continue basefile = self.fpath_meta(path) + basefile_lp = long_path(basefile) if k == "removed": - os.remove(basefile) + os.remove(basefile_lp) elif k == "added": - self.geodiff.make_copy_sqlite(self.fpath(path), basefile) + self.geodiff.make_copy_sqlite(long_path(self.fpath(path)), basefile_lp) elif k == "updated": # in case for geopackage cannot be created diff (e.g. forced update with committed changes from wal file) diff = item.get("diff") if not diff: self.log.info("updating basefile (copy) for: " + path) - self.geodiff.make_copy_sqlite(self.fpath(path), basefile) + self.geodiff.make_copy_sqlite(long_path(self.fpath(path)), basefile_lp) else: self.log.info("updating basefile (diff) for: " + path) # better to apply diff to previous basefile to avoid issues with geodiff tmp files @@ -960,7 +964,7 @@ def apply_push_changes(self, changes): if patch_error: # in case of local sync issues it is safier to remove basefile, next time it will be downloaded from server self.log.warning("removing basefile (because of apply diff error) for: " + path) - os.remove(basefile) + os.remove(basefile_lp) else: pass @@ -974,7 +978,8 @@ def create_conflicted_copy(self, file: str, user_name: str): :rtype: str """ src = self.fpath(file) - if not os.path.exists(src): + src_lp = long_path(src) + if not os.path.exists(src_lp): return backup_path = unique_path_name( @@ -982,9 +987,9 @@ def create_conflicted_copy(self, file: str, user_name: str): ) if self.is_versioned_file(file): - self.geodiff.make_copy_sqlite(src, backup_path) + self.geodiff.make_copy_sqlite(src_lp, long_path(backup_path)) else: - shutil.copy(src, backup_path) + shutil.copy(src_lp, long_path(backup_path)) return backup_path def apply_diffs(self, basefile, diffs): @@ -1003,9 +1008,10 @@ def apply_diffs(self, basefile, diffs): if not self.is_versioned_file(basefile): return error + basefile_lp = long_path(basefile) for index, diff in enumerate(diffs): try: - self.geodiff.apply_changeset(basefile, diff) + self.geodiff.apply_changeset(basefile_lp, long_path(diff)) except (pygeodiff.GeoDiffLibError, pygeodiff.GeoDiffLibConflictError) as e: self.log.warning("failed to apply changeset " + diff + " to " + basefile) error = str(e) @@ -1055,12 +1061,12 @@ def resolve_unfinished_pull(self, user_name): self.log.info("resolving unfinished pull") - for root, dirs, files in os.walk(self.unfinished_pull_dir): + for root, dirs, files in os.walk(long_path(self.unfinished_pull_dir)): for file_name in files: - src = os.path.join(root, file_name) - file_path = os.path.relpath(src, self.unfinished_pull_dir) - dest = self.fpath(file_path) - basefile = self.fpath_meta(file_path) + src = os.path.join(root, file_name) # already long-path-prefixed, root came from os.walk above + file_path = os.path.relpath(src, long_path(self.unfinished_pull_dir)) + dest = long_path(self.fpath(file_path)) + basefile = long_path(self.fpath_meta(file_path)) self.log.info("trying to resolve unfinished pull for: " + file_path) @@ -1123,7 +1129,7 @@ def get_geodiff_changes_count(self, diff_rel_path: str): Never raises – diagnostics/logging must not fail. """ - diff_abs = self.fpath_meta(diff_rel_path) + diff_abs = long_path(self.fpath_meta(diff_rel_path)) try: return pygeodiff.GeoDiff().changes_count(diff_abs) except ( diff --git a/mergin/report.py b/mergin/report.py index 5b9cae4..abbf451 100644 --- a/mergin/report.py +++ b/mergin/report.py @@ -7,7 +7,7 @@ from . import ClientError from .merginproject import MerginProject, pygeodiff -from .utils import int_version +from .utils import int_version, long_path try: from qgis.core import ( @@ -243,7 +243,7 @@ def create_report(mc, directory, since, to, out_file): mc.download_file_diffs(directory, f["path"], history_keys) # download full gpkg in "to" version to analyze its schema to determine which col is geometry - full_gpkg = mp.fpath_cache(f["path"], version=to) + full_gpkg = long_path(mp.fpath_cache(f["path"], version=to)) if not os.path.exists(full_gpkg): mc.download_file(directory, f["path"], full_gpkg, to) @@ -263,7 +263,7 @@ def create_report(mc, directory, since, to, out_file): warnings.append(f"Missing diff: {f['path']} was {f['history'][version]['change']} in {version}") continue - v_diff_file = mp.fpath_cache(f["history"][version]["diff"]["path"], version=version) + v_diff_file = long_path(mp.fpath_cache(f["history"][version]["diff"]["path"], version=version)) version_data = versions_map[version] cr = mp.geodiff.read_changeset(v_diff_file) report = changeset_report(cr, schema, mp) diff --git a/mergin/test/test_client_pull.py b/mergin/test/test_client_pull.py index c20fe44..fda8bd0 100644 --- a/mergin/test/test_client_pull.py +++ b/mergin/test/test_client_pull.py @@ -4,6 +4,7 @@ from mergin.common import DeltaChangeType, CHUNK_SIZE from mergin.models import ProjectDeltaChange, ProjectDeltaItemDiff from mergin.client_pull import get_download_diff_files, get_download_items +from mergin.utils import long_path def test_get_diff_download_files(): @@ -25,7 +26,7 @@ def test_get_diff_download_files(): # Check diff f2 = files[0] - assert f2.dest_file == os.path.join(tmp_dir, "diff2") + assert f2.dest_file == long_path(os.path.join(tmp_dir, "diff2")) assert len(f2.downloaded_items) == 1 assert f2.downloaded_items[0].file_path == "data.gpkg" assert f2.downloaded_items[0].size == 20 @@ -41,7 +42,7 @@ def test_get_download_items(): assert items[0].file_path == "small.txt" assert items[0].size == 100 assert items[0].part_index == 0 - assert items[0].download_file_path == os.path.join(tmp_dir, "small.txt.0") + assert items[0].download_file_path == long_path(os.path.join(tmp_dir, "small.txt.0")) # Case 2: Large file (multiple chunks) file_size = int(CHUNK_SIZE * 2.5) @@ -51,17 +52,17 @@ def test_get_download_items(): # Chunk 0 assert items[0].size == CHUNK_SIZE assert items[0].part_index == 0 - assert items[0].download_file_path == os.path.join(tmp_dir, "large.txt.0") + assert items[0].download_file_path == long_path(os.path.join(tmp_dir, "large.txt.0")) # Chunk 1 assert items[1].size == CHUNK_SIZE assert items[1].part_index == 1 - assert items[1].download_file_path == os.path.join(tmp_dir, "large.txt.1") + assert items[1].download_file_path == long_path(os.path.join(tmp_dir, "large.txt.1")) # Chunk 2 assert items[2].size == int(CHUNK_SIZE * 0.5) assert items[2].part_index == 2 - assert items[2].download_file_path == os.path.join(tmp_dir, "large.txt.2") + assert items[2].download_file_path == long_path(os.path.join(tmp_dir, "large.txt.2")) # Case 3: Empty file items = get_download_items("empty.txt", 0, "v1", tmp_dir) @@ -73,4 +74,4 @@ def test_get_download_items(): assert items[0].diff_only is True assert items[0].file_path == "base.gpkg" assert items[0].size == 50 - assert items[0].download_file_path == os.path.join(tmp_dir, "diff_file.0") + assert items[0].download_file_path == long_path(os.path.join(tmp_dir, "diff_file.0")) diff --git a/mergin/utils.py b/mergin/utils.py index 02bee94..d1cdafc 100644 --- a/mergin/utils.py +++ b/mergin/utils.py @@ -88,10 +88,11 @@ def do_sqlite_checkpoint(path, log=None): """ new_size = None new_checksum = None - if ".gpkg" in path and os.path.exists(f"{path}-wal"): + path_lp = long_path(path) + if ".gpkg" in path and os.path.exists(f"{path_lp}-wal"): if log: log.info("checkpoint - going to add it in " + path) - conn = sqlite3.connect(path) + conn = sqlite3.connect(path_lp) cursor = conn.cursor() cursor.execute("PRAGMA wal_checkpoint=FULL") if log: @@ -99,8 +100,8 @@ def do_sqlite_checkpoint(path, log=None): cursor.execute("VACUUM") conn.commit() conn.close() - new_size = os.path.getsize(path) - new_checksum = generate_checksum(path) + new_size = os.path.getsize(path_lp) + new_checksum = generate_checksum(path_lp) if log: log.info("checkpoint - new size {} checksum {}".format(new_size, new_checksum)) @@ -278,6 +279,26 @@ def is_path_too_long(path: str) -> bool: return os.name == "nt" and len(path) >= WINDOWS_MAX_PATH +def long_path(path: str) -> str: + """ + Prefix an absolute path with the Windows "\\?\" extended-length marker, + so file APIs used by geodiff/SQLite and Python's own open() can handle paths longer + than MAX_PATH (260 characters) without raising an error. + + :param path: absolute or relative path, with either posix or windows separators + :type path: str + :returns: extended-length path on Windows, the unchanged path otherwise + :rtype: str + """ + if os.name != "nt": + return path + backslash = chr(92) + prefix = backslash + backslash + "?" + backslash + if path.startswith(prefix): + return path + return prefix + os.path.abspath(path) + + def is_qgis_file(path: str) -> bool: """ Check if file is a QGIS project file. From 096fa0a84a9aeac0b5bddb30b177bc1155bc5242 Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Mon, 17 Aug 2026 23:04:58 +0200 Subject: [PATCH 05/13] black --- mergin/merginproject.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mergin/merginproject.py b/mergin/merginproject.py index d49870b..0757a96 100644 --- a/mergin/merginproject.py +++ b/mergin/merginproject.py @@ -839,7 +839,9 @@ def update_with_rebase(self, path, src, dest, basefile, temp_dir, user_name): dest_lp = long_path(dest) basefile_lp = long_path(basefile) - server_diff = long_path(self.fpath(f"{path}-server_diff", temp_dir)) # diff between server file and local basefile + server_diff = long_path( + self.fpath(f"{path}-server_diff", temp_dir) + ) # diff between server file and local basefile local_diff = long_path(self.fpath(f"{path}-local_diff", temp_dir)) # temporary backup of file pulled from server for recovery From d2bf5f255edbdb5360de9aa0f84fb446464caa74 Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Mon, 17 Aug 2026 23:08:20 +0200 Subject: [PATCH 06/13] cleanup --- mergin/common.py | 3 --- mergin/utils.py | 19 +++---------------- 2 files changed, 3 insertions(+), 19 deletions(-) diff --git a/mergin/common.py b/mergin/common.py index c3d11ca..3d3f7e8 100644 --- a/mergin/common.py +++ b/mergin/common.py @@ -33,9 +33,6 @@ # Maximum changes uploading to server MAX_UPLOAD_CHANGES = 100 -# maximum length of a path supported by Windows without long paths enabled (MAX_PATH) -WINDOWS_MAX_PATH = 260 - # default URL for submitting logs MERGIN_DEFAULT_LOGS_URL = "https://g4pfq226j0.execute-api.eu-west-1.amazonaws.com/mergin_client_log_submit" diff --git a/mergin/utils.py b/mergin/utils.py index d1cdafc..251de54 100644 --- a/mergin/utils.py +++ b/mergin/utils.py @@ -9,7 +9,7 @@ import tempfile from enum import Enum from typing import Optional, Type, Union, ByteString -from .common import ClientError, WINDOWS_MAX_PATH +from .common import ClientError def generate_checksum(file, chunk_size=4096): @@ -267,23 +267,10 @@ def is_versioned_file(path: str) -> bool: return f_extension.lower() in diff_extensions -def is_path_too_long(path: str) -> bool: - """ - Check whether an absolute path is too long to be reliably created/opened on this OS. - - :param path: absolute path to check - :type path: str - :returns: whether the path is likely to be rejected by the OS - :rtype: bool - """ - return os.name == "nt" and len(path) >= WINDOWS_MAX_PATH - - def long_path(path: str) -> str: """ - Prefix an absolute path with the Windows "\\?\" extended-length marker, - so file APIs used by geodiff/SQLite and Python's own open() can handle paths longer - than MAX_PATH (260 characters) without raising an error. + Prefix an absolute path with the Windows "\\?\" extended-length marker, so file APIs used by + geodiff/SQLite and Python's own open() can handle long paths without raising an error. :param path: absolute or relative path, with either posix or windows separators :type path: str From e8c2f75d66378f57849e65862bd87c14fcf8d564 Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Mon, 17 Aug 2026 23:17:42 +0200 Subject: [PATCH 07/13] black 2 --- mergin/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mergin/utils.py b/mergin/utils.py index 251de54..b5bcac9 100644 --- a/mergin/utils.py +++ b/mergin/utils.py @@ -269,7 +269,7 @@ def is_versioned_file(path: str) -> bool: def long_path(path: str) -> str: """ - Prefix an absolute path with the Windows "\\?\" extended-length marker, so file APIs used by + Prefix an absolute path with the Windows "\\?\" extended-length marker, so file APIs used by geodiff/SQLite and Python's own open() can handle long paths without raising an error. :param path: absolute or relative path, with either posix or windows separators From 6bd80e30af4b8ee3e1a8781fcef2e9c01dc3c94b Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Tue, 18 Aug 2026 08:14:32 +0200 Subject: [PATCH 08/13] rm long path from tests --- mergin/test/test_client_pull.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/mergin/test/test_client_pull.py b/mergin/test/test_client_pull.py index fda8bd0..6bb8784 100644 --- a/mergin/test/test_client_pull.py +++ b/mergin/test/test_client_pull.py @@ -4,7 +4,6 @@ from mergin.common import DeltaChangeType, CHUNK_SIZE from mergin.models import ProjectDeltaChange, ProjectDeltaItemDiff from mergin.client_pull import get_download_diff_files, get_download_items -from mergin.utils import long_path def test_get_diff_download_files(): @@ -26,7 +25,7 @@ def test_get_diff_download_files(): # Check diff f2 = files[0] - assert f2.dest_file == long_path(os.path.join(tmp_dir, "diff2")) + assert f2.dest_file == os.path.join(tmp_dir, "diff2") assert len(f2.downloaded_items) == 1 assert f2.downloaded_items[0].file_path == "data.gpkg" assert f2.downloaded_items[0].size == 20 @@ -42,7 +41,7 @@ def test_get_download_items(): assert items[0].file_path == "small.txt" assert items[0].size == 100 assert items[0].part_index == 0 - assert items[0].download_file_path == long_path(os.path.join(tmp_dir, "small.txt.0")) + assert items[0].download_file_path == os.path.join(tmp_dir, "small.txt.0") # Case 2: Large file (multiple chunks) file_size = int(CHUNK_SIZE * 2.5) @@ -52,17 +51,17 @@ def test_get_download_items(): # Chunk 0 assert items[0].size == CHUNK_SIZE assert items[0].part_index == 0 - assert items[0].download_file_path == long_path(os.path.join(tmp_dir, "large.txt.0")) + assert items[0].download_file_path == os.path.join(tmp_dir, "large.txt.0") # Chunk 1 assert items[1].size == CHUNK_SIZE assert items[1].part_index == 1 - assert items[1].download_file_path == long_path(os.path.join(tmp_dir, "large.txt.1")) + assert items[1].download_file_path == (os.path.join(tmp_dir, "large.txt.1") # Chunk 2 assert items[2].size == int(CHUNK_SIZE * 0.5) assert items[2].part_index == 2 - assert items[2].download_file_path == long_path(os.path.join(tmp_dir, "large.txt.2")) + assert items[2].download_file_path == os.path.join(tmp_dir, "large.txt.2") # Case 3: Empty file items = get_download_items("empty.txt", 0, "v1", tmp_dir) @@ -74,4 +73,4 @@ def test_get_download_items(): assert items[0].diff_only is True assert items[0].file_path == "base.gpkg" assert items[0].size == 50 - assert items[0].download_file_path == long_path(os.path.join(tmp_dir, "diff_file.0")) + assert items[0].download_file_path == os.path.join(tmp_dir, "diff_file.0") From 782603c2b80c952d5d1ec113b419b21b037463fd Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Tue, 18 Aug 2026 08:15:05 +0200 Subject: [PATCH 09/13] rm long path from tests 2 --- mergin/test/test_client_pull.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mergin/test/test_client_pull.py b/mergin/test/test_client_pull.py index 6bb8784..c20fe44 100644 --- a/mergin/test/test_client_pull.py +++ b/mergin/test/test_client_pull.py @@ -56,7 +56,7 @@ def test_get_download_items(): # Chunk 1 assert items[1].size == CHUNK_SIZE assert items[1].part_index == 1 - assert items[1].download_file_path == (os.path.join(tmp_dir, "large.txt.1") + assert items[1].download_file_path == os.path.join(tmp_dir, "large.txt.1") # Chunk 2 assert items[2].size == int(CHUNK_SIZE * 0.5) From bcc7ae60ce855045b2f6b7e52456b48a55d0c664 Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Wed, 26 Aug 2026 14:32:35 +0200 Subject: [PATCH 10/13] introduce GeoDiffLongPath handler --- mergin/client.py | 4 +- mergin/client_pull.py | 2 +- mergin/merginproject.py | 148 +++++++++++++++++++++++++--------------- mergin/report.py | 2 +- 4 files changed, 95 insertions(+), 61 deletions(-) diff --git a/mergin/client.py b/mergin/client.py index 8467a1c..8d4bdc8 100644 --- a/mergin/client.py +++ b/mergin/client.py @@ -1384,9 +1384,7 @@ def reset_local_changes(self, directory: str, files_to_reset: typing.List[str] = for file in push_changes["updated"]: if all_files or file["path"] in files_to_reset: if mp.is_versioned_file(file["path"]): - mp.geodiff.make_copy_sqlite( - long_path(mp.fpath_meta(file["path"])), long_path(mp.fpath(file["path"])) - ) + mp.geodiff.make_copy_sqlite(mp.fpath_meta(file["path"]), mp.fpath(file["path"])) else: files_download.append(file["path"]) diff --git a/mergin/client_pull.py b/mergin/client_pull.py index 2030457..08b48af 100644 --- a/mergin/client_pull.py +++ b/mergin/client_pull.py @@ -423,7 +423,7 @@ def apply(self, directory, mp): # Make a copy of the file to meta dir only if there is no user-specified path for the file. # destination_file is None for full project download and takes a meaningful value for a single file download. if mp.is_versioned_file(self.file_path) and self.destination_file is None: - mp.geodiff.make_copy_sqlite(long_path(mp.fpath(self.file_path)), long_path(mp.fpath_meta(self.file_path))) + mp.geodiff.make_copy_sqlite(mp.fpath(self.file_path), mp.fpath_meta(self.file_path)) class PullJob: diff --git a/mergin/merginproject.py b/mergin/merginproject.py index 0757a96..68d16dd 100644 --- a/mergin/merginproject.py +++ b/mergin/merginproject.py @@ -40,6 +40,54 @@ import pygeodiff +class GeoDiffLongPath: + """ + Wraps a pygeodiff.GeoDiff instance so that every filesystem path passed to it is prefixed + with the Windows extended-length ("\\?\") marker. + + Only the geodiff methods that take path arguments are listed explicitly. + """ + + def __init__(self, geodiff): + self._geodiff = geodiff + + def create_changeset(self, base, modified, changeset): + return self._geodiff.create_changeset(long_path(base), long_path(modified), long_path(changeset)) + + def apply_changeset(self, base, changeset): + return self._geodiff.apply_changeset(long_path(base), long_path(changeset)) + + def rebase(self, base, modified_their, modified, conflict): + return self._geodiff.rebase( + long_path(base), long_path(modified_their), long_path(modified), long_path(conflict) + ) + + def make_copy_sqlite(self, src, dst): + return self._geodiff.make_copy_sqlite(long_path(src), long_path(dst)) + + def has_changes(self, changeset): + return self._geodiff.has_changes(long_path(changeset)) + + def changes_count(self, changeset): + return self._geodiff.changes_count(long_path(changeset)) + + def read_changeset(self, changeset): + return self._geodiff.read_changeset(long_path(changeset)) + + def list_changes_summary(self, changeset, json): + return self._geodiff.list_changes_summary(long_path(changeset), long_path(json)) + + def concat_changes(self, list_changesets, output_changeset): + return self._geodiff.concat_changes([long_path(p) for p in list_changesets], long_path(output_changeset)) + + def schema(self, driver, driver_info, src, json): + return self._geodiff.schema(driver, driver_info, long_path(src), long_path(json)) + + def __getattr__(self, name): + # everything without path arguments (set_logger_callback, level/table setters, version(), ...) + return getattr(self._geodiff, name) + + class MerginProject: """Base class for Mergin Maps local projects. @@ -70,7 +118,7 @@ def __init__(self, directory): # make sure we can load correct pygeodiff try: - self.geodiff = pygeodiff.GeoDiff() + self.geodiff = GeoDiffLongPath(pygeodiff.GeoDiff()) except pygeodiff.geodifflib.GeoDiffLibVersionError: # this is a fatal error, we can't live without geodiff self.log.error("Unable to load geodiff! (lib version error)") @@ -627,8 +675,8 @@ def get_local_delta(self, diff_directory: str) -> List[ProjectDeltaChange]: diff_location = self.fpath(diff_file, diff_directory) diff_location_lp = long_path(diff_location) try: - self.geodiff.create_changeset(long_path(origin_file), long_path(current_file), diff_location_lp) - if not self.geodiff.has_changes(diff_location_lp): + self.geodiff.create_changeset(origin_file, current_file, diff_location) + if not self.geodiff.has_changes(diff_location): os.remove(diff_location_lp) continue @@ -681,8 +729,8 @@ def get_push_changes(self): diff_file = self.fpath_meta(diff_name) diff_file_lp = long_path(diff_file) try: - self.geodiff.create_changeset(long_path(origin_file), long_path(current_file), diff_file_lp) - if self.geodiff.has_changes(diff_file_lp): + self.geodiff.create_changeset(origin_file, current_file, diff_file) + if self.geodiff.has_changes(diff_file): diff_size = os.path.getsize(diff_file_lp) file["checksum"] = file["origin_checksum"] # need to match basefile on server file["chunks"] = [str(uuid.uuid4()) for i in range(math.ceil(diff_size / UPLOAD_CHUNK_SIZE))] @@ -716,7 +764,7 @@ def copy_versioned_file_for_upload(self, f: FileChange, tmp_dir: str) -> str: tmp_file = os.path.join(tmp_dir, path) os.makedirs(os.path.dirname(tmp_file), exist_ok=True) tmp_file_lp = long_path(tmp_file) - self.geodiff.make_copy_sqlite(long_path(self.fpath(path)), tmp_file_lp) + self.geodiff.make_copy_sqlite(self.fpath(path), tmp_file) f.size = os.path.getsize(tmp_file_lp) f.checksum = generate_checksum(tmp_file_lp) f.chunks = [str(uuid.uuid4()) for i in range(math.ceil(f.size / UPLOAD_CHUNK_SIZE))] @@ -731,7 +779,7 @@ def get_list_of_push_changes(self, push_changes): changeset = self.fpath_meta(changeset_path) result_file = self.fpath("change_list" + str(idx), self.meta_dir) try: - self.geodiff.list_changes_summary(long_path(changeset), result_file) + self.geodiff.list_changes_summary(changeset, result_file) with open(result_file, "r") as f: change = f.read() changes[file["path"]] = json.loads(change) @@ -772,8 +820,8 @@ def apply_pull_actions(self, actions: List[PullAction], download_dir: str, serve if action_type == PullActionType.COPY: # simply copy the file from server if is_versioned_file(path): - self.geodiff.make_copy_sqlite(server_file_lp, live_file_lp) - self.geodiff.make_copy_sqlite(server_file_lp, basefile_lp) + self.geodiff.make_copy_sqlite(server_file, live_file) + self.geodiff.make_copy_sqlite(server_file, basefile) else: shutil.copy(server_file_lp, live_file_lp) elif action_type == PullActionType.APPLY_DIFF_NO_REBASE: @@ -793,13 +841,13 @@ def apply_pull_actions(self, actions: List[PullAction], download_dir: str, serve conflicts.append(conflict) if self.is_versioned_file(path): try: - self.geodiff.make_copy_sqlite(server_file_lp, live_file_lp) - self.geodiff.make_copy_sqlite(server_file_lp, basefile_lp) + self.geodiff.make_copy_sqlite(server_file, live_file) + self.geodiff.make_copy_sqlite(server_file, basefile) except pygeodiff.GeoDiffLibError: self.log.info("failed to create SQLite copy for file: " + path) # create unfinished pull copy instead - f_server_unfinished = long_path(self.fpath_unfinished_pull(path)) - self.geodiff.make_copy_sqlite(server_file_lp, f_server_unfinished) + f_server_unfinished = self.fpath_unfinished_pull(path) + self.geodiff.make_copy_sqlite(server_file, f_server_unfinished) else: shutil.copy(server_file_lp, live_file_lp) @@ -835,57 +883,51 @@ def update_with_rebase(self, path, src, dest, basefile, temp_dir, user_name): """ self.log.info("updating file with rebase: " + path) - src_lp = long_path(src) - dest_lp = long_path(dest) - basefile_lp = long_path(basefile) - - server_diff = long_path( - self.fpath(f"{path}-server_diff", temp_dir) - ) # diff between server file and local basefile - local_diff = long_path(self.fpath(f"{path}-local_diff", temp_dir)) + server_diff = self.fpath(f"{path}-server_diff", temp_dir) # diff between server file and local basefile + local_diff = self.fpath(f"{path}-local_diff", temp_dir) # temporary backup of file pulled from server for recovery - f_server_backup = long_path(self.fpath(f"{path}-server_backup", temp_dir)) - self.geodiff.make_copy_sqlite(src_lp, f_server_backup) + f_server_backup = self.fpath(f"{path}-server_backup", temp_dir) + self.geodiff.make_copy_sqlite(src, f_server_backup) # create temp backup (ideally with geodiff) of locally modified file if needed later - f_conflict_file = long_path(self.fpath(f"{path}-local_backup", temp_dir)) + f_conflict_file = self.fpath(f"{path}-local_backup", temp_dir) try: - self.geodiff.create_changeset(basefile_lp, dest_lp, local_diff) - self.geodiff.make_copy_sqlite(basefile_lp, f_conflict_file) + self.geodiff.create_changeset(basefile, dest, local_diff) + self.geodiff.make_copy_sqlite(basefile, f_conflict_file) self.geodiff.apply_changeset(f_conflict_file, local_diff) except (pygeodiff.GeoDiffLibError, pygeodiff.GeoDiffLibConflictError): self.log.info("backup of local file with geodiff failed - need to do hard copy") - self.geodiff.make_copy_sqlite(dest_lp, f_conflict_file) + self.geodiff.make_copy_sqlite(dest, f_conflict_file) # in case there will be any conflicting operations found during rebase, # they will be stored in a JSON file - if there are no conflicts, the file # won't even be created - rebase_conflicts = long_path( - unique_path_name(edit_conflict_file_name(self.fpath(path), user_name, int_version(self.version()))) + rebase_conflicts = unique_path_name( + edit_conflict_file_name(self.fpath(path), user_name, int_version(self.version())) ) # try to do rebase magic try: - self.geodiff.create_changeset(basefile_lp, src_lp, server_diff) - self.geodiff.rebase(basefile_lp, src_lp, dest_lp, rebase_conflicts) + self.geodiff.create_changeset(basefile, src, server_diff) + self.geodiff.rebase(basefile, src, dest, rebase_conflicts) # make sure basefile is in the same state as remote server file (for calc of push changes) - self.geodiff.apply_changeset(basefile_lp, server_diff) + self.geodiff.apply_changeset(basefile, server_diff) self.log.info("rebase successful!") except (pygeodiff.GeoDiffLibError, pygeodiff.GeoDiffLibConflictError) as err: self.log.warning("rebase failed! going to create conflict file") try: # it would not be possible to commit local changes, they need to end up in new conflict file - self.geodiff.make_copy_sqlite(f_conflict_file, dest_lp) + self.geodiff.make_copy_sqlite(f_conflict_file, dest) conflict = self.create_conflicted_copy(path, user_name) # original file synced with server - self.geodiff.make_copy_sqlite(f_server_backup, basefile_lp) - self.geodiff.make_copy_sqlite(f_server_backup, dest_lp) + self.geodiff.make_copy_sqlite(f_server_backup, basefile) + self.geodiff.make_copy_sqlite(f_server_backup, dest) return conflict except pygeodiff.GeoDiffLibError as err: self.log.warning("creation of conflicted copy failed! going to create an unfinished pull") - f_server_unfinished = long_path(self.fpath_unfinished_pull(path)) + f_server_unfinished = self.fpath_unfinished_pull(path) self.geodiff.make_copy_sqlite(f_server_backup, f_server_unfinished) return "" @@ -911,27 +953,22 @@ def update_without_rebase(self, path, src, dest, basefile, temp_dir): :type temp_dir: str """ self.log.info("updating file without rebase: " + path) - src_lp = long_path(src) - dest_lp = long_path(dest) - basefile_lp = long_path(basefile) try: - server_diff = long_path( - self.fpath(f"{path}-server_diff", temp_dir) - ) # diff between server file and local basefile + server_diff = self.fpath(f"{path}-server_diff", temp_dir) # diff between server file and local basefile # TODO: it could happen that basefile does not exist. # It was either never created (e.g. when pushing without geodiff) # or it was deleted by mistake(?) by the user. We should detect that # when starting pull and download it as well - self.geodiff.create_changeset(basefile_lp, src_lp, server_diff) - self.geodiff.apply_changeset(dest_lp, server_diff) - self.geodiff.apply_changeset(basefile_lp, server_diff) + self.geodiff.create_changeset(basefile, src, server_diff) + self.geodiff.apply_changeset(dest, server_diff) + self.geodiff.apply_changeset(basefile, server_diff) self.log.info("update successful") except (pygeodiff.GeoDiffLibError, pygeodiff.GeoDiffLibConflictError): self.log.warning("update failed! going to copy file") # something bad happened and we have failed to patch our local files - this should not happen if there # wasn't a schema change or something similar that geodiff can't handle. - self.geodiff.make_copy_sqlite(src_lp, dest_lp) - self.geodiff.make_copy_sqlite(src_lp, basefile_lp) + self.geodiff.make_copy_sqlite(src, dest) + self.geodiff.make_copy_sqlite(src, basefile) def apply_push_changes(self, changes): """ @@ -951,13 +988,13 @@ def apply_push_changes(self, changes): if k == "removed": os.remove(basefile_lp) elif k == "added": - self.geodiff.make_copy_sqlite(long_path(self.fpath(path)), basefile_lp) + self.geodiff.make_copy_sqlite(self.fpath(path), basefile) elif k == "updated": # in case for geopackage cannot be created diff (e.g. forced update with committed changes from wal file) diff = item.get("diff") if not diff: self.log.info("updating basefile (copy) for: " + path) - self.geodiff.make_copy_sqlite(long_path(self.fpath(path)), basefile_lp) + self.geodiff.make_copy_sqlite(self.fpath(path), basefile) else: self.log.info("updating basefile (diff) for: " + path) # better to apply diff to previous basefile to avoid issues with geodiff tmp files @@ -989,7 +1026,7 @@ def create_conflicted_copy(self, file: str, user_name: str): ) if self.is_versioned_file(file): - self.geodiff.make_copy_sqlite(src_lp, long_path(backup_path)) + self.geodiff.make_copy_sqlite(src, backup_path) else: shutil.copy(src_lp, long_path(backup_path)) return backup_path @@ -1010,10 +1047,9 @@ def apply_diffs(self, basefile, diffs): if not self.is_versioned_file(basefile): return error - basefile_lp = long_path(basefile) for index, diff in enumerate(diffs): try: - self.geodiff.apply_changeset(basefile_lp, long_path(diff)) + self.geodiff.apply_changeset(basefile, diff) except (pygeodiff.GeoDiffLibError, pygeodiff.GeoDiffLibConflictError) as e: self.log.warning("failed to apply changeset " + diff + " to " + basefile) error = str(e) @@ -1067,8 +1103,8 @@ def resolve_unfinished_pull(self, user_name): for file_name in files: src = os.path.join(root, file_name) # already long-path-prefixed, root came from os.walk above file_path = os.path.relpath(src, long_path(self.unfinished_pull_dir)) - dest = long_path(self.fpath(file_path)) - basefile = long_path(self.fpath_meta(file_path)) + dest = self.fpath(file_path) + basefile = self.fpath_meta(file_path) self.log.info("trying to resolve unfinished pull for: " + file_path) @@ -1131,9 +1167,9 @@ def get_geodiff_changes_count(self, diff_rel_path: str): Never raises – diagnostics/logging must not fail. """ - diff_abs = long_path(self.fpath_meta(diff_rel_path)) + diff_abs = self.fpath_meta(diff_rel_path) try: - return pygeodiff.GeoDiff().changes_count(diff_abs) + return GeoDiffLongPath(pygeodiff.GeoDiff()).changes_count(diff_abs) except ( pygeodiff.GeoDiffLibError, pygeodiff.GeoDiffLibConflictError, diff --git a/mergin/report.py b/mergin/report.py index abbf451..6e24e51 100644 --- a/mergin/report.py +++ b/mergin/report.py @@ -263,7 +263,7 @@ def create_report(mc, directory, since, to, out_file): warnings.append(f"Missing diff: {f['path']} was {f['history'][version]['change']} in {version}") continue - v_diff_file = long_path(mp.fpath_cache(f["history"][version]["diff"]["path"], version=version)) + v_diff_file = mp.fpath_cache(f["history"][version]["diff"]["path"], version=version) version_data = versions_map[version] cr = mp.geodiff.read_changeset(v_diff_file) report = changeset_report(cr, schema, mp) From 2051d4fe96595c76e1ba782b5eabab7e7ccc95c0 Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Wed, 26 Aug 2026 15:36:38 +0200 Subject: [PATCH 11/13] introduce fs handler for long paths --- mergin/client.py | 9 +++--- mergin/client_pull.py | 41 ++++++++++++------------- mergin/client_push.py | 12 ++++---- mergin/fs.py | 51 +++++++++++++++++++++++++++++++ mergin/merginproject.py | 68 +++++++++++++++++++---------------------- mergin/report.py | 11 ++++--- mergin/utils.py | 8 ++--- 7 files changed, 121 insertions(+), 79 deletions(-) create mode 100644 mergin/fs.py diff --git a/mergin/client.py b/mergin/client.py index 8d4bdc8..e73e758 100644 --- a/mergin/client.py +++ b/mergin/client.py @@ -2,7 +2,6 @@ import math import os import json -import shutil import zlib import base64 import urllib.parse @@ -67,8 +66,8 @@ int_version, is_version_acceptable, normalize_role, - long_path, ) +from . import fs from .version import __version__ try: @@ -1238,7 +1237,7 @@ def get_file_diff(self, project_dir, file_path, output_diff, version_from, versi # collect required versions from the cache diffs = [] for v in versions_to_fetch[1:]: - diffs.append(long_path(mp.fpath_cache(file_history["history"][v]["diff"]["path"], v))) + diffs.append(mp.fpath_cache(file_history["history"][v]["diff"]["path"], v)) # concatenate diffs, if needed output_dir = os.path.dirname(output_diff) @@ -1247,7 +1246,7 @@ def get_file_diff(self, project_dir, file_path, output_diff, version_from, versi if len(diffs) > 1: mp.geodiff.concat_changes(diffs, output_diff) elif len(diffs) == 1: - shutil.copy(diffs[0], output_diff) + fs.copy(diffs[0], output_diff) def download_file_diffs(self, project_dir, file_path, versions): """Download file diffs for specified versions if they are not present @@ -1378,7 +1377,7 @@ def reset_local_changes(self, directory: str, files_to_reset: typing.List[str] = # remove all added files for file in push_changes["added"]: if all_files or file["path"] in files_to_reset: - os.remove(long_path(mp.fpath(file["path"]))) + fs.remove(mp.fpath(file["path"])) # update files get override with previous version for file in push_changes["updated"]: diff --git a/mergin/client_pull.py b/mergin/client_pull.py index 08b48af..a1e9c9a 100644 --- a/mergin/client_pull.py +++ b/mergin/client_pull.py @@ -25,7 +25,8 @@ from .common import CHUNK_SIZE, ClientError, DeltaChangeType, PullActionType from .models import ProjectDelta, ProjectDeltaChange, PullAction from .merginproject import MerginProject -from .utils import cleanup_tmp_dir, save_to_file, long_path +from .utils import cleanup_tmp_dir, save_to_file +from . import fs from typing import List, Optional # status = download_project_async(...) @@ -93,9 +94,7 @@ def __init__(self, file_path, size, version, diff_only, part_index, download_fil self.version = version # version of the file ("v123") self.diff_only = diff_only # whether downloading diff or full version self.part_index = part_index # index of the chunk - self.download_file_path = long_path( - download_file_path - ) # full path to a temporary file which will receive the content + self.download_file_path = download_file_path # full path to a temporary file which will receive the content def __repr__(self): return "".format( @@ -130,9 +129,7 @@ class DownloadDiffQueueItem: def __init__(self, diff_id, download_file_path): self.diff_id = diff_id # relative path to the file within project - self.download_file_path = long_path( - download_file_path - ) # full path to a temporary file which will receive the content + self.download_file_path = download_file_path # full path to a temporary file which will receive the content self.size = 0 # size of the item in bytes def __repr__(self): @@ -146,7 +143,7 @@ def download_blocking(self, mc, mp): if resp.status in [200, 206]: mp.log.debug(f"Download finished: {self.diff_id}") save_to_file(resp, self.download_file_path) - self.size = os.path.getsize(self.download_file_path) + self.size = fs.getsize(self.download_file_path) else: mp.log.error(f"Download failed: {self.diff_id}") raise ClientError(f"Failed to download of diff file {self.diff_id} to {self.download_file_path}") @@ -161,26 +158,26 @@ class DownloadFile: """ def __init__(self, dest_file, downloaded_items: typing.List[DownloadQueueItem], size_check=True): - self.dest_file = long_path(dest_file) # full path to the destination file to be created + self.dest_file = dest_file # full path to the destination file to be created self.downloaded_items = downloaded_items # list of pieces of the destination file to be merged self.size_check = size_check # whether we want to do merged file size check def from_chunks(self): """Merges downloaded chunks into a single file at dest_file path""" file_dir = os.path.dirname(self.dest_file) - os.makedirs(file_dir, exist_ok=True) + fs.makedirs(file_dir, exist_ok=True) - with open(self.dest_file, "wb") as final: + with fs.open_file(self.dest_file, "wb") as final: for item in self.downloaded_items: - with open(item.download_file_path, "rb") as chunk: + with fs.open_file(item.download_file_path, "rb") as chunk: shutil.copyfileobj(chunk, final) - os.remove(item.download_file_path) + fs.remove(item.download_file_path) if not self.size_check: return expected_size = sum(item.size for item in self.downloaded_items) - if os.path.getsize(self.dest_file) != expected_size: - os.remove(self.dest_file) + if fs.getsize(self.dest_file) != expected_size: + fs.remove(self.dest_file) raise ClientError("Download of file {} failed. Please try it again.".format(self.dest_file)) @@ -200,7 +197,7 @@ def get_download_items( items = [] for part_index in range(chunks): - download_file_path = long_path(os.path.join(file_dir, basename + ".{}".format(part_index))) + download_file_path = os.path.join(file_dir, basename + ".{}".format(part_index)) size = min(CHUNK_SIZE, file_size - part_index * CHUNK_SIZE) items.append(DownloadQueueItem(file_path, size, file_version, diff_only, part_index, download_file_path)) @@ -483,7 +480,7 @@ def get_download_diff_files(delta_item: ProjectDeltaChange, target_dir: str) -> result = [] for diff in delta_item.diffs: - dest_file_path = long_path(os.path.normpath(os.path.join(target_dir, diff.id))) + dest_file_path = os.path.normpath(os.path.join(target_dir, diff.id)) download_items = get_download_items(delta_item.path, diff.size, diff.version, target_dir, diff.id, True) result.append(DownloadFile(dest_file_path, download_items)) return result @@ -559,7 +556,7 @@ def pull_project_async(mc, directory) -> Optional[PullJob]: pull_action_type == PullActionType.COPY_CONFLICT and change.type == DeltaChangeType.UPDATE_DIFF ): basefile = mp.fpath_meta(change.path) - if not os.path.exists(long_path(basefile)): + if not fs.exists(basefile): # The basefile does not exist for some reason. This should not happen normally (maybe user removed the file # or we removed it within previous pull because we failed to apply patch the older version for some reason). # But it's not a problem - we will download the newest version and we're sorted. @@ -726,7 +723,7 @@ def pull_project_finalize(job: PullJob): basefile = job.mp.fpath_meta(file_path) server_file = job.mp.fpath(file_path, job.tmp_dir.name) - shutil.copy(long_path(basefile), long_path(server_file)) + fs.copy(basefile, server_file) diffs = [job.mp.fpath(f, job.tmp_dir.name) for f in file_diffs] patch_error = job.mp.apply_diffs(server_file, diffs) if patch_error: @@ -739,7 +736,7 @@ def pull_project_finalize(job: PullJob): job.mp.log.error("Diffs we were applying: " + str(diffs)) job.mp.log.error("Removing basefile because it would be corrupted anyway...") job.mp.log.info("--- pull aborted") - os.remove(long_path(basefile)) + fs.remove(basefile) raise ClientError("Cannot patch basefile {}! Please try syncing again.".format(basefile)) conflicts = [] job.mp.log.info(f"--- applying pull actions {job.pull_actions}") @@ -833,8 +830,8 @@ def download_diffs_async(mc, project_directory, file_path, versions): download_path=diff.get("path"), diff_only=True, ) - dest_file_path = long_path(mp.fpath_cache(diff["path"], version=file["version"])) - if os.path.exists(dest_file_path): + dest_file_path = mp.fpath_cache(diff["path"], version=file["version"]) + if fs.exists(dest_file_path): continue download_files.append(DownloadFile(dest_file_path, items)) download_list.extend(items) diff --git a/mergin/client_push.py b/mergin/client_push.py index bdcf0d1..d918844 100644 --- a/mergin/client_push.py +++ b/mergin/client_push.py @@ -18,7 +18,6 @@ import pprint import tempfile import concurrent.futures -import os import time from typing import List, Tuple, Optional, ByteString @@ -34,7 +33,8 @@ ) from .merginproject import MerginProject, pygeodiff from .editor import filter_changes -from .utils import get_data_checksum, cleanup_tmp_dir, long_path +from .utils import get_data_checksum, cleanup_tmp_dir +from . import fs POST_JSON_HEADERS = {"Content-Type": "application/json"} @@ -114,7 +114,7 @@ def upload_chunk_v2_api(self, data: ByteString, checksum: str): self.mc.upload_chunks_cache.add(checksum, self.server_chunk_id) def upload_blocking(self): - with open(long_path(self.file_path), "rb") as file_handle: + with fs.open_file(self.file_path, "rb") as file_handle: file_handle.seek(self.chunk_index * UPLOAD_CHUNK_SIZE) data = file_handle.read(UPLOAD_CHUNK_SIZE) checksum_str = get_data_checksum(data) @@ -507,9 +507,9 @@ def remove_diff_files(job: UploadJob) -> None: for change in job.changes.updated: diff = change.get_diff() if diff: - diff_file = long_path(job.mp.fpath_meta(diff.path)) - if os.path.exists(diff_file): - os.remove(diff_file) + diff_file = job.mp.fpath_meta(diff.path) + if fs.exists(diff_file): + fs.remove(diff_file) def get_push_changes_batch(mc, directory: str) -> Tuple[LocalProjectChanges, int]: diff --git a/mergin/fs.py b/mergin/fs.py new file mode 100644 index 0000000..0608396 --- /dev/null +++ b/mergin/fs.py @@ -0,0 +1,51 @@ +""" +Thin wrappers around the standard-library filesystem calls used by the sync code. + +Every wrapper applies utils.long_path() to its path argument(s) so that Windows paths +longer than MAX_PATH are handled transparently. + +Use these instead of calling os.* / shutil.* / open() / sqlite3.connect() directly on +project file paths. +""" + +import os +import shutil +import sqlite3 + +from .utils import long_path + + +def remove(path): + os.remove(long_path(path)) + + +def exists(path) -> bool: + return os.path.exists(long_path(path)) + + +def getsize(path) -> int: + return os.path.getsize(long_path(path)) + + +def getmtime(path) -> float: + return os.path.getmtime(long_path(path)) + + +def copy(src, dst): + return shutil.copy(long_path(src), long_path(dst)) + + +def walk(path): + return os.walk(long_path(path)) + + +def makedirs(path, exist_ok=False): + os.makedirs(long_path(path), exist_ok=exist_ok) + + +def connect(path): + return sqlite3.connect(long_path(path)) + + +def open_file(path, *args, **kwargs): + return open(long_path(path), *args, **kwargs) diff --git a/mergin/merginproject.py b/mergin/merginproject.py index 68d16dd..e4d4c55 100644 --- a/mergin/merginproject.py +++ b/mergin/merginproject.py @@ -27,6 +27,7 @@ edit_conflict_file_name, ) from .local_changes import FileChange +from . import fs this_dir = os.path.dirname(os.path.realpath(__file__)) @@ -328,7 +329,7 @@ def is_gpkg_open(self, path): f_extension = os.path.splitext(path)[1] if f_extension != ".gpkg": return False - if os.path.exists(f"{long_path(path)}-wal"): + if fs.exists(f"{path}-wal"): return True return False @@ -358,21 +359,20 @@ def inspect_files(self): :rtype: list[dict] """ files_meta = [] - for root, dirs, files in os.walk(self.dir, topdown=True): + for root, dirs, files in fs.walk(self.dir): dirs[:] = [d for d in dirs if d not in [".mergin"]] for file in files: if self.ignore_file(file): continue - - abs_path = os.path.abspath(os.path.join(root, file)) - rel_path = os.path.relpath(abs_path, start=self.dir) + abs_path = os.path.join(root, file) + rel_path = os.path.relpath(abs_path, start=long_path(self.dir)) proj_path = "/".join(rel_path.split(os.path.sep)) # we need posix path files_meta.append( { "path": proj_path, "checksum": generate_checksum(abs_path), - "size": os.path.getsize(abs_path), - "mtime": datetime.fromtimestamp(os.path.getmtime(abs_path), tzlocal()), + "size": fs.getsize(abs_path), + "mtime": datetime.fromtimestamp(fs.getmtime(abs_path), tzlocal()), } ) return files_meta @@ -673,16 +673,15 @@ def get_local_delta(self, diff_directory: str) -> List[ProjectDeltaChange]: delta_item.checksum = checkpoint_checksum diff_location = self.fpath(diff_file, diff_directory) - diff_location_lp = long_path(diff_location) try: self.geodiff.create_changeset(origin_file, current_file, diff_location) if not self.geodiff.has_changes(diff_location): - os.remove(diff_location_lp) + fs.remove(diff_location) continue delta_item.checksum = change.get("origin_checksum") delta_item.type = DeltaChangeType.UPDATE_DIFF - os.remove(diff_location_lp) + fs.remove(diff_location) except (pygeodiff.GeoDiffLibError, pygeodiff.GeoDiffLibConflictError) as e: self.log.warning("failed to create changeset for " + path) # probably the database schema has been modified if geodiff cannot create changeset. @@ -727,23 +726,22 @@ def get_push_changes(self): diff_id = str(uuid.uuid4()) diff_name = path + "-diff-" + diff_id diff_file = self.fpath_meta(diff_name) - diff_file_lp = long_path(diff_file) try: self.geodiff.create_changeset(origin_file, current_file, diff_file) if self.geodiff.has_changes(diff_file): - diff_size = os.path.getsize(diff_file_lp) + diff_size = fs.getsize(diff_file) file["checksum"] = file["origin_checksum"] # need to match basefile on server file["chunks"] = [str(uuid.uuid4()) for i in range(math.ceil(diff_size / UPLOAD_CHUNK_SIZE))] - file["mtime"] = datetime.fromtimestamp(os.path.getmtime(long_path(current_file)), tzlocal()) + file["mtime"] = datetime.fromtimestamp(fs.getmtime(current_file), tzlocal()) file["diff"] = { "path": diff_name, - "checksum": generate_checksum(diff_file_lp), + "checksum": generate_checksum(diff_file), "size": diff_size, - "mtime": datetime.fromtimestamp(os.path.getmtime(diff_file_lp), tzlocal()), + "mtime": datetime.fromtimestamp(fs.getmtime(diff_file), tzlocal()), } else: - if os.path.exists(diff_file_lp): - os.remove(diff_file_lp) + if fs.exists(diff_file): + fs.remove(diff_file) not_updated.append(file) except (pygeodiff.GeoDiffLibError, pygeodiff.GeoDiffLibConflictError) as e: self.log.warning("failed to create changeset for " + path) @@ -763,10 +761,9 @@ def copy_versioned_file_for_upload(self, f: FileChange, tmp_dir: str) -> str: self.log.info("Making a temporary copy (full upload): " + path) tmp_file = os.path.join(tmp_dir, path) os.makedirs(os.path.dirname(tmp_file), exist_ok=True) - tmp_file_lp = long_path(tmp_file) self.geodiff.make_copy_sqlite(self.fpath(path), tmp_file) - f.size = os.path.getsize(tmp_file_lp) - f.checksum = generate_checksum(tmp_file_lp) + f.size = fs.getsize(tmp_file) + f.checksum = generate_checksum(tmp_file) f.chunks = [str(uuid.uuid4()) for i in range(math.ceil(f.size / UPLOAD_CHUNK_SIZE))] f.upload_file = tmp_file return tmp_file @@ -813,9 +810,6 @@ def apply_pull_actions(self, actions: List[PullAction], download_dir: str, serve server_file = self.fpath(path, download_dir) live_file = self.fpath(path) basefile = self.fpath_meta(path) - server_file_lp = long_path(server_file) - live_file_lp = long_path(live_file) - basefile_lp = long_path(basefile) action_type = action.type if action_type == PullActionType.COPY: # simply copy the file from server @@ -823,7 +817,7 @@ def apply_pull_actions(self, actions: List[PullAction], download_dir: str, serve self.geodiff.make_copy_sqlite(server_file, live_file) self.geodiff.make_copy_sqlite(server_file, basefile) else: - shutil.copy(server_file_lp, live_file_lp) + fs.copy(server_file, live_file) elif action_type == PullActionType.APPLY_DIFF_NO_REBASE: # simply apply the diff without rebase (no local changes or non-conflicting local changes) self.update_without_rebase(path, server_file, live_file, basefile, download_dir) @@ -849,14 +843,14 @@ def apply_pull_actions(self, actions: List[PullAction], download_dir: str, serve f_server_unfinished = self.fpath_unfinished_pull(path) self.geodiff.make_copy_sqlite(server_file, f_server_unfinished) else: - shutil.copy(server_file_lp, live_file_lp) + fs.copy(server_file, live_file) elif action_type == PullActionType.DELETE: # remove local file - if os.path.exists(live_file_lp): - os.remove(live_file_lp) - if self.is_versioned_file(path) and os.path.exists(basefile_lp): - os.remove(basefile_lp) + if fs.exists(live_file): + fs.remove(live_file) + if self.is_versioned_file(path) and fs.exists(basefile): + fs.remove(basefile) return conflicts @@ -984,9 +978,8 @@ def apply_push_changes(self, changes): continue basefile = self.fpath_meta(path) - basefile_lp = long_path(basefile) if k == "removed": - os.remove(basefile_lp) + fs.remove(basefile) elif k == "added": self.geodiff.make_copy_sqlite(self.fpath(path), basefile) elif k == "updated": @@ -1003,7 +996,7 @@ def apply_push_changes(self, changes): if patch_error: # in case of local sync issues it is safier to remove basefile, next time it will be downloaded from server self.log.warning("removing basefile (because of apply diff error) for: " + path) - os.remove(basefile_lp) + fs.remove(basefile) else: pass @@ -1017,8 +1010,7 @@ def create_conflicted_copy(self, file: str, user_name: str): :rtype: str """ src = self.fpath(file) - src_lp = long_path(src) - if not os.path.exists(src_lp): + if not fs.exists(src): return backup_path = unique_path_name( @@ -1028,7 +1020,7 @@ def create_conflicted_copy(self, file: str, user_name: str): if self.is_versioned_file(file): self.geodiff.make_copy_sqlite(src, backup_path) else: - shutil.copy(src_lp, long_path(backup_path)) + fs.copy(src, backup_path) return backup_path def apply_diffs(self, basefile, diffs): @@ -1099,9 +1091,11 @@ def resolve_unfinished_pull(self, user_name): self.log.info("resolving unfinished pull") - for root, dirs, files in os.walk(long_path(self.unfinished_pull_dir)): + for root, dirs, files in fs.walk(self.unfinished_pull_dir): for file_name in files: - src = os.path.join(root, file_name) # already long-path-prefixed, root came from os.walk above + # fs.walk() traverses the long-path-prefixed dir, so root (and thus src) is prefixed too; + src = os.path.join(root, file_name) + # the relpath base must be prefixed as well to strip it correctly. file_path = os.path.relpath(src, long_path(self.unfinished_pull_dir)) dest = self.fpath(file_path) basefile = self.fpath_meta(file_path) diff --git a/mergin/report.py b/mergin/report.py index 6e24e51..44d4113 100644 --- a/mergin/report.py +++ b/mergin/report.py @@ -7,7 +7,8 @@ from . import ClientError from .merginproject import MerginProject, pygeodiff -from .utils import int_version, long_path +from .utils import int_version +from . import fs try: from qgis.core import ( @@ -243,15 +244,15 @@ def create_report(mc, directory, since, to, out_file): mc.download_file_diffs(directory, f["path"], history_keys) # download full gpkg in "to" version to analyze its schema to determine which col is geometry - full_gpkg = long_path(mp.fpath_cache(f["path"], version=to)) - if not os.path.exists(full_gpkg): + full_gpkg = mp.fpath_cache(f["path"], version=to) + if not fs.exists(full_gpkg): mc.download_file(directory, f["path"], full_gpkg, to) # get gpkg schema schema_file = full_gpkg + "-schema.json" # geodiff writes schema into a file - if not os.path.exists(schema_file): + if not fs.exists(schema_file): mp.geodiff.schema("sqlite", "", full_gpkg, schema_file) - with open(schema_file, "r") as sf: + with fs.open_file(schema_file, "r") as sf: schema = json.load(sf).get("geodiff_schema") # add records for every version (diff) and all tables within geopackage diff --git a/mergin/utils.py b/mergin/utils.py index b5bcac9..67174a7 100644 --- a/mergin/utils.py +++ b/mergin/utils.py @@ -21,7 +21,7 @@ def generate_checksum(file, chunk_size=4096): :return: sha1 checksum """ checksum = hashlib.sha1() # nosec B324 - usedforsecurity=False flag is compatible with python 3.9+ - with open(file, "rb") as f: + with open(long_path(file), "rb") as f: while True: chunk = f.read(chunk_size) if not chunk: @@ -37,9 +37,9 @@ def save_to_file(stream, path): """ directory = os.path.abspath(os.path.dirname(path)) - os.makedirs(directory, exist_ok=True) + os.makedirs(long_path(directory), exist_ok=True) - with open(path, "wb") as output: + with open(long_path(path), "wb") as output: writer = io.BufferedWriter(output, buffer_size=32768) while True: part = stream.read(4096) @@ -101,7 +101,7 @@ def do_sqlite_checkpoint(path, log=None): conn.commit() conn.close() new_size = os.path.getsize(path_lp) - new_checksum = generate_checksum(path_lp) + new_checksum = generate_checksum(path) if log: log.info("checkpoint - new size {} checksum {}".format(new_size, new_checksum)) From 8a342cf61eac7786378798511bd1e7dc9f124d59 Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Wed, 26 Aug 2026 16:06:36 +0200 Subject: [PATCH 12/13] cleanup --- mergin/merginproject.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mergin/merginproject.py b/mergin/merginproject.py index e4d4c55..961b2bc 100644 --- a/mergin/merginproject.py +++ b/mergin/merginproject.py @@ -1093,7 +1093,6 @@ def resolve_unfinished_pull(self, user_name): for root, dirs, files in fs.walk(self.unfinished_pull_dir): for file_name in files: - # fs.walk() traverses the long-path-prefixed dir, so root (and thus src) is prefixed too; src = os.path.join(root, file_name) # the relpath base must be prefixed as well to strip it correctly. file_path = os.path.relpath(src, long_path(self.unfinished_pull_dir)) From fdfdb7500b08986fb363e45c2ec7eb893685ce17 Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Wed, 26 Aug 2026 16:32:21 +0200 Subject: [PATCH 13/13] explicit topdown walk --- mergin/fs.py | 4 ++-- mergin/merginproject.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mergin/fs.py b/mergin/fs.py index 0608396..ee8af6c 100644 --- a/mergin/fs.py +++ b/mergin/fs.py @@ -35,8 +35,8 @@ def copy(src, dst): return shutil.copy(long_path(src), long_path(dst)) -def walk(path): - return os.walk(long_path(path)) +def walk(path, **kwargs): + return os.walk(long_path(path), **kwargs) def makedirs(path, exist_ok=False): diff --git a/mergin/merginproject.py b/mergin/merginproject.py index 961b2bc..ea55e1b 100644 --- a/mergin/merginproject.py +++ b/mergin/merginproject.py @@ -359,7 +359,7 @@ def inspect_files(self): :rtype: list[dict] """ files_meta = [] - for root, dirs, files in fs.walk(self.dir): + for root, dirs, files in fs.walk(self.dir, topdown=True): dirs[:] = [d for d in dirs if d not in [".mergin"]] for file in files: if self.ignore_file(file):