From c5e683f263d0ee8abf0b3bfc13bdcc0e57e2362a Mon Sep 17 00:00:00 2001 From: Alex Wolf Date: Thu, 22 Oct 2026 05:04:46 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=9A=B8=20Enable=20saving=20project=20note?= =?UTF-8?q?s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lamin_cli/__main__.py | 19 +++-- lamin_cli/_load.py | 109 ++++++++++++++++++++------ lamin_cli/_notes.py | 47 +++++++++++- lamin_cli/_save.py | 110 +++++++++++++++++--------- tests/core/test_save_notes.py | 141 ++++++++++++++++++++++++++++++++++ 5 files changed, 357 insertions(+), 69 deletions(-) diff --git a/lamin_cli/__main__.py b/lamin_cli/__main__.py index cb85c20..4d51f5a 100644 --- a/lamin_cli/__main__.py +++ b/lamin_cli/__main__.py @@ -663,7 +663,7 @@ def load(entity: str | None = None, uid: str | None = None, key: str | None = No lamin load --key myanalyses/analysis.ipynb --with-env # notes via name and topic/type hierarchy lamin load README.md - lamin load my-topic/my-note.md + lamin load My-topic/my-note.md # anything via URL lamin load https://lamin.ai/account/instance/artifact/e2G7k9EVul4JbfsE # anything via registry and --uid @@ -989,9 +989,9 @@ def update( @click.option("--branch", type=str, default=None, help="A valid branch name or uid.") @click.option( "--registry", - type=click.Choice(["artifact", "transform", "record"]), + type=click.Choice(["artifact", "transform", "record", "project"]), default=None, - help="Either 'artifact', 'transform', or 'record'. If not passed, chooses based on path suffix.", + help="Either 'artifact', 'transform', 'record', or 'project'. If not passed, chooses based on path.", ) @click.option( "--batch-size", @@ -1014,11 +1014,11 @@ def save( project: str, space: str, branch: str, - registry: Literal["artifact", "transform", "record"] | None, + registry: Literal["artifact", "transform", "record", "project"] | None, batch_size: int | None, store_kwargs: str | None, ): - """Save a file or folder as an artifact, transform, or record. + """Save a file or folder as an artifact, transform, record, or project. Save a **dataset** or **model** as {class}`~lamindb.Artifact`: @@ -1044,6 +1044,13 @@ def save( lamin save my_script.py --key my_scripts/my_script.py ``` + Save a **markdown note** in `Projects/` as {class}`~lamindb.Project`: + + ``` + lamin save Projects/my-project.md + lamin save Projects/my-type/my-project.md # `my-type` is an existing project type + ``` + Save a **markdown note** as {class}`~lamindb.Record`: ``` @@ -1058,7 +1065,7 @@ def save( The `save` command defaults to saving `.py`, `.ipynb`, `.R`, `.Rmd`, and `.qmd` files as {class}`~lamindb.Transform` - and - if ommitting `--key` - `.md` files as {class}`~lamindb.Record`. + and - if ommitting `--key` - `.md` files as {class}`~lamindb.Record` or {class}`~lamindb.Project`. You can enforce saving a file as an {class}`~lamindb.Artifact` by passing `--registry artifact`. You can pass a project to `--project` to label the artifact by project. diff --git a/lamin_cli/_load.py b/lamin_cli/_load.py index 0b70e6d..d6ff5f4 100644 --- a/lamin_cli/_load.py +++ b/lamin_cli/_load.py @@ -10,8 +10,12 @@ from ._context import get_current_run_file from ._notes import ( + PROJECTS_FOLDER, is_path_within, + is_project_note_target, parse_note_target, + project_type_chain, + resolve_note_project, resolve_note_record, upsert_lamindb_record_uid, ) @@ -23,6 +27,35 @@ from .urls import decompose_url +def _write_note_file( + *, + content: str, + uid: str, + note_name: str, + relative_parents: list[str], + active_dev_dir: Path | None, +) -> Path: + cwd = Path.cwd().resolve() + filename = f"{note_name}.md" + if active_dev_dir is not None and is_path_within(cwd, active_dev_dir): + target_path = active_dev_dir.joinpath(*relative_parents, filename) + else: + target_path = cwd / filename + + target_path.parent.mkdir(parents=True, exist_ok=True) + if target_path.exists(): + response = input(f"! {target_path} exists: replace? (y/n)") + if response != "y": + raise SystemExit("Aborted.") + + target_path.write_text( + upsert_lamindb_record_uid(content or "", uid), + encoding="utf-8", + ) + logger.important(f"note is here: {target_path}") + return target_path + + def load( entity: str | None = None, uid: str | None = None, @@ -65,7 +98,7 @@ def _note_type_chain_from_record(note_record) -> list[str]: if entity is not None and uid is None and key is None: note_target = parse_note_target(entity) if note_target is not None: - entity = "record" + entity = "project" if is_project_note_target(note_target[0]) else "record" if entity is None: if key is None: @@ -76,7 +109,7 @@ def _note_type_chain_from_record(note_record) -> list[str]: if entity.startswith("https://") and "lamin" in entity: url = entity instance, entity, uid = decompose_url(url) - elif entity not in {"artifact", "transform", "collection", "record"}: + elif entity not in {"artifact", "transform", "collection", "record", "project"}: raise SystemExit( "Entity has to be a laminhub URL or 'artifact', 'collection', or 'transform'" ) @@ -188,29 +221,61 @@ def script_to_notebook( raise click.ClickException( f"Record note '{note_name}' has no readme block to load." ) - - cwd = Path.cwd().resolve() - if active_dev_dir is not None and is_path_within(cwd, active_dev_dir): - target_path = ( - active_dev_dir / Path(*type_chain) / f"{note_name}.md" - if type_chain - else active_dev_dir / f"{note_name}.md" + return _write_note_file( + content=readme_block.content or "", + uid=note_record.uid, + note_name=note_name, + relative_parents=type_chain, + active_dev_dir=active_dev_dir, + ) + case "project": + if note_target is not None: + type_chain, note_name = note_target + project = resolve_note_project( + ln=ln, + type_chain=project_type_chain(type_chain), + note_name=note_name, + create_if_missing=False, ) + if project is None: + note_path = ( + "/".join([*type_chain, note_name]) if type_chain else note_name + ) + raise click.ClickException( + f"Project note '{note_path}' does not exist. Save it first with `lamin save`." + ) + relative_parents = type_chain + elif uid is not None: + projects = ln.Project.objects.filter(uid__startswith=uid) + if (n_projects := len(projects)) == 0: + raise click.ClickException( + f"Project with uid={uid} does not exist." + ) + if n_projects > 1: + projects = projects.order_by("-created_at") + project = projects.first() + type_chain = _note_type_chain_from_record(project) + note_name = project.name + relative_parents = [PROJECTS_FOLDER, *type_chain] else: - target_path = cwd / f"{note_name}.md" - - target_path.parent.mkdir(parents=True, exist_ok=True) - if target_path.exists(): - response = input(f"! {target_path} exists: replace? (y/n)") - if response != "y": - raise SystemExit("Aborted.") - - target_path.write_text( - upsert_lamindb_record_uid(readme_block.content or "", note_record.uid), - encoding="utf-8", + raise click.ClickException( + "For project note loads, pass a note target like " + "Projects/.md or Projects//.md." + ) + readme_block = ( + project.ablocks.filter(kind="readme").order_by("-created_at").first() + ) + if readme_block is None: + raise click.ClickException( + f"Project note '{note_name}' has no readme block to load." + ) + return _write_note_file( + content=readme_block.content or "", + uid=project.uid, + note_name=note_name, + relative_parents=relative_parents, + active_dev_dir=active_dev_dir, ) - logger.important(f"note is here: {target_path}") - return target_path case "transform": if query_by_uid: # we don't use .get here because DoesNotExist is hard to catch diff --git a/lamin_cli/_notes.py b/lamin_cli/_notes.py index 70b7374..88260cb 100644 --- a/lamin_cli/_notes.py +++ b/lamin_cli/_notes.py @@ -219,11 +219,30 @@ def extract_note_target_from_path( return parse_note_target(relative_path.as_posix(), allow_extensionless_single=True) -def resolve_note_type_parent(ln, type_chain: list[str]): +PROJECTS_FOLDER = "Projects" + + +def is_projects_folder(name: str) -> bool: + return name.casefold() == PROJECTS_FOLDER.casefold() + + +def is_project_note_target(type_chain: list[str]) -> bool: + """True when the note lives under the dev-dir `Projects` folder.""" + return bool(type_chain) and is_projects_folder(type_chain[0]) + + +def project_type_chain(type_chain: list[str]) -> list[str]: + """Project type path with the leading `Projects` folder removed.""" + if is_project_note_target(type_chain): + return type_chain[1:] + return list(type_chain) + + +def _resolve_type_parent(model, type_chain: list[str], *, kind: str): parent_type = None resolved_path: list[str] = [] for type_name in type_chain: - type_record = ln.Record.filter( + type_record = model.filter( name__iexact=type_name, is_type=True, type=parent_type, @@ -232,7 +251,7 @@ def resolve_note_type_parent(ln, type_chain: list[str]): parent_label = "/".join(resolved_path) if resolved_path else "" expected_path = "/".join([*resolved_path, type_name]) raise click.ClickException( - f"Record type '{type_name}' not found under '{parent_label}'. " + f"{kind} type '{type_name}' not found under '{parent_label}'. " f"Expected hierarchy segment '{expected_path}'. Create it first." ) parent_type = type_record @@ -240,6 +259,28 @@ def resolve_note_type_parent(ln, type_chain: list[str]): return parent_type +def resolve_note_type_parent(ln, type_chain: list[str]): + return _resolve_type_parent(ln.Record, type_chain, kind="Record") + + +def resolve_project_type_parent(ln, type_chain: list[str]): + return _resolve_type_parent(ln.Project, type_chain, kind="Project") + + +def resolve_note_project( + *, + ln, + type_chain: list[str], + note_name: str, + create_if_missing: bool = False, +): + parent_type = resolve_project_type_parent(ln, type_chain) + project = ln.Project.filter(name=note_name, type=parent_type).one_or_none() + if project is None and create_if_missing: + project = ln.Project(name=note_name, type=parent_type).save() + return project + + def resolve_note_record( *, ln, diff --git a/lamin_cli/_save.py b/lamin_cli/_save.py index 0921f6a..bc92085 100644 --- a/lamin_cli/_save.py +++ b/lamin_cli/_save.py @@ -13,7 +13,10 @@ from lamin_cli._context import get_current_run_file from lamin_cli._notes import ( extract_note_target_from_path, + is_project_note_target, parse_lamindb_record_uid, + project_type_chain, + resolve_note_project, resolve_note_record, strip_lamindb_record_uid, upsert_lamindb_record_uid, @@ -177,47 +180,56 @@ def parse_plan_markdown(content: str) -> str | None: return overview or None -def _note_record_for_save(ln, *, content: str, type_chain: list[str], note_name: str): - """Resolve the note record, preferring `lamindb.uid` over the filename.""" +def _note_for_save( + ln, + *, + registry: str, + content: str, + type_chain: list[str], + note_name: str, +): + """Resolve the note, preferring `lamindb.uid` over the filename.""" + kind = "project" if registry == "project" else "record" + model = ln.Project if registry == "project" else ln.Record + resolve = resolve_note_project if registry == "project" else resolve_note_record record_uid = parse_lamindb_record_uid(content) if record_uid is None: - note_record = resolve_note_record( + note = resolve( ln=ln, type_chain=type_chain, note_name=note_name, create_if_missing=True, ) - assert note_record is not None - return note_record + assert note is not None + return note - note_record = ln.Record.filter(uid=record_uid).one_or_none() - if note_record is None: + note = model.filter(uid=record_uid).one_or_none() + if note is None: raise click.ClickException( - f"Record with uid '{record_uid}' from lamindb.uid does not exist." + f"{kind.capitalize()} with uid '{record_uid}' from lamindb.uid does not exist." ) - if note_record.name == note_name: - return note_record + if note.name == note_name: + return note conflict = ( - ln.Record.filter(name=note_name, type=note_record.type) - .exclude(uid=note_record.uid) - .first() + model.filter(name=note_name, type=note.type).exclude(uid=note.uid).first() ) if conflict is not None: raise click.ClickException( - f"Cannot rename record '{note_record.name}' to '{note_name}': " - "a record with that name already exists under the same type." + f"Cannot rename {kind} '{note.name}' to '{note_name}': " + f"a {kind} with that name already exists under the same type." ) - previous_name = note_record.name - note_record.name = note_name - note_record.save() - logger.important(f"renamed record: {previous_name} → {note_name}") - return note_record + previous_name = note.name + note.name = note_name + note.save() + logger.important(f"renamed {kind}: {previous_name} → {note_name}") + return note def _save_note_markdown( ppath: Path, *, + registry: str, type_chain: list[str], note_name: str, ) -> None: @@ -226,30 +238,42 @@ def _save_note_markdown( from lamin_cli._annotate import _add_block content = ppath.read_text(encoding="utf-8") - note_record = _note_record_for_save( - ln, content=content, type_chain=type_chain, note_name=note_name + note = _note_for_save( + ln, + registry=registry, + content=content, + type_chain=type_chain, + note_name=note_name, ) - local_content = upsert_lamindb_record_uid(content, note_record.uid) + local_content = upsert_lamindb_record_uid(content, note.uid) if local_content != content: ppath.write_text(local_content, encoding="utf-8") block = _add_block( - note_record, - "record", + note, + registry, strip_lamindb_record_uid(local_content), kind="readme", ) - logger.important(f"saved note content: RecordBlock('{block.uid}')") + block_label = "ProjectBlock" if registry == "project" else "RecordBlock" + logger.important(f"saved note content: {block_label}('{block.uid}')") if ln.setup.settings.instance.is_remote: slug = ln.setup.settings.instance.slug ui_url = ln.setup.settings.instance.ui_url - logger.important(f"go to: {ui_url}/{slug}/record/{note_record.uid}") + logger.important(f"go to: {ui_url}/{slug}/{registry}/{note.uid}") -def _resolve_note_target_or_raise(ppath: Path) -> tuple[list[str], str]: +def _resolve_note_target_or_raise( + ppath: Path, *, registry: str +) -> tuple[list[str], str]: note_target = extract_note_target_from_path( ppath, dev_dir=ln_setup.settings.dev_dir ) if note_target is None: + if registry == "project": + raise click.ClickException( + "To save as project, pass a markdown path under dev-dir in the form " + "Projects/.md or Projects//.md." + ) raise click.ClickException( "To save as record, pass a markdown path under dev-dir in the form " "//.../.md or .md." @@ -368,35 +392,45 @@ def save( branch_record = ln_setup.settings.branch is_cloud_path = not isinstance(ppath, LocalPathClasses) + # Markdown under /Projects/ is a project. Other dev-dir notes are records. if ( isinstance(ppath, LocalPathClasses) and key is None and not saving_plan and not user_passed_registry and ppath.name != "README.md" - and extract_note_target_from_path( + ): + note_target = extract_note_target_from_path( Path(ppath), dev_dir=ln_setup.settings.dev_dir ) - is not None - ): - registry = "record" + if note_target is not None: + registry = "project" if is_project_note_target(note_target[0]) else "record" if store_kwargs is not None and registry != "artifact": raise click.ClickException( "--store-kwargs and --batch-size are only supported when saving artifacts" ) - if registry == "record": + if registry in {"record", "project"}: if ppath.suffix.lower() != ".md": raise click.ClickException( - "Saving as registry='record' requires a .md file." + f"Saving as registry='{registry}' requires a .md file." ) if key is not None: logger.warning( - "key is ignored for records, record identity comes from path" + f"key is ignored for {registry}s, {registry} identity comes from path" ) - type_chain, note_name = _resolve_note_target_or_raise(Path(ppath)) - _save_note_markdown(Path(ppath), type_chain=type_chain, note_name=note_name) + type_chain, note_name = _resolve_note_target_or_raise( + Path(ppath), registry=registry + ) + if registry == "project": + type_chain = project_type_chain(type_chain) + _save_note_markdown( + Path(ppath), + registry=registry, + type_chain=type_chain, + note_name=note_name, + ) return None if registry == "artifact": @@ -644,5 +678,5 @@ def save( return return_code else: raise click.ClickException( - "Allowed values for '--registry' are: 'artifact', 'transform', 'record'" + "Allowed values for '--registry' are: 'artifact', 'transform', 'record', 'project'" ) diff --git a/tests/core/test_save_notes.py b/tests/core/test_save_notes.py index 651b81d..b5e51df 100644 --- a/tests/core/test_save_notes.py +++ b/tests/core/test_save_notes.py @@ -1232,3 +1232,144 @@ def test_load_markdown_note_inserts_record_uid_when_missing(): type_record.refresh_from_db() type_record.delete(permanent=True) branch.delete(permanent=True) + + +def test_save_and_load_markdown_note_in_projects_folder_creates_project(): + unique = time.time_ns() + note_name = f"Cure51 data sharing {unique}" + renamed_name = f"Cure51 data sharing renamed {unique}" + branch = ln.Branch(name=f"cli_project_notes_branch_{unique}").save() + record_type = ln.Record(name="Projects", is_type=True).save() + notes_root = Path(__file__).parent / f"notes_projects_{unique}" + projects_dir = notes_root / "Projects" + projects_dir.mkdir(parents=True, exist_ok=True) + note_path = projects_dir / f"{note_name}.md" + note_path.write_text("# First version\n\nhello\n") + project = None + try: + ln.setup.switch(branch.name) + set_dev_dir = run_lamin("settings", "dev-dir", "set", str(notes_root)) + assert set_dev_dir.returncode == 0, set_dev_dir.stderr + + result = run_lamin("save", str(note_path), cwd=notes_root) + assert result.returncode == 0, ( + f"stdout: {result.stdout}\nstderr: {result.stderr}" + ) + assert "ProjectBlock" in result.stdout + + project = ln.Project.filter(name=note_name, type=None, branch=branch).one() + assert project.is_type is False + assert ( + ln.Record.filter( + name=note_name, type=record_type, branch=branch + ).one_or_none() + is None + ) + readmes = project.ablocks.filter(kind="readme").order_by("created_at") + assert readmes.count() == 1 + assert readmes.first().content.startswith("# First version") + assert "lamindb:" not in readmes.first().content + saved_text = note_path.read_text() + assert f"uid: {project.uid}" in saved_text + + note_path.write_text(saved_text.replace("# First version", "# Second version")) + result = run_lamin("save", str(note_path), cwd=notes_root) + assert result.returncode == 0, ( + f"stdout: {result.stdout}\nstderr: {result.stderr}" + ) + readmes = project.ablocks.filter(kind="readme").order_by("created_at") + assert readmes.count() == 2 + assert "Second version" in readmes.last().content + assert readmes.last().is_latest + + renamed_path = projects_dir / f"{renamed_name}.md" + note_path.rename(renamed_path) + result = run_lamin("save", str(renamed_path), cwd=notes_root) + assert result.returncode == 0, ( + f"stdout: {result.stdout}\nstderr: {result.stderr}" + ) + assert "renamed project" in result.stdout + project.refresh_from_db() + assert project.name == renamed_name + + renamed_path.unlink() + load_result = run_lamin("load", f"Projects/{renamed_name}.md", cwd=notes_root) + assert load_result.returncode == 0, ( + f"stdout: {load_result.stdout}\nstderr: {load_result.stderr}" + ) + loaded = renamed_path.read_text() + assert f"uid: {project.uid}" in loaded + assert "Second version" in loaded + assert "lamindb:" not in readmes.last().content + finally: + ln.setup.switch("main") + run_lamin("settings", "dev-dir", "unset", cwd=notes_root) + if notes_root.exists(): + shutil.rmtree(notes_root) + if project is not None and ( + ln.Project.filter(uid=project.uid, branch=branch).one_or_none() is not None + ): + project.refresh_from_db() + project.delete(permanent=True) + if ln.Record.filter(uid=record_type.uid).one_or_none() is not None: + record_type.refresh_from_db() + record_type.delete(permanent=True) + branch.delete(permanent=True) + + +def test_save_markdown_note_under_project_type(): + unique = time.time_ns() + note_name = f"typed-project-note-{unique}" + branch = ln.Branch(name=f"cli_project_type_notes_branch_{unique}").save() + project_type = ln.Project(name=f"Program{unique}", is_type=True).save() + notes_root = Path(__file__).parent / f"notes_project_type_{unique}" + note_path = notes_root / "Projects" / project_type.name / f"{note_name}.md" + note_path.parent.mkdir(parents=True, exist_ok=True) + note_path.write_text("# Typed project\n\ncontent\n") + project = None + try: + ln.setup.switch(branch.name) + set_dev_dir = run_lamin("settings", "dev-dir", "set", str(notes_root)) + assert set_dev_dir.returncode == 0, set_dev_dir.stderr + result = run_lamin("save", str(note_path), cwd=notes_root) + assert result.returncode == 0, ( + f"stdout: {result.stdout}\nstderr: {result.stderr}" + ) + project = ln.Project.filter( + name=note_name, type=project_type, branch=branch + ).one() + assert project.is_type is False + readme = project.ablocks.filter(kind="readme").one() + assert "Typed project" in readme.content + finally: + ln.setup.switch("main") + run_lamin("settings", "dev-dir", "unset", cwd=notes_root) + if notes_root.exists(): + shutil.rmtree(notes_root) + if project is not None and ( + ln.Project.filter(uid=project.uid, branch=branch).one_or_none() is not None + ): + project.refresh_from_db() + project.delete(permanent=True) + if ln.Project.filter(uid=project_type.uid).one_or_none() is not None: + project_type.refresh_from_db() + project_type.delete(permanent=True) + branch.delete(permanent=True) + + +def test_save_markdown_note_missing_project_type_errors(): + unique = time.time_ns() + notes_root = Path(__file__).parent / f"notes_missing_project_type_{unique}" + note_path = notes_root / "Projects" / f"MissingType{unique}" / "note.md" + note_path.parent.mkdir(parents=True, exist_ok=True) + note_path.write_text("# Missing project type\n") + try: + set_dev_dir = run_lamin("settings", "dev-dir", "set", str(notes_root)) + assert set_dev_dir.returncode == 0, set_dev_dir.stderr + result = run_lamin("save", str(note_path), cwd=notes_root) + assert result.returncode == 1 + assert f"Project type 'MissingType{unique}' not found" in result.stderr + finally: + run_lamin("settings", "dev-dir", "unset", cwd=notes_root) + if notes_root.exists(): + shutil.rmtree(notes_root)