From 548862c3e3d96d3095d3f471df4b15fd20540fb1 Mon Sep 17 00:00:00 2001 From: Felipe Coutinho Date: Tue, 19 May 2026 11:54:35 -0300 Subject: [PATCH 1/4] Implement branches Fix freeze --- pyproject.toml | 1 + src/noworkflow/now/cmd/__init__.py | 4 + src/noworkflow/now/cmd/cmd_branch.py | 72 ++++++ src/noworkflow/now/cmd/cmd_restore.py | 5 +- src/noworkflow/now/cmd/cmd_run.py | 6 +- .../now/persistence/content/base.py | 2 +- .../now/persistence/content/dulwich_engine.py | 229 +++++++++++++++++- .../now/persistence/content/gitbase.py | 4 +- .../now/persistence/content/parallel.py | 22 +- .../now/persistence/content/plain_engine.py | 2 +- .../now/persistence/models/trial.py | 27 ++- 11 files changed, 356 insertions(+), 18 deletions(-) create mode 100644 src/noworkflow/now/cmd/cmd_branch.py diff --git a/pyproject.toml b/pyproject.toml index f2b1de92..248b4d44 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,7 @@ dependencies = [ "setuptools<72", "nbformat", "graphviz", + "dulwich" ] [project.optional-dependencies] diff --git a/src/noworkflow/now/cmd/__init__.py b/src/noworkflow/now/cmd/__init__.py index a561982e..e13a2765 100644 --- a/src/noworkflow/now/cmd/__init__.py +++ b/src/noworkflow/now/cmd/__init__.py @@ -29,6 +29,7 @@ from .cmd_gc import GC from .cmd_evaluation import Evaluation from .cmd_clean import Clean +from .cmd_branch import Branch from ..utils.io import print_msg @@ -59,7 +60,9 @@ def main(): GC(), Evaluation(), Clean(), + Branch() ] + for cmd in commands: cmd.create_parser(subparsers) @@ -95,4 +98,5 @@ def main(): "Push", "Pull", "Import", + "Branch", ] diff --git a/src/noworkflow/now/cmd/cmd_branch.py b/src/noworkflow/now/cmd/cmd_branch.py new file mode 100644 index 00000000..8c9f72e0 --- /dev/null +++ b/src/noworkflow/now/cmd/cmd_branch.py @@ -0,0 +1,72 @@ +# Copyright (c) 2026 Universidade Federal Fluminense (UFF) +# This file is part of noWorkflow. +# Please, consult the license terms in the LICENSE file. +"""'now branch' command""" +from __future__ import (absolute_import, print_function, + division, unicode_literals) + +import os + +from ..persistence import persistence_config, content +from ..persistence.models import Trial +from ..utils.io import print_msg + +from .command import Command + + +class Branch(Command): + """Manage noWorkflow Git branches""" + + def add_arguments(self): + add_arg = self.add_argument + add_arg("--dir", type=str, + help="set project path where is the database. Default to " + "current directory") + add_arg("--content-engine", type=str, + help="set the content database engine") + + subparsers = self.parser.add_subparsers(dest="branch_cmd") + + subparsers.add_parser("list", help="list branches") + subparsers.add_parser("current", help="show current branch") + + switch = subparsers.add_parser("switch", help="switch branch") + switch.add_argument("name", type=str) + + rename = subparsers.add_parser("rename", help="rename branch") + rename.add_argument("old", type=str) + rename.add_argument("new", type=str) + + def _connect(self, args): + persistence_config.content_engine = args.content_engine + persistence_config.connect_existing(args.dir or os.getcwd()) + if not hasattr(content.content_database_engine, "branches"): + raise RuntimeError( + "Git branch prototype requires a Git content engine" + ) + + def execute(self, args): + self._connect(args) + command = args.branch_cmd or "list" + + if command == "list": + current = content.current_branch() + for name in content.branches(): + marker = "*" if name == current else " " + trial_id = content.branch_trial(name) or "" + print("{} {} {}".format(marker, name, trial_id)) + elif command == "current": + trial_id = content.current_branch_trial() or "" + print("{} {}".format(content.current_branch(), trial_id)) + elif command == "switch": + content.checkout_branch(args.name) + trial_id = content.branch_trial(args.name) + if trial_id: + Trial(trial_id).create_head() + print_msg("Switched to branch {}".format(args.name), True) + elif command == "rename": + content.rename_branch(args.old, args.new) + print_msg( + "Renamed branch {} to {}".format(args.old, args.new), + True + ) diff --git a/src/noworkflow/now/cmd/cmd_restore.py b/src/noworkflow/now/cmd/cmd_restore.py index a1b1e472..cc4cc829 100644 --- a/src/noworkflow/now/cmd/cmd_restore.py +++ b/src/noworkflow/now/cmd/cmd_restore.py @@ -126,7 +126,10 @@ def create_backup(self, metascript, files, args): print_msg("Backup Trial {} created".format(metascript.trial_id), self.print_msg) - content.commit_content(metascript.message or "Backup Trial {}".format(metascript.trial_id)) + content.commit_content( + metascript.message or "Backup Trial {}".format(metascript.trial_id), + trial_id=metascript.trial_id + ) def restore(self, path, code_hash, trial_id, mode="normal"): """Restore file with from """ diff --git a/src/noworkflow/now/cmd/cmd_run.py b/src/noworkflow/now/cmd/cmd_run.py index 0d827c95..d7bd4dde 100644 --- a/src/noworkflow/now/cmd/cmd_run.py +++ b/src/noworkflow/now/cmd/cmd_run.py @@ -67,9 +67,13 @@ def run(metascript, args=None): Tag.create_automatic_tag(*metascript.create_automatic_tag_args()) Trial.set_user_based_on_env(metascript.trial_id) metaprofiler.meta_profiler.save() - content.commit_content(metascript.message or "Trial {}".format(metascript.trial_id)) + content.commit_content( + metascript.message or "Trial {}".format(metascript.trial_id), + trial_id=metascript.trial_id + ) finally: metascript.create_last() + content.close() class Run(Command): """Run a script collecting its provenance""" diff --git a/src/noworkflow/now/persistence/content/base.py b/src/noworkflow/now/persistence/content/base.py index a14a8cee..403f6426 100644 --- a/src/noworkflow/now/persistence/content/base.py +++ b/src/noworkflow/now/persistence/content/base.py @@ -60,7 +60,7 @@ def gc(self, content_hash): """Collect garbage from database""" raise NotImplementedError("Implement in subclass") - def commit_content(self, message): + def commit_content(self, message, trial_id=None): """Commit content""" raise NotImplementedError("Implement in subclass") diff --git a/src/noworkflow/now/persistence/content/dulwich_engine.py b/src/noworkflow/now/persistence/content/dulwich_engine.py index fe13a108..4e8c7cd8 100644 --- a/src/noworkflow/now/persistence/content/dulwich_engine.py +++ b/src/noworkflow/now/persistence/content/dulwich_engine.py @@ -2,6 +2,7 @@ import hashlib import time import io +import posixpath from dulwich import file from dulwich.repo import Repo @@ -29,9 +30,12 @@ def connect(self, config): os.makedirs(self.content_path) Repo.init_bare(self.content_path) self.repo = Repo(self.content_path) + self._commit_ref = self.branch_ref(self._default_branch) + self._set_head(self._commit_ref) self.create_initial_commit() else: self.repo = Repo(self.content_path) + self._commit_ref = self._current_branch_ref() @staticmethod def do_put(content_path, object_hashes, lock, content, filename): @@ -81,12 +85,12 @@ def create_initial_commit(self): object_store.add_object(empty_tree) self.create_commit_object(self._initial_message, empty_tree.id) - def create_commit_object(self, message, tree): + def create_commit_object(self, message, tree, trial_id=None): """Create a commit object""" with self.use_safe_open(): - master_ref = self.repo.get_refs().get( - self._commit_ref.encode("utf-8"), None - ) + self._commit_ref = self._current_branch_ref() + branch_ref = self._commit_ref.encode("utf-8") + master_ref = self.repo.get_refs().get(branch_ref, None) commit = Commit() if master_ref is not None: @@ -102,12 +106,19 @@ def create_commit_object(self, message, tree): commit.message = message.encode() self.repo.object_store.add_object(commit) - self.repo.refs[ - self._commit_ref.encode("utf-8") - ] = commit.id + self.repo.refs[branch_ref] = commit.id + if trial_id: + self.repo.refs[self.trial_ref(trial_id).encode("utf-8")] = commit.id return commit.id + def commit_content(self, message, trial_id=None): + """Commit the current files and update branch/trial refs""" + commit_id = super(DulwichEngine, self).commit_content(message) + if trial_id: + self.repo.refs[self.trial_ref(trial_id).encode("utf-8")] = commit_id + return commit_id + def new_tree(self, parent): """Create new git tree""" return Tree() @@ -126,6 +137,210 @@ def write_tree(self, tree): self.repo.object_store.add_object(tree) return tree.id + # Branch prototype API + + @staticmethod + def _to_text(value): + if value is None: + return None + if isinstance(value, bytes): + return value.decode("utf-8") + return value + + @staticmethod + def _to_hex(value): + if value is None: + return None + if isinstance(value, bytes): + return value.decode("ascii") + return str(value) + + @staticmethod + def _valid_branch_name(name): + return ( + name and + ".." not in name and + not name.startswith("/") and + not name.endswith("/") and + not name.endswith(".lock") and + all(ch not in name for ch in " ~^:?*[\\") + ) + + def branch_ref(self, name): + """Return a full Git branch ref""" + if not self._valid_branch_name(name): + raise RuntimeError("invalid branch name: {}".format(name)) + return "refs/heads/{}".format(name) + + def trial_ref(self, trial_id): + """Return a full noWorkflow trial ref""" + return self._trial_ref_prefix + str(trial_id) + + def _set_head(self, branch_ref): + """Point HEAD at a branch ref""" + ref = branch_ref.encode("utf-8") + try: + self.repo.refs.set_symbolic_ref(b"HEAD", ref) + except AttributeError: + with open(os.path.join(self.content_path, "HEAD"), "wb") as fil: + fil.write(b"ref: " + ref + b"\n") + self._commit_ref = branch_ref + + def _current_branch_ref(self): + """Return current branch ref, upgrading old stores lazily""" + try: + head = self.repo.refs.read_ref(b"HEAD") + except (AttributeError, KeyError): + head = None + head = self._to_text(head) + if head and head.startswith("refs/heads/"): + return head + + refs = self.repo.get_refs() + for candidate in ("refs/heads/master", "refs/heads/main"): + if candidate.encode("utf-8") in refs: + self._set_head(candidate) + return candidate + + branch_ref = self.branch_ref(self._default_branch) + self._set_head(branch_ref) + return branch_ref + + def current_branch(self): + """Return the current branch name""" + ref = self._current_branch_ref() + return ref.rsplit("/", 1)[-1] + + def branches(self): + """Return branch names in content.git""" + result = [] + for ref in self.repo.get_refs(): + ref = self._to_text(ref) + if ref.startswith("refs/heads/"): + result.append(ref.rsplit("/", 1)[-1]) + return sorted(result) + + def branch_commit(self, name=None): + """Return commit id for a branch""" + branch_ref = self.branch_ref(name) if name else self._current_branch_ref() + commit_id = self.repo.get_refs().get(branch_ref.encode("utf-8")) + return self._to_hex(commit_id) + + def trial_commit(self, trial_id): + """Return commit id for a trial ref""" + commit_id = self.repo.get_refs().get(self.trial_ref(trial_id).encode("utf-8")) + print(commit_id) + return self._to_hex(commit_id) + + def trial_for_commit(self, commit_id): + """Find trial id by commit id""" + if not commit_id: + return None + wanted = self._to_hex(commit_id) + prefix = self._trial_ref_prefix + for ref, value in self.repo.get_refs().items(): + ref = self._to_text(ref) + if ref.startswith(prefix) and self._to_hex(value) == wanted: + return ref[len(prefix):] + return None + + def current_branch_head(self): + """Return trial id at the current branch head""" + return self.trial_for_commit(self.branch_commit()) + + def branch_trial(self, name): + """Return trial id at the named branch head""" + return self.trial_for_commit(self.branch_commit(name)) + + def create_branch(self, name, commit_id): + """Create branch pointing to commit id""" + if not commit_id: + raise RuntimeError("cannot create branch without a commit") + branch_ref = self.branch_ref(name) + encoded_ref = branch_ref.encode("utf-8") + if encoded_ref in self.repo.get_refs(): + raise RuntimeError("branch already exists: {}".format(name)) + self.repo.refs[encoded_ref] = commit_id.encode("ascii") + + def switch_branch(self, name): + """Switch HEAD to branch""" + branch_ref = self.branch_ref(name) + if branch_ref.encode("utf-8") not in self.repo.get_refs(): + raise RuntimeError("branch not found: {}".format(name)) + self._set_head(branch_ref) + + def rename_branch(self, old, new): + """Rename a branch ref""" + old_ref = self.branch_ref(old) + new_ref = self.branch_ref(new) + refs = self.repo.get_refs() + old_key = old_ref.encode("utf-8") + new_key = new_ref.encode("utf-8") + if old_key not in refs: + raise RuntimeError("branch not found: {}".format(old)) + if new_key in refs: + raise RuntimeError("branch already exists: {}".format(new)) + self.repo.refs[new_key] = refs[old_key] + del self.repo.refs[old_key] + if self._current_branch_ref() == old_ref: + self._set_head(new_ref) + + def ensure_branch_for_trial(self, trial_id): + """Create and switch to an automatic branch for a restored trial""" + current_trial = self.current_branch_trial() + if current_trial == str(trial_id): + return self.current_branch() + + commit_id = self.trial_commit(trial_id) + if not commit_id: + return self.current_branch() + + base = "now-diverge-{}".format(str(trial_id)[:8]) + name = base + index = 1 + existing = set(self.branches()) + while name in existing: + index += 1 + name = "{}-{}".format(base, index) + self.create_branch(name, commit_id) + self.switch_branch(name) + return name + + def checkout_branch(self, name): + """Switch branch and restore versioned files from its tree""" + self.switch_branch(name) + commit_id = self.branch_commit(name) + if not commit_id: + return + commit = self.repo[commit_id.encode("ascii")] + self._checkout_tree(commit.tree, "") + + def _tree_items(self, tree): + if hasattr(tree, "iteritems"): + return tree.iteritems() + return tree.items() + + def _checkout_tree(self, tree_id, prefix): + tree = self.repo[tree_id] + for item in self._tree_items(tree): + if len(item) == 3: + name, mode, sha = item + else: + name, entry = item + mode, sha = entry.mode, entry.sha + name = self._to_text(name) + relative = posixpath.join(prefix, name) if prefix else name + obj = self.repo[sha] + if isinstance(obj, Tree): + self._checkout_tree(sha, relative) + else: + path = os.path.join(self.base_path, *relative.split("/")) + parent = os.path.dirname(path) + if parent: + os.makedirs(parent, exist_ok=True) + with self.std_open(path, "wb") as fil: + fil.write(obj.as_raw_string()) + DistributedDulwichEngine = create_distributed(DulwichEngine) PoolDulwichEngine = create_pool(DulwichEngine) diff --git a/src/noworkflow/now/persistence/content/gitbase.py b/src/noworkflow/now/persistence/content/gitbase.py index eb224d04..54fbb4b4 100644 --- a/src/noworkflow/now/persistence/content/gitbase.py +++ b/src/noworkflow/now/persistence/content/gitbase.py @@ -17,6 +17,8 @@ def __init__(self, config): self._commit_name = 'Noworkflow' self._commit_email = 'noworkflow@noworkflow.com' self._commit_ref = 'refs/heads/master' + self._default_branch = 'main' + self._trial_ref_prefix = 'refs/noworkflow/trials/' self._initial_message = "Initial Commit" self.name_counter = Counter() self.base_path = None @@ -32,7 +34,7 @@ def set_path(self, config): def gc(self, aggressive=False): git_system.garbage_collection(self.content_path, aggressive) - def commit_content(self, message): + def commit_content(self, message, trial_id=None): """Commit the current files of content database""" self.close() trees = {'': self.new_tree('')} diff --git a/src/noworkflow/now/persistence/content/parallel.py b/src/noworkflow/now/persistence/content/parallel.py index 61b1579c..5d76b31e 100644 --- a/src/noworkflow/now/persistence/content/parallel.py +++ b/src/noworkflow/now/persistence/content/parallel.py @@ -24,8 +24,10 @@ def run(self): self.task_queue.task_done() break - self.engine.do_put(*queue_content) - self.task_queue.task_done() + try: + self.engine.do_put(*queue_content) + finally: + self.task_queue.task_done() class Distributed(cls): @@ -50,6 +52,7 @@ def start_processes(self): with safeopen.use_safe_open(): for _ in range(self.num_consumers): consumer = Worker(self.tasks, self) + consumer.daemon = True self.consumers.append(consumer) consumer.start() @@ -69,9 +72,18 @@ def close(self): for _ in range(self.num_consumers): self.tasks.put(None) - # Wait for all of the tasks to finish - self.tasks.join() - self.processes_started = False + try: + # Wait for all of the tasks to finish + self.tasks.join() + for consumer in self.consumers: + consumer.join() + finally: + for consumer in self.consumers: + if consumer.is_alive(): + consumer.terminate() + consumer.join() + self.consumers = [] + self.processes_started = False Distributed.__name__ = name or ("Distributed" + cls.__name__) return Distributed diff --git a/src/noworkflow/now/persistence/content/plain_engine.py b/src/noworkflow/now/persistence/content/plain_engine.py index a7e0143e..544abd9a 100644 --- a/src/noworkflow/now/persistence/content/plain_engine.py +++ b/src/noworkflow/now/persistence/content/plain_engine.py @@ -68,7 +68,7 @@ def gc(self, content_hash): """Do nothing for plain storage""" pass - def commit_content(self, message): + def commit_content(self, message, trial_id=None): """Do nothing for plain storage""" pass diff --git a/src/noworkflow/now/persistence/models/trial.py b/src/noworkflow/now/persistence/models/trial.py index 1d021121..f86ecf3e 100644 --- a/src/noworkflow/now/persistence/models/trial.py +++ b/src/noworkflow/now/persistence/models/trial.py @@ -1062,6 +1062,31 @@ def load_parent(cls, script, remove=True, check=False, session=None): session=session) return proxy(trial) + @classmethod + def load_branch_parent(cls, script, session=None): + """Load parent from Git branch metadata, falling back to Head/last trial""" + session = session or relational.session + head = Head.load_head(script, session=session) + if head: + trial = head.trial + try: + content.ensure_branch_for_trial(trial.id) + except AttributeError: + pass + Head.remove(head.id, session=relational.make_session()) + return trial + + try: + trial_id = content.current_branch_trial() + except AttributeError: + trial_id = None + if trial_id: + trial = cls.load_trial(trial_id, session=session) + if trial: + return proxy(trial) + + return proxy(cls.last_trial(script=script, check=True, session=session)) + @classmethod # query def fast_last_trial_id(cls, session=None): """Load last trial id that did not bypass modules @@ -1228,7 +1253,7 @@ def create(cls, script, start, command, path, bypass_modules, session=None): session = session or relational.session # ToDo: use core query - parent = cls.load_parent(script, check=True) + parent = cls.load_branch_parent(script, session=session) parent_id = parent.id if parent else None inherited_id = None From b8f518c6eb303e2d6d32b1501a28077aaa2e319a Mon Sep 17 00:00:00 2001 From: Felipe Coutinho Date: Thu, 21 May 2026 17:21:16 -0300 Subject: [PATCH 2/4] Refactor --- src/noworkflow/now/cmd/cmd_branch.py | 5 +-- .../now/persistence/content/dulwich_engine.py | 44 ++++++++----------- .../now/persistence/content/gitbase.py | 2 +- .../now/persistence/models/trial.py | 36 +++------------ 4 files changed, 28 insertions(+), 59 deletions(-) diff --git a/src/noworkflow/now/cmd/cmd_branch.py b/src/noworkflow/now/cmd/cmd_branch.py index 8c9f72e0..c459ed95 100644 --- a/src/noworkflow/now/cmd/cmd_branch.py +++ b/src/noworkflow/now/cmd/cmd_branch.py @@ -66,7 +66,4 @@ def execute(self, args): print_msg("Switched to branch {}".format(args.name), True) elif command == "rename": content.rename_branch(args.old, args.new) - print_msg( - "Renamed branch {} to {}".format(args.old, args.new), - True - ) + print_msg("Renamed branch {} to {}".format(args.old, args.new), True) diff --git a/src/noworkflow/now/persistence/content/dulwich_engine.py b/src/noworkflow/now/persistence/content/dulwich_engine.py index 4e8c7cd8..6db53eba 100644 --- a/src/noworkflow/now/persistence/content/dulwich_engine.py +++ b/src/noworkflow/now/persistence/content/dulwich_engine.py @@ -192,13 +192,13 @@ def _current_branch_ref(self): head = self.repo.refs.read_ref(b"HEAD") except (AttributeError, KeyError): head = None - head = self._to_text(head) + head = head.decode("utf-8") if head and head.startswith("refs/heads/"): return head refs = self.repo.get_refs() for candidate in ("refs/heads/master", "refs/heads/main"): - if candidate.encode("utf-8") in refs: + if candidate in refs: self._set_head(candidate) return candidate @@ -215,24 +215,16 @@ def branches(self): """Return branch names in content.git""" result = [] for ref in self.repo.get_refs(): - ref = self._to_text(ref) if ref.startswith("refs/heads/"): result.append(ref.rsplit("/", 1)[-1]) return sorted(result) - def branch_commit(self, name=None): - """Return commit id for a branch""" - branch_ref = self.branch_ref(name) if name else self._current_branch_ref() - commit_id = self.repo.get_refs().get(branch_ref.encode("utf-8")) - return self._to_hex(commit_id) - def trial_commit(self, trial_id): + def get_commit_id_by_trial_id(self, trial_id): """Return commit id for a trial ref""" - commit_id = self.repo.get_refs().get(self.trial_ref(trial_id).encode("utf-8")) - print(commit_id) - return self._to_hex(commit_id) + return self.repo.get_refs().get(self.trial_ref(trial_id).encode("utf-8")) - def trial_for_commit(self, commit_id): + def get_trial_id_by_commit_id(self, commit_id): """Find trial id by commit id""" if not commit_id: return None @@ -240,32 +232,35 @@ def trial_for_commit(self, commit_id): prefix = self._trial_ref_prefix for ref, value in self.repo.get_refs().items(): ref = self._to_text(ref) - if ref.startswith(prefix) and self._to_hex(value) == wanted: + if ref.startswith(prefix) and value == commit_id: return ref[len(prefix):] return None - def current_branch_head(self): - """Return trial id at the current branch head""" - return self.trial_for_commit(self.branch_commit()) + def get_branch_head_trial_id(self, name=None): + """Return trial_id at the named branch head""" + """If no name is given, return trial_id current branch head""" + return self.get_trial_id_by_commit_id(self.get_branch_head_commit_id(name)) - def branch_trial(self, name): - """Return trial id at the named branch head""" - return self.trial_for_commit(self.branch_commit(name)) + def get_branch_head_commit_id(self, name=None): + """Return commit_id at the named branch head""" + """If no name is given, return commit_id current branch head""" + branch_ref = self.branch_ref(name) if name else self._current_branch_ref() + commit_id = self.repo.get_refs().get(branch_ref.encode("utf-8")) + return commit_id def create_branch(self, name, commit_id): """Create branch pointing to commit id""" if not commit_id: raise RuntimeError("cannot create branch without a commit") branch_ref = self.branch_ref(name) - encoded_ref = branch_ref.encode("utf-8") - if encoded_ref in self.repo.get_refs(): + if branch_ref in self.repo.get_refs(): raise RuntimeError("branch already exists: {}".format(name)) - self.repo.refs[encoded_ref] = commit_id.encode("ascii") + self.repo.refs[branch_ref] = commit_id.encode("ascii") def switch_branch(self, name): """Switch HEAD to branch""" branch_ref = self.branch_ref(name) - if branch_ref.encode("utf-8") not in self.repo.get_refs(): + if branch_ref not in self.repo.get_refs(): raise RuntimeError("branch not found: {}".format(name)) self._set_head(branch_ref) @@ -328,7 +323,6 @@ def _checkout_tree(self, tree_id, prefix): else: name, entry = item mode, sha = entry.mode, entry.sha - name = self._to_text(name) relative = posixpath.join(prefix, name) if prefix else name obj = self.repo[sha] if isinstance(obj, Tree): diff --git a/src/noworkflow/now/persistence/content/gitbase.py b/src/noworkflow/now/persistence/content/gitbase.py index 54fbb4b4..8e34fc11 100644 --- a/src/noworkflow/now/persistence/content/gitbase.py +++ b/src/noworkflow/now/persistence/content/gitbase.py @@ -17,7 +17,7 @@ def __init__(self, config): self._commit_name = 'Noworkflow' self._commit_email = 'noworkflow@noworkflow.com' self._commit_ref = 'refs/heads/master' - self._default_branch = 'main' + self._default_branch = 'master' self._trial_ref_prefix = 'refs/noworkflow/trials/' self._initial_message = "Initial Commit" self.name_counter = Counter() diff --git a/src/noworkflow/now/persistence/models/trial.py b/src/noworkflow/now/persistence/models/trial.py index f86ecf3e..a62a563d 100644 --- a/src/noworkflow/now/persistence/models/trial.py +++ b/src/noworkflow/now/persistence/models/trial.py @@ -1052,40 +1052,18 @@ def load_parent(cls, script, remove=True, check=False, session=None): """ session = session or relational.session head = Head.load_head(script, session=session) + if head: + content.ensure_branch_for_trial(head.trial.id) trial = head.trial if remove: Head.remove(head.id, session=relational.make_session()) - elif not head: - trial = cls.last_trial( - script=script, check=check, - session=session) - return proxy(trial) - - @classmethod - def load_branch_parent(cls, script, session=None): - """Load parent from Git branch metadata, falling back to Head/last trial""" - session = session or relational.session - head = Head.load_head(script, session=session) - if head: - trial = head.trial - try: - content.ensure_branch_for_trial(trial.id) - except AttributeError: - pass - Head.remove(head.id, session=relational.make_session()) - return trial + return proxy(trial) - try: - trial_id = content.current_branch_trial() - except AttributeError: - trial_id = None - if trial_id: - trial = cls.load_trial(trial_id, session=session) - if trial: - return proxy(trial) + trial_id = content.get_branch_head_trial_id() + trial = cls.load_trial(trial_id, session=session) + return proxy(trial) - return proxy(cls.last_trial(script=script, check=True, session=session)) @classmethod # query def fast_last_trial_id(cls, session=None): @@ -1253,7 +1231,7 @@ def create(cls, script, start, command, path, bypass_modules, session=None): session = session or relational.session # ToDo: use core query - parent = cls.load_branch_parent(script, session=session) + parent = cls.load_parent(script, check=True) parent_id = parent.id if parent else None inherited_id = None From 24c30a6fea3a13341412af80b9bc488fd4d13520 Mon Sep 17 00:00:00 2001 From: Felipe Coutinho Date: Mon, 1 Jun 2026 15:42:24 -0300 Subject: [PATCH 3/4] Fix implementation --- src/noworkflow/now/cmd/cmd_run.py | 3 +- .../now/persistence/content/dulwich_engine.py | 247 +++++++----------- .../now/persistence/content/gitbase.py | 2 + 3 files changed, 102 insertions(+), 150 deletions(-) diff --git a/src/noworkflow/now/cmd/cmd_run.py b/src/noworkflow/now/cmd/cmd_run.py index d7bd4dde..6c9bf547 100644 --- a/src/noworkflow/now/cmd/cmd_run.py +++ b/src/noworkflow/now/cmd/cmd_run.py @@ -67,6 +67,7 @@ def run(metascript, args=None): Tag.create_automatic_tag(*metascript.create_automatic_tag_args()) Trial.set_user_based_on_env(metascript.trial_id) metaprofiler.meta_profiler.save() + content.commit_content( metascript.message or "Trial {}".format(metascript.trial_id), trial_id=metascript.trial_id @@ -159,7 +160,7 @@ def add_arguments(self): help="add a message to the commit of the trial") add_arg("--content-engine", type=str, help="set the content database engine") - + # Internal add_cmd("--create_last", action="store_true", help=argparse.SUPPRESS) diff --git a/src/noworkflow/now/persistence/content/dulwich_engine.py b/src/noworkflow/now/persistence/content/dulwich_engine.py index 6db53eba..955c8d6c 100644 --- a/src/noworkflow/now/persistence/content/dulwich_engine.py +++ b/src/noworkflow/now/persistence/content/dulwich_engine.py @@ -15,6 +15,7 @@ from .gitbase import GitContentDatabaseEngine from .parallel import create_distributed, create_pool, create_threading, NullLock from . import safeopen +from ..models import Trial class DulwichEngine(GitContentDatabaseEngine): @@ -83,18 +84,30 @@ def create_initial_commit(self): object_store = self.repo.object_store empty_tree = Tree() object_store.add_object(empty_tree) - self.create_commit_object(self._initial_message, empty_tree.id) + commit_id = self.create_commit_object(self._initial_message, empty_tree.id) + self.repo.refs[(self._commit_ref).encode("utf-8")] = commit_id + self.switch_branch(self._default_branch) def create_commit_object(self, message, tree, trial_id=None): """Create a commit object""" with self.use_safe_open(): - self._commit_ref = self._current_branch_ref() - branch_ref = self._commit_ref.encode("utf-8") - master_ref = self.repo.get_refs().get(branch_ref, None) - commit = Commit() - if master_ref is not None: - commit.parents = [master_ref] + + if trial_id != None: + current_branch, current_branch_ref = self.current_branch() + branch_head_commit_id = self.get_branch_head_commit_id(current_branch) + branch_head_trial_id = self.get_branch_head_trial_id(current_branch) + + parent_trial_id = Trial(trial_id).parent_id + + if parent_trial_id and branch_head_trial_id != parent_trial_id: + new_branch = self.create_branch(branch_head_commit_id) + self.switch_branch(new_branch) + else: + self._set_head(current_branch_ref) + + if branch_head_commit_id is not None: + commit.parents = [branch_head_commit_id] commit.tree = tree author = (self._commit_name + " <" + self._commit_email + ">").encode() @@ -106,17 +119,21 @@ def create_commit_object(self, message, tree, trial_id=None): commit.message = message.encode() self.repo.object_store.add_object(commit) - self.repo.refs[branch_ref] = commit.id - if trial_id: - self.repo.refs[self.trial_ref(trial_id).encode("utf-8")] = commit.id + + self.repo.refs[ + self._commit_ref.encode("utf-8") + ] = commit.id + + self.repo.refs[ + self.trial_ref(trial_id).encode("utf-8") + ] = commit.id return commit.id def commit_content(self, message, trial_id=None): """Commit the current files and update branch/trial refs""" - commit_id = super(DulwichEngine, self).commit_content(message) - if trial_id: - self.repo.refs[self.trial_ref(trial_id).encode("utf-8")] = commit_id + empty_tree = Tree() + commit_id = self.create_commit_object(message, empty_tree.id, trial_id) return commit_id def new_tree(self, parent): @@ -137,8 +154,6 @@ def write_tree(self, tree): self.repo.object_store.add_object(tree) return tree.id - # Branch prototype API - @staticmethod def _to_text(value): if value is None: @@ -155,21 +170,53 @@ def _to_hex(value): return value.decode("ascii") return str(value) - @staticmethod - def _valid_branch_name(name): - return ( - name and - ".." not in name and - not name.startswith("/") and - not name.endswith("/") and - not name.endswith(".lock") and - all(ch not in name for ch in " ~^:?*[\\") - ) + def get_commit_id_by_trial_id(self, trial_id): + """Return commit id for a trial ref""" + return self.repo.get_refs().get(self.trial_ref(trial_id).encode("utf-8")) + + def get_trial_id_by_commit_id(self, commit_id): + """Find trial id by commit id""" + if not commit_id: + return None + wanted = self._to_hex(commit_id) + + prefix = self._trial_ref_prefix + + with self.use_safe_open(): + for ref, value in self.repo.get_refs().items(): + ref = self._to_text(ref) + if ref.startswith(prefix) and value == commit_id: + return ref[len(prefix):] + return None + + def get_branch_head_trial_id(self, name=None): + """Return trial_id at the named branch head""" + """If no name is given, return trial_id current branch head""" + return self.get_trial_id_by_commit_id(self.get_branch_head_commit_id(name)) + + def get_branch_head_commit_id(self, name=None): + """Return commit_id at the named branch head""" + """If no name is given, return commit_id current branch head""" + with self.use_safe_open(): + if name != None: + branch_ref = self.branch_ref(name) + else: + branch_name, branch_ref = self.current_branch() + + commit_id = self.repo.get_refs().get(branch_ref.encode("utf-8")) + return commit_id + + def branches(self): + """Return branch names in content.git""" + result = [] + for ref in self.repo.get_refs(): + ref = self._to_text(ref) + if ref.startswith("refs/heads/"): + result.append(ref.rsplit("/", 1)[-1]) + return sorted(result) def branch_ref(self, name): """Return a full Git branch ref""" - if not self._valid_branch_name(name): - raise RuntimeError("invalid branch name: {}".format(name)) return "refs/heads/{}".format(name) def trial_ref(self, trial_id): @@ -186,81 +233,37 @@ def _set_head(self, branch_ref): fil.write(b"ref: " + ref + b"\n") self._commit_ref = branch_ref - def _current_branch_ref(self): - """Return current branch ref, upgrading old stores lazily""" - try: - head = self.repo.refs.read_ref(b"HEAD") - except (AttributeError, KeyError): - head = None - head = head.decode("utf-8") - if head and head.startswith("refs/heads/"): - return head - - refs = self.repo.get_refs() - for candidate in ("refs/heads/master", "refs/heads/main"): - if candidate in refs: - self._set_head(candidate) - return candidate - - branch_ref = self.branch_ref(self._default_branch) - self._set_head(branch_ref) - return branch_ref - def current_branch(self): - """Return the current branch name""" - ref = self._current_branch_ref() - return ref.rsplit("/", 1)[-1] - - def branches(self): - """Return branch names in content.git""" - result = [] - for ref in self.repo.get_refs(): - if ref.startswith("refs/heads/"): - result.append(ref.rsplit("/", 1)[-1]) - return sorted(result) - - - def get_commit_id_by_trial_id(self, trial_id): - """Return commit id for a trial ref""" - return self.repo.get_refs().get(self.trial_ref(trial_id).encode("utf-8")) + """Return the current branch name and ref""" + with self.use_safe_open(): + ref_chain, commit_sha = self.repo.refs.follow(b'HEAD') + full_name = ref_chain[1].decode('utf-8') + return full_name.rsplit("/")[-1], full_name - def get_trial_id_by_commit_id(self, commit_id): - """Find trial id by commit id""" + def create_branch(self, commit_id): + """Create branch pointing to commit id""" if not commit_id: - return None - wanted = self._to_hex(commit_id) - prefix = self._trial_ref_prefix - for ref, value in self.repo.get_refs().items(): - ref = self._to_text(ref) - if ref.startswith(prefix) and value == commit_id: - return ref[len(prefix):] - return None + raise RuntimeError("cannot create branch without a commit") - def get_branch_head_trial_id(self, name=None): - """Return trial_id at the named branch head""" - """If no name is given, return trial_id current branch head""" - return self.get_trial_id_by_commit_id(self.get_branch_head_commit_id(name)) + trial_id = self.get_trial_id_by_commit_id(commit_id) - def get_branch_head_commit_id(self, name=None): - """Return commit_id at the named branch head""" - """If no name is given, return commit_id current branch head""" - branch_ref = self.branch_ref(name) if name else self._current_branch_ref() - commit_id = self.repo.get_refs().get(branch_ref.encode("utf-8")) - return commit_id + name = "now-diverge-{}".format(str(trial_id)[:8]) + index = 1 + existing = set(self.branches()) - def create_branch(self, name, commit_id): - """Create branch pointing to commit id""" - if not commit_id: - raise RuntimeError("cannot create branch without a commit") - branch_ref = self.branch_ref(name) - if branch_ref in self.repo.get_refs(): - raise RuntimeError("branch already exists: {}".format(name)) - self.repo.refs[branch_ref] = commit_id.encode("ascii") + while name in existing: + index += 1 + name = "{}-{}".format(name, index) + + encoded_ref = (self.branch_ref(name)).encode("utf-8") + self.repo.refs[encoded_ref] = commit_id + return name def switch_branch(self, name): """Switch HEAD to branch""" branch_ref = self.branch_ref(name) - if branch_ref not in self.repo.get_refs(): + encoded_ref = (branch_ref).encode("utf-8") + if encoded_ref not in self.repo.get_refs(): raise RuntimeError("branch not found: {}".format(name)) self._set_head(branch_ref) @@ -277,64 +280,10 @@ def rename_branch(self, old, new): raise RuntimeError("branch already exists: {}".format(new)) self.repo.refs[new_key] = refs[old_key] del self.repo.refs[old_key] - if self._current_branch_ref() == old_ref: - self._set_head(new_ref) - - def ensure_branch_for_trial(self, trial_id): - """Create and switch to an automatic branch for a restored trial""" - current_trial = self.current_branch_trial() - if current_trial == str(trial_id): - return self.current_branch() - - commit_id = self.trial_commit(trial_id) - if not commit_id: - return self.current_branch() - - base = "now-diverge-{}".format(str(trial_id)[:8]) - name = base - index = 1 - existing = set(self.branches()) - while name in existing: - index += 1 - name = "{}-{}".format(base, index) - self.create_branch(name, commit_id) - self.switch_branch(name) - return name - - def checkout_branch(self, name): - """Switch branch and restore versioned files from its tree""" - self.switch_branch(name) - commit_id = self.branch_commit(name) - if not commit_id: - return - commit = self.repo[commit_id.encode("ascii")] - self._checkout_tree(commit.tree, "") - - def _tree_items(self, tree): - if hasattr(tree, "iteritems"): - return tree.iteritems() - return tree.items() - - def _checkout_tree(self, tree_id, prefix): - tree = self.repo[tree_id] - for item in self._tree_items(tree): - if len(item) == 3: - name, mode, sha = item - else: - name, entry = item - mode, sha = entry.mode, entry.sha - relative = posixpath.join(prefix, name) if prefix else name - obj = self.repo[sha] - if isinstance(obj, Tree): - self._checkout_tree(sha, relative) - else: - path = os.path.join(self.base_path, *relative.split("/")) - parent = os.path.dirname(path) - if parent: - os.makedirs(parent, exist_ok=True) - with self.std_open(path, "wb") as fil: - fil.write(obj.as_raw_string()) + _, current_name_ref = self.current_branch() + if current_name_ref == old_ref: + self._set_head(new_ref) DistributedDulwichEngine = create_distributed(DulwichEngine) PoolDulwichEngine = create_pool(DulwichEngine) diff --git a/src/noworkflow/now/persistence/content/gitbase.py b/src/noworkflow/now/persistence/content/gitbase.py index 8e34fc11..922941ee 100644 --- a/src/noworkflow/now/persistence/content/gitbase.py +++ b/src/noworkflow/now/persistence/content/gitbase.py @@ -20,6 +20,8 @@ def __init__(self, config): self._default_branch = 'master' self._trial_ref_prefix = 'refs/noworkflow/trials/' self._initial_message = "Initial Commit" + self._default_branch = 'master' + self._trial_ref_prefix = 'refs/noworkflow/trials/' self.name_counter = Counter() self.base_path = None self.user_path = os.path.expanduser("~") From f85a9f94d4c16c87f5624b0f9a72500d035d7f66 Mon Sep 17 00:00:00 2001 From: Felipe Coutinho Date: Mon, 1 Jun 2026 15:54:22 -0300 Subject: [PATCH 4/4] Refactor --- src/noworkflow/now/cmd/cmd_branch.py | 17 ++++----- .../now/persistence/content/dulwich_engine.py | 37 ++++++++++--------- .../now/persistence/content/gitbase.py | 2 - .../now/persistence/content/parallel.py | 22 +++-------- .../now/persistence/models/trial.py | 11 ++---- 5 files changed, 37 insertions(+), 52 deletions(-) diff --git a/src/noworkflow/now/cmd/cmd_branch.py b/src/noworkflow/now/cmd/cmd_branch.py index c459ed95..0de5002b 100644 --- a/src/noworkflow/now/cmd/cmd_branch.py +++ b/src/noworkflow/now/cmd/cmd_branch.py @@ -48,21 +48,20 @@ def _connect(self, args): def execute(self, args): self._connect(args) command = args.branch_cmd or "list" + current, _ = content.current_branch() if command == "list": - current = content.current_branch() for name in content.branches(): marker = "*" if name == current else " " - trial_id = content.branch_trial(name) or "" - print("{} {} {}".format(marker, name, trial_id)) + print("{} {}".format(marker, name)) elif command == "current": - trial_id = content.current_branch_trial() or "" - print("{} {}".format(content.current_branch(), trial_id)) + print("{}".format(current)) elif command == "switch": - content.checkout_branch(args.name) - trial_id = content.branch_trial(args.name) - if trial_id: - Trial(trial_id).create_head() + # TODO: IMPLEMENT this + # content.checkout_branch(args.name) + # trial_id = content.get_branch_head_trial_id(args.name) + # if trial_id: + # Trial(trial_id).create_head() print_msg("Switched to branch {}".format(args.name), True) elif command == "rename": content.rename_branch(args.old, args.new) diff --git a/src/noworkflow/now/persistence/content/dulwich_engine.py b/src/noworkflow/now/persistence/content/dulwich_engine.py index 955c8d6c..9dd54f29 100644 --- a/src/noworkflow/now/persistence/content/dulwich_engine.py +++ b/src/noworkflow/now/persistence/content/dulwich_engine.py @@ -2,7 +2,6 @@ import hashlib import time import io -import posixpath from dulwich import file from dulwich.repo import Repo @@ -36,7 +35,7 @@ def connect(self, config): self.create_initial_commit() else: self.repo = Repo(self.content_path) - self._commit_ref = self._current_branch_ref() + _, self._commit_ref = self.current_branch() @staticmethod def do_put(content_path, object_hashes, lock, content, filename): @@ -154,21 +153,9 @@ def write_tree(self, tree): self.repo.object_store.add_object(tree) return tree.id - @staticmethod - def _to_text(value): - if value is None: - return None - if isinstance(value, bytes): - return value.decode("utf-8") - return value - - @staticmethod - def _to_hex(value): - if value is None: - return None - if isinstance(value, bytes): - return value.decode("ascii") - return str(value) + ##################################################################### + ## Branch Implementation + ##################################################################### def get_commit_id_by_trial_id(self, trial_id): """Return commit id for a trial ref""" @@ -285,6 +272,22 @@ def rename_branch(self, old, new): if current_name_ref == old_ref: self._set_head(new_ref) + @staticmethod + def _to_text(value): + if value is None: + return None + if isinstance(value, bytes): + return value.decode("utf-8") + return value + + @staticmethod + def _to_hex(value): + if value is None: + return None + if isinstance(value, bytes): + return value.decode("ascii") + return str(value) + DistributedDulwichEngine = create_distributed(DulwichEngine) PoolDulwichEngine = create_pool(DulwichEngine) ThreadingDulwichEngine = create_threading(DulwichEngine) diff --git a/src/noworkflow/now/persistence/content/gitbase.py b/src/noworkflow/now/persistence/content/gitbase.py index 922941ee..8e34fc11 100644 --- a/src/noworkflow/now/persistence/content/gitbase.py +++ b/src/noworkflow/now/persistence/content/gitbase.py @@ -20,8 +20,6 @@ def __init__(self, config): self._default_branch = 'master' self._trial_ref_prefix = 'refs/noworkflow/trials/' self._initial_message = "Initial Commit" - self._default_branch = 'master' - self._trial_ref_prefix = 'refs/noworkflow/trials/' self.name_counter = Counter() self.base_path = None self.user_path = os.path.expanduser("~") diff --git a/src/noworkflow/now/persistence/content/parallel.py b/src/noworkflow/now/persistence/content/parallel.py index 5d76b31e..61b1579c 100644 --- a/src/noworkflow/now/persistence/content/parallel.py +++ b/src/noworkflow/now/persistence/content/parallel.py @@ -24,10 +24,8 @@ def run(self): self.task_queue.task_done() break - try: - self.engine.do_put(*queue_content) - finally: - self.task_queue.task_done() + self.engine.do_put(*queue_content) + self.task_queue.task_done() class Distributed(cls): @@ -52,7 +50,6 @@ def start_processes(self): with safeopen.use_safe_open(): for _ in range(self.num_consumers): consumer = Worker(self.tasks, self) - consumer.daemon = True self.consumers.append(consumer) consumer.start() @@ -72,18 +69,9 @@ def close(self): for _ in range(self.num_consumers): self.tasks.put(None) - try: - # Wait for all of the tasks to finish - self.tasks.join() - for consumer in self.consumers: - consumer.join() - finally: - for consumer in self.consumers: - if consumer.is_alive(): - consumer.terminate() - consumer.join() - self.consumers = [] - self.processes_started = False + # Wait for all of the tasks to finish + self.tasks.join() + self.processes_started = False Distributed.__name__ = name or ("Distributed" + cls.__name__) return Distributed diff --git a/src/noworkflow/now/persistence/models/trial.py b/src/noworkflow/now/persistence/models/trial.py index a62a563d..1d021121 100644 --- a/src/noworkflow/now/persistence/models/trial.py +++ b/src/noworkflow/now/persistence/models/trial.py @@ -1052,19 +1052,16 @@ def load_parent(cls, script, remove=True, check=False, session=None): """ session = session or relational.session head = Head.load_head(script, session=session) - if head: - content.ensure_branch_for_trial(head.trial.id) trial = head.trial if remove: Head.remove(head.id, session=relational.make_session()) - return proxy(trial) - - trial_id = content.get_branch_head_trial_id() - trial = cls.load_trial(trial_id, session=session) + elif not head: + trial = cls.last_trial( + script=script, check=check, + session=session) return proxy(trial) - @classmethod # query def fast_last_trial_id(cls, session=None): """Load last trial id that did not bypass modules